diff --git a/.github/workflows/desktop.yaml b/.github/workflows/desktop.yaml index 0892f0bf1..896d178bf 100644 --- a/.github/workflows/desktop.yaml +++ b/.github/workflows/desktop.yaml @@ -10,6 +10,11 @@ on: - "src/**" - "scripts/build_hivemind_windows.py" - "scripts/hivemind-win32.patch" + - "scripts/gate13_linux_packaged_lifecycle.py" + - "scripts/gateq38_linux_host_runtime.py" + - "tests/test_runtime_archive_hardlinks.py" + - "tests/test_windows_download_helper.py" + - "tests/test_windows_online_integration.py" - "docs/REVIVAL.md" - "docs/REVIVAL_TEST_RESULTS.md" workflow_dispatch: @@ -21,7 +26,7 @@ jobs: matrix: # Public-alpha packaging is Windows/Linux only; macOS is explicitly deferred. include: - - os: ubuntu-latest + - os: ubuntu-22.04 platform: linux install_archive: communityai-desktop-linux.tar.gz - os: windows-latest @@ -49,7 +54,10 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install --no-install-recommends --yes libdbus-1-3 libegl1 libgl1 libxkbcommon0 + sudo apt-get install --no-install-recommends --yes \ + libdbus-1-3 libegl1 libgl1 libglib2.0-0 libfontconfig1 libfreetype6 \ + libxkbcommon0 libxkbcommon-x11-0 libxcb-cursor0 libxcb-icccm4 \ + libxcb-image0 libxcb-keysyms1 libxcb-render-util0 libxcb-shape0 libwayland-cursor0 xvfb xauth - name: Build patched Windows Hivemind runtime if: runner.os == 'Windows' run: | @@ -58,8 +66,7 @@ jobs: $wheel = Get-ChildItem ../dist/hivemind-1.1.12-*-win_amd64.whl | Select-Object -Last 1 if (-not $wheel) { throw "Hivemind wheel build produced no artifact" } python -m pip install $wheel.FullName - - name: Install qualified Windows CUDA runtime - if: runner.os == 'Windows' + - name: Install qualified CUDA runtime run: >- python -m pip install --index-url https://download.pytorch.org/whl/cu124 @@ -68,35 +75,115 @@ jobs: run: | python -m pip install -e "..[api]" python -m pip install -e ".[dev]" - - name: Verify qualified Windows CUDA runtime - if: runner.os == 'Windows' + - name: Verify qualified CUDA runtime run: python -c "import torch; assert torch.__version__ == '2.6.0+cu124'" - name: Run protocol and runtime-boundary tests working-directory: . run: | python -m unittest discover -s desktop/tests -v communityai-desktop --self-test + - name: Verify normalized archive consumers + working-directory: . + run: python -m unittest discover -s tests -p test_runtime_archive_hardlinks.py -v + - name: Verify Linux installer process ownership as package-maintenance user + if: runner.os == 'Linux' + working-directory: . + run: sudo "$(command -v python)" -m unittest discover -s desktop/tests -p test_installers.py -v - name: Reverify qualified CUDA runtime before bundling run: python -c "import torch; assert torch.__version__ == '2.6.0+cu124'" - name: Build and smoke unsigned public-alpha bundle run: >- python build_desktop.py - --publication-bundle ../public-alpha/catalog-v1 + --publication-bundle ../public-alpha/catalog-qwen-v2 --source-commit "${{ github.sha }}" --build-workflow "${{ github.workflow_ref }}" - name: Independently verify release checksums and expected provenance run: >- python build_desktop.py --verify-release-output dist/desktop - --publication-bundle ../public-alpha/catalog-v1 + --publication-bundle ../public-alpha/catalog-qwen-v2 --source-commit "${{ github.sha }}" --build-workflow "${{ github.workflow_ref }}" --verify-build-environment + - name: Exercise frozen native CPU runtime without model downloads + run: ./dist/desktop/CommunityAI/node/CommunityAI-Node --native-self-test + - name: Exercise frozen desktop through X11 + if: runner.os == 'Linux' + env: + QT_QPA_PLATFORM: xcb + run: | + xvfb-run --auto-servernum dist/desktop/CommunityAI/CommunityAI --ui-self-test + xvfb-run --auto-servernum dist/desktop/CommunityAI/CommunityAI --onboarding-ui-self-test - name: Exercise packaged node, native credentials, and public seed if: runner.os == 'Windows' run: >- python ../scripts/smoke_desktop_managed_node.py --node-command dist/desktop/CommunityAI/node/CommunityAI-Node.exe + - name: Build unsigned Windows engineering installer + if: runner.os == 'Windows' + shell: pwsh + run: | + $innoInstaller = Join-Path $env:RUNNER_TEMP 'innosetup-6.7.3.exe' + $innoDirectory = Join-Path $env:RUNNER_TEMP 'inno-compiler' + Invoke-WebRequest -Uri 'https://github.com/jrsoftware/issrc/releases/download/is-6_7_3/innosetup-6.7.3.exe' -OutFile $innoInstaller + $signature = Get-AuthenticodeSignature -LiteralPath $innoInstaller + if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'Pyrsys') { + throw 'Unexpected Inno Setup publisher signature' + } + $process = Start-Process -FilePath $innoInstaller -ArgumentList @('/VERYSILENT','/SUPPRESSMSGBOXES','/NORESTART','/CURRENTUSER','/NOICONS',"/DIR=$innoDirectory") -WindowStyle Hidden -Wait -PassThru + if ($process.ExitCode -ne 0) { throw 'Inno Setup compiler installation failed' } + ./installers/build_windows_installer.ps1 -Bundle dist/desktop/CommunityAI -OutputDirectory dist/installers -Version '0.1.0-alpha.${{ github.run_number }}' -Compiler (Join-Path $innoDirectory 'ISCC.exe') -UnsignedEngineering + - name: Compile online downloader with inert fixture metadata + if: runner.os == 'Windows' + shell: pwsh + run: | + # Compile the Pascal boundary without publishing or running a downloader. + # This intentionally unreachable fixture is kept outside uploaded artifacts. + $fixtureDirectory = Join-Path $env:RUNNER_TEMP 'communityai-online-compile-fixture' + New-Item -ItemType Directory -Path $fixtureDirectory -Force | Out-Null + $fixtureManifest = Join-Path $fixtureDirectory 'release-downloads.json' + @{ + schema_version = 1 + artifacts = @{ + 'windows-x64' = @{ + platform = 'windows-x64' + kind = 'offline-installer' + format = 'exe' + version = '0.0.0-test' + filename = 'communityai-0.0.0-test-windows-setup.exe' + url = 'https://example.invalid/releases/0.0.0-test/communityai-0.0.0-test-windows-setup.exe' + sha256 = ('a' * 64) + size_bytes = 2519046440 + } + } + } | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $fixtureManifest -Encoding utf8 + ./installers/build_windows_online_installer.ps1 -Manifest $fixtureManifest -OutputDirectory (Join-Path $fixtureDirectory 'output') -Compiler (Join-Path $env:RUNNER_TEMP 'inno-compiler/ISCC.exe') -UnsignedAlpha + - name: Verify resumable Windows downloader and owned setup handoff + if: runner.os == 'Windows' + working-directory: . + shell: pwsh + run: | + $env:COMMUNITYAI_INNO_COMPILER = Join-Path $env:RUNNER_TEMP 'inno-compiler/ISCC.exe' + python -m unittest discover -s tests -p test_windows_download_helper.py -v + if ($LASTEXITCODE -ne 0) { throw 'Resumable downloader tests failed' } + python -m unittest discover -s tests -p test_windows_online_integration.py -v + if ($LASTEXITCODE -ne 0) { throw 'Online setup integration tests failed' } + - name: Build Debian engineering package + if: runner.os == 'Linux' + run: >- + python installers/build_deb.py + --bundle dist/desktop/CommunityAI + --output dist/installers + --version '0.1.0~alpha.${{ github.run_number }}' + --maintainer 'CommunityAI engineering ' + - name: Upload engineering setup artifacts + uses: actions/upload-artifact@v6 + with: + name: communityai-setup-${{ matrix.platform }} + path: desktop/dist/installers/* + if-no-files-found: error + compression-level: 0 + retention-days: 7 - name: Upload self-contained install archive uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 867029569..42d2c9df4 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -55,14 +55,22 @@ jobs: tests/test_api_keys.py \ tests/test_cache.py \ tests/test_catalog_bootstrap.py \ + tests/test_catalog_desktop_cleanup.py \ tests/test_catalog_publication.py \ + tests/test_catalog_refresh.py \ desktop/tests/test_build_desktop.py \ + desktop/tests/test_native_runtime_self_test.py \ + desktop/tests/test_release_downloads.py \ + desktop/tests/test_runtime_packaging.py \ desktop/tests/test_startup.py \ + desktop/tests/test_windows_online_installer.py \ tests/test_deepseek_v3.py \ tests/test_discovery.py \ tests/test_edge_benchmark.py \ tests/test_external_qualification.py \ tests/test_device_portability.py \ + tests/test_gate16_catalog_channel.py \ + tests/test_gate16_catalog_drill.py \ tests/test_gemma4_block.py \ tests/test_gemma4_multimodal_loading.py \ tests/test_gemma4_unified.py \ @@ -70,6 +78,11 @@ jobs: tests/test_join_token.py \ tests/test_kv_cache_strategy.py \ tests/test_loading_diagnostics.py \ + tests/test_linux_online_installer.py \ + tests/test_login_startup_linux.py \ + tests/test_login_startup_windows_cycle.py \ + tests/test_login_startup_windows_safety.py \ + tests/test_login_startup_windows_state.py \ tests/test_model_catalog.py \ tests/test_model_manager.py \ tests/test_model_manifest.py \ @@ -87,9 +100,11 @@ jobs: tests/test_qualification_matrix.py \ tests/test_qualification_cost_guard.py \ tests/test_qualification_image_contract.py \ + tests/test_qualify_catalog_desktop.py \ tests/test_qwen3_5_block.py \ tests/test_qwen3_5_multimodal_loading.py \ tests/test_route_health.py \ + tests/test_runtime_archive_hardlinks.py \ tests/test_server_registry.py \ tests/test_startup_guard.py \ tests/test_tied_embeddings_from_pretrained.py \ @@ -102,7 +117,10 @@ jobs: # Atomic config exchange and shared admission are Windows/Linux public-alpha contracts. # The production desktop workflow separately exercises the packaged Windows node. # macOS remains outside the supported release matrix. + # Gate 16 additionally uses actual loopback-only TLS p2pd and the real handler, + # with no model allocation, external peers or public route mutation. python -m pytest -v --durations=0 --durations-min=1.0 \ + tests/test_gate16_live_rpc.py \ tests/test_policy_store.py \ tests/test_server_admission.py diff --git a/.gitignore b/.gitignore index 455eef566..e8b51922b 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ dmypy.json .idea/ /.gate* + +# Owner-requested local emergency publisher backup; never commit key material. +/.publisher-secrets/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c1bff885..617ff2d86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,327 @@ and qualification evidence remains in `docs/REVIVAL_TEST_RESULTS.md`. ### Added +- Linux desktop packages include the required X11 shape-library dependency and + avoid initializing the optional Triton compiler for the approved eager/native + inference profiles, allowing GPU sharing without a development toolchain. + Installer maintenance refuses replacement when process ownership cannot be + inspected, and continues discovering helpers created during shutdown until the + installed process tree stays stopped. + +- A sharing worker whose selected blocks exceed the VRAM budget now waits with + a clear explanation instead of repeatedly restarting. Raising the resource + limit can resume the previously selected worker; Pause remains authoritative. + +- Catalog migration keeps per-model cache and resource preferences when the same + verified manifest moves to the application's managed directory. A different + manifest identity does not inherit those settings through a reused model name. + +- Windows Inno Setup and Debian engineering package builders now package the + verified desktop runtime. Windows upgrade/removal requests same-user desktop + shutdown and waits for owned-node cleanup; Linux package maintenance stops the + installed process tree. Settings/cache remain outside installer ownership. + Trusted signing, full installer qualification and repository publication remain open. + +- Model health now shows a clickable block grid with serving replicas, joining + announcements, signed reservations, offline announcements and local worker + failures. Peer details show observed block ranges and reported runtime metadata. + Local client and sharing-worker downloads show file progress, verified bytes, + transfer speed, retries and resumed bytes, separately from model loading. + +- Desktop sharing now has separate VRAM and processing-usage sliders, defaulting + to 100% on fresh installs while sharing stays opt-in. Applying limits pauses + workers before persistence and resumes only previously selected workers. Failed + saves leave sharing paused. Processing limits pace contribution compute between + steps and share a budget across workers; brief bursts, downloads, model loading + and local inference are outside an instantaneous usage guarantee. + +- Verified standalone Qwen3.5-0.8B inference now provides local fallback for + `auto` requests, with local-only preferences, token/context budgets and stream + cancellation. Packaged offline inference passed on Windows GPU and Linux CPU. +- The Windows engineering package now has real Qwen3.8 community completion, + short chat, worker-loss fallback/rejoin and HTTP-blocked cache-restart results + through all 64 blocks under the signed public policy on an assigned L4/T4/C3 route. + Full CPU inference, same-session VM replacement and stock-reference parity + also passed. These bounded results do not complete desktop formation, + installation lifecycle or broader hardware/performance qualification; see + [current readiness](docs/RELEASE_READINESS.md). +- Chat requests can explicitly set `enable_thinking: false` when supported by + the verified model template. Independent discovery observation maintains + readiness tracking during long generations. Interrupted artifact downloads + resume verified partial files, and disk admission includes owned manifest + cache files with hardlinks deduplicated. +- Signed Qwen catalog sequence 2 is published at its qualification path. The + packaged bootstrap supports explicit migration to the replacement bundled + trust root while preserving preferences and cached data. Existing installations + need the updated application; a network catalog cannot replace its trust root. + +- Qwen3.8-27B FP8 now has exact candidate and BF16 reference manifests plus an + explicit `fp8_dequant` execution profile. Source quantization metadata is + preserved and validated bidirectionally; block loading expands fine-grained + scale grids into the manifested BF16/FP16 dtype; server advertisement, + memory accounting, CLI selection, and the product qualification worker carry + the same profile. A synthetic FP8 checkpoint passes the production + config/load/forward path, and independent offline verification passes 158 + primary and 146 regression tests. The source-bound checkpoint is partial: + official artifact verification, a real Qwen3.8 layer, the complete 64-block + route, parity, recovery, packaging, and hardware measurements remain open. +- Automatic Qwen3.8 workers now plan and enforce the exact startup metadata and + deduplicated checkpoint shards for their contiguous block span instead of + admitting against the full model. Server loading is restricted to that set, + and cached signed intents are invalidated by selected-set, budget, throughput, + proposal, identity-key, or lease-expiry changes. Public v1 intent claims keep + the same privacy-safe shape. This checkpoint passes 142 focused, 132 adjacent, + and 1,564 offline unit tests without downloading model weights or using cloud. +- Acknowledged automatic worker plans now bind the exact manifest, canonical + span/cache, byte count, and artifact-set digest into the real source or frozen + server command. The supervisor rejects executable or flag substitution and + unsafe config/module/training/credential options; the server ignores ambient + config and revalidates the exact plan before announcement or weight access. + This no-spend checkpoint passes 147 focused, 259 related, and 1,568 offline + tests without downloading model weights or claiming real Qwen execution. +- The exact Qwen3.8 `0:1` worker plan now has a real local Windows/CUDA + outcome: a new isolated cache anonymously acquired and rehashed the official + 384,054,133-byte config/index/layer selection, loaded it in manifested BF16, + and returned a finite changed `[1,1,5120]` exact-peer result in 0.363 + seconds. A later network-disabled cache-reuse replay bound 19 production + paths to pushed commit `af7d887`, repeated the same deterministic result, + exited cleanly, and retained an exact PID/listener/tagged-process cleanup + audit. Complete-route parity, recovery, packaging, and RTX 30/40/50 + qualification remain open. +- Qwen3.8 complete-route attempts now have a durable, provider-neutral + controller that rederives the exact four-span artifact plan from the official + layer index, emits canonical GCP resource specifications, binds protected + authorization to the stable plan and exact sources, journals start issuance + before the action, and recovers or cleans up without replaying paid work. + Independent bypass review, 102 focused, 423 adjacent, and 1,651 offline unit + tests pass. This is a USD 0 controller checkpoint: the ledger contains no + Q3.8 reservation, so no cloud create or complete-route result is claimed. +- Qwen3.8 complete-route attempts now have a source-bound GCP adapter that + compiles the exact private eleven-resource start specification, validates + run-prefixed inventory and plan-scoped network isolation, and performs + retry-safe best-effort cleanup including terminal resources. Paid start and + collection reject before any provider call until the protected host runtime + and fresh status/evidence transport are bound. This USD 0 checkpoint passes + 137 focused, 168 adjacent, and 1,686 offline unit tests. +- Qwen3.8 Linux staging now has a source-bound production-package validator + that binds the exact archive/audit, physical and semantic manifest, complete + packaged node onedir inventory, and controller protection pass into one + canonical record. It rejects unsafe links/modes, packaged model weights, + verifier mutation, and archive pathname replacement. This USD 0 checkpoint + passes 26 focused, 184 adjacent, and 1,712 offline unit tests; native Linux + validator execution and privileged host staging remain open. +- Qwen3.8 runtime-package validation now feeds the final route contract through + a strict protected source context instead of circularly consuming that plan. + The complete immutable package record is bound into the stable plan, + execution inventory, action IDs, reservation, and preflight evidence. This + USD 0 checkpoint passes 156 focused, 184 Gate Q3.8, 205 adjacent, and 1,733 + offline unit tests; native Linux host staging and execution remain open. +- Qwen3.8 Linux hosts now have a privileged, source-bound runtime preparation + and cleanup contract. It validates the exact archive and release inventory, + safely extracts a protected root-owned runtime, runs an offline nonroot + packaged preflight, proves process-group cleanup on every failure path, and + publishes durable no-replace digest-only prepared state. The final local + candidate passes 55 host-runtime, 240 Gate Q3.8, 206 adjacent, and 1,789 + offline tests; native Linux execution, bootstrap/status integration, and any + paid route remain open. +- Qwen3.8 route observation now latches the exact GCP instance generation set + before entering active phases. Provider ID and creation-time substitutions, + same-name instance recreation, or partial generation loss force cleanup, while + disks and firewalls cannot carry instance-generation metadata. Paid start and + collection remain fail-closed until the protected host bootstrap/status bridge + is complete. This USD 0 checkpoint passes 170 focused and 264 complete Gate + Q3.8 tests with four native-platform skips. +- Qwen3.8 Linux host status now has a bounded canonical authenticated-envelope + primitive. Controller contexts and HMAC-signed worker/job records bind the exact + source, plan, actions, provider generation, boot UUID, monotonic revision, and + prepared-record digest while rejecting stale, replayed, substituted, deeply nested, + or oversized input. This USD 0 checkpoint passes 41 transport, 211 controller/adapter, + and 305 complete Gate Q3.8 tests with four native-platform skips; protected key and + context installation, prepared-record equality, adapter consumption, and native + Linux bootstrap remain open. +- Qwen3.8 Linux runtime preparation now loads protected controller context and + per-instance key inputs, binds provider generation and boot identity into + prepared state, and derives the authenticated status envelope from the + reopened record. Preparation and cleanup share one lifecycle lock, and + cleanup publishes a durable terminal marker before deleting state. This USD 0 + checkpoint passes 119 focused, 328 complete Gate Q3.8, and 1,877 offline + tests; key/context delivery, external publication, adapter consumption, and + native Linux execution remain open. +- Qwen3.8 route plans now bind a separate exact run-scoped IAP SSH + firewall, and the GCP adapter can consume canonical authenticated guest + attributes only through paired protected key/replay resolvers and only + between generation-stable complete provider inventories. Broadened or + substituted firewall state, malformed or ambiguous carriers, wrong keys, + replayed revisions, instance recreation, and protected-bootstrap loss fail + closed; cleanup never depends on status material. The firewall candidate + passes 180 focused and 368 complete Gate Q3.8 tests, and the consumer + candidate passes 101 focused and 376 complete Gate Q3.8 tests, with four + native-platform skips in each complete matrix. Both are USD 0 checkpoints; + key generation, vaulting, protected delivery, and native Linux probes remain + open, and paid start/collection remain disabled. +- Qwen3.8 controller secrets and protected host delivery are now generation-bound + end to end. The controller vaults one private key per exact instance generation + and epoch with crash-safe rotation, revocation tombstones, and idempotent cleanup. + A bounded authenticated bundle installs atomically as one root-private host file, + while fixed IAP SSH delivery streams it through stdin and accepts a receipt only + between stable provider inventories. Receipt authentication precedes freshness + policy, key material never enters argv, environment, logs, or ordinary state, and + paid start/collection remain blocked. The final USD 0 candidate passes 412 complete + Gate Q3.8 tests with five native-platform skips plus Black, isort, compilation, and + whitespace checks; native Linux and provider-read probes remain open. +- Qwen3.8 protected-host behavior now has a native Linux root result. An + ephemeral network-disabled container ran the exact pushed controller, adapter, + transport, runtime, and staging matrix from read-only mounts: 415 tests passed + and only two Windows-only checks skipped. Linux ownership, private modes, + nonroot traversal, symlink rejection, atomic delivery, receipt/replay, + generation-bracketed fake-provider reads, and terminal cleanup executed at + USD 0. No live GCP, IAP, metadata, capacity, or paid-route result is claimed; + paid start/collection remain blocked pending a committed reservation. +- Gate 14 hardware acceptance now has a fail-closed, source-bound verifier, durable + lifecycle controller, thin Windows/Linux host probes, and an exact GCP action executor. They + pin the Gate 13 packaged lifecycle and Windows/Qwen plus Linux/Gemma Gate 9 envelopes, enforce + the USD 100 aggregate ledger ceiling, serialize fresh L4 hosts, and require cleanup proof for + the exact authorization bytes, controller source, provider plan, project, zone, resources, + and successful terminal state while excluding the protected bootstrap. Host evidence now binds + fresh calibrated bandwidth and physical-power samples before, across, and below the configured + limits. Every calibration is bound to a controller-issued, one-time 15-minute challenge and + measured start/end timestamps inside a maximum 120-second sample window. Controller state + persists the issued digest and one-time consumption, while interrupted issuance reattaches only + the exact still-valid file; stale, missing, future-dated, or cross-challenge evidence fails closed. + Every provider mutation is bound + to a valid controller action and a fresh authenticated inventory; exact disks must retain the full + authorized image-project path, and instances must prove that no service account is attached. + The focused 67-test suite covers challenge freshness, calibration, deadline, rollback, + orphan/returned resources, + forged state, evidence substitution, cross-platform, hidden/excess spend, cleanup binding, foreign + image projects, attached service accounts, and stale direct actions. The original verifier/controller + source `c0f2342e15aa7e12ca7c2980deca64d613204143` passed independent adversarial review, + CodeQL, style, Linux/Windows tests, and production-package provenance verification. After native + GCP authentication was refreshed, a read-only preflight proved one active account, an active + project, the protected bootstrap running, global GPU quota 1 with usage 0, ready L4/G2 and + Windows/Linux image inputs, and zero Gate 14 instances, disks, firewalls, or running L4s. + No reservation or provider resource was created, so the current-epoch USD 44 remainder is intact. + Automatic workers now remain policy-blocked until their signed placement intent has been published + and acknowledged by a remote DHT peer. The privileged worker contract carries both facts into the + source-bound host evidence path; failed publication cannot admit an unacknowledged assignment. + The broader placement, discovery, supervisor, and control-API verification passes 141 tests. + Gate 14 also has a separate exact-once native Scheduled Task/systemd host-job namespace that + validates only the final strict platform evidence. Its shared process-safety core is canonical- + digest-bound by the already hashed adapter and compiled directly from the verified in-memory + bytes, preserving Gate 13 defaults while rejecting changed, linked, or raced core input. Exact + source `5632cc528b7a3e39296fd3fef8a0b3fd6dd620d0` passed both Windows and Linux production-package + jobs in run `33679402658`, including exact-source build/smoke, independent checksum/provenance + verification, and archive-bound uploads. A final no-spend audit also replaced a stale Gate 13 + Linux HOME/runtime hardcode with the configured host-job values, so the Gate 14 desktop session + accepts its isolated paths. The combined Gate 13/Gate 14 host-job and Gate 14 contract suite now + passes 107 tests. A shared packaged-lifecycle sequencer now rejects user-supplied pass claims, + binds the exact production archive and release metadata, completes the non-calibration drills, + and publishes an immutable challenge-ready checkpoint before accepting a controller challenge. + The challenge covers that exact checkpoint digest; retained checkpoints cannot replay the + lifecycle. Controller-owned staging is separated from lifecycle outputs, validated with exact + POSIX ownership or native Windows owner/protected-DACL and per-right access probes, and every + staged file is read without following a raced path and revalidated around action phases. + Failure cleanup is unconditional, private facts are removed before final evidence publication, + and the persisted evidence is strictly reread and compared. Gate 14 now also aligns the + controller and lifecycle on the exact `gate14-lifecycle.json` basename and adds a + source-digest-bound persistent Windows PowerShell action bridge. The bridge keeps one native + host and state nonce across prepare/calibrate/cleanup, accepts only bounded canonical + controller-bound frames, rejects duplicate keys, replayed or out-of-order request IDs, and + private response material, and force-cleans the product process tree on EOF or failure. Its + production prepare/calibrate handlers intentionally fail closed until the release audit + companions and Gate 9 warm cache are controller-bound. The expanded no-spend Gate 14 contract + matrix passes 154 tests, including native and client-side rejection of Boolean, numeric, + array-coerced, and changed controller bindings. The controller-owned lifecycle input now binds + the complete Actions audit ZIP by exact artifact name, digest, size, and the exact four extracted + `SHA256SUMS`, desktop-metrics, provenance, and release-metadata members. Their package, source, + platform, archive, checksum inventory, smoke, and incomplete unsigned-alpha claims are validated + together. A separate warm-cache binding preserves the historical Gate 9 acquisition and envelope + identities while requiring a fresh direct-upstream, no-mirror, empty-cache materialization record + with the exact sorted artifact digests, roles, counts, and bytes for Windows/Qwen and Linux/Gemma. + Outer archive, extracted member, and materialization-record drift is rechecked at every lifecycle + boundary, and the immutable challenge-ready checkpoint binds all three identities. The lifecycle/ + controller focus passes 75 tests, including unknown nested cache fields, wrong repository, + non-finite timing, historical-artifact mutation, and source/member substitution. Exact source + `d4586f02530be7ce052ce9298ae95207e4368f05` passed both jobs in production desktop run + `33695073372`. Its retained Actions artifacts are Windows install + `sha256:3eea3254309fac149de210fa7c397cc94d0bf3f38c8a302e2f8d4b52671caa4e` + (2,695,093,223 bytes), Windows audit + `sha256:afb5a6201f0a1d5788c43ead11f1fcb0994f8749d6d64525b8aa5ae4e78a2c4f` + (468,612 bytes), Linux install + `sha256:ee1a9dccba6dbdb800e9b65cca6f0abbe7a8f38cb8f425802cebdb3cbffff47c` + (3,360,731,719 bytes), and Linux audit + `sha256:210e65d8b7fbe6517813638eecbcc8eddfe639a48f5180e5e9ef8dac80b658d0` + (514,751 bytes), expiring 2026-09-09. Gate 14 now has a native cross-platform lifecycle + entrypoint and an equivalent source-digest-bound persistent Linux action host alongside the + Windows PowerShell host. Both carry the complete safe controller challenge summary into + calibration, enforce operation-specific bounded timeouts, retain one process/state identity + across prepare, challenge wait, calibration, and cleanup, reject malformed/replayed/coerced + frames, and clean on EOF or failure. The Linux parent also kills its owned process group if + graceful host cleanup times out. Production prepare/calibrate remain fail-closed until their + real package/cache/control handlers are installed; the transport self-test cannot manufacture + acceptance evidence. The cross-platform action/entrypoint focus passes 64 tests. No physical + Gate 9 cache survives the cleaned historical hosts, so the paid run still requires fresh direct + host materialization. This is software and preflight evidence only: no Gate 14 hardware or + cloud-run pass is claimed yet. Restart-safe Linux and Windows product-action checkpoints now + source-bind concrete package, warm-cache, policy, automatic-placement, recovery, calibration, + and cleanup operations through their persistent hosts. The Windows implementation adds native + Job Object membership cleanup, no-follow locked cache identity, action-specific persistent + launch paths, release/config/source binding, and phased retryable teardown that retains failed + process, burn, and credential cleanup state. The Windows product/transport focus passes 50 tests, + its Gate 13 lifecycle regression passes 16, and the complete Gate 14 matrix passes 211 tests. + Gate 14 now also has a source-bound two-phase official-cache boundary. The ordinary process + verifies a controller-owned canonical plan and lifecycle template before transfer, writes only + the exact manifest cache and canonical handoff below its work root, and cannot mutate protected + staging. A privileged promoter revalidates source, plan, record, binding, handoff, and physical + cache identity before creating the lifecycle inputs. Structural controller ownership is separate + from the qualification token's write-denial proof. A bootstrap-boundary review then caught that + the first promoted-file policy also denied the ordinary host job its required read path. Promoted + Windows files now use a protected DACL with SYSTEM/Administrators full control and Authenticated + Users generic read; POSIX files remain root-owned mode 0644. The ordinary identity can read but + receives no write, delete, owner-change, or DACL-change grant. Promotion has a retryable commit + point, and invalid templates fail before any acquisition. The cache/lifecycle focus passes 97 + tests and the complete Gate 14 matrix passes 254 under independent adversarial review. No + production model bytes, reservation, or cloud resource were used (USD 0). The clean-host + materializer now invokes the exact packaged node without reopening a verified manifest or + executable by pathname: controller mode sends the once-read bounded manifest through stdin + under its exact digest, Windows holds restrictive read-only-share handles through process exit, + and Linux executes the verified descriptor through procfs while preserving the original onedir + path as argv zero. Native mutation/launch probes, malformed-input tests, the 64-test focused + cache/acquisition suite, the 264-test Gate 14 matrix, and packaged dispatch regressions pass + locally with the Linux-native descriptor test deferred to Linux CI. Real fresh native + materialization, complete onedir sidecar-inventory verification, bootstrap integration, + packaged execution, and physical Windows/Linux L4 qualification remain open. +- Gate 13's successful manual desktop flow now has a bounded automated replay. The + production package can open its real Qt window in a hidden qualification mode, perform + localhost inference, save the actual sharing-policy dialog, click Start, exit, relaunch, + prove sharing resumed, click Pause, and infer again without retaining prompt, response, + credential, endpoint, or path data. A standard-library outer runner verifies the exact + production archive, runs all four packaged self-tests, executes both sessions, validates + canonical evidence, and removes its run temporaries. The durable host-job boundary now + accepts the Python replay entrypoint on Windows and Linux. Local real-window and contract + tests pass; no paid clean-host replay or new release artifact is claimed by this change. +- The owner raised the current combined GCP/Fly public-alpha accounting epoch to USD 500 on + 2026-08-31. The existing USD 52 committed maximum remains charged. Run + `gate13-20260831-b` reserved USD 56 but failed before VM creation when its two IAP tags reached + gcloud as one value; exact cleanup passed. Fresh run `gate13-20260831-c` reserves USD 56 with + corrected explicit tag arguments, leaving USD 336. Per-run preflight, hard deadlines, exact + cleanup, protected-resource, and evidence requirements are unchanged. +- Gate 13 paid qualification now has durable source-bound native host jobs: an exact-current-user + Windows Scheduled Task and a non-root transient Linux systemd service persist one attempt across + operator disconnects, bound output, terminate the complete process tree on timeout or overflow, + revalidate canonical lifecycle evidence before collection, and never re-arm consumed route or + client intents. Exact Windows lifecycle-config co-location and full Linux `ExecStart` structure + matching close the final independent-review gaps. Source `0e16ac2` passes the 217-test Gate 13 and + desktop matrix independently. This is a software prerequisite only: it created no cloud resources, + authorizes no paid run, and does not claim a completed clean-host lifecycle or Gate 13 pass. +- Gate 13 paid qualification now has a persisted authorization-bound run-state contract that + inventories exact resources before every transition, accepts the product route before any + client, runs Windows/Qwen before Linux/Gemma, permanently consumes a failed or ambiguous + lifecycle host, rejects stale/foreign/deadline-expired observations, validates digest-bound + canonical 16-phase records, and permits a pass only after both records and exact provider + absence. The failed `gate13-20260831-a` attempt is cleanup-proved: its route, clients, disks, + and firewalls are absent while the protected bootstrap remains running. No lifecycle pass is + claimed, and the run's USD 52 maximum remains committed in the current budget epoch. - Gate 13 packaged-lifecycle prerequisites now emit deterministic self-contained Windows ZIP and Linux tar.gz archives, preserve or reject platform filesystem semantics fail-closed, and bind the archive plus strict desktop metrics into exact-type release provenance. The local diff --git a/README.md b/README.md index 535fdb73a..54cb907a9 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,21 @@ **AI powered by people.** +[Download CommunityAI for Windows or Linux](https://github.com/flujo-app/CommunityAI/releases/tag/v0.1.0-alpha.20260909.3) +— available for closed alpha testing. Already using the app's updater? Check for +updates in the sidebar, then choose **Restart to update** when the download finishes. + +**Latest release: 0.1.0-alpha.20260909.3** + +| Platform | Online installer | Offline installer | +| --- | --- | --- | +| Windows | [Download setup](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-online-setup.exe) | [Download full setup](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe) | +| Ubuntu/Debian | [Download installer (Python)](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-linux-online.py) | [Download .deb](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai_0.1.0~alpha.20260909.3_amd64.deb) | + +The online installer downloads and verifies the full package during setup. + +image -![CommunityAI sharing screen](desktop/dist/communityai-sharing-final.png) ## How it works @@ -20,13 +33,32 @@ Community-AI is a shared Large-Language-Model, by the people, for the people. Community-AI takes care of everything else. -The application ships one model-agnostic runtime. Its signed catalog approves exact model -manifests; when a model is selected, CommunityAI downloads only the upstream Hugging Face -checkpoint files needed by the local client components or contributed block range, verifies -their declared size and SHA-256, and keeps them in a persistent shared cache. It does not need -one installer or container image per model. Download minimization is currently limited to -whole upstream checkpoint shards. See -[`ADR 0003`](docs/adr/0003-direct-manifested-artifact-delivery.md). -CommunityAI is still working toward its first public inference alpha. Credits, -earnings, payments, and payouts are planned later and are not currently available. +image + +image + +image + +## System requirements + +A dedicated GPU is optional. These are practical starting guidelines for the +alpha; the lowest supported hardware configuration has not been certified. + +| Component | What you need | +| --- | --- | +| **CPU** | 64-bit Intel or AMD (x86-64). Four cores recommended. | +| **RAM** | 8 GB as a starting point; 16 GB recommended. Sharing needs extra memory for the model parts you contribute. | +| **GPU** | Optional. NVIDIA CUDA is supported for acceleration and sharing; an RTX 2070 SUPER with 8 GB VRAM has been tested. AMD and Intel GPU acceleration is not included. | +| **Disk space** | Start with 20 GB free, preferably on an SSD. The app uses about 4.3 GB on Windows or 5.2 GB on Linux; the small local model adds about 1.8 GB. Sharing requires additional space. | +| **Operating system** | 64-bit Windows 10 (1809+)/11, Ubuntu 22.04+ or Debian 12+ with a desktop environment. | +| **Internet** | Required for community inference and initial downloads. The small local model works offline after downloading. | + +When the community model is available, your app sends text to peers without +downloading its weights or processing model layers locally. The small local +fallback remains available when the mesh cannot answer. Sharing your GPU is optional. + +For NVIDIA use, install a CUDA 12.4-compatible driver (551.61+ on Windows, +550.54.14+ on Linux). Python, PyTorch and the CUDA runtime are included. +See the [installation guide](https://github.com/flujo-app/CommunityAI/blob/codex/gate14-20260902-b/docs/ALPHA_INSTALL.md) +for setup, updates and the current alpha limitations. diff --git a/Run Gate 13 GCP.cmd b/Run Gate 13 GCP.cmd new file mode 100644 index 000000000..22bdb64f2 --- /dev/null +++ b/Run Gate 13 GCP.cmd @@ -0,0 +1,19 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion +cd /d "%~dp0" +set "GATE13_PYTHON=" +if exist ".venv-cuda\Scripts\python.exe" set "GATE13_PYTHON=.venv-cuda\Scripts\python.exe" +if not defined GATE13_PYTHON where py.exe >nul 2>nul && set "GATE13_PYTHON=py.exe -3" +if not defined GATE13_PYTHON where python.exe >nul 2>nul && set "GATE13_PYTHON=python.exe" +if not defined GATE13_PYTHON ( + echo Python 3 was not found. + set "GATE13_EXIT=2" +) else ( + call %GATE13_PYTHON% scripts\run_gate13_gcp.py + set "GATE13_EXIT=!ERRORLEVEL!" +) +echo. +echo Gate 13 finished with exit code !GATE13_EXIT!. +echo This window may now be closed. +pause >nul +exit /b !GATE13_EXIT! diff --git a/Run Qwen Formation Modal.cmd b/Run Qwen Formation Modal.cmd new file mode 100644 index 000000000..00beac0e9 --- /dev/null +++ b/Run Qwen Formation Modal.cmd @@ -0,0 +1,18 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion +cd /d "%~dp0" +set "MODAL_PYTHON=%LOCALAPPDATA%\Programs\Python\Python313\python.exe" +if defined COMMUNITYAI_MODAL_PYTHON set "MODAL_PYTHON=%COMMUNITYAI_MODAL_PYTHON%" +set "PYTHONIOENCODING=utf-8" +if exist "%MODAL_PYTHON%" ( + "%MODAL_PYTHON%" scripts\run_qwen_formation_modal.py %* + set "QWEN_EXIT=!ERRORLEVEL!" +) else ( + echo Modal Python was not found. Set COMMUNITYAI_MODAL_PYTHON to its Python executable. + set "QWEN_EXIT=2" +) +echo. +echo Qwen Modal formation finished with exit code !QWEN_EXIT!. +echo This window may now be closed. +pause >nul +exit /b !QWEN_EXIT! diff --git a/Run Qwen Formation.cmd b/Run Qwen Formation.cmd new file mode 100644 index 000000000..659e9fd54 --- /dev/null +++ b/Run Qwen Formation.cmd @@ -0,0 +1,22 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion +cd /d "%~dp0" +set "QWEN_PYTHON=" +set "QWEN_PYTHON_ARGS=" +if exist ".gate13-runs\qwen-product-venv\Scripts\python.exe" set "QWEN_PYTHON=.gate13-runs\qwen-product-venv\Scripts\python.exe" +if not defined QWEN_PYTHON if exist ".venv-cuda\Scripts\python.exe" set "QWEN_PYTHON=.venv-cuda\Scripts\python.exe" +if defined COMMUNITYAI_TEST_PYTHON set "QWEN_PYTHON=%COMMUNITYAI_TEST_PYTHON%" +if not defined QWEN_PYTHON where py.exe >nul 2>nul && set "QWEN_PYTHON=py.exe" && set "QWEN_PYTHON_ARGS=-3" +if not defined QWEN_PYTHON where python.exe >nul 2>nul && set "QWEN_PYTHON=python.exe" +if not defined QWEN_PYTHON ( + echo Python 3 was not found. + set "QWEN_EXIT=2" +) else ( + call "%QWEN_PYTHON%" %QWEN_PYTHON_ARGS% scripts\run_qwen_formation.py %* + set "QWEN_EXIT=!ERRORLEVEL!" +) +echo. +echo Qwen formation finished with exit code !QWEN_EXIT!. +echo This window may now be closed. +pause >nul +exit /b !QWEN_EXIT! diff --git a/Run Qwen Full Inference GCP.cmd b/Run Qwen Full Inference GCP.cmd new file mode 100644 index 000000000..e2ae3d967 --- /dev/null +++ b/Run Qwen Full Inference GCP.cmd @@ -0,0 +1,9 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0" +python scripts\run_qwen_full_inference_gcp.py +set "Q38_EXIT=%ERRORLEVEL%" +echo. +echo Qwen full inference finished with exit code %Q38_EXIT%. +pause >nul +exit /b %Q38_EXIT% diff --git a/Run Qwen Mixed Inference.cmd b/Run Qwen Mixed Inference.cmd new file mode 100644 index 000000000..f1066c9e6 --- /dev/null +++ b/Run Qwen Mixed Inference.cmd @@ -0,0 +1,9 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0" +python scripts\run_qwen_mixed_inference.py +set "Q38_EXIT=%ERRORLEVEL%" +echo. +echo Qwen mixed inference finished with exit code %Q38_EXIT%. +pause >nul +exit /b %Q38_EXIT% diff --git a/Run Qwen Product Test.cmd b/Run Qwen Product Test.cmd new file mode 100644 index 000000000..12dae1eed --- /dev/null +++ b/Run Qwen Product Test.cmd @@ -0,0 +1,16 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0" +set "QWEN_PYTHON=.gate13-runs\qwen-product-venv\Scripts\python.exe" +if defined COMMUNITYAI_TEST_PYTHON set "QWEN_PYTHON=%COMMUNITYAI_TEST_PYTHON%" +if not exist "%QWEN_PYTHON%" ( + echo Set COMMUNITYAI_TEST_PYTHON to the Python executable with desktop test dependencies installed. + pause >nul + exit /b 2 +) +"%QWEN_PYTHON%" scripts\run_qwen_product_test.py %* +set "QWEN_EXIT=%ERRORLEVEL%" +echo. +echo Qwen product test finished with exit code %QWEN_EXIT%. +pause >nul +exit /b %QWEN_EXIT% diff --git a/Run Qwen Qualification.cmd b/Run Qwen Qualification.cmd new file mode 100644 index 000000000..82669d59f --- /dev/null +++ b/Run Qwen Qualification.cmd @@ -0,0 +1,22 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion +cd /d "%~dp0" +set "QWEN_PYTHON=" +set "QWEN_PYTHON_ARGS=" +if exist ".venv-cuda\Scripts\python.exe" set "QWEN_PYTHON=.venv-cuda\Scripts\python.exe" +if not defined QWEN_PYTHON if exist ".gate13-runs\qwen-product-venv\Scripts\python.exe" set "QWEN_PYTHON=.gate13-runs\qwen-product-venv\Scripts\python.exe" +if defined COMMUNITYAI_TEST_PYTHON set "QWEN_PYTHON=%COMMUNITYAI_TEST_PYTHON%" +if not defined QWEN_PYTHON where py.exe >nul 2>nul && set "QWEN_PYTHON=py.exe" && set "QWEN_PYTHON_ARGS=-3" +if not defined QWEN_PYTHON where python.exe >nul 2>nul && set "QWEN_PYTHON=python.exe" +if not defined QWEN_PYTHON ( + echo Python 3 was not found. + set "QWEN_EXIT=2" +) else ( + call "%QWEN_PYTHON%" %QWEN_PYTHON_ARGS% scripts\run_qwen_qualification.py %* + set "QWEN_EXIT=!ERRORLEVEL!" +) +echo. +echo Qwen qualification finished with exit code !QWEN_EXIT!. +echo This window may now be closed. +pause >nul +exit /b !QWEN_EXIT! diff --git a/config/gate13_gcp.json b/config/gate13_gcp.json new file mode 100644 index 000000000..ac00e497d --- /dev/null +++ b/config/gate13_gcp.json @@ -0,0 +1,28 @@ +{ + "acceptance_helper_commit": "b84f9a1c29487559e2472c65eb8994a5a2a23240", + "catalog_source_commit": "1476d67f3887dfd0de2acfb1305cbcca9975614f", + "configure_helper_commit": "54498193b75438aa6f051dc1552391fd4b5e79a4", + "linux_image": "ubuntu-2404-noble-amd64-v20260826", + "linux_image_project": "ubuntu-os-cloud", + "linux_machine_type": "e2-standard-8", + "linux_startup_commit": "0981c2829ffb42c7136ac5bb8e9dc78d888253f4", + "network": "communityai-discovery", + "project": "community-ai-506321", + "protected_instance": "communityai-bootstrap-1", + "protected_zone": "us-central1-a", + "region": "us-central1", + "route_image": "common-cu129-ubuntu-2404-nvidia-580-v20260831", + "route_image_project": "deeplearning-platform-release", + "route_machine_type": "g2-standard-8", + "route_setup_commit": "a08f20789f230d99f722940e1d7c4e64f5374cbc", + "route_source_commit": "d2c93af74311bddbee516b5f35e65789449b8b07", + "route_wheel_bytes": 389449, + "route_wheel_path": "../petals-revival-route-build/dist/g13-worker-lifecycle/drift-2.3.0.dev2-py3-none-any.whl", + "route_wheel_sha256": "edfd4598c293719d4d7701c9613b64f47f9fd20c3a2dc2e4c0fcacacad3c493a", + "subnet": "communityai-us-central1", + "windows_image": "windows-server-2025-dc-v20260814", + "windows_image_project": "windows-cloud", + "windows_machine_type": "e2-standard-8", + "windows_startup_commit": "e60c3577c7205ff434cad6e9396f89555626aceb", + "zone": "us-central1-b" +} diff --git a/config/qwen_formation.json b/config/qwen_formation.json new file mode 100644 index 000000000..525a831a1 --- /dev/null +++ b/config/qwen_formation.json @@ -0,0 +1,14 @@ +{ + "project": "community-ai-506321", + "region": "us-central1", + "zone": "us-central1-b", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "worker_machine_type": "n2-highmem-4", + "client_machine_type": "e2-standard-4", + "capacity_blocks": 16, + "disk_gb": 80, + "max_duration_seconds": 21600 +} diff --git a/config/qwen_full_inference_gcp.json b/config/qwen_full_inference_gcp.json new file mode 100644 index 000000000..b52cb2430 --- /dev/null +++ b/config/qwen_full_inference_gcp.json @@ -0,0 +1,14 @@ +{ + "project": "community-ai-506321", + "region": "us-central1", + "zone": "us-central1-b", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "worker_machine_type": "e2-highmem-4", + "client_machine_type": "e2-standard-4", + "disk_gb": 80, + "max_duration_seconds": 21600, + "spans": ["0:16", "16:32", "32:48", "48:64"] +} diff --git a/config/qwen_mixed_inference.json b/config/qwen_mixed_inference.json new file mode 100644 index 000000000..61c5b9287 --- /dev/null +++ b/config/qwen_mixed_inference.json @@ -0,0 +1,22 @@ +{ + "project": "community-ai-506321", + "region": "us-central1", + "zone": "us-central1-b", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "gpu_image": "ubuntu-2404-noble-amd64-v20260826", + "gpu_image_project": "ubuntu-os-cloud", + "ubuntu_driver_version": "580.173.02-0ubuntu0.24.04.1", + "gpu_machine_type": "g2-standard-8", + "worker_machine_type": "e2-highmem-4", + "client_machine_type": "e2-standard-4", + "disk_gb": 80, + "max_duration_seconds": 21600, + "spans": ["0:16", "16:32", "32:48", "48:64"], + "azure_subscription": "49af46a3-e748-4cdf-b5e2-fdc381b2333a", + "azure_location": "eastus", + "azure_size": "Standard_NC4as_T4_v3", + "azure_image": "Canonical:ubuntu-24_04-lts:server:24.04.202608270" +} diff --git a/config/qwen_product_test.json b/config/qwen_product_test.json new file mode 100644 index 000000000..807913851 --- /dev/null +++ b/config/qwen_product_test.json @@ -0,0 +1,11 @@ +{ + "node": ".gate13-runs/qwen-product-build-v9/CommunityAI/node/CommunityAI-Node.exe", + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "package_provenance": ".gate13-runs/qwen-product-build-v9/provenance.json", + "local_cache": ".gate13-runs/qwen-product-model-cache", + "remote_cache": ".gate13-runs/qwen-product-client-acquisition-v7/cache", + "cache_provenance": ".gate13-runs/qwen-product-client-acquisition-v7/result.json", + "cloud_config": "config/qwen_mixed_inference.json", + "device": "cuda:0", + "port": 18089 +} diff --git a/desktop/README.md b/desktop/README.md index b9708d4d2..8ffa24d55 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -10,7 +10,9 @@ The desktop currently provides the promoted milestone-5 vertical slice: - a modern Home, Models, Sharing, and API-access experience; - available models, total peers, optional peer-region counts, and current contribution status; - one-click model selection plus start and pause controls for contribution workers; -- a persistent GPU-memory target slider ready for node-side budget enforcement; +- separate VRAM and processing sliders, both initially 100%, with stop/save/resume + application and contribution-side enforcement; +- block-health grids, available peer details and verified local download progress; - create, relabel, and revoke controls for OpenAI client keys; - one-time display and clipboard copy for newly created client-key secrets; and - native credential ownership with verified generation or automatic import of an existing headless-node key; @@ -39,9 +41,12 @@ it automatically when `~/.drift/node/node-config.json` is absent. It authenticat bounded HTTPS catalog against a bundled root, enforces expiry and persistent rollback state, installs only exact digest-matched manifests, generates the seed-backed node configuration, and retains an unexpired last-known-good catalog for offline recovery. -The first signed public-alpha bootstrap and its exact Qwen/Gemma manifests are -published under [`public-alpha/catalog-v1`](../public-alpha/catalog-v1). Production -desktop CI verifies and bundles those inputs; an input-free local engineering build +The original Qwen/Gemma bootstrap remains under +[`public-alpha/catalog-v1`](../public-alpha/catalog-v1). Desktop CI now verifies and +bundles the separately published [Qwen sequence 2](../public-alpha/catalog-qwen-v2), +with local Qwen3.5-0.8B, community Qwen3.8 and an explicit former-root migration. +This candidate's [product qualification](../docs/QWEN_DESKTOP_PRODUCT_RESULTS.md) +remains open. An input-free local engineering build remains available and honestly renders the missing-catalog state on a truly clean install. See [`CATALOG_BOOTSTRAP_V1.md`](../docs/CATALOG_BOOTSTRAP_V1.md). @@ -61,6 +66,10 @@ signed installers, and update/rollback behavior remain later milestone-5 gates. ## Development +Inno Setup and Debian engineering installers are built after archive verification. +See [installer commands and lifecycle limits](installers/README.md) and the +[free-signing application draft](../docs/WINDOWS_SIGNING.md). + Create a disposable environment and install the package: ```shell @@ -113,7 +122,7 @@ bundle into the product with: ```shell python build_desktop.py \ - --publication-bundle ../public-alpha/catalog-v1 \ + --publication-bundle ../public-alpha/catalog-qwen-v2 \ --source-commit \ --build-workflow local ``` @@ -134,7 +143,7 @@ completed output in a fresh process with: ```shell python build_desktop.py \ --verify-release-output dist/desktop \ - --publication-bundle ../public-alpha/catalog-v1 \ + --publication-bundle ../public-alpha/catalog-qwen-v2 \ --source-commit \ --build-workflow local \ --verify-build-environment diff --git a/desktop/build_desktop.py b/desktop/build_desktop.py index 2dac1013d..fb6cd75c1 100644 --- a/desktop/build_desktop.py +++ b/desktop/build_desktop.py @@ -5,6 +5,7 @@ import argparse import gzip import hashlib +import importlib.metadata import json import os import platform @@ -21,6 +22,11 @@ from communityai_desktop.acceptance import run_self_test from communityai_desktop.pyside_shell import check_runtime +try: # Direct script execution and repository test imports use different roots. + from runtime_packaging import normalize_runtime +except ModuleNotFoundError: + from desktop.runtime_packaging import normalize_runtime + APP_NAME = "CommunityAI" NODE_NAME = "CommunityAI-Node" NODE_DIRECTORY = "node" @@ -43,11 +49,13 @@ ".gitattributes", ".github/workflows/desktop.yaml", "desktop/build_desktop.py", + "desktop/runtime_packaging.py", "desktop/launch_desktop.py", "desktop/launch_node.py", "desktop/pyproject.toml", "desktop/src", "public-alpha/catalog-v1", + "public-alpha/catalog-qwen-v2", "pyproject.toml", "scripts/build_hivemind_windows.py", "scripts/hivemind-win32.patch", @@ -56,6 +64,30 @@ _EXPECTED_UNSET = object() +def _check_build_storage(output_root: Path, build_root: Path) -> None: + """Reserve conservative staging/archive capacity on each actual volume.""" + gib = 1024**3 + volumes: dict[int, tuple[Path, int]] = {} + # Observed Windows output is 4.5 GB unpacked plus a 2.7 GB archive. + # Work staging also holds the sidecar before it is moved into the bundle. + for target, required in ((output_root, 8 * gib), (build_root, 5 * gib)): + existing = target.resolve() + while not existing.exists(): + existing = existing.parent + volume = existing.stat().st_dev + prior = volumes.get(volume, (existing, 2 * gib)) # reserve per volume + volumes[volume] = (prior[0], prior[1] + required) + for location, required in volumes.values(): + free = shutil.disk_usage(location).free + if free < required: + raise RuntimeError( + f"Insufficient build space on the volume containing {location}: " + f"{free / gib:.1f} GiB free; an estimated {required / gib:.1f} GiB is required " + "for staging, unpacked output, archive and reserve. Free space or choose " + "--output-root and --build-root on a volume with sufficient capacity." + ) + + def _canonical_json(payload: object) -> str: return json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n" @@ -352,6 +384,7 @@ def _normalized_tar_info(name: str, *, mode: int) -> tarfile.TarInfo: def _write_tar_install_archive(archive_path: Path, entries: Sequence[dict[str, object]]) -> None: + regular_inodes: dict[tuple[int, int], dict[str, object]] = {} with archive_path.open("wb") as raw_stream: with gzip.GzipFile(filename="", mode="wb", fileobj=raw_stream, compresslevel=9, mtime=0) as compressed: with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive: @@ -367,10 +400,23 @@ def _write_tar_install_archive(archive_path: Path, entries: Sequence[dict[str, o archive.addfile(info) elif entry["kind"] == "file": source = Path(entry["_source"]) + source_stat = source.lstat() + if not stat.S_ISREG(source_stat.st_mode): + raise RuntimeError("install archive regular source changed type") + identity = source_stat.st_dev, source_stat.st_ino + prior = regular_inodes.get(identity) + if prior is not None: + if any(entry[key] != prior[key] for key in ("sha256", "size_bytes", "mode")): + raise RuntimeError("install archive hardlink identity changed") + info.type = tarfile.LNKTYPE + info.linkname = str(prior["path"]) + archive.addfile(info) + continue info.type = tarfile.REGTYPE info.size = int(entry["size_bytes"]) with source.open("rb") as source_stream: archive.addfile(info, source_stream) + regular_inodes[identity] = entry else: raise RuntimeError(f"unsupported install archive entry kind: {entry['kind']!r}") @@ -454,8 +500,9 @@ def _verify_tar_install_archive( actual[member_path] = member if set(actual) != set(expected): raise RuntimeError("install archive members do not match the release bundle") - for member_path, entry in expected.items(): - member = actual[member_path] + verified_regular: set[str] = set() + for member_path, member in actual.items(): + entry = expected[member_path] if entry["kind"] == "directory": if not member.isdir() or stat.S_IMODE(member.mode) != int(entry["mode"]): raise RuntimeError(f"install archive directory mode or type mismatch: {member_path}") @@ -466,16 +513,32 @@ def _verify_tar_install_archive( if canonical_target != entry["link_target"]: raise RuntimeError(f"install archive symlink target mismatch: {member_path}") elif entry["kind"] == "file": + effective_size = member.size + if member.islnk(): + target = _validate_install_member_path(member.linkname) + target_entry = expected.get(target, {}) + if ( + member.linkname != target + or target not in verified_regular + or member.size != 0 + or any(entry[key] != target_entry.get(key) for key in ("sha256", "size_bytes", "mode")) + ): + raise RuntimeError( + "install archive hardlink target is not a verified identical regular file" + ) + effective_size = int(target_entry["size_bytes"]) if ( - not member.isfile() + not (member.isfile() or member.islnk()) or member.issparse() - or member.size != int(entry["size_bytes"]) + or effective_size != int(entry["size_bytes"]) or stat.S_IMODE(member.mode) != int(entry["mode"]) ): raise RuntimeError(f"install archive file size, mode, or type mismatch: {member_path}") stream = archive.extractfile(member) if stream is None or _sha256_archive_stream(stream) != entry["sha256"]: raise RuntimeError(f"install archive file digest mismatch: {member_path}") + if member.isfile(): + verified_regular.add(member_path) else: raise RuntimeError(f"unsupported install archive entry kind: {entry['kind']!r}") except (OSError, tarfile.TarError) as exc: @@ -1135,7 +1198,23 @@ def _run_bundle( def _run_pyinstaller(arguments: list[str]) -> None: - subprocess.run([sys.executable, "-m", "PyInstaller", *arguments], check=True) + environment = os.environ.copy() + if os.name == "nt": + # Qt links Windows' ICU ABI. An unrelated tool on PATH may ship another + # icuuc.dll with the same basename and incompatible exports. Restrict + # dependency lookup to this Python environment and Windows; PyInstaller's + # package hooks still discover Torch and Qt's own runtime directories. + windows = Path(os.environ["SystemRoot"]) + environment["PATH"] = os.pathsep.join( + str(path) + for path in ( + Path(sys.executable).parent, + Path(sys.base_prefix), + windows / "System32", + windows, + ) + ) + subprocess.run([sys.executable, "-m", "PyInstaller", *arguments], check=True, env=environment) def _prepare_release_inputs(publication_bundle: Path | None) -> dict[str, object] | None: @@ -1180,6 +1259,7 @@ def _verify_packaged_release_inputs( def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--output-root", type=Path) + parser.add_argument("--build-root", type=Path) parser.add_argument("--publication-bundle", type=Path) parser.add_argument("--source-commit") parser.add_argument("--build-workflow") @@ -1234,7 +1314,8 @@ def main() -> int: source_commit, source_tree = _source_identity(repository, args.source_commit) build_workflow = args.build_workflow or os.environ.get("GITHUB_WORKFLOW_REF", "local") output_root = (args.output_root or project / "dist" / "desktop").resolve() - build_root = project / "build" / "desktop" + build_root = (args.build_root or project / "build" / "desktop").resolve() + _check_build_storage(output_root, build_root) bundle_root = output_root / APP_NAME icon_path = project / "src" / "communityai_desktop" / "assets" / "communityai.ico" if not icon_path.is_file(): @@ -1269,6 +1350,8 @@ def main() -> int: str(build_root / "spec"), "--hidden-import", "communityai_desktop.pyside_shell", + "--hidden-import", + "communityai_desktop.gate13_playthrough", "--add-data", f"{icon_path}{os.pathsep}communityai_desktop/assets", ] @@ -1325,6 +1408,13 @@ def main() -> int: "--exclude-module", "PySide6", ] + if platform.system() == "Linux": + # Approved desktop profiles use eager/native kernels. Optional PEFT/bitsandbytes + # imports otherwise initialize Triton's JIT on GPU hosts, requiring a compiler + # and Python development headers that ordinary frozen-app users do not have. + # Retain PyTorch and bitsandbytes native kernels; source deployments can opt + # into Triton separately when their execution profile requires it. + node_args.extend(("--exclude-module", "triton")) credential_backend = { "Windows": "keyring.backends.Windows", "Darwin": "keyring.backends.macOS", @@ -1346,6 +1436,11 @@ def main() -> int: if not node_executable.is_file(): raise RuntimeError(f"packaged node executable was not staged: {node_executable}") + normalization = normalize_runtime( + node_root, target_platform=platform.system(), torch_version=importlib.metadata.version("torch") + ) + (bundle_root / "runtime-packaging.json").write_text(_canonical_json(normalization), encoding="utf-8") + environment = os.environ.copy() environment.setdefault("QT_QPA_PLATFORM", "offscreen") runtime = check_runtime() diff --git a/desktop/installers/ONLINE_LINUX.md b/desktop/installers/ONLINE_LINUX.md new file mode 100644 index 000000000..6a79d18f0 --- /dev/null +++ b/desktop/installers/ONLINE_LINUX.md @@ -0,0 +1,75 @@ +# Linux online installer + +This builder creates a small standalone Python 3.9+ script. It downloads one +explicitly pinned full `.deb`, including the packaged CPU/NVIDIA runtime, and +installs it through APT. It uses the same package, worker shutdown and retained +settings/cache behavior as the offline installer. It does not resolve new pip +dependencies, download model weights, add a repository or install a GPU driver. + +Build from the repository with the shared release manifest: + +```sh +python3 desktop/installers/build_linux_online.py \ + --manifest release-downloads.json \ + --output communityai-linux-online.py +``` + +The manifest must contain a validated `linux-amd64` offline-installer entry from +`release_downloads.py`. The generated script embeds its immutable HTTPS URL, +version, filename, byte size and SHA-256. Its adjacent `.py.json` records the +script's own identity and the pinned offline entry. Existing output files are +never replaced. Use the actual release's immutable object URL; placeholder +fixtures are not publication artifacts. + +Download the generated script as a file from the official release, verify its +published checksum, then run it as your ordinary desktop user: + +```sh +python3 communityai-linux-online.py +``` + +Do not pipe a remote response into a shell or run the downloader itself with +sudo. It requests sudo only after checking the complete download. The elevated +helper runs isolated system Python, copies the package into its own root-owned +directory, and rechecks the pinned size/hash while copying. APT receives that +protected copy, so changing the original file during the password prompt cannot +change what APT installs. APT keeps its normal dependency resolution and prompts. + +The default staging parent is `/var/tmp`. Allow temporary space for **two copies +of the compressed package**, plus the unpacked application and any dependency or +upgrade overhead. For the qualified September 8 package (2,302,428,788 bytes), +this means about 4.60 GB of +package staging, in addition to the installed runtime. The wrapper checks two +package sizes plus a 64 MiB margin in the selected staging directory; the helper +also checks room for its copy in `/var/tmp`. These checks do not promise enough +space for all APT operations. `--directory PATH` selects another existing parent +for the ordinary-user download; the protected copy remains in `/var/tmp`. + +`--download-only` retains the exact verified `.deb` and prints its path without +requesting sudo. It needs one package's temporary space. The same file can later +be used with the documented offline installation procedure. + +Downloads use HTTPS certificate validation, reject redirects and encoded bodies, +and require the exact byte count and SHA-256 before elevation. They have a +30-second blocking network-operation timeout and a two-hour Linux signal +deadline. The protected local copy also has a two-hour deadline. These deadlines +are disarmed before APT starts. Cancellation or verification failure removes +only staging created by that invocation. If APT's exit cannot be confirmed, its +input is retained rather than removed or the package manager killed; the path is +printed for later review. Package-manager recovery follows APT's own messages. + +Fixture validation covers download, cancellation, protected-copy ordering, +changed-source rejection, APT exit handling, and a generated script's standalone +`--help` using inert fixtures. The focused suites also passed inside the cached +Ubuntu 22.04 container with two CPU cores, a 2 GiB memory cap, read-only sources +and no network. Linux's source-symlink rejection passed there. These checks +never invoked real sudo or APT. + +The release's actual hosted download, real sudo/protected-copy/APT installation +and removal subsequently passed on Ubuntu 22.04 with two CPUs and 3 GiB memory. +The [scoped acceptance](../../docs/evidence/alpha-online-linux-hosted-20260908.md) +binds the downloaded package hash, root-owned protected input, actual APT path +and dpkg installed/removal transaction. It preserves the original diagnostic +harness's missing process observation; its planned post-online native check did +not run. Existing CPU/CUDA acceptance covers the identical offline package. +The published online script itself was also downloaded anonymously and hashed. diff --git a/desktop/installers/ONLINE_WINDOWS.md b/desktop/installers/ONLINE_WINDOWS.md new file mode 100644 index 000000000..5a5cf61fa --- /dev/null +++ b/desktop/installers/ONLINE_WINDOWS.md @@ -0,0 +1,99 @@ +# Windows online setup + +The small online setup downloads the complete, version-pinned offline setup. It +does not choose CPU/GPU components or install dependencies with pip. Both download +options therefore install the same self-contained runtime; the online option +reduces the initial download, not the total bytes needed. + +Build only after the offline setup has a release-manifest entry with its exact +HTTPS URL, filename, version, SHA-256 and byte size: + +```powershell +desktop/installers/build_windows_online_installer.ps1 ` + -Manifest ` + -OutputDirectory ` + -PythonCommand ` + -Compiler ` + -UnsignedAlpha +``` + +Use Inno Setup 6.7.3 and the Windows .NET Framework v4 C# compiler. The +builder compiles the small helper without adding a .NET runtime to the payload. +Windows 10 includes the required .NET Framework. The native process bridge +is pinned to Inno 6.7.3's 32-bit Setup engine until another engine is qualified. `-ValidateOnly` checks metadata and the explicit +signing choice without creating output or calling the compiler. A signing command +can replace `-UnsignedAlpha`. Existing output executables are not overwritten. +There is no default download URL or mutable remote manifest lookup. + +The embedded helper streams the pinned HTTPS object and resumes interrupted +connections within the current setup session. It makes at most eight requests, +with bounded backoff, 30-second network-operation timeouts and a two-hour +monotonic deadline. It rejects redirects, encoded content, unexpected lengths, +and resumed responses whose `Content-Range` does not match the exact offset and +total. A server that ignores a resume request is rejected. Failure or cancellation +removes the partial file after the helper has stopped; restarting setup begins +a fresh session. + +The helper verifies the complete size and SHA-256. Inno independently checks both +again before executing the offline setup. The wizard displays byte progress, +retry and verification status. Cancellation stops only the owned download +helper. Inno starts it suspended, assigns it to a job configured to terminate its +processes when closed, and then resumes it. The helper additionally watches the +exact parent process and cancellation signal. Forced cleanup is bounded, and an +unconfirmed process exit cannot launch the offline installer. Progress records +serve only the display; locked or unwritable records cannot fail a valid download. +Publication attempts have a short bounded retry and are throttled even on failure. +The wizard also reads the growing local file's byte length, so stale records do +not freeze byte progress. The final helper exit and Inno's independent size/hash +checks authorize execution. + +The child setup runs directly under the same user, and the online process waits +for it before its temporary directory is removed. The child setup's exit code is +returned; a child cancellation or failure cannot become a successful exit. + +The online setup creates no application directory, shortcuts or uninstall entry. +The offline setup retains ownership of worker shutdown, upgrades, installation +location, uninstall and preservation of settings/cache. It also offers the normal +interactive choice to open CommunityAI after installation. + +Accepted and forwarded command-line options are `/SILENT`, `/VERYSILENT`, +`/SUPPRESSMSGBOXES`, `/NORESTART`, `/SP-`, `/NOICONS`, `/NOCANCEL`, +`/CURRENTUSER`, `/DIR=`, `/GROUP=`, `/LANG=` and `/RESTARTEXITCODE=`. +Arguments are forwarded individually; Inno's internal loader arguments are +excluded. Unsupported options are rejected before downloading. Use the offline +setup for other options. + +The offline alpha creates its normal Start Menu shortcuts. Its configuration +does not enable Inno's `AllowNoIcons` checkbox and disables the program-group +page, so forwarding `/NOICONS` or `/GROUP=` does not change that behavior. +Uninstall removes the shortcuts it created. See Inno's +[shortcut option](https://jrsoftware.org/ishelp/topic_setup_allownoicons.htm) and +[command-line behavior](https://jrsoftware.org/ishelp/topic_setupcmdline.htm). + +For unattended installation, use +`/VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP-`. The online setup logs its own +argument, download, verification and child-launch failures without showing a +dialog in either silent mode. A failed download returns false from Inno's +`NextButtonClick`, which aborts a silent installation before execution. The +additional `/SUPPRESSMSGBOXES` remains necessary for Inno's built-in errors and +the offline child installer; `/VERYSILENT` alone does not suppress those messages. +This follows Inno's [silent-mode parameters](https://jrsoftware.org/ishelp/topic_setupcmdline.htm) +and [silent event behavior](https://jrsoftware.org/ishelp/topic_scriptevents.htm). + +`/LOG` or `/LOG=` selects the online setup log and gives the offline setup +its own automatically named log. To name the offline log separately, use +`/INSTALLERLOG=`. Do not use the same path for both logs. + +The builder writes a companion `.exe.json` binding the small executable to its +offline installer metadata. `live_download_verified` remains false: compilation +and source tests do not establish a successful hosted download or actual install. +Qualification must separately verify the published download, failure/cancel +behavior and handoff to the offline setup before advertising the online option. + +The implementation uses Windows +[job objects](https://learn.microsoft.com/en-us/windows/win32/procthread/job-objects), +[.NET Framework](https://learn.microsoft.com/en-us/dotnet/framework/get-started/system-requirements), +and Inno's +[64-bit file-size check](https://jrsoftware.org/ishelp/topic_isxfunc_filesize64.htm), +[filtered command-line parameters](https://jrsoftware.org/ishelp/topic_isxfunc_paramstr.htm) +and [waited execution/exit-code API](https://jrsoftware.org/ishelp/topic_isxfunc_exec.htm). diff --git a/desktop/installers/README.md b/desktop/installers/README.md new file mode 100644 index 000000000..242f2c048 --- /dev/null +++ b/desktop/installers/README.md @@ -0,0 +1,117 @@ +# CommunityAI engineering installers + +These builders package the already verified output of `desktop/build_desktop.py`. +Run its independent checksum/provenance verification before invoking either +builder. CI does this and uploads separate `communityai-setup-windows` and +`communityai-setup-linux` artifacts. They are **unsigned engineering builds**, +not a release qualification or Store submission. + +## Windows + +Use Inno Setup 6 and PowerShell, from the repository root: + +```powershell +./desktop/installers/build_windows_installer.ps1 ` + -Bundle desktop/dist/desktop/CommunityAI ` + -OutputDirectory desktop/dist/installers ` + -Version 0.1.0-alpha.1 -Compiler 'C:/path/to/ISCC.exe' -UnsignedEngineering +``` + +The installer defaults to the current user's LocalAppData Programs directory and +requires no elevation. Silent install uses `/VERYSILENT /SUPPRESSMSGBOXES +/NORESTART`; the uninstaller accepts the same options. Re-run the next installer +to upgrade. Before replacement/removal, the installed `CommunityAI.exe +--prepare-update` asks the same user's desktop to quit and waits for owned-node +cleanup and release of its instance lock. Failure stops replacement. Existing +unmarked application directories are refused; use a new directory for a prior +unpacked engineering archive. + +The installer removes obsolete `_internal` and `node` files only inside its +marked installation directory. Settings, credentials and model cache live +outside this directory and are retained. The alpha uses the explicit +[manual retention and deletion choices](../../docs/DESKTOP_UNINSTALL.md): disable +the sign-in toggle before uninstalling, keep state for reinstall, remove only +reviewed model-cache folders to reclaim disk space, or explicitly reset the +native credential and node state. The installer does not automatically delete +cache or login entries. + +`-AppIdentifier` exists for isolated engineering installations. Keep the default +`CommunityAI.Desktop` stable for public upgrades. The owner has approved unsigned +direct-download installers for the alpha, labelled accordingly and accompanied +by verified checksums/provenance. Later signed builds supply an approved +publisher/signing command instead of `-UnsignedEngineering`; Inno uses the +command for setup and uninstaller signing. See the +[signing decision/application draft](../../docs/WINDOWS_SIGNING.md). Payload +signing and provider enrollment remain required before Store submission. + +## Ubuntu/Debian + +Build on Ubuntu 22.04 or a compatible baseline with `dpkg-deb`: + +```sh +python desktop/installers/build_deb.py \ + --bundle desktop/dist/desktop/CommunityAI \ + --output desktop/dist/installers --version '0.1.0~alpha.1' \ + --maintainer 'Maintainer Name ' +sudo apt install ./desktop/dist/installers/communityai_0.1.0~alpha.1_amd64.deb +``` + +The package owns `/opt/communityai`, `/usr/bin/communityai` and the system menu +entry. `preinst`/`prerm` stop processes whose executables belong to the marked +installation and their observed descendants, using kernel PID handles and +process start times. Shutdown continually discovers new installed helpers and +requires repeated quiet observations before allowing file replacement. They fail +if matching processes remain or process ownership +cannot be inspected. Container qualification must grant root `SYS_PTRACE` so it +can inspect ordinary-user executables through `/proc`, as on the target desktop +systems; missing permission must not silently skip a running installation. They never enumerate +or remove home-directory settings/cache. Per-user login entries and optional +cache/state deletion follow the same +[manual choices](../../docs/DESKTOP_UNINSTALL.md), including when using `apt purge`. +Python 3.9+ and Linux PID handles are +required; the supported baseline is Ubuntu 22.04+/Debian 12+ on amd64. +The declared Qt/X11 dependencies include `libxcb-shape0`; omitting it prevented +the frozen desktop opening on a minimal Debian host even though offscreen tests +passed. CI now also opens the frozen UI and onboarding through X11/Xvfb. +Frozen Linux packages use the approved eager/native inference kernels and exclude +the optional Triton JIT, so importing adapter support does not require a compiler +or Python development headers on contributors' computers. Source deployments can +install Triton separately for execution profiles that need it. + +CI's maintainer address is explicitly an engineering placeholder. Replace it +before publication. A real package must pass installation/upgrade/removal with +the complete runtime on both target distributions. + +## Signed APT repository + +APT repository signing uses GPG and does not require a commercial certificate. +Use a separate protected repository signing key, not the model-catalog key. With +Python 3.11+, `apt-utils`, GnuPG and the key available in the operator's GPG home: + +```sh +python desktop/installers/build_apt_repository.py \ + --package desktop/dist/installers/communityai_0.1.0~alpha.1_amd64.deb \ + --output desktop/dist/apt-release-1 --signing-key FULL_KEY_FINGERPRINT +``` + +Repeat `--package` for retained versions. The output directory must be new. The +builder writes versioned package paths, `Packages`, `Packages.gz`, `Release`, +`InRelease`, `Release.gpg`, the exported public key and a hash inventory. Both +signatures are independently checked with `gpgv`. It never publishes output or +creates a signing key. Repository metadata expires after 30 days; re-sign and +publish it before expiry even when package versions do not change. + +Publish immutable package paths first, then index files, then signed release +metadata at the chosen HTTPS origin. Preserve existing versioned package URLs. +Distribute the public key and full fingerprint through the project download +page; users install it as `/etc/apt/keyrings/communityai.gpg` and use a deb822 +`.sources` entry with `Types: deb`, the actual HTTPS `URIs`, `Suites: alpha`, +`Components: main`, `Architectures: amd64` and +`Signed-By: /etc/apt/keyrings/communityai.gpg`. Keep the key readable by APT and +scope package pinning to this repository and the `communityai` package as in +[Debian's third-party guidance](https://wiki.debian.org/DebianRepository/UseThirdParty). + +Production key custody/backup, a permanent HTTPS origin, final setup commands and +publication remain open. A disposable container test proved signed index +acceptance, package candidate/download/hash verification and tampered-signature +rejection; it created no production key or public repository. diff --git a/desktop/installers/WindowsDownload.cs b/desktop/installers/WindowsDownload.cs new file mode 100644 index 000000000..a98cd4b4a --- /dev/null +++ b/desktop/installers/WindowsDownload.cs @@ -0,0 +1,390 @@ +// Standalone .NET Framework 4 downloader. Never launches the downloaded file. +// CLI: URL output size sha256 progress cancel parentPid parentCreationFileTime +// Atomic progress: downloaded|total|status (0 download, 1 retry, 2 hash, +// 3 verified, 4 failed, 5 cancelled). Only exit 0 means verified success. +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Net; +using System.Runtime.InteropServices; +using System.Security; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; + +internal static class WindowsDownload +{ + private const int Attempts = 8; +#if WINDOWS_DOWNLOAD_TEST + private const int IoTimeout = 500; + private const int BackoffBase = 20; +#else + private const int IoTimeout = 30000; + private const int BackoffBase = 1000; +#endif +#if WINDOWS_DOWNLOAD_SHORT_DEADLINE && WINDOWS_DOWNLOAD_TEST + private const long DeadlineMilliseconds = 700; +#else + private const long DeadlineMilliseconds = 2L * 60 * 60 * 1000; +#endif + private static readonly Stopwatch Clock = new Stopwatch(); + private static readonly ManualResetEvent Finished = new ManualResetEvent(false); + private static readonly ManualResetEvent Cancelled = new ManualResetEvent(false); + private static readonly object RequestLock = new object(); + private static HttpWebRequest CurrentRequest; + private static Process Parent; + private static IntPtr ParentHandle; + private static string OutputPath, ProgressPath, CancelPath; + private static long ExpectedSize, Downloaded, LastProgress = -1000; + private static int StopReason; + private static bool OwnsOutput; + private static string FailureDetail = "Download failed"; + private static string LastTransferError = "Incomplete response"; + + [DllImport("kernel32.dll", ExactSpelling = true)] + private static extern uint WaitForSingleObject(IntPtr handle, uint milliseconds); + + private sealed class InvalidDownload : Exception + { + internal InvalidDownload(string message) : base(message) { } + } + + private sealed class Stopped : Exception { } + + private static void Require(bool condition, string message) + { + if (!condition) throw new InvalidDownload(message); + } + + private static void CheckStop() + { + if (Clock.ElapsedMilliseconds >= DeadlineMilliseconds) + Interlocked.CompareExchange(ref StopReason, 3, 0); + if (Interlocked.CompareExchange(ref StopReason, 0, 0) != 0) throw new Stopped(); + } + + private static void Watch() + { + while (!Finished.WaitOne(100)) + { + int reason = 0; + try + { + if (File.Exists(CancelPath)) reason = 1; + // A failed parent-handle query also stops the transfer. + else if (WaitForSingleObject(ParentHandle, 0) != 258) reason = 2; + else if (Clock.ElapsedMilliseconds >= DeadlineMilliseconds) reason = 3; + } + catch { reason = 2; } + if (reason == 0) continue; + Interlocked.CompareExchange(ref StopReason, reason, 0); + Cancelled.Set(); + lock (RequestLock) + { + if (CurrentRequest != null) + { + try { CurrentRequest.Abort(); } catch { } + } + } + return; + } + } + + private static void Progress(int status, bool final) + { + long remaining = 250 - (Clock.ElapsedMilliseconds - LastProgress); + if (remaining > 0) + { + if (!final) return; + Thread.Sleep((int)remaining); + } + string temporary = ProgressPath + ".new"; + bool ownsTemporary = false; + try + { + using (FileStream stream = new FileStream(temporary, FileMode.CreateNew, + FileAccess.Write, FileShare.None)) + { + ownsTemporary = true; + byte[] bytes = Encoding.ASCII.GetBytes( + Downloaded.ToString(CultureInfo.InvariantCulture) + "|" + + ExpectedSize.ToString(CultureInfo.InvariantCulture) + "|" + + status.ToString(CultureInfo.InvariantCulture)); + stream.Write(bytes, 0, bytes.Length); + } + Stopwatch publication = Stopwatch.StartNew(); + while (true) + { + try + { + if (File.Exists(ProgressPath)) File.Replace(temporary, ProgressPath, null); + else File.Move(temporary, ProgressPath); + break; + } + catch (IOException error) + { + int code = Marshal.GetHRForException(error) & 0xffff; + if ((code != 32 && code != 33) || publication.ElapsedMilliseconds >= 75) throw; + // Inno's progress reader may briefly deny delete sharing. + if (!final) CheckStop(); + Thread.Sleep(25); + } + } + ownsTemporary = false; + LastProgress = Clock.ElapsedMilliseconds; + } + catch (Stopped) { throw; } + // Presentation never authorizes execution: Run independently verifies + // the complete size/hash, and Inno checks both again after exit 0. + catch (IOException error) { ProgressWarning(error); } + catch (UnauthorizedAccessException error) { ProgressWarning(error); } + catch (SecurityException error) { ProgressWarning(error); } + finally + { + if (ownsTemporary) { try { File.Delete(temporary); } catch { } } + LastProgress = Clock.ElapsedMilliseconds; + } + } + + private static void ProgressWarning(Exception error) + { + try + { + // No path, URL, stack or response headers are written here. + string warning = "Skipped progress frame: " + error.GetType().Name + + " HRESULT " + Marshal.GetHRForException(error).ToString("X8", CultureInfo.InvariantCulture); + File.WriteAllText(ProgressPath + ".warning", warning, new UTF8Encoding(false)); + } + catch { } + } + + private static Uri ValidateUrl(string text) + { + Uri uri; + Require(text.Length <= 2048 && Uri.TryCreate(text, UriKind.Absolute, out uri), "Invalid URL"); + uri = new Uri(text, UriKind.Absolute); + bool allowed = uri.Scheme == "https" && uri.Port == 443; +#if WINDOWS_DOWNLOAD_TEST + allowed = allowed || (uri.Scheme == "http" && uri.Host == "127.0.0.1"); +#endif + Require(allowed && uri.UserInfo.Length == 0 && uri.Query.Length == 0 && + uri.Fragment.Length == 0 && !String.IsNullOrEmpty(uri.Host), "Invalid HTTPS URL"); + return uri; + } + + private static void ValidateRangeHeader(string header, long offset, long expectedSize) + { + Match match = Regex.Match(header ?? "", @"\Abytes ([0-9]+)-([0-9]+)/([0-9]+)\z"); + long first, last, total; + Require(match.Success && + Int64.TryParse(match.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out first) && + Int64.TryParse(match.Groups[2].Value, NumberStyles.None, CultureInfo.InvariantCulture, out last) && + Int64.TryParse(match.Groups[3].Value, NumberStyles.None, CultureInfo.InvariantCulture, out total), + "Invalid Content-Range"); + first = Int64.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture); + last = Int64.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture); + total = Int64.Parse(match.Groups[3].Value, CultureInfo.InvariantCulture); + Require(first == offset && last == expectedSize - 1 && total == expectedSize, "Mismatched Content-Range"); + } + + private static void ValidateResponse(HttpWebResponse response, long offset) + { + string encoding = response.ContentEncoding; + Require(String.IsNullOrEmpty(encoding) || + String.Equals(encoding, "identity", StringComparison.OrdinalIgnoreCase), "Unexpected encoding"); + if (response.StatusCode == HttpStatusCode.OK) + { + Require(offset == 0, "Server ignored the resume range"); + Require(String.IsNullOrEmpty(response.Headers["Content-Range"]), "Unexpected range on 200 response"); + } + else if (response.StatusCode == HttpStatusCode.PartialContent) + { + ValidateRangeHeader(response.Headers["Content-Range"], offset, ExpectedSize); + } + else throw new InvalidDownload("Unexpected HTTP status"); + Require(response.ContentLength == ExpectedSize - offset, "Mismatched Content-Length"); + } + + private static void Transfer(Uri uri, FileStream output) + { + for (int attempt = 0; attempt < Attempts && output.Length < ExpectedSize; attempt++) + { + CheckStop(); + long offset = output.Length; + output.Position = offset; + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri); + request.AllowAutoRedirect = false; + request.AutomaticDecompression = DecompressionMethods.None; + request.Headers[HttpRequestHeader.AcceptEncoding] = "identity"; + request.UserAgent = "CommunityAI-Online-Installer/1"; + request.Timeout = IoTimeout; + request.ReadWriteTimeout = IoTimeout; + request.KeepAlive = false; + if (offset > 0) request.AddRange(offset); + lock (RequestLock) { CurrentRequest = request; } + bool retry = false; + try + { + CheckStop(); + using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) + { + ValidateResponse(response, offset); + using (Stream input = response.GetResponseStream()) + { + byte[] buffer = new byte[1024 * 1024]; + int count; + while ((count = input.Read(buffer, 0, buffer.Length)) != 0) + { + CheckStop(); + Require(count <= ExpectedSize - output.Position, "Response exceeded expected size"); + output.Write(buffer, 0, count); + Downloaded = output.Position; + Progress(0, false); + } + } + } + if (output.Length != ExpectedSize) throw new IOException("Incomplete response"); + } + catch (WebException error) + { + CheckStop(); + HttpWebResponse failed = error.Response as HttpWebResponse; + if (failed != null) + { + using (failed) + { + int status = (int)failed.StatusCode; + Require(status == 408 || status == 429 || status == 500 || status == 502 || + status == 503 || status == 504, "Non-retryable HTTP status"); + LastTransferError = "HTTP " + status.ToString(CultureInfo.InvariantCulture); + } + } + else LastTransferError = "Network " + error.Status.ToString(); + retry = true; + } + catch (IOException) { CheckStop(); LastTransferError = "Interrupted response or file I/O"; retry = true; } + finally + { + lock (RequestLock) { CurrentRequest = null; } + request.Abort(); + } + if (output.Length == ExpectedSize) break; + Require(retry && attempt + 1 < Attempts, "Retry budget exhausted: " + LastTransferError); + Progress(1, false); + Cancelled.WaitOne(Math.Min(30000, BackoffBase * (1 << attempt))); + } + CheckStop(); + Require(output.Length == ExpectedSize, "Incomplete download"); + } + + private static int Run(string[] args) + { + Require(args.Length == 8, "Expected eight arguments"); + Uri uri = ValidateUrl(args[0]); + OutputPath = Path.GetFullPath(args[1]); + Require(Int64.TryParse(args[2], NumberStyles.None, CultureInfo.InvariantCulture, out ExpectedSize) && + ExpectedSize > 0 && ExpectedSize <= 32L * 1024 * 1024 * 1024, "Invalid size"); + Require(Regex.IsMatch(args[3], @"\A[0-9a-f]{64}\z"), "Invalid SHA-256"); + string progress = Path.GetFullPath(args[4]); + string cancel = Path.GetFullPath(args[5]); + string[] paths = { OutputPath, progress, cancel, progress + ".new", progress + ".error", progress + ".warning" }; + for (int i = 0; i < paths.Length; i++) + for (int j = 0; j < i; j++) + Require(!String.Equals(paths[i], paths[j], StringComparison.OrdinalIgnoreCase), "Paths must differ"); + ProgressPath = progress; + CancelPath = cancel; + int parentPid; + long parentCreation; + Require(Int32.TryParse(args[6], out parentPid) && parentPid > 0 && parentPid != Process.GetCurrentProcess().Id, + "Invalid parent PID"); + Require(Int64.TryParse(args[7], NumberStyles.None, CultureInfo.InvariantCulture, out parentCreation) && + parentCreation > 0, "Invalid parent creation time"); + Parent = Process.GetProcessById(parentPid); + ParentHandle = Parent.Handle; // Retain the actual process handle through all I/O and cleanup. + Require(Parent.StartTime.ToFileTimeUtc() == parentCreation && + WaitForSingleObject(ParentHandle, 0) == 258, "Parent identity mismatch"); + if (File.Exists(CancelPath)) { StopReason = 1; throw new Stopped(); } + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + Clock.Start(); + Thread watcher = new Thread(Watch); + watcher.IsBackground = true; + watcher.Start(); + try + { + using (FileStream output = new FileStream(OutputPath, FileMode.CreateNew, + FileAccess.ReadWrite, FileShare.Read)) + { + OwnsOutput = true; + Progress(0, true); + Transfer(uri, output); + output.Flush(); + output.Position = 0; + Progress(2, true); + using (SHA256 hash = SHA256.Create()) + { + byte[] buffer = new byte[1024 * 1024]; + int count; + while ((count = output.Read(buffer, 0, buffer.Length)) != 0) + { + CheckStop(); + hash.TransformBlock(buffer, 0, count, buffer, 0); + } + hash.TransformFinalBlock(new byte[0], 0, 0); + string digest = BitConverter.ToString(hash.Hash).Replace("-", "").ToLowerInvariant(); + Require(digest == args[3] && output.Length == ExpectedSize, "SHA-256 or size mismatch"); + } + CheckStop(); + Progress(3, true); + CheckStop(); + } + return 0; + } + finally + { + Finished.Set(); + watcher.Join(); + } + } + + [STAThread] + private static int Main(string[] args) + { + int result = 1; + try { result = Run(args); } + catch (Stopped) + { + result = StopReason == 3 ? 2 : 3; + FailureDetail = StopReason == 3 ? "Two-hour download deadline exceeded" : + (StopReason == 2 ? "The parent installer exited" : "Download cancelled"); + } + catch (Exception error) + { + result = 1; + FailureDetail = error is InvalidDownload ? error.Message : + "Download failed (" + error.GetType().Name + ")"; + } + finally + { + Finished.Set(); + if (result != 0) + { + if (OwnsOutput) { try { File.Delete(OutputPath); } catch { } } + if (ProgressPath != null) + { + try { Progress(result == 3 ? 5 : 4, true); } catch { } + try + { + string detail = FailureDetail.Replace('\r', ' ').Replace('\n', ' '); + if (detail.Length > 1024) detail = detail.Substring(0, 1024); + File.WriteAllText(ProgressPath + ".error", detail, new UTF8Encoding(false)); + } + catch { } + } + } + if (Parent != null) { try { Parent.Dispose(); } catch { } } + } + return result; + } +} diff --git a/desktop/installers/build_apt_repository.py b/desktop/installers/build_apt_repository.py new file mode 100644 index 000000000..39278fdb9 --- /dev/null +++ b/desktop/installers/build_apt_repository.py @@ -0,0 +1,101 @@ +"""Build a new, signed APT snapshot from verified CommunityAI packages; never publish it.""" + +import argparse +import datetime +import email.utils +import gzip +import hashlib +import json +import re +import shutil +import subprocess +from pathlib import Path + + +def build(packages, output, signing_key): + if not re.fullmatch(r"[A-Fa-f0-9]{40}|[A-Fa-f0-9]{64}", signing_key): + raise ValueError("Use the complete repository signing-key fingerprint") + if not packages: + raise ValueError("At least one verified CommunityAI package is required") + output = output.resolve() + output.mkdir(parents=True, exist_ok=False) + pool = output / "pool/main/c/communityai" + pool.mkdir(parents=True) + selected = set() + for package in packages: + package = package.resolve(strict=True) + fields = subprocess.check_output( + ["dpkg-deb", "--show", "--showformat=${Package}\t${Version}\t${Architecture}", str(package)], text=True + ).split("\t") + if len(fields) != 3 or fields[0] != "communityai" or fields[2] != "amd64": + raise ValueError("Only the CommunityAI amd64 package belongs in this repository") + version = fields[1] + if not re.fullmatch(r"[0-9][A-Za-z0-9.+~\-]*", version) or version in selected: + raise ValueError("Invalid or duplicate package version") + selected.add(version) + shutil.copyfile(package, pool / f"communityai_{version}_amd64.deb") + distribution = output / "dists/alpha" + index_root = distribution / "main/binary-amd64" + index_root.mkdir(parents=True) + index = subprocess.check_output(["apt-ftparchive", "packages", "pool"], cwd=output) + (index_root / "Packages").write_bytes(index) + (index_root / "Packages.gz").write_bytes(gzip.compress(index, mtime=0)) + valid_until = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=30) + options = { + "Origin": "CommunityAI", + "Label": "CommunityAI", + "Suite": "alpha", + "Codename": "alpha", + "Architectures": "amd64", + "Components": "main", + "Description": "CommunityAI alpha packages", + "Valid-Until": email.utils.format_datetime(valid_until, usegmt=True), + } + command = ["apt-ftparchive"] + for name, value in options.items(): + command.extend(["-o", f"APT::FTPArchive::Release::{name}={value}"]) + release = distribution / "Release" + release.write_bytes(subprocess.check_output([*command, "release", "dists/alpha"], cwd=output)) + keyring = output / "communityai-archive-keyring.gpg" + exported = subprocess.check_output(["gpg", "--batch", "--export", signing_key]) + if not exported: + raise ValueError("The repository public signing key is unavailable") + keyring.write_bytes(exported) + for action, filename in (("--clearsign", "InRelease"), ("--detach-sign", "Release.gpg")): + target = distribution / filename + subprocess.run( + ["gpg", "--batch", "--local-user", signing_key, "--output", str(target), action, str(release)], check=True + ) + verification = ["gpgv", "--keyring", str(keyring), str(target)] + if action == "--detach-sign": + verification.append(str(release)) + subprocess.run(verification, check=True) + artifacts = {} + for path in sorted(output.rglob("*")): + if path.is_file(): + with path.open("rb") as source: + artifacts[path.relative_to(output).as_posix()] = hashlib.file_digest(source, "sha256").hexdigest() + (output / "repository.json").write_text( + json.dumps( + { + "schema_version": 1, + "signing_key_fingerprint": signing_key.upper(), + "valid_until": valid_until.isoformat(), + "sha256": artifacts, + "published": False, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--signing-key", required=True) + args = parser.parse_args() + print(build(args.package, args.output, args.signing_key)) diff --git a/desktop/installers/build_deb.py b/desktop/installers/build_deb.py new file mode 100644 index 000000000..9661fbe39 --- /dev/null +++ b/desktop/installers/build_deb.py @@ -0,0 +1,113 @@ +"""Package an already verified Linux runtime; keep per-user state outside dpkg ownership.""" + +import argparse +import errno +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path + + +def copy_bundle(source, destination): + """Preserve runtime hardlink groups, including the cross-device fallback.""" + copied = {} + + def link_or_copy(source, destination): + info = os.stat(source, follow_symlinks=False) + identity = info.st_dev, info.st_ino + prior = copied.get(identity) + if prior is not None: + os.link(prior, destination, follow_symlinks=False) + else: + try: + os.link(source, destination, follow_symlinks=False) + except OSError as exc: + if exc.errno != errno.EXDEV: + raise + shutil.copy2(source, destination) + copied[identity] = destination + return destination + + shutil.copytree(source, destination, symlinks=True, copy_function=link_or_copy) + + +def installed_size_kib(root): + """Count each hardlinked payload once; symlinks do not copy target contents.""" + unique = {} + for path in root.rglob("*"): + info = path.lstat() + if stat.S_ISREG(info.st_mode): + unique[(info.st_dev, info.st_ino)] = info.st_size + return (sum(unique.values()) + 1023) // 1024 + + +def build(bundle, output, version, maintainer): + bundle, output = bundle.resolve(), output.resolve() + if not (bundle / "CommunityAI").is_file() or not (bundle / "node/CommunityAI-Node").is_file(): + raise ValueError("Expected the complete verified Linux CommunityAI bundle") + if not re.fullmatch(r"[0-9][A-Za-z0-9.+~\-]*", version): + raise ValueError("Invalid Debian version") + if not re.fullmatch(r"[^<>\r\n]+ <[^<>\s]+@[^<>\s]+>", maintainer): + raise ValueError("Maintainer must have a name and email address") + output.mkdir(parents=True, exist_ok=True) + scripts = Path(__file__).resolve().parent + + with tempfile.TemporaryDirectory(prefix="communityai-deb-", dir=bundle.parent) as temporary: + root = Path(temporary) + root.chmod(0o755) + app = root / "opt/communityai" + # Hardlinks avoid duplicating several GiB of verified CUDA libraries. + copy_bundle(bundle, app) + (app / ".communityai-installation").unlink(missing_ok=True) + shutil.copyfile(scripts / "installation-marker.txt", app / ".communityai-installation") + shutil.copyfile(scripts / "linux_online_root.py", app / "update_install.py") + control = root / "DEBIAN" + control.mkdir() + size_kib = installed_size_kib(app) + (control / "control").write_text( + f"Package: communityai\nVersion: {version}\nArchitecture: amd64\n" + f"Maintainer: {maintainer}\nInstalled-Size: {size_kib}\n" + "Section: science\nPriority: optional\nPre-Depends: python3 (>= 3.9)\n" + "Depends: libc6 (>= 2.35), libstdc++6, libdbus-1-3, libegl1, libgl1, libglib2.0-0, " + "libfontconfig1, libfreetype6, libxkbcommon0, libxkbcommon-x11-0, libxcb-cursor0, " + "libxcb-icccm4, libxcb-image0, libxcb-keysyms1, libxcb-render-util0, libxcb-shape0, libwayland-cursor0\n" + "Recommends: gnome-keyring, policykit-1\nHomepage: https://github.com/flujo-app/CommunityAI\n" + "Description: CommunityAI public inference alpha\n" + " Local OpenAI-compatible inference with optional community model sharing.\n" + " Model weights download on demand. Settings and cache survive removal.\n", + encoding="utf-8", + ) + for name in ("preinst", "prerm"): + shutil.copyfile(scripts / "linux_maintenance.py", control / name) + (control / name).chmod(0o755) + executable = root / "usr/bin/communityai" + executable.parent.mkdir(parents=True) + executable.write_text('#!/bin/sh\nexec /opt/communityai/CommunityAI "$@"\n', encoding="utf-8") + executable.chmod(0o755) + desktop = root / "usr/share/applications/communityai.desktop" + desktop.parent.mkdir(parents=True) + desktop.write_text( + "[Desktop Entry]\nType=Application\nName=CommunityAI\n" + "Comment=Local and community AI inference\nExec=communityai\n" + "Terminal=false\nCategories=Science;Utility;\n", + encoding="utf-8", + ) + target = output / f"communityai_{version}_amd64.deb" + subprocess.run(["dpkg-deb", "--root-owner-group", "-Zxz", "-z1", "--build", str(root), str(target)], check=True) + subprocess.run(["dpkg-deb", "--info", str(target)], check=True) + print(json.dumps({"artifact": str(target), "version": version, "settings_cache_policy": "preserved"})) + return target + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--maintainer", required=True) + args = parser.parse_args() + build(args.bundle, args.output, args.version, args.maintainer) diff --git a/desktop/installers/build_linux_online.py b/desktop/installers/build_linux_online.py new file mode 100644 index 000000000..fec2a9a61 --- /dev/null +++ b/desktop/installers/build_linux_online.py @@ -0,0 +1,64 @@ +"""Generate a standalone Python 3.9+ Linux installer pinned to one release manifest entry.""" + +import argparse +import hashlib +import json +from pathlib import Path + +from release_downloads import load_release_manifest, select_artifact + + +def build(manifest_path, output): + output = Path(output) + metadata_path = output.with_suffix(output.suffix + ".json") + if output.exists() or output.is_symlink() or metadata_path.exists() or metadata_path.is_symlink(): + raise FileExistsError("Online installer output or metadata already exists") + artifact = select_artifact(load_release_manifest(manifest_path), "linux-amd64") + template = Path(__file__).with_name("linux_online_template.py").read_text(encoding="utf-8") + marker = "ARTIFACT = None # Replaced with a validated release entry by build_linux_online.py." + if template.count(marker) != 1: + raise ValueError("Linux online installer template marker is missing or ambiguous") + # Python repr produces a literal, never executable text from a manifest URL. + rendered = template.replace(marker, "ARTIFACT = " + repr(artifact)) + helper_marker = "ROOT_HELPER = None # Replaced with the isolated protected-copy helper by the builder." + if rendered.count(helper_marker) != 1: + raise ValueError("Linux protected installation helper marker is missing or ambiguous") + helper = Path(__file__).with_name("linux_online_root.py").read_text(encoding="utf-8") + rendered = rendered.replace(helper_marker, "ROOT_HELPER = " + repr(helper)) + compile(rendered, str(output), "exec") + with output.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(rendered) + output.chmod(0o755) + encoded = rendered.encode("utf-8") + metadata = { + "schema_version": 1, + "kind": "online-installer", + "platform": "linux-amd64", + "format": "python3-script", + "minimum_python": "3.9", + "version": artifact["version"], + "filename": output.name, + "sha256": hashlib.sha256(encoded).hexdigest(), + "size_bytes": len(encoded), + "offline_installer": artifact, + "publisher_signature": False, + "live_download_verified": False, + } + with metadata_path.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(metadata, indent=2) + "\n") + print(json.dumps({"artifact": str(output), "package_url": artifact["url"], "package_sha256": artifact["sha256"]})) + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument( + "--output", type=Path, required=True, help="new output .py file; existing files are never replaced" + ) + args = parser.parse_args() + build(args.manifest, args.output) + + +if __name__ == "__main__": + main() diff --git a/desktop/installers/build_windows_installer.ps1 b/desktop/installers/build_windows_installer.ps1 new file mode 100644 index 000000000..cb576c50c --- /dev/null +++ b/desktop/installers/build_windows_installer.ps1 @@ -0,0 +1,48 @@ +param( + [Parameter(Mandatory=$true)][string]$Bundle, + [Parameter(Mandatory=$true)][string]$OutputDirectory, + [Parameter(Mandatory=$true)][string]$Version, + [string]$Compiler = 'ISCC.exe', + [string]$PublisherName = 'CommunityAI contributors', + [string]$AppIdentifier = 'CommunityAI.Desktop', + [string]$SigningToolCommand, + [switch]$UnsignedEngineering +) +$ErrorActionPreference = 'Stop' +$bundlePath = (Resolve-Path -LiteralPath $Bundle).Path +if (-not (Test-Path -LiteralPath (Join-Path $bundlePath 'CommunityAI.exe') -PathType Leaf)) { + throw 'Bundle must contain CommunityAI.exe' +} +if (-not (Test-Path -LiteralPath (Join-Path $bundlePath 'node\CommunityAI-Node.exe') -PathType Leaf)) { + throw 'Bundle must contain the packaged node runtime' +} +if ($Version -notmatch '^\d+\.\d+\.\d+(?:[.-][A-Za-z0-9]+)*$') { throw 'Invalid version' } +if ($PublisherName -match '["\r\n]') { throw 'Invalid publisher name' } +if ($AppIdentifier -notmatch '^[A-Za-z0-9.-]+$') { throw 'Invalid application identifier' } +if (-not $SigningToolCommand -and -not $UnsignedEngineering) { + throw 'A signing tool is required; explicitly select UnsignedEngineering for a test build' +} +if ($SigningToolCommand -and $UnsignedEngineering) { throw 'Choose signed or unsigned engineering output' } +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +$outputPath = (Resolve-Path -LiteralPath $OutputDirectory).Path +$arguments = @('/Qp', "/DBundleDir=$bundlePath", "/DOutputPath=$outputPath", "/DAppVersion=$Version", "/DPublisherName=$PublisherName") +$arguments += "/DAppIdentifier=$AppIdentifier" +if ($SigningToolCommand) { + $arguments += @('/DSigningTool=communityai', "/Scommunityai=$SigningToolCommand") +} +$arguments += (Join-Path $PSScriptRoot 'communityai.iss') +& $Compiler @arguments +if ($LASTEXITCODE -ne 0) { throw "Installer compiler failed: $LASTEXITCODE" } +$installer = Join-Path $outputPath "communityai-$Version-windows-setup.exe" +$signature = Get-AuthenticodeSignature -LiteralPath $installer +if ($SigningToolCommand -and $signature.Status -ne 'Valid') { throw 'The installer has no valid Authenticode signature' } +$metadata = @{ + version = $Version + publisher = $PublisherName + unsigned_engineering = [bool]$UnsignedEngineering + sha256 = (Get-FileHash -LiteralPath $installer -Algorithm SHA256).Hash.ToLowerInvariant() + authenticode_status = [string]$signature.Status + settings_cache_policy = 'Preserved on upgrade and uninstall' +} +$metadata | ConvertTo-Json | Set-Content -LiteralPath "$installer.json" -Encoding utf8 +Write-Output $installer diff --git a/desktop/installers/build_windows_online_installer.ps1 b/desktop/installers/build_windows_online_installer.ps1 new file mode 100644 index 000000000..c10755415 --- /dev/null +++ b/desktop/installers/build_windows_online_installer.ps1 @@ -0,0 +1,91 @@ +param( + [Parameter(Mandatory=$true)][string]$Manifest, + [Parameter(Mandatory=$true)][string]$OutputDirectory, + [string]$Compiler = 'ISCC.exe', + [string]$PythonCommand = 'python', + [string]$SigningToolCommand, + [switch]$UnsignedAlpha, + [switch]$ValidateOnly +) +$ErrorActionPreference = 'Stop' +$manifestPath = (Resolve-Path -LiteralPath $Manifest).Path +$selector = Join-Path $PSScriptRoot 'release_downloads.py' +$selectedJson = & $PythonCommand $selector select $manifestPath --platform windows-x64 +if ($LASTEXITCODE -ne 0) { throw 'The release download manifest was rejected' } +$artifact = ($selectedJson -join "`n") | ConvertFrom-Json + +# Defense at the preprocessor boundary as well as the common manifest validator. +if ($artifact.platform -cne 'windows-x64' -or $artifact.kind -cne 'offline-installer' -or $artifact.format -cne 'exe') { + throw 'The selected artifact is not a Windows offline installer' +} +if ($artifact.version -notmatch '^\d+\.\d+\.\d+(?:[.-][A-Za-z0-9]+)*$') { throw 'Invalid release version' } +if ($artifact.filename -notmatch '^[A-Za-z0-9][A-Za-z0-9._-]*\.exe$') { throw 'Invalid installer filename' } +if ($artifact.sha256 -cnotmatch '^[0-9a-f]{64}$') { throw 'Invalid installer SHA-256' } +if ($artifact.size_bytes -isnot [long] -and $artifact.size_bytes -isnot [int]) { throw 'Invalid installer byte size' } +if ($artifact.size_bytes -le 0) { throw 'Invalid installer byte size' } +if ($artifact.url -notmatch '^https://[^\s"''{}\\]+$') { throw 'Invalid HTTPS installer URL' } +$publisher = if ($artifact.publisher) { $artifact.publisher } else { 'CommunityAI contributors' } +if ($publisher -match '["''{}\r\n]' -or $publisher.Length -gt 160) { throw 'Invalid publisher name' } +if ($SigningToolCommand -and $UnsignedAlpha) { throw 'Choose signed or explicitly unsigned output' } +if (-not $SigningToolCommand -and -not $UnsignedAlpha) { throw 'Specify a signing command or explicitly select UnsignedAlpha' } + +if ($ValidateOnly) { + $artifact | ConvertTo-Json -Depth 10 + exit 0 +} +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +$outputPath = (Resolve-Path -LiteralPath $OutputDirectory).Path +$filename = "communityai-$($artifact.version)-windows-online-setup.exe" +$installerPath = Join-Path $outputPath $filename +if (Test-Path -LiteralPath $installerPath) { throw 'Refusing to overwrite an existing online installer' } +$helperSource = Join-Path $PSScriptRoot 'WindowsDownload.cs' +$frameworkCompiler = Join-Path $env:WINDIR 'Microsoft.NET\Framework64\v4.0.30319\csc.exe' +if (-not (Test-Path -LiteralPath $frameworkCompiler -PathType Leaf)) { + $frameworkCompiler = Join-Path $env:WINDIR 'Microsoft.NET\Framework\v4.0.30319\csc.exe' +} +if (-not (Test-Path -LiteralPath $frameworkCompiler -PathType Leaf)) { + throw 'The Windows .NET Framework C# compiler is required to build the downloader helper' +} +$helperBuild = Join-Path $outputPath 'downloader-build' +New-Item -ItemType Directory -Path $helperBuild -Force | Out-Null +$helperPath = Join-Path $helperBuild 'WindowsDownload.exe' +& $frameworkCompiler /nologo /target:exe /platform:anycpu /optimize+ "/out:$helperPath" $helperSource +if ($LASTEXITCODE -ne 0) { throw "Downloader helper compiler failed: $LASTEXITCODE" } +$arguments = @( + '/Qp', + "/DOutputPath=$outputPath", + "/DAppVersion=$($artifact.version)", + "/DPublisherName=$publisher", + "/DInstallerUrl=$($artifact.url)", + "/DInstallerFilename=$($artifact.filename)", + "/DInstallerSha256=$($artifact.sha256)", + "/DInstallerSize=$($artifact.size_bytes)", + "/DDownloadHelper=$helperPath" +) +if ($SigningToolCommand) { $arguments += @('/DSigningTool=communityai', "/Scommunityai=$SigningToolCommand") } +$arguments += (Join-Path $PSScriptRoot 'communityai-online.iss') +& $Compiler @arguments +if ($LASTEXITCODE -ne 0) { throw "Online installer compiler failed: $LASTEXITCODE" } +if (-not (Test-Path -LiteralPath $installerPath -PathType Leaf)) { throw 'The compiler did not produce the online installer' } +$signature = Get-AuthenticodeSignature -LiteralPath $installerPath +if ($SigningToolCommand -and $signature.Status -ne 'Valid') { throw 'The online installer has no valid Authenticode signature' } +$metadata = @{ + schema_version = 1 + kind = 'online-installer' + version = $artifact.version + platform = 'windows-x64' + filename = $filename + size_bytes = (Get-Item -LiteralPath $installerPath).Length + sha256 = (Get-FileHash -LiteralPath $installerPath -Algorithm SHA256).Hash.ToLowerInvariant() + authenticode_status = [string]$signature.Status + unsigned_alpha = [bool]$UnsignedAlpha + offline_installer = $artifact + release_manifest_sha256 = (Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant() + download_helper_sha256 = (Get-FileHash -LiteralPath $helperPath -Algorithm SHA256).Hash.ToLowerInvariant() + download_helper_source_sha256 = (Get-FileHash -LiteralPath $helperSource -Algorithm SHA256).Hash.ToLowerInvariant() + installer_script_sha256 = (Get-FileHash -LiteralPath (Join-Path $PSScriptRoot 'communityai-online.iss') -Algorithm SHA256).Hash.ToLowerInvariant() + builder_script_sha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant() + live_download_verified = $false +} +$metadata | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath "$installerPath.json" -Encoding utf8 +Write-Output $installerPath diff --git a/desktop/installers/communityai-online.iss b/desktop/installers/communityai-online.iss new file mode 100644 index 000000000..2ee5ad774 --- /dev/null +++ b/desktop/installers/communityai-online.iss @@ -0,0 +1,471 @@ +; Small verified downloader. The offline setup owns every product installation. +; Build through build_windows_online_installer.ps1 with validated release metadata. +#if VER != EncodeVer(6, 7, 3) + #error The native process bridge is qualified only for Inno Setup 6.7.3 (32-bit Setup engine). +#endif +#ifndef AppVersion + #error AppVersion is required +#endif +#ifndef InstallerUrl + #error InstallerUrl is required +#endif +#ifndef InstallerFilename + #error InstallerFilename is required +#endif +#ifndef InstallerSha256 + #error InstallerSha256 is required +#endif +#ifndef InstallerSize + #error InstallerSize is required +#endif +#ifndef DownloadHelper + #error DownloadHelper is required +#endif +#ifndef OutputPath + #error OutputPath is required +#endif +#ifndef PublisherName + #define PublisherName "CommunityAI contributors" +#endif + +[Setup] +AppId=CommunityAI.OnlineDownloader +AppName=CommunityAI Online Setup +AppVersion={#AppVersion} +AppPublisher={#PublisherName} +AppPublisherURL=https://github.com/flujo-app/CommunityAI +PrivilegesRequired=lowest +ArchitecturesAllowed=x64compatible +MinVersion=10.0 +CreateAppDir=no +Uninstallable=no +DisableDirPage=yes +DisableProgramGroupPage=yes +DisableWelcomePage=no +DisableFinishedPage=yes +CloseApplications=no +RestartApplications=no +OutputDir={#OutputPath} +OutputBaseFilename=communityai-{#AppVersion}-windows-online-setup +Compression=lzma2/fast +WizardStyle=modern +SetupLogging=yes +#ifdef SigningTool +SignTool={#SigningTool} +#endif + +[Messages] +WelcomeLabel2=This downloads the complete CommunityAI setup, including its CPU and NVIDIA GPU runtime.%n%nDownload size: {#InstallerSize} bytes. Model files are downloaded separately when needed.%n%nAfter verification, the regular CommunityAI installer will open. +ReadyLabel1=Ready to download CommunityAI {#AppVersion}. +ReadyLabel2a=Setup will verify the complete download before opening the CommunityAI installer. +ButtonInstall=&Download and install + +[Files] +Source: "{#DownloadHelper}"; Flags: dontcopy + +[Code] +type + // Inno 6's Setup engine uses the Win32 ABI, including on x64 Windows. + TStartupInfo32 = record + cb, lpReserved, lpDesktop, lpTitle: LongWord; + dwX, dwY, dwXSize, dwYSize, dwXCountChars, dwYCountChars: LongWord; + dwFillAttribute, dwFlags: LongWord; + wShowWindow, cbReserved2: Word; + lpReserved2, hStdInput, hStdOutput, hStdError: LongWord; + end; + TProcessInformation32 = record + hProcess, hThread, dwProcessId, dwThreadId: LongWord; + end; + TJobExtendedLimit32 = record + PerProcessUserTimeLimit, PerJobUserTimeLimit: Int64; + LimitFlags, MinimumWorkingSetSize, MaximumWorkingSetSize: LongWord; + ActiveProcessLimit, Affinity, PriorityClass, SchedulingClass, Padding: LongWord; + ReadOperationCount, WriteOperationCount, OtherOperationCount: Int64; + ReadTransferCount, WriteTransferCount, OtherTransferCount: Int64; + ProcessMemoryLimit, JobMemoryLimit, PeakProcessMemoryUsed, PeakJobMemoryUsed: LongWord; + end; + +function CreateProcessW(ApplicationName, CommandLine: String; + ProcessAttributes, ThreadAttributes, InheritHandles, CreationFlags, + Environment: LongWord; CurrentDirectory: String; + var StartupInfo: TStartupInfo32; var ProcessInformation: TProcessInformation32): Boolean; + external 'CreateProcessW@kernel32.dll stdcall'; +function CreateJobObjectW(Attributes, Name: LongWord): LongWord; + external 'CreateJobObjectW@kernel32.dll stdcall'; +function SetInformationJobObject(Job: LongWord; InformationClass: Integer; + var Information: TJobExtendedLimit32; InformationLength: LongWord): Boolean; + external 'SetInformationJobObject@kernel32.dll stdcall'; +function AssignProcessToJobObject(Job, Process: LongWord): Boolean; + external 'AssignProcessToJobObject@kernel32.dll stdcall'; +function ResumeThread(Thread: LongWord): LongWord; + external 'ResumeThread@kernel32.dll stdcall'; +function WaitForSingleObject(Handle, Milliseconds: LongWord): LongWord; + external 'WaitForSingleObject@kernel32.dll stdcall'; +function GetExitCodeProcess(Process: LongWord; var ExitCode: LongWord): Boolean; + external 'GetExitCodeProcess@kernel32.dll stdcall'; +function TerminateProcess(Process, ExitCode: LongWord): Boolean; + external 'TerminateProcess@kernel32.dll stdcall'; +function TerminateJobObject(Job, ExitCode: LongWord): Boolean; + external 'TerminateJobObject@kernel32.dll stdcall'; +function CloseHandle(Handle: LongWord): Boolean; + external 'CloseHandle@kernel32.dll stdcall'; +function GetCurrentProcess: LongWord; + external 'GetCurrentProcess@kernel32.dll stdcall'; +function GetCurrentProcessId: LongWord; + external 'GetCurrentProcessId@kernel32.dll stdcall'; +function GetProcessTimes(Process: LongWord; var Created, Exited, KernelTime, UserTime: Int64): Boolean; + external 'GetProcessTimes@kernel32.dll stdcall'; + +var + DownloadPage: TOutputProgressWizardPage; + DownloadCancelButton: TNewButton; + CancelRequested, DownloadActive, NoCancel, HelperStopped: Boolean; + ProgressPath, CancelPath: String; + LastLoggedPercent, LastStatus, ScaledProgress: Integer; + // Pascal Script initializes globals to zero; these templates remain untouched. + EmptyStartup: TStartupInfo32; + EmptyLimits: TJobExtendedLimit32; + DownloadVerified: Boolean; + DownloadStartedTick: Int64; + ChildStarted: Boolean; + ChildExitCode: Integer; + ChildParameters: String; + +function GetTickCount64: Int64; + external 'GetTickCount64@kernel32.dll stdcall'; + +procedure ReportFailure(const Message: String); +begin + Log(Message); + // /VERYSILENT alone does not suppress Inno message boxes. Our own failures + // must reach the nonzero exit without waiting for an unattended dialog. + if not WizardSilent then + SuppressibleMsgBox(Message, mbError, MB_OK, IDOK); +end; + +function SafeArgument(const Value: String): Boolean; +var + Index: Integer; +begin + Result := False; + if Pos('"', Value) <> 0 then + Exit; + for Index := 1 to Length(Value) do + if (Ord(Value[Index]) < 32) or (Ord(Value[Index]) = 127) then + Exit; + Result := True; +end; + +function ForwardedSwitch(const Value: String): Boolean; +begin + Result := (Value = '/SILENT') or (Value = '/VERYSILENT') or + (Value = '/SUPPRESSMSGBOXES') or (Value = '/NORESTART') or + (Value = '/SP-') or (Value = '/NOICONS') or + (Value = '/NOCANCEL') or (Value = '/CURRENTUSER'); +end; + +function ForwardedValue(const Value: String): Boolean; +begin + Result := (Pos('/DIR=', Value) = 1) or (Pos('/GROUP=', Value) = 1) or + (Pos('/LANG=', Value) = 1) or (Pos('/RESTARTEXITCODE=', Value) = 1); +end; + +function InitializeSetup: Boolean; +var + Index: Integer; + Argument, UpperArgument: String; +begin + Result := False; + ChildExitCode := 1; + ChildParameters := ''; + // ParamStr excludes Inno's private loader/notification arguments. GetCmdTail + // includes them, so forwarding that raw string to another setup is unsafe. + for Index := 1 to ParamCount do begin + Argument := ParamStr(Index); + UpperArgument := UpperCase(Argument); + if not SafeArgument(Argument) then begin + ReportFailure('An installer argument contains unsupported characters.'); + Exit; + end; + if UpperArgument = '/NOCANCEL' then + NoCancel := True; + if ForwardedSwitch(UpperArgument) or ForwardedValue(UpperArgument) then + ChildParameters := ChildParameters + ' "' + Argument + '"' + else if (UpperArgument = '/LOG') or (Pos('/LOG=', UpperArgument) = 1) then begin + // /LOG belongs to this downloader. Give the full setup its own unique log + // instead of making two processes overwrite the same explicitly named log. + ChildParameters := ChildParameters + ' /LOG'; + end + else if Pos('/INSTALLERLOG=', UpperArgument) = 1 then + ChildParameters := ChildParameters + ' "/LOG=' + Copy(Argument, 15, MaxInt) + '"' + else begin + ReportFailure('Unsupported online setup argument: ' + Argument + + Chr(13) + Chr(10) + 'Use the full offline installer for additional setup options.'); + Exit; + end; + end; + Result := True; +end; + +procedure DownloadCancelClick(Sender: TObject); +begin + CancelRequested := True; + DownloadCancelButton.Enabled := False; + SaveStringToFile(CancelPath, '1', False); +end; + +procedure CancelButtonClick(CurPageID: Integer; var Cancel, Confirm: Boolean); +begin + if DownloadActive then begin + Cancel := False; + Confirm := False; + if not NoCancel then + DownloadCancelClick(WizardForm.CancelButton); + end; +end; + +procedure InitializeWizard; +begin + DownloadPage := CreateOutputProgressPage('Downloading CommunityAI', + 'Interrupted connections resume automatically. The complete file is verified before setup opens.'); + DownloadCancelButton := TNewButton.Create(DownloadPage); + DownloadCancelButton.Parent := DownloadPage.Surface; + DownloadCancelButton.Top := DownloadPage.ProgressBar.Top + DownloadPage.ProgressBar.Height + ScaleY(12); + DownloadCancelButton.Width := WizardForm.CancelButton.Width; + DownloadCancelButton.Height := WizardForm.CancelButton.Height; + DownloadCancelButton.Caption := 'Cancel'; + DownloadCancelButton.OnClick := @DownloadCancelClick; +end; + +procedure UpdateDownloadProgress; +var + Raw: AnsiString; + Value, StatusText: String; + Separator, Status, Percent: Integer; + Received, Total, FileBytes: Int64; +begin + // An atomic helper record can briefly be unavailable while being replaced. + // It is only presentation; exit status and independent size/hash checks gate execution. + FileBytes := -1; + FileSize64(ExpandConstant('{tmp}\{#InstallerFilename}'), FileBytes); + if (FileBytes >= 0) and (FileBytes <= {#InstallerSize}) then begin + ScaledProgress := FileBytes * 10000 div {#InstallerSize}; + DownloadPage.SetText('Downloading the complete runtime...', + IntToStr(FileBytes) + ' / {#InstallerSize} bytes'); + end; + if LoadStringFromFile(ProgressPath, Raw) then begin + Value := Trim(String(Raw)); + Separator := Pos('|', Value); + if Separator > 0 then begin + Received := StrToInt64Def(Copy(Value, 1, Separator - 1), -1); + Delete(Value, 1, Separator); + Separator := Pos('|', Value); + if Separator > 0 then begin + Total := StrToInt64Def(Copy(Value, 1, Separator - 1), -1); + Status := StrToIntDef(Copy(Value, Separator + 1, MaxInt), -1); + if (Received >= 0) and (Received <= {#InstallerSize}) and + (Total = {#InstallerSize}) and (Status >= 0) and (Status <= 5) then begin + // A locked progress record can be stale. The growing local file still + // supplies byte progress, without participating in verification. + if (FileBytes > Received) and (FileBytes <= Total) then + Received := FileBytes; + StatusText := 'Downloading the complete runtime...'; + if Status = 1 then + StatusText := 'Connection interrupted. Resuming the download...' + else if Status = 2 then + StatusText := 'Verifying the complete download...' + else if Status = 3 then + StatusText := 'Download verified.'; + DownloadPage.SetText(StatusText, IntToStr(Received) + ' / ' + IntToStr(Total) + ' bytes'); + ScaledProgress := Received * 10000 div Total; + DownloadPage.SetProgress(ScaledProgress, 10000); + Percent := Received * 100 div Total; + if (Percent >= LastLoggedPercent + 5) or (Status <> LastStatus) then begin + Log(IntToStr(Received) + ' of ' + IntToStr(Total) + ' bytes done. Download status ' + IntToStr(Status) + '.'); + LastLoggedPercent := Percent; + LastStatus := Status; + end; + end; + end; + end; + end; + // SetText/SetProgress dispatch pending wizard input, including cancellation. + if CancelRequested then + DownloadPage.SetText('Cancelling the download...', 'No installer will be launched.') + else + DownloadPage.SetProgress(ScaledProgress, 10000); +end; + +procedure RunDownloadHelper; +var + Startup: TStartupInfo32; + ProcessInfo: TProcessInformation32; + Limits: TJobExtendedLimit32; + Job, ExitCode, WaitCode: LongWord; + Created, Exited, KernelTime, UserTime, CancelTick: Int64; + Helper, Arguments: String; + ErrorText: AnsiString; +begin + ExtractTemporaryFile('WindowsDownload.exe'); + Helper := ExpandConstant('{tmp}\WindowsDownload.exe'); + ProgressPath := ExpandConstant('{tmp}\download-progress'); + CancelPath := ExpandConstant('{tmp}\download-cancel'); + DeleteFile(CancelPath); + DeleteFile(ProgressPath); + DeleteFile(ProgressPath + '.error'); + Startup := EmptyStartup; + Startup.cb := 68; + Startup.dwFlags := 1; + Startup.wShowWindow := 0; + Limits := EmptyLimits; + Limits.LimitFlags := $2000; // JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + Job := CreateJobObjectW(0, 0); + if Job = 0 then + RaiseException('Could not create the download process container.'); + try + if not SetInformationJobObject(Job, 9, Limits, 112) then + RaiseException('Could not configure download process cleanup.'); + if not GetProcessTimes(GetCurrentProcess, Created, Exited, KernelTime, UserTime) then + RaiseException('Could not identify this setup process.'); + Arguments := ' "{#InstallerUrl}" "' + ExpandConstant('{tmp}\{#InstallerFilename}') + + '" {#InstallerSize} {#InstallerSha256} "' + ProgressPath + '" "' + CancelPath + + '" ' + IntToStr(GetCurrentProcessId) + ' ' + IntToStr(Created); + Log('Downloading pinned installer: {#InstallerUrl}'); + // Assign the suspended helper to its job before it can perform any I/O. + if not CreateProcessW(Helper, '"' + Helper + '"' + Arguments, 0, 0, 0, + $08000004, 0, ExpandConstant('{tmp}'), Startup, ProcessInfo) then + RaiseException('Could not start the download helper.'); + HelperStopped := False; + try + if not AssignProcessToJobObject(Job, ProcessInfo.hProcess) then + RaiseException('Could not contain the download helper.'); + if ResumeThread(ProcessInfo.hThread) = $FFFFFFFF then + RaiseException('Could not resume the download helper.'); + CancelTick := 0; + repeat + UpdateDownloadProgress; + if GetTickCount64 - DownloadStartedTick >= 7200000 then begin + CancelRequested := True; + SaveStringToFile(CancelPath, '1', False); + end; + if CancelRequested then begin + if CancelTick = 0 then + CancelTick := GetTickCount64; + if GetTickCount64 - CancelTick >= 5000 then begin + TerminateJobObject(Job, 1); + if WaitForSingleObject(ProcessInfo.hProcess, 5000) <> 0 then + RaiseException('Could not confirm that the cancelled download stopped.'); + end; + end; + WaitCode := WaitForSingleObject(ProcessInfo.hProcess, 100); + until WaitCode <> 258; + UpdateDownloadProgress; + if (WaitCode <> 0) or not GetExitCodeProcess(ProcessInfo.hProcess, ExitCode) then + RaiseException('Could not confirm the download helper exit.'); + HelperStopped := True; + if CancelRequested then + RaiseException('Download cancelled or its two-hour limit expired.'); + if ExitCode <> 0 then begin + ErrorText := ''; + LoadStringFromFile(ProgressPath + '.error', ErrorText); + RaiseException('Download verification failed (helper exit ' + IntToStr(ExitCode) + '). ' + + Copy(String(ErrorText), 1, 1024)); + end; + finally + // This exact process belongs to this download only. The offline installer + // is launched later and is never placed in this kill-on-close job. + if WaitForSingleObject(ProcessInfo.hProcess, 0) <> 0 then begin + TerminateProcess(ProcessInfo.hProcess, 1); + end; + // Close the containing job before the final bounded wait, retaining the + // exact process handle until its exit has been checked. + CloseHandle(Job); + Job := 0; + HelperStopped := WaitForSingleObject(ProcessInfo.hProcess, 5000) = 0; + CloseHandle(ProcessInfo.hThread); + CloseHandle(ProcessInfo.hProcess); + if not HelperStopped then + RaiseException('Download cleanup could not confirm process exit. Temporary input was retained.'); + end; + finally + if Job <> 0 then + CloseHandle(Job); + end; +end; + +function NextButtonClick(CurPageID: Integer): Boolean; +var + DownloadedSize: Int64; + DownloadedFile: String; +begin + Result := True; + if CurPageID <> wpReady then + Exit; + DownloadVerified := False; + HelperStopped := True; + CancelRequested := False; + DownloadCancelButton.Enabled := not NoCancel; + LastLoggedPercent := -5; + LastStatus := -1; + ScaledProgress := 0; + DownloadedFile := ExpandConstant('{tmp}\{#InstallerFilename}'); + DownloadPage.SetText('Connecting to the download server...', ''); + DownloadPage.SetProgress(0, 10000); + DownloadPage.Show; + DownloadActive := True; + try + try + DownloadStartedTick := GetTickCount64; + RunDownloadHelper; + if not FileSize64(DownloadedFile, DownloadedSize) then + RaiseException('The downloaded installer could not be read.'); + if DownloadedSize <> {#InstallerSize} then + RaiseException('The downloaded installer has an unexpected size.'); + DownloadPage.SetText('Checking the installer before opening it...', ''); + if GetSHA256OfFile(DownloadedFile) <> '{#InstallerSha256}' then + RaiseException('The downloaded installer has an unexpected SHA-256.'); + if CancelRequested then + RaiseException('Download cancelled.'); + DownloadVerified := True; + Log('{#InstallerSize} of {#InstallerSize} bytes done. Independent size and SHA-256 verified.'); + except + if HelperStopped then + DeleteFile(DownloadedFile) + else + Log('Download helper exit unconfirmed; retaining temporary package: ' + DownloadedFile); + ReportFailure('CommunityAI could not be downloaded and verified. No installer was launched.' + + Chr(13) + Chr(10) + GetExceptionMessage); + Result := False; + end; + finally + DownloadActive := False; + DownloadPage.Hide; + end; +end; + +procedure CurStepChanged(CurStep: TSetupStep); +begin + if CurStep <> ssPostInstall then + Exit; + if not DownloadVerified then begin + ChildExitCode := 1; + ReportFailure('The installer has not passed download verification.'); + Exit; + end; + WizardForm.Hide; + // Execute the verified file directly, never through a shell or fetched script. + // Wait before exiting so Inno keeps {tmp} alive throughout the child install. + ChildStarted := Exec(ExpandConstant('{tmp}\{#InstallerFilename}'), ChildParameters, + ExpandConstant('{tmp}'), SW_SHOWNORMAL, ewWaitUntilTerminated, ChildExitCode); + if not ChildStarted then begin + ChildExitCode := 1; + ReportFailure('The verified CommunityAI installer could not be started.'); + end + else if ChildExitCode <> 0 then + Log(Format('The CommunityAI installer returned exit code %d.', [ChildExitCode])); +end; + +function GetCustomSetupExitCode: Integer; +begin + Result := ChildExitCode; +end; diff --git a/desktop/installers/communityai.iss b/desktop/installers/communityai.iss new file mode 100644 index 000000000..9f58645b1 --- /dev/null +++ b/desktop/installers/communityai.iss @@ -0,0 +1,101 @@ +#ifndef BundleDir + #error BundleDir must name the verified CommunityAI bundle +#endif +#ifndef AppVersion + #define AppVersion "0.1.0" +#endif +#ifndef PublisherName + #define PublisherName "CommunityAI contributors" +#endif +#ifndef OutputPath + #define OutputPath "dist" +#endif +#ifndef AppIdentifier + #define AppIdentifier "CommunityAI.Desktop" +#endif + +[Setup] +AppId={#AppIdentifier} +AppName=CommunityAI +AppVersion={#AppVersion} +AppPublisher={#PublisherName} +AppPublisherURL=https://github.com/flujo-app/CommunityAI +DefaultDirName={localappdata}\Programs\CommunityAI +DefaultGroupName=CommunityAI +PrivilegesRequired=lowest +ArchitecturesAllowed=x64compatible +ArchitecturesInstallIn64BitMode=x64compatible +MinVersion=10.0 +OutputDir={#OutputPath} +OutputBaseFilename=communityai-{#AppVersion}-windows-setup +Compression=lzma2/fast +SolidCompression=yes +WizardStyle=modern +DisableProgramGroupPage=yes +UninstallDisplayIcon={app}\CommunityAI.exe +CloseApplications=yes +RestartApplications=no +SetupLogging=yes +#ifdef SigningTool +SignTool={#SigningTool} +SignedUninstaller=yes +#else +SignedUninstaller=no +#endif + +[Files] +Source: "{#BundleDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs +Source: "installation-marker.txt"; DestDir: "{app}"; DestName: ".communityai-installation"; Flags: ignoreversion + +[Icons] +Name: "{group}\CommunityAI"; Filename: "{app}\CommunityAI.exe" +Name: "{group}\Uninstall CommunityAI"; Filename: "{uninstallexe}" + +[Run] +Filename: "{app}\CommunityAI.exe"; Description: "Open CommunityAI"; Flags: nowait postinstall skipifsilent +Filename: "{app}\CommunityAI.exe"; Flags: nowait runasoriginaluser; Check: IsSilentUpdate + +[InstallDelete] +Type: filesandordirs; Name: "{app}\_internal"; Check: HasInstallationMarker +Type: filesandordirs; Name: "{app}\node"; Check: HasInstallationMarker + +[Code] +function IsSilentUpdate: Boolean; +begin + Result := WizardSilent and (ExpandConstant('{param:UPDATE|0}') = '1'); +end; + +function HasInstallationMarker: Boolean; +begin + Result := FileExists(ExpandConstant('{app}\.communityai-installation')); +end; + +function StopInstalledApplication: Boolean; +var + ExitCode: Integer; + ApplicationPath: String; +begin + ApplicationPath := ExpandConstant('{app}\CommunityAI.exe'); + Result := True; + if FileExists(ApplicationPath) then + Result := Exec(ApplicationPath, '--prepare-update', ExpandConstant('{app}'), SW_HIDE, + ewWaitUntilTerminated, ExitCode) and (ExitCode = 0); +end; + +function PrepareToInstall(var NeedsRestart: Boolean): String; +begin + Result := ''; + if FileExists(ExpandConstant('{app}\CommunityAI.exe')) and not HasInstallationMarker then begin + Result := 'This folder contains an application not managed by this installer. Choose a new installation folder.'; + Exit; + end; + if not StopInstalledApplication then + Result := 'CommunityAI could not finish shutting down. Close CommunityAI and retry. No application files were replaced.'; +end; + +function InitializeUninstall: Boolean; +begin + Result := StopInstalledApplication; + if not Result then + MsgBox('CommunityAI could not finish shutting down. Close CommunityAI and retry uninstall.', mbError, MB_OK); +end; diff --git a/desktop/installers/installation-marker.txt b/desktop/installers/installation-marker.txt new file mode 100644 index 000000000..d72eab2fc --- /dev/null +++ b/desktop/installers/installation-marker.txt @@ -0,0 +1 @@ +CommunityAI installer-managed application files. User settings and model cache are stored separately. diff --git a/desktop/installers/linux_maintenance.py b/desktop/installers/linux_maintenance.py new file mode 100644 index 000000000..6c480a360 --- /dev/null +++ b/desktop/installers/linux_maintenance.py @@ -0,0 +1,89 @@ +#!/usr/bin/python3 +"""Stop only processes belonging to the installed CommunityAI tree before dpkg changes it.""" + +import os +import signal +import sys +import time +from pathlib import Path + + +def process_snapshot(): + result = {} + for directory in Path("/proc").iterdir(): + if not directory.name.isdigit(): + continue + try: + fields = (directory / "stat").read_text().rsplit(")", 1)[1].split() + executable = Path(os.readlink(directory / "exe").removesuffix(" (deleted)")) + result[int(directory.name)] = (int(fields[1]), fields[19], executable) + except PermissionError as exc: + raise RuntimeError( + "Cannot inspect process ownership; package replacement is refused. " + "Package maintenance needs permission to inspect all processes." + ) from exc + except (OSError, ValueError, IndexError): + continue + return result + + +def stop_installation(root=Path("/opt/communityai"), timeout=30): + root = root.resolve() + if not root.exists(): + return + marker = root / ".communityai-installation" + if not marker.is_file() or not marker.read_text().startswith("CommunityAI installer-managed"): + raise RuntimeError("The installation directory is not marked as CommunityAI-owned") + identities = {} + + def remaining(): + current = process_snapshot() + selected = { + pid + for pid, (_, started, exe) in current.items() + if pid != os.getpid() and (exe.is_relative_to(root) or identities.get(pid) == started) + } + while True: + descendants = {pid for pid, (parent, _, _) in current.items() if parent in selected and pid != os.getpid()} + if descendants <= selected: + break + selected |= descendants + identities.update({pid: current[pid][1] for pid in selected}) + return selected + + def send(pids, signum): + for pid in pids: + try: + descriptor = os.pidfd_open(pid) + try: + current = process_snapshot() + if pid in current and current[pid][1] == identities[pid]: + signal.pidfd_send_signal(descriptor, signum) + finally: + os.close(descriptor) + except ProcessLookupError: + pass + + for signum, duration in ((signal.SIGTERM, timeout), (signal.SIGKILL, 5)): + deadline = time.monotonic() + duration + signaled = set() + quiet_since = None + while time.monotonic() < deadline: + live = remaining() + now = time.monotonic() + if live: + quiet_since = None + new = {pid for pid in live if (pid, identities[pid]) not in signaled} + send(new, signum) + signaled.update((pid, identities[pid]) for pid in new) + else: + quiet_since = now if quiet_since is None else quiet_since + if now - quiet_since >= 0.3: + return + time.sleep(0.1) + raise RuntimeError("CommunityAI processes did not stay stopped; package replacement is refused") + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] in ("install", "upgrade", "remove", "deconfigure"): + stop_installation() diff --git a/desktop/installers/linux_online_root.py b/desktop/installers/linux_online_root.py new file mode 100644 index 000000000..b3d747953 --- /dev/null +++ b/desktop/installers/linux_online_root.py @@ -0,0 +1,117 @@ +"""Embedded, isolated root helper: verify a protected copy before handing it to APT.""" + +import contextlib +import hashlib +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + +MAX_PACKAGE_BYTES = 32 * 1024**3 +CHUNK_BYTES = 1024 * 1024 + + +@contextlib.contextmanager +def copy_deadline(): + def interrupted(signum, frame): + raise ValueError("Protected package copy cancelled or timed out") + + alarm = signal.signal(signal.SIGALRM, interrupted) + term = signal.signal(signal.SIGTERM, interrupted) + signal.setitimer(signal.ITIMER_REAL, 2 * 60 * 60) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, alarm) + signal.signal(signal.SIGTERM, term) + + +def protected_copy(source, destination, expected_size, expected_sha256): + descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + digest = hashlib.sha256() + received = 0 + with os.fdopen(descriptor, "rb") as incoming, destination.open("xb") as output: + metadata = os.fstat(incoming.fileno()) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_size: + raise ValueError("Source package is not the expected regular file") + while True: + block = incoming.read(min(CHUNK_BYTES, expected_size - received + 1)) + if not block: + break + received += len(block) + if received > expected_size: + raise ValueError("Package grew during protected copy") + digest.update(block) + output.write(block) + output.flush() + os.fsync(output.fileno()) + if received != expected_size or digest.hexdigest() != expected_sha256: + raise ValueError("Protected package size or SHA-256 verification failed") + + +def run( + source, + filename, + expected_size, + expected_sha256, + *, + directory=Path("/var/tmp"), + popen=subprocess.Popen, + noninteractive=False +): + if os.geteuid() != 0: + raise ValueError("The protected installation helper requires root") + if not re.fullmatch(r"communityai_[0-9][A-Za-z0-9.+~\-]*_amd64\.deb", filename): + raise ValueError("Invalid package filename") + if not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) or not 0 < expected_size <= MAX_PACKAGE_BYTES: + raise ValueError("Invalid pinned package identity") + if shutil.disk_usage(directory).free < expected_size + 64 * 1024 * 1024: + raise ValueError("Insufficient disk space for the protected package copy") + staging = Path(tempfile.mkdtemp(prefix="communityai-install-", dir=directory)) + package = staging / filename + keep_staging = False + try: + # The copy's root-owned directory prevents an ordinary-user writer from + # changing bytes after verification, including through an old open fd. + with copy_deadline(): + protected_copy(source, package, expected_size, expected_sha256) + package.chmod(0o644) + staging.chmod(0o755) + print("Protected package size and SHA-256 verified. Starting APT.") + keep_staging = True + child = popen(["/usr/bin/apt", "install", *(["--yes"] if noninteractive else []), str(package)]) + try: + code = child.wait() + except KeyboardInterrupt: + print("Waiting for APT to finish safely; do not kill the package manager.", file=sys.stderr) + code = child.wait() + keep_staging = False + return code + finally: + if keep_staging: + print("APT exit was not confirmed; retained protected package: " + str(package), file=sys.stderr) + else: + shutil.rmtree(staging) + + +def main(): + noninteractive = len(sys.argv) == 6 and sys.argv[5] == "--noninteractive" + if len(sys.argv) != 5 and not noninteractive: + raise SystemExit("Expected source package, filename, byte count and SHA-256") + try: + return run(Path(sys.argv[1]), sys.argv[2], int(sys.argv[3]), sys.argv[4], noninteractive=noninteractive) + except KeyboardInterrupt: + return 130 + except (OSError, ValueError) as exc: + print("Protected installation stopped: " + str(exc), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/desktop/installers/linux_online_template.py b/desktop/installers/linux_online_template.py new file mode 100644 index 000000000..cc8c675dd --- /dev/null +++ b/desktop/installers/linux_online_template.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Download the pinned CommunityAI Debian package, verify it, then install through APT.""" + +import argparse +import contextlib +import hashlib +import os +import platform +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +ARTIFACT = None # Replaced with a validated release entry by build_linux_online.py. +ROOT_HELPER = None # Replaced with the isolated protected-copy helper by the builder. +CHUNK_BYTES = 1024 * 1024 +READ_TIMEOUT_SECONDS = 30 +DOWNLOAD_TIMEOUT_SECONDS = 2 * 60 * 60 +DISK_MARGIN_BYTES = 64 * 1024 * 1024 + + +class InstallError(RuntimeError): + pass + + +class Cancelled(InstallError): + pass + + +class NoRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, new_url): + raise InstallError("The download origin redirected the request; no redirected content was downloaded.") + + +@contextlib.contextmanager +def download_deadline(): + """Bound DNS/connect/read time too; the supported installer platform is Linux.""" + + def expired(signum, frame): + raise InstallError("The two-hour download deadline expired; rerun the installer to retry.") + + def cancelled(signum, frame): + raise Cancelled("Download cancelled.") + + previous_alarm = signal.signal(signal.SIGALRM, expired) + previous_term = signal.signal(signal.SIGTERM, cancelled) + signal.setitimer(signal.ITIMER_REAL, DOWNLOAD_TIMEOUT_SECONDS) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_alarm) + signal.signal(signal.SIGTERM, previous_term) + + +def download(artifact, destination, *, opener=None, clock=time.monotonic, progress=print): + """Stream one exact object; no resume, redirects, credentials, or remote manifest.""" + opener = opener or urllib.request.build_opener(NoRedirects()) + request = urllib.request.Request( + artifact["url"], headers={"Accept-Encoding": "identity", "User-Agent": "CommunityAI-Online-Installer/1"} + ) + started = clock() + digest = hashlib.sha256() + received = 0 + next_progress = 0 + with opener.open(request, timeout=READ_TIMEOUT_SECONDS) as response: + if response.status != 200 or response.geturl() != artifact["url"]: + raise InstallError("The server did not return the exact pinned download URL.") + encoding = response.headers.get("Content-Encoding", "identity").lower() + if encoding != "identity": + raise InstallError("The server returned an encoded download instead of the pinned package bytes.") + length = response.headers.get("Content-Length") + if length is not None and (not length.isdecimal() or int(length) != artifact["size_bytes"]): + raise InstallError("The server's package size does not match this installer.") + with destination.open("xb") as output: + while True: + if clock() - started >= DOWNLOAD_TIMEOUT_SECONDS: + raise InstallError("The download deadline expired.") + chunk = response.read(min(CHUNK_BYTES, artifact["size_bytes"] - received + 1)) + if not chunk: + break + received += len(chunk) + if received > artifact["size_bytes"]: + raise InstallError("The download exceeds the pinned package size.") + digest.update(chunk) + output.write(chunk) + percent = received * 100 // artifact["size_bytes"] + if percent >= next_progress: + progress("Downloaded {}% ({:,} / {:,} bytes)".format(percent, received, artifact["size_bytes"])) + next_progress = (percent // 5 + 1) * 5 + output.flush() + os.fsync(output.fileno()) + if received != artifact["size_bytes"]: + raise InstallError("The download is incomplete; rerun the installer to retry.") + if digest.hexdigest() != artifact["sha256"]: + raise InstallError("SHA-256 verification failed; the package will not be installed.") + return destination + + +def install_command(package, artifact): + # Fixed system paths avoid resolving a privileged command through a user PATH. + return [ + "/usr/bin/sudo", + "/usr/bin/python3", + "-I", + "-c", + ROOT_HELPER, + str(package.resolve()), + artifact["filename"], + str(artifact["size_bytes"]), + artifact["sha256"], + ] + + +def install_verified(package, artifact, *, popen=subprocess.Popen): + """APT keeps its normal prompts and owns package-manager recovery on interruption.""" + child = popen(install_command(package, artifact)) + try: + return child.wait() + except KeyboardInterrupt: + # The terminal also sends SIGINT to sudo/APT. Do not kill dpkg or remove + # its input while it may still be active. A second interruption escapes + # and the caller conservatively keeps this one owned staging directory. + print("Installation interrupted; waiting for the package manager to finish safely.", file=sys.stderr) + return child.wait() + + +def run(artifact, *, directory=Path("/var/tmp"), download_only=False): + if sys.platform != "linux" or platform.machine().lower() not in ("x86_64", "amd64"): + raise InstallError("This installer requires amd64 Debian 12+ or Ubuntu 22.04+.") + if os.geteuid() == 0: + raise InstallError("Run this script as your ordinary user. It requests sudo only after verifying the package.") + if not directory.is_dir(): + raise InstallError("The download directory must already exist.") + if not download_only and not all( + Path(path).is_file() for path in ("/usr/bin/sudo", "/usr/bin/apt", "/usr/bin/python3") + ): + raise InstallError( + "APT, sudo and system Python 3 are required. Use --download-only to retain the verified package." + ) + download_space = artifact["size_bytes"] * (1 if download_only else 2) + DISK_MARGIN_BYTES + if shutil.disk_usage(directory).free < download_space: + raise InstallError( + "The download directory needs at least {:,} free bytes for download and protected staging.".format( + download_space + ) + ) + print("CommunityAI {} for Debian/Ubuntu amd64".format(artifact["version"])) + print( + "Download: {:,} bytes. The complete runtime is included; model weights download separately.".format( + artifact["size_bytes"] + ) + ) + print("SHA-256: " + artifact["sha256"]) + print("Settings and model cache follow the existing package's normal retention policy.") + if not download_only: + print( + "Installation verifies a protected copy and needs temporary space for a second package, plus the installed runtime." + ) + staging = Path(tempfile.mkdtemp(prefix="communityai-online-", dir=directory)) + keep_staging = False + package = staging / artifact["filename"] + try: + with download_deadline(): + download(artifact, package) + print("Package size and SHA-256 verified.") + if download_only: + keep_staging = True + print("Verified package saved to: " + str(package)) + return 0 + keep_staging = True + code = install_verified(package, artifact) + keep_staging = False # wait() confirmed that the package manager exited. + if code != 0: + raise InstallError( + "APT exited with status {}. Follow its recovery instructions before retrying.".format(code) + ) + print("CommunityAI is installed. Launch it from the application menu as your ordinary user.") + return 0 + finally: + if keep_staging: + print("Retained installer package: " + str(package)) + else: + # Only the directory returned by this invocation's mkdtemp is removed. + shutil.rmtree(staging) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--download-only", action="store_true", help="retain the verified package without invoking APT") + parser.add_argument( + "--directory", + type=Path, + default=Path("/var/tmp"), + help="existing staging directory with enough download space (default: /var/tmp)", + ) + args = parser.parse_args(argv) + if ARTIFACT is None: + parser.error("This is an unconfigured template; generate an installer with build_linux_online.py.") + try: + return run(ARTIFACT, directory=args.directory, download_only=args.download_only) + except KeyboardInterrupt: + print("Cancelled. No unverified package was installed.", file=sys.stderr) + return 130 + except Cancelled as exc: + print(str(exc), file=sys.stderr) + return 143 + except (InstallError, OSError, urllib.error.URLError) as exc: + print("Installation stopped: " + str(exc), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/desktop/installers/release_downloads.py b/desktop/installers/release_downloads.py new file mode 100644 index 000000000..bb35d133c --- /dev/null +++ b/desktop/installers/release_downloads.py @@ -0,0 +1,201 @@ +"""Pinned offline-package metadata shared by the Windows and Linux online builders. + +The online installers embed this metadata at build time. They never resolve a +mutable remote manifest or install an arbitrary latest dependency wheel. +""" + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path +from urllib.parse import quote, unquote, urlsplit + +MAX_MANIFEST_BYTES = 64 * 1024 +MAX_PACKAGE_BYTES = 32 * 1024**3 +PLATFORMS = {"windows-x64": "exe", "linux-amd64": "deb"} +REQUIRED_ARTIFACT_KEYS = { + "platform", + "kind", + "format", + "version", + "filename", + "url", + "sha256", + "size_bytes", +} + + +def _require(condition, message): + if not condition: + raise ValueError(message) + + +def _unique_pairs(pairs): + result = {} + for key, value in pairs: + _require(key not in result, "Duplicate JSON field") + result[key] = value + return result + + +def validate_https_url(value): + """Accept direct public HTTPS object URLs, without credentials or query tokens.""" + _require(isinstance(value, str) and 1 <= len(value) <= 2048, "Invalid download URL") + _require(re.fullmatch(r"[A-Za-z0-9._~:/%+\-]+", value) is not None, "Invalid download URL characters") + parsed = urlsplit(value) + _require(parsed.scheme == "https" and parsed.hostname, "Download URL must use HTTPS") + _require(parsed.username is None and parsed.password is None, "Download URL cannot contain credentials") + _require(not parsed.query and not parsed.fragment, "Download URL must be immutable without query or fragment") + _require(parsed.port in (None, 443), "Download URL must use port 443") + _require( + re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9.\-]*[A-Za-z0-9])?", parsed.hostname) is not None, + "Invalid download hostname", + ) + for segment in parsed.path.split("/"): + decoded = unquote(segment, errors="strict") + _require(decoded not in (".", ".."), "Download URL cannot traverse directories") + _require( + all(character.isascii() and (character.isalnum() or character in "-._~+") for character in decoded), + "Invalid download object path", + ) + return value + + +def validate_artifact(value, expected_platform=None): + _require(isinstance(value, dict), "Artifact must be an object") + _require(REQUIRED_ARTIFACT_KEYS <= value.keys(), "Missing artifact fields") + _require(value.keys() <= REQUIRED_ARTIFACT_KEYS | {"publisher"}, "Unknown artifact fields") + platform = value["platform"] + _require(isinstance(platform, str) and platform in PLATFORMS, "Unsupported platform") + _require(expected_platform is None or platform == expected_platform, "Artifact platform mismatch") + _require(value["kind"] == "offline-installer", "Online installers must use a pinned offline installer") + _require(value["format"] == PLATFORMS[platform], "Artifact format mismatch") + version = value["version"] + _require(isinstance(version, str) and len(version) <= 128, "Invalid version") + pattern = r"[0-9]+\.[0-9]+\.[0-9]+(?:[.-][A-Za-z0-9]+)*" if platform == "windows-x64" else r"[0-9][A-Za-z0-9.+~\-]*" + _require(re.fullmatch(pattern, version) is not None, "Invalid version") + expected_name = ( + f"communityai-{version}-windows-setup.exe" if platform == "windows-x64" else f"communityai_{version}_amd64.deb" + ) + _require(value["filename"] == expected_name, "Filename must match the pinned platform and version") + url = validate_https_url(value["url"]) + _require(unquote(urlsplit(url).path.rsplit("/", 1)[-1]) == expected_name, "URL filename mismatch") + _require( + isinstance(value["sha256"], str) and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is not None, + "Invalid SHA-256 digest", + ) + _require( + type(value["size_bytes"]) is int and 0 < value["size_bytes"] <= MAX_PACKAGE_BYTES, + "Invalid package byte size", + ) + if "publisher" in value: + publisher = value["publisher"] + _require( + isinstance(publisher, str) + and 1 <= len(publisher) <= 128 + and all(character.isprintable() and character not in '\\"' for character in publisher), + "Invalid publisher label", + ) + return dict(value) + + +def validate_release_manifest(value): + _require(isinstance(value, dict) and set(value) == {"schema_version", "artifacts"}, "Invalid manifest fields") + _require(type(value["schema_version"]) is int and value["schema_version"] == 1, "Unsupported manifest version") + artifacts = value["artifacts"] + _require(isinstance(artifacts, dict) and 1 <= len(artifacts) <= len(PLATFORMS), "Invalid artifact inventory") + return { + "schema_version": 1, + "artifacts": {platform: validate_artifact(artifact, platform) for platform, artifact in artifacts.items()}, + } + + +def load_release_manifest(path): + with Path(path).open("rb") as stream: + raw = stream.read(MAX_MANIFEST_BYTES + 1) + _require(len(raw) <= MAX_MANIFEST_BYTES, "Manifest exceeds the size limit") + return validate_release_manifest(json.loads(raw, object_pairs_hook=_unique_pairs)) + + +def select_artifact(manifest, platform): + checked = validate_release_manifest(manifest) + _require(platform in checked["artifacts"], "Selected platform is absent from the release") + return checked["artifacts"][platform] + + +def artifact_from_file(platform, path, version, base_url, publisher=None): + """Hash the exact offline package once using bounded memory; never alter it.""" + path = Path(path) + _require(path.is_file() and not path.is_symlink(), "Offline package must be a regular file") + _require(base_url == base_url.rstrip("/"), "Base URL must not end with a slash") + validate_https_url(base_url) + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + before = os.fstat(stream.fileno()) + for block in iter(lambda: stream.read(1024 * 1024), b""): + size += len(block) + _require(size <= MAX_PACKAGE_BYTES, "Offline package exceeds size limit") + digest.update(block) + after = os.fstat(stream.fileno()) + named = path.stat(follow_symlinks=False) + _require( + (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) + == (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) + == (named.st_dev, named.st_ino, named.st_size, named.st_mtime_ns) + and size == before.st_size, + "Offline package changed while being hashed", + ) + artifact = { + "platform": platform, + "kind": "offline-installer", + "format": PLATFORMS.get(platform), + "version": version, + "filename": path.name, + "url": base_url + "/" + quote(path.name, safe="-._~+"), + "sha256": digest.hexdigest(), + "size_bytes": size, + } + if publisher is not None: + artifact["publisher"] = publisher + return validate_artifact(artifact, platform) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + for name in ("validate", "select"): + command = commands.add_parser(name) + command.add_argument("manifest", type=Path) + if name == "select": + command.add_argument("--platform", choices=PLATFORMS, required=True) + create = commands.add_parser("create") + create.add_argument("--base-url", required=True) + create.add_argument("--artifact", nargs=3, action="append", metavar=("PLATFORM", "FILE", "VERSION"), required=True) + create.add_argument("--publisher") + create.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + if args.command == "create": + _require(not args.output.exists(), "Manifest output already exists") + artifacts = {} + for platform, path, version in args.artifact: + _require(platform not in artifacts, "Duplicate artifact platform") + artifacts[platform] = artifact_from_file(platform, path, version, args.base_url, args.publisher) + result = validate_release_manifest({"schema_version": 1, "artifacts": artifacts}) + with args.output.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(result, indent=2, sort_keys=True) + "\n") + else: + result = load_release_manifest(args.manifest) + if args.command == "select": + result = select_artifact(result, args.platform) + except (OSError, ValueError, UnicodeError) as exc: + parser.exit(1, f"Release download metadata rejected: {exc}\n") + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/desktop/installers/sign_app_update.py b/desktop/installers/sign_app_update.py new file mode 100644 index 000000000..ba70fc435 --- /dev/null +++ b/desktop/installers/sign_app_update.py @@ -0,0 +1,54 @@ +"""Sign a complete release manifest for the desktop updater; key supplied on stdin.""" + +import argparse +import base64 +import json +import sys +import time +from pathlib import Path + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from release_downloads import load_release_manifest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from communityai_desktop.updater import PUBLIC_KEY, SIGNATURE_DOMAIN, canonical, verify_feed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--sequence", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + manifest = load_release_manifest(args.manifest) + artifacts = {} + for target, item in manifest["artifacts"].items(): + expected = args.version.replace("-alpha.", "~alpha.") if target == "linux-amd64" else args.version + if item["version"] != expected: + raise ValueError("Every installer must match the update release") + artifacts[target] = {key: item[key] for key in ("filename", "url", "size_bytes", "sha256")} + now = int(time.time()) + signed = { + "schema_version": 1, + "channel": "alpha", + "version": args.version, + "sequence": args.sequence, + "published_at": now, + "expires_at": now + 90 * 86400, + "artifacts": artifacts, + } + key = Ed25519PrivateKey.from_private_bytes(base64.b64decode(sys.stdin.read().strip(), validate=True)) + document = { + "signed": signed, + "signature": base64.b64encode(key.sign(SIGNATURE_DOMAIN + canonical(signed))).decode(), + } + raw = canonical(document) + verify_feed(raw, public_key=PUBLIC_KEY) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(raw + b"\n") + print(json.dumps({"signed_update": str(args.output), "version": args.version, "sequence": args.sequence})) + + +if __name__ == "__main__": + main() diff --git a/desktop/launch_node.py b/desktop/launch_node.py index b4400c842..e5d556311 100644 --- a/desktop/launch_node.py +++ b/desktop/launch_node.py @@ -9,11 +9,13 @@ from __future__ import annotations import json +import math import multiprocessing import os import sys from importlib.metadata import version from importlib.resources import files +from pathlib import Path def _runtime_contract() -> dict[str, object]: @@ -82,6 +84,100 @@ def _worker_runtime_contract() -> dict[str, object]: } +def _verify_frozen_module_path(module) -> None: + if getattr(sys, "frozen", False): + try: + Path(module.__file__).resolve().relative_to(Path(sys._MEIPASS).resolve(strict=True)) + except (AttributeError, OSError, TypeError, ValueError) as exc: + raise RuntimeError("native self-test imported a module outside the frozen runtime") from exc + + +def _bitsandbytes_native_path(cextension) -> str: + """Require the actual loaded CUDA library, not merely a successful Python import.""" + _verify_frozen_module_path(cextension) + native = cextension.lib + if native is None or getattr(native, "compiled_with_cuda", False) is not True: + raise RuntimeError("bitsandbytes did not load a native CUDA backend") + name = "libbitsandbytes_cuda124" + (".dll" if os.name == "nt" else ".so") + try: + loaded = Path(native._lib._name).resolve(strict=True) + expected = Path(cextension.__file__).resolve().with_name(name) + if loaded != expected or not loaded.is_file(): + raise RuntimeError("bitsandbytes loaded an unexpected native library") + except (AttributeError, OSError, TypeError, ValueError) as exc: + raise RuntimeError("bitsandbytes native library path could not be verified") from exc + return name + + +def _native_runtime_contract(*, require_cuda: bool = False) -> dict[str, object]: + """Finite native math with tiny tensors; never fetch weights or join a network. + + Run this dedicated process under the qualification runner's timeout. The CPU + mode does not initialize CUDA. Required-CUDA mode also exercises lazy linalg + loading and the pruned bitsandbytes profile, and fails if no GPU is available. + """ + import torch + + _verify_frozen_module_path(torch) + if torch.__version__ != "2.6.0+cu124" or torch.version.cuda != "12.4": + raise RuntimeError("native self-test requires the pinned torch 2.6.0+cu124 runtime") + torch.set_num_threads(1) + result = { + "schema_version": 1, + "application": "CommunityAI-Native-Runtime", + "frozen": bool(getattr(sys, "frozen", False)), + "torch": torch.__version__, + "cuda_build": torch.version.cuda, + "model_loading_performed": False, + "network_join_performed": False, + "cuda_required": require_cuda, + "cuda_test_performed": False, + } + with torch.inference_mode(): + matrix = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32, device="cpu") + expected = torch.tensor([[5.0, 11.0], [11.0, 25.0]], dtype=torch.float32, device="cpu") + if not torch.allclose(matrix @ matrix.T, expected, rtol=0, atol=0): + raise RuntimeError("native CPU matrix multiplication returned an incorrect result") + result["cpu_matmul_passed"] = True + if not require_cuda: + return result + if not torch.cuda.is_available(): + raise RuntimeError("native self-test requires an available CUDA GPU") + device = "cuda:0" + cuda_matrix = matrix.to(device) + if not torch.allclose(cuda_matrix @ cuda_matrix.T, expected.to(device), rtol=1e-5, atol=1e-5): + raise RuntimeError("native CUDA matrix multiplication returned an incorrect result") + diagonal = torch.diag(torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float32, device=device)) + left, singular, right = torch.linalg.svd(diagonal, full_matrices=False) + if not torch.allclose(left @ torch.diag(singular) @ right, diagonal, rtol=1e-4, atol=1e-4): + raise RuntimeError("native CUDA linalg reconstruction returned an incorrect result") + + from bitsandbytes import cextension, functional + + native_name = _bitsandbytes_native_path(cextension) + _verify_frozen_module_path(functional) + values = torch.linspace(-1.0, 1.0, 64, dtype=torch.float16, device=device) + packed, state = functional.quantize_4bit(values, blocksize=64, quant_type="nf4") + restored = functional.dequantize_4bit(packed, quant_state=state, blocksize=64, quant_type="nf4") + if restored.shape != values.shape or restored.device != values.device: + raise RuntimeError("bitsandbytes NF4 roundtrip changed tensor shape or device") + maximum_error = float((restored - values).abs().max().item()) + if not math.isfinite(maximum_error) or maximum_error > 0.2: + raise RuntimeError("bitsandbytes NF4 roundtrip returned an incorrect result") + torch.cuda.synchronize() + result.update( + { + "cuda_test_performed": True, + "cuda_matmul_passed": True, + "cuda_linalg_passed": True, + "bitsandbytes_native_library": native_name, + "bitsandbytes_nf4_roundtrip_passed": True, + "bitsandbytes_nf4_maximum_absolute_error": maximum_error, + } + ) + return result + + def main() -> int: # PyInstaller's multiprocessing children must be intercepted before importing # Torch, Hivemind, or any application modules. @@ -93,6 +189,11 @@ def main() -> int: if argv == ["server", "--self-test"]: print(json.dumps(_worker_runtime_contract(), sort_keys=True)) return 0 + if argv in (["--native-self-test"], ["--native-self-test", "--require-cuda"]): + print(json.dumps(_native_runtime_contract(require_cuda="--require-cuda" in argv), sort_keys=True)) + return 0 + if "--native-self-test" in argv or "--require-cuda" in argv: + raise RuntimeError("Use --native-self-test with only the optional --require-cuda flag") if argv[:1] == ["server"]: sys.argv = ["CommunityAI-Node server", *argv[1:]] from drift.cli.run_server import main as run diff --git a/desktop/pyproject.toml b/desktop/pyproject.toml index da4f04be4..37961beb1 100644 --- a/desktop/pyproject.toml +++ b/desktop/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.10" dependencies = [ "keyring>=25,<27", "PySide6>=6.8,<7", + "cryptography>=42", ] [project.optional-dependencies] diff --git a/desktop/runtime_packaging.py b/desktop/runtime_packaging.py new file mode 100644 index 000000000..1b79063a1 --- /dev/null +++ b/desktop/runtime_packaging.py @@ -0,0 +1,162 @@ +"""Normalize a fresh frozen runtime without changing native-library loader paths. + +PyInstaller's torch hook deliberately suppresses some Linux symlinks because a +library can use its own location to find dependencies. Hardlinks retain those +locations while sharing identical contents. This module never imports torch or +detects the build machine's GPU: the supported frozen profile is explicit. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import stat +import uuid +from collections import defaultdict +from pathlib import Path + +TORCH_PROFILE = "2.6.0+cu124" +_BNB_CUDA = re.compile(r"libbitsandbytes_cuda(?P\d+)(?:_nocublaslt)?\.(?:dll|so(?:\.\d+)*)$") +_LINUX_LIBRARY = re.compile(r".+\.so(?:\.\d+)*$") + + +def _files(root: Path) -> list[Path]: + if root.is_symlink() or getattr(root, "is_junction", lambda: False)() or not stat.S_ISDIR(root.lstat().st_mode): + raise RuntimeError("runtime root must be an ordinary directory") + result = [] + for directory, directories, files in os.walk(root, followlinks=False): + base = Path(directory) + for name in directories: + child = base / name + if child.is_symlink() or getattr(child, "is_junction", lambda: False)(): + raise RuntimeError("runtime contains a linked directory") + for name in files: + child = base / name + mode = child.lstat().st_mode + if not (stat.S_ISREG(mode) or stat.S_ISLNK(mode)): + raise RuntimeError("runtime contains a non-file entry") + result.append(child) + return sorted(result) + + +def storage_metrics(root: Path) -> dict[str, int]: + """Count regular pathname bytes separately from unique inode content bytes. + + These are content lengths, not filesystem block allocation. Symlinks have no + copied payload and are counted separately; old release metrics stay logical. + """ + unique = {} + logical = regular = links = 0 + for path in _files(root): + info = path.lstat() + if stat.S_ISLNK(info.st_mode): + links += 1 + continue + regular += 1 + logical += info.st_size + unique[(info.st_dev, info.st_ino)] = info.st_size + return { + "regular_file_count": regular, + "symlink_count": links, + "logical_file_bytes": logical, + "unique_file_bytes": sum(unique.values()), + "unique_file_count": len(unique), + } + + +def _identity(path: Path) -> tuple[int, int, int, int, int]: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise RuntimeError("native library changed type during normalization") + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, stat.S_IMODE(info.st_mode) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def normalize_runtime(node_root: Path, *, target_platform: str, torch_version: str) -> dict[str, object]: + """Prune unsupported BNB builds and hardlink equal Linux native libraries. + + Call only on a fresh, exclusively owned build output, before its frozen + checks and release attestation. A failure invalidates that build output. + """ + if target_platform not in ("Linux", "Windows") or torch_version != TORCH_PROFILE: + raise RuntimeError("runtime normalization requires the pinned torch 2.6.0+cu124 profile") + override = os.environ.get("BNB_CUDA_VERSION", "") + if override and override != "124": + raise RuntimeError("BNB_CUDA_VERSION conflicts with the frozen CUDA 12.4 profile") + node_root = Path(node_root) + before = storage_metrics(node_root) + files = _files(node_root) + variants = [(path, _BNB_CUDA.fullmatch(path.name)) for path in files] + variants = [(path, match) for path, match in variants if match is not None] + suffix = "so" if target_platform == "Linux" else "dll" + required_library = node_root / "_internal/bitsandbytes" / f"libbitsandbytes_cuda124.{suffix}" + if required_library not in files or not stat.S_ISREG(required_library.lstat().st_mode): + raise RuntimeError("frozen runtime is missing its CUDA 12.4 bitsandbytes library") + removed = [] + for path, match in variants: + if match["version"] != "124": + info = path.lstat() + removed.append({"path": path.relative_to(node_root).as_posix(), "size_bytes": info.st_size}) + path.unlink() + + replacements = [] + if target_platform == "Linux": + candidates = defaultdict(list) + for path in _files(node_root): + if _LINUX_LIBRARY.fullmatch(path.name) and not path.is_symlink(): + identity = _identity(path) + candidates[(identity[2], identity[4])].append((path, identity)) + for group in candidates.values(): + if len(group) < 2: + continue + canonical = {} + for path, identity in group: + digest = _sha256(path) + if _identity(path) != identity: + raise RuntimeError("native library changed during normalization") + if digest not in canonical: + canonical[digest] = (path, identity) + continue + source, source_identity = canonical[digest] + if identity[:2] == source_identity[:2]: + continue + if _identity(source) != source_identity or _identity(path) != identity: + raise RuntimeError("native library changed during normalization") + temporary = path.with_name(f".{path.name}.hardlink-{uuid.uuid4().hex}") + try: + os.link(source, temporary, follow_symlinks=False) + if _identity(source) != source_identity or _identity(path) != identity: + raise RuntimeError("native library changed during normalization") + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + if _identity(path) != source_identity: + raise RuntimeError("native library hardlink was not preserved") + replacements.append( + { + "path": path.relative_to(node_root).as_posix(), + "target": source.relative_to(node_root).as_posix(), + "size_bytes": identity[2], + "sha256": digest, + "mode": identity[4], + } + ) + return { + "schema_version": 1, + "scope": "frozen-node-runtime", + "platform": target_platform, + "torch_version": torch_version, + "bitsandbytes_cuda_version": "124", + "before": before, + "after": storage_metrics(node_root), + "removed_bitsandbytes_variants": removed, + "hardlinked_native_libraries": replacements, + } diff --git a/desktop/src/communityai_desktop/acceptance.py b/desktop/src/communityai_desktop/acceptance.py index 35009ad6f..567d757d2 100644 --- a/desktop/src/communityai_desktop/acceptance.py +++ b/desktop/src/communityai_desktop/acceptance.py @@ -308,8 +308,10 @@ def do_DELETE(self): # noqa: N802 @contextmanager -def fake_node() -> Iterator[Tuple[str, str]]: +def fake_node(*, all_workers_paused: bool = False) -> Iterator[Tuple[str, str]]: state = _FakeNodeState() + if all_workers_paused: + state.worker_states = {worker_id: (model, "paused") for worker_id, (model, _) in state.worker_states.items()} server = ThreadingHTTPServer(("127.0.0.1", 0), _handler(state)) thread = threading.Thread(target=server.serve_forever, name="desktop-acceptance-node", daemon=True) thread.start() diff --git a/desktop/src/communityai_desktop/app.py b/desktop/src/communityai_desktop/app.py index 91c56717a..68b6102d7 100644 --- a/desktop/src/communityai_desktop/app.py +++ b/desktop/src/communityai_desktop/app.py @@ -9,7 +9,6 @@ from pathlib import Path from typing import Any, Optional, Sequence -from communityai_desktop import __version__ from communityai_desktop.acceptance import fake_node, run_self_test from communityai_desktop.client import NodeClient, NodeClientError, normalize_loopback_url from communityai_desktop.controller import DesktopController @@ -26,12 +25,13 @@ NodeLifecycleSupervisor, default_bootstrap_config_path, ) +from communityai_desktop.release import RELEASE_VERSION from communityai_desktop.startup import LOGIN_STARTUP_FLAG, SingleInstanceError def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="CommunityAI desktop") - parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + parser.add_argument("--version", action="version", version=f"%(prog)s {RELEASE_VERSION}") parser.add_argument("--node-url", default="http://127.0.0.1:8080") parser.add_argument("--timeout", type=float, default=5.0) parser.add_argument("--credential-service", default=DEFAULT_CREDENTIAL_SERVICE, help=argparse.SUPPRESS) @@ -44,6 +44,9 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--no-manage-node", action="store_true", help=argparse.SUPPRESS) parser.add_argument(LOGIN_STARTUP_FLAG, action="store_true", help=argparse.SUPPRESS) parser.add_argument("--capture-page", type=int, default=0, help=argparse.SUPPRESS) + parser.add_argument("--gate13-ui-evidence", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--gate13-ui-screenshot", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--resource-ui-evidence", type=Path, help=argparse.SUPPRESS) action = parser.add_mutually_exclusive_group() action.add_argument("--store-control-key", action="store_true") action.add_argument("--delete-control-key", action="store_true") @@ -53,6 +56,11 @@ def build_parser() -> argparse.ArgumentParser: action.add_argument("--onboarding-ui-self-test", action="store_true", help=argparse.SUPPRESS) action.add_argument("--capture-ui", type=Path, help=argparse.SUPPRESS) action.add_argument("--probe-only", action="store_true", help=argparse.SUPPRESS) + action.add_argument( + "--prepare-update", action="store_true", help="Stop this user's desktop and owned node for installation" + ) + action.add_argument("--gate13-ui-playthrough", type=Path, help=argparse.SUPPRESS) + action.add_argument("--resource-ui-playthrough", type=Path, help=argparse.SUPPRESS) return parser @@ -69,7 +77,23 @@ def _write_json(value: Any) -> None: def main(argv: Optional[Sequence[str]] = None) -> int: parser = build_parser() args = parser.parse_args(argv) + if args.gate13_ui_playthrough is None: + if args.gate13_ui_evidence is not None or args.gate13_ui_screenshot is not None: + parser.error("Gate 13 evidence options require --gate13-ui-playthrough") + elif args.gate13_ui_evidence is None: + parser.error("--gate13-ui-playthrough requires --gate13-ui-evidence") + if bool(args.resource_ui_playthrough) != bool(args.resource_ui_evidence): + parser.error("Resource UI playthrough requires both plan and evidence paths") try: + if args.prepare_update: + try: + from communityai_desktop.maintenance import prepare_update + + return prepare_update() + except Exception as exc: + # A windowed PyInstaller traceback dialog would hold the installer + # indefinitely if, for example, the installed Qt runtime is broken. + parser.exit(2, f"CommunityAI shutdown failed: {exc}\n") if args.self_test: _write_json(run_self_test()) return 0 @@ -150,6 +174,20 @@ def connect() -> DesktopController: token = credential_store.get_or_migrate() return DesktopController(NodeClient(node_url, token, timeout=args.timeout)) + qualification_automation = None + if args.gate13_ui_playthrough is not None: + from communityai_desktop.gate13_playthrough import Gate13Playthrough, PlaythroughPlan + + qualification_automation = Gate13Playthrough( + PlaythroughPlan.load(args.gate13_ui_playthrough), + args.gate13_ui_evidence, + screenshot_path=args.gate13_ui_screenshot, + ) + elif args.resource_ui_playthrough is not None: + from communityai_desktop.resource_playthrough import ResourcePlaythrough + + qualification_automation = ResourcePlaythrough(args.resource_ui_playthrough, args.resource_ui_evidence) + if args.probe_only: try: _write_json(connect().snapshot()) @@ -160,6 +198,18 @@ def connect() -> DesktopController: from communityai_desktop.pyside_shell import run + updater = None + if qualification_automation is None: + from communityai_desktop.updater import UpdateManager, installed_root + from PySide6.QtCore import QStandardPaths + + root = installed_root() + if root is not None: + cache = ( + Path(QStandardPaths.writableLocation(QStandardPaths.GenericCacheLocation)) / "CommunityAI/updates" + ) + updater = UpdateManager(cache, root=root) + # Credential and connection errors belong in the window for normal desktop # startup. Existing headless installations migrate automatically. try: @@ -169,6 +219,9 @@ def connect() -> DesktopController: start_minimized=args.started_at_login, activate_existing_instance=not args.started_at_login, before_termination_restore=None if lifecycle is None else lifecycle.close, + qualification_automation=qualification_automation, + single_instance=qualification_automation is None, + updater=updater, ) or 0 ) diff --git a/desktop/src/communityai_desktop/client.py b/desktop/src/communityai_desktop/client.py index 328ee0ae2..9bbfd0f93 100644 --- a/desktop/src/communityai_desktop/client.py +++ b/desktop/src/communityai_desktop/client.py @@ -12,6 +12,8 @@ from urllib.parse import quote, urlsplit, urlunsplit from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener +from communityai_desktop.telemetry import download_view + MAX_RESPONSE_BYTES = 4 * 1024 * 1024 SUPPORTED_CONTROL_API_VERSION = 1 CONTRIBUTION_STATUS_SCHEMA_VERSION = 3 @@ -76,18 +78,21 @@ def normalize_loopback_url(value: str) -> str: return urlunsplit((parsed.scheme, parsed.netloc, "", "", "")) -def _normalize_model_download(value: Any) -> Dict[str, int]: +def _normalize_model_download(value: Any) -> Dict[str, Any]: expected_keys = {"schema_version", "selected_whole_shard_bytes"} - if not isinstance(value, dict) or set(value) != expected_keys: + if not isinstance(value, dict) or set(value) - {"progress"} != expected_keys: raise NodeClientError("Local node model download estimate has an invalid schema") if type(value["schema_version"]) is not int or value["schema_version"] != MODEL_DOWNLOAD_SCHEMA_VERSION: raise NodeClientError("Local node model download estimate has an unsupported schema version") size = value["selected_whole_shard_bytes"] - if isinstance(size, bool) or not isinstance(size, int) or not 1 <= size <= MAX_SELECTED_WHOLE_SHARD_BYTES: + if size is not None and ( + isinstance(size, bool) or not isinstance(size, int) or not 0 <= size <= MAX_SELECTED_WHOLE_SHARD_BYTES + ): raise NodeClientError("Local node model download estimate has invalid selected whole-shard bytes") return { "schema_version": MODEL_DOWNLOAD_SCHEMA_VERSION, "selected_whole_shard_bytes": size, + **({"progress": download_view(value["progress"])} if "progress" in value else {}), } @@ -137,7 +142,10 @@ def _normalize_auto_selection(value: Any) -> Dict[str, Any]: raise NodeClientError("Local node status has invalid auto selection manifest") covered = _optional_number(value.get("covered_blocks"), "auto covered blocks", integer=True, positive=True) total = _optional_number(value.get("total_blocks"), "auto total blocks", integer=True, positive=True) - peers = _optional_number(value.get("peer_count"), "auto peer count", integer=True, positive=True) + local = value.get("source") == "local" + peers = _optional_number(value.get("peer_count"), "auto peer count", integer=True, positive=not local) + if local and peers != 0: + raise NodeClientError("Standalone inference must not claim remote peers") if covered is None or total is None or peers is None: raise NodeClientError("Local node status omitted automatic route evidence") if covered != total: @@ -210,8 +218,18 @@ def _normalize_policy(value: Any) -> Dict[str, Any]: "pause_timeout", "schedule", } - if not isinstance(value, dict) or set(value) != fields or not isinstance(value["sharing_enabled"], bool): + if ( + not isinstance(value, dict) + or set(value) not in (fields, fields | {"max_processing_percent"}) + or not isinstance(value["sharing_enabled"], bool) + ): raise NodeClientError("Local node contribution policy is malformed") + processing = {} + if "max_processing_percent" in value: + percent = _optional_number(value["max_processing_percent"], "processing percentage", positive=True) + if percent is None or not 1 <= percent <= 100: + raise NodeClientError("Local node has invalid processing percentage") + processing["max_processing_percent"] = percent allowed = _normalize_model_selectors(value["allowed_models"], "allowed models") preferred = _normalize_model_selectors(value["preferred_models"], "preferred models") denied = _normalize_model_selectors(value["denied_models"], "denied models") @@ -270,6 +288,7 @@ def optional_text(field: str): clean_schedule = {"timezone": timezone, "windows": clean_windows} return { "sharing_enabled": value["sharing_enabled"], + **processing, "allowed_models": allowed, "preferred_models": preferred, "denied_models": denied, @@ -379,10 +398,12 @@ def _normalize_contribution_status(value: Any) -> Dict[str, Any]: "model": model, "state": state, "desired_running": worker["desired_running"], + "operator_paused": worker.get("operator_paused") is True, "placement": placement, "policy": policy, "schedule": schedule, "resources": resources, + "download_progress": download_view(worker.get("download_progress")), } ) if not configured and normalized_workers: @@ -396,6 +417,21 @@ def _normalize_contribution_status(value: Any) -> Dict[str, Any]: } +def _normalize_hardware(value: Any) -> Dict[str, Any]: + if not isinstance(value, dict): + return {} + result = {} + for field in ("cpu_name", "gpu_name", "gpu_device", "device"): + item = value.get(field) + result[field] = ( + " ".join(item.split())[:160] if isinstance(item, str) and item.isprintable() and item.strip() else None + ) + for field in ("gpu_total_bytes", "sharing_vram_bytes", "sharing_vram_available_bytes"): + item = value.get(field) + result[field] = item if type(item) is int and 0 <= item <= 64 * 1024**4 else None + return result + + class NodeClient: """Synchronous control client; GUI adapters must call it off their event loop.""" @@ -476,6 +512,7 @@ def status(self) -> Dict[str, Any]: ] result["auto_selection"] = _normalize_auto_selection(result.get("auto_selection")) result["contribution"] = _normalize_contribution_status(result.get("contribution")) + result["hardware"] = _normalize_hardware(result.get("hardware")) return result def get_contribution_policy(self) -> Dict[str, Any]: @@ -484,6 +521,16 @@ def get_contribution_policy(self) -> Dict[str, Any]: require_revision=True, ) + def set_inference_mode(self, mode: str) -> Dict[str, Any]: + if mode not in ("auto", "local_only"): + raise ValueError("inference mode must be auto or local_only") + policy = self.get_contribution_policy() + return self._request( + "PUT", + "/control/v1/inference-mode", + payload={"inference_mode": mode, "expected_config_revision": policy["config_revision"]}, + ) + def update_contribution_policy(self, policy: Mapping[str, Any], *, expected_revision: str) -> Dict[str, Any]: if not isinstance(policy, Mapping): raise ValueError("contribution policy must be a mapping") diff --git a/desktop/src/communityai_desktop/controller.py b/desktop/src/communityai_desktop/controller.py index 444c08c2a..d1fe89a15 100644 --- a/desktop/src/communityai_desktop/controller.py +++ b/desktop/src/communityai_desktop/controller.py @@ -4,10 +4,13 @@ from typing import Any, Dict -from communityai_desktop.client import NodeClient +from communityai_desktop.client import NodeApiError, NodeClient, NodeClientError +from communityai_desktop.telemetry import route_view -def _download_storage_estimate(size_bytes: int) -> str: +def _download_storage_estimate(size_bytes: int | None) -> str: + if size_bytes is None: + return "Pending verified shard selection" return f"{size_bytes / 1_000_000_000:.1f} GB ({size_bytes:,} bytes)" @@ -23,17 +26,23 @@ def snapshot(self) -> Dict[str, Any]: model["auto_selected"] = model["id"] == auto_selection["model"] contribution = status["contribution"] workers = [self._worker_view(worker) for worker in contribution["workers"]] + hardware = dict(status.get("hardware") or {}) + selected = next((model for model in models if model["auto_selected"]), {}) + hardware["inference_device"] = selected.get("device") return { "node_status": status.get("status", "unknown"), "openai_base_url": status["openai_base_url"], "started_at": status.get("started_at"), "runtime_budget": status.get("runtime_budget", {}), + "hardware": hardware, + "inference_mode": status.get("inference_mode", "auto"), + "inference_mode_editable": status.get("inference_mode_editable", False), "models": models, "auto_selection": auto_selection, "workers": workers, "keys": [self._key_view(key) for key in self.client.list_keys()], "network": self._network_view(status.get("network"), models), - "contribution": self._contribution_view(contribution, workers), + "contribution": self._contribution_view(contribution, workers, hardware), } @staticmethod @@ -49,6 +58,7 @@ def _model_view(model: Dict[str, Any]) -> Dict[str, Any]: and isinstance(total, int) and total > 0 and covered == total + and route.get("chat_ready", True) ) return { "id": str(model.get("id", "unknown")), @@ -57,11 +67,17 @@ def _model_view(model: Dict[str, Any]) -> Dict[str, Any]: "covered_blocks": covered, "total_blocks": total, "route_complete": route_complete, + "chat_ready": route.get("chat_ready"), + "text_peer_count": route.get("text_peer_count"), "peer_count": route.get("peer_count"), + "execution": "local" if route.get("source") == "local" else "distributed", + "device": route.get("device"), "selected_whole_shard_bytes": selected_whole_shard_bytes, "download_storage_estimate": _download_storage_estimate(selected_whole_shard_bytes), "active_requests": model.get("active_requests", 0), "last_error": model.get("last_error"), + "health": route_view(route), + "download_progress": None if selected_whole_shard_bytes == 0 else model["download"].get("progress"), } @staticmethod @@ -100,8 +116,21 @@ def _worker_view(worker: Dict[str, Any]) -> Dict[str, Any]: ) state = worker["state"] desired_running = worker["desired_running"] - if state in ("running", "starting"): + progress_state = (worker.get("download_progress") or {}).get("state") + preparing = state in ("running", "starting") and progress_state in { + "waiting", + "checking", + "downloading", + "retrying", + "verifying", + "loading", + } + if preparing: + display_status = "Downloading model" if progress_state in ("downloading", "retrying") else "Preparing model" + elif state == "running": display_status = "Sharing" + elif state == "starting": + display_status = "Starting sharing" elif desired_running and blocked_reason: display_status = f"Waiting: {blocked_reason}" elif not admitted: @@ -115,7 +144,9 @@ def _worker_view(worker: Dict[str, Any]) -> Dict[str, Any]: "model": worker["model"], "state": state, "desired_running": desired_running, - "sharing_active": state in ("running", "starting"), + "operator_paused": worker.get("operator_paused", False), + "sharing_active": state == "running" and not preparing, + "preparing": preparing, "can_start": admitted, "blocked_reason": blocked_reason, "display_status": display_status, @@ -130,6 +161,8 @@ def _worker_view(worker: Dict[str, Any]) -> Dict[str, Any]: "resource_suspended": resources["suspended"], "limits": resources["limits"], "measurements": resources["measurements"], + "placement": worker.get("placement", {}), + "download_progress": worker.get("download_progress"), } @staticmethod @@ -160,7 +193,11 @@ def _network_view(network: Any, models: list[Dict[str, Any]]) -> Dict[str, Any]: return {"peer_count": peer_count, "regions": clean_regions} @staticmethod - def _contribution_view(contribution: Dict[str, Any], workers: list[Dict[str, Any]]) -> Dict[str, Any]: + def _contribution_view( + contribution: Dict[str, Any], workers: list[Dict[str, Any]], hardware: Dict[str, Any] | None = None + ) -> Dict[str, Any]: + policy_snapshot = contribution["policy"] + policy = policy_snapshot["policy"] active_models = sorted({worker["model"] for worker in workers if worker["sharing_active"]}) selected_models = sorted({worker["model"] for worker in workers if worker["desired_running"]}) blocked_reasons = [] @@ -169,7 +206,12 @@ def _contribution_view(contribution: Dict[str, Any], workers: list[Dict[str, Any reason = worker["blocked_reason"] if reason and reason not in blocked_reasons: blocked_reasons.append(reason) - if worker["desired_running"] and reason and reason not in selected_blocked_reasons: + selected = worker["desired_running"] or ( + policy.get("sharing_enabled") + and worker.get("placement", {}).get("automatic") + and not worker.get("operator_paused", False) + ) + if selected and reason and reason not in selected_blocked_reasons: selected_blocked_reasons.append(reason) vram_pairs = { @@ -188,16 +230,22 @@ def _contribution_view(contribution: Dict[str, Any], workers: list[Dict[str, Any vram_bytes = vram_pool_bytes = vram_percent = None vram_status = "unavailable" - policy_snapshot = contribution["policy"] + hardware = hardware or {} + if hardware.get("sharing_vram_bytes") is not None: + vram_bytes = hardware["sharing_vram_bytes"] + vram_pool_bytes = hardware.get("gpu_total_bytes") + vram_percent = round(vram_bytes * 100 / vram_pool_bytes) if vram_pool_bytes else None + vram_status = "configured" + intent_enabled = policy.get("sharing_enabled", False) or bool(selected_models) return { "configured": contribution["configured"], "editable": contribution["editable"], "config_revision": policy_snapshot["config_revision"], - "policy": policy_snapshot["policy"], + "policy": policy, "enabled": bool(active_models), - "intent_enabled": bool(selected_models), - "can_start": any(worker["can_start"] for worker in workers), - "can_pause": any(worker["desired_running"] for worker in workers), + "intent_enabled": intent_enabled, + "can_start": contribution["editable"] and bool(workers), + "can_pause": intent_enabled, "active_models": active_models, "selected_models": selected_models, "blocked_reasons": blocked_reasons, @@ -206,14 +254,88 @@ def _contribution_view(contribution: Dict[str, Any], workers: list[Dict[str, Any "vram_bytes": vram_bytes, "vram_pool_bytes": vram_pool_bytes, "vram_percent": vram_percent, + "processing_percent": policy.get("max_processing_percent", 100), + "vram_available_bytes": hardware.get("sharing_vram_available_bytes"), } + def set_sharing_enabled(self, enabled: bool) -> Dict[str, Any]: + """Persist the user's sharing choice before starting or stopping workers.""" + current = self.client.status()["contribution"] + if not current["editable"]: + raise NodeClientError("Sharing settings are unavailable. Restart CommunityAI and try again.") + if enabled and not current["workers"]: + raise NodeClientError("No community model is available for sharing yet.") + saved = current["policy"] + for worker in current["workers"]: + self.client.worker_action(worker["id"], "pause") + policy = {**saved["policy"], "sharing_enabled": enabled} + if enabled: + policy["max_disk_space"] = policy.get("max_disk_space") or "20GiB" + policy["max_vram"] = policy.get("max_vram") or "100%" + policy["max_processing_percent"] = policy.get("max_processing_percent", 100) + result = self.client.update_contribution_policy(policy, expected_revision=saved["config_revision"]) + if not enabled: + return {**result, "message": "Sharing paused."} + waiting = False + for worker in current["workers"]: + try: + self.client.worker_action(worker["id"], "start") + except NodeApiError as exc: + if exc.status_code != 409: + raise + waiting = True + return {**result, "message": "Sharing is waiting to start." if waiting else "Sharing enabled."} + def worker_action(self, worker_id: str, action: str) -> Dict[str, Any]: return self.client.worker_action(worker_id, action) def update_contribution_policy(self, policy: Dict[str, Any], *, expected_revision: str) -> Dict[str, Any]: return self.client.update_contribution_policy(policy, expected_revision=expected_revision) + def update_resource_limits(self, changes: Dict[str, Any], *, expected_revision: str) -> Dict[str, Any]: + if not changes or set(changes) - {"max_vram", "max_processing_percent"}: + raise ValueError("Only VRAM and processing percentages can change here") + for field, value in changes.items(): + if field == "max_vram": + if not isinstance(value, str) or not value.endswith("%") or not value[:-1].isdigit(): + raise ValueError("VRAM must be a whole percentage") + value = int(value[:-1]) + if type(value) is not int or not 1 <= value <= 100: + raise ValueError("Resource percentages must be whole numbers from 1 to 100") + current = self.client.status()["contribution"] + saved = current["policy"] + if saved["config_revision"] != expected_revision: + raise NodeClientError("Settings changed elsewhere. Refresh before applying limits.") + if not current["editable"] or "max_processing_percent" not in saved["policy"]: + raise NodeClientError("Update the local node to use these resource controls.") + resume = [ + worker["id"] + for worker in current["workers"] + if worker["desired_running"] + or ( + saved["policy"].get("sharing_enabled") + and worker.get("placement", {}).get("automatic") + and not worker.get("operator_paused", False) + ) + ] + # Pause all configured workers, including automatic workers with no current + # process. This prevents the placement service racing the policy transaction. + for worker in current["workers"]: + self.client.worker_action(worker["id"], "pause") + result = self.client.update_contribution_policy( + {**saved["policy"], **changes}, expected_revision=expected_revision + ) + # Failed persistence deliberately leaves workers stopped. Never restart a + # worker with an old, more permissive limit after a rejected save. + errors = [] + for worker_id in resume: + try: + self.client.worker_action(worker_id, "start") + except NodeClientError as exc: + errors.append(str(exc)) + result["message"] = "Changes saved. Sharing is waiting to start." if errors else "Changes saved." + return result + def set_workers_enabled(self, worker_ids: list[str], enabled: bool) -> list[Dict[str, Any]]: action = "start" if enabled else "pause" return [self.client.worker_action(worker_id, action) for worker_id in worker_ids] diff --git a/desktop/src/communityai_desktop/gate13_playthrough.py b/desktop/src/communityai_desktop/gate13_playthrough.py new file mode 100644 index 000000000..e76fc8b59 --- /dev/null +++ b/desktop/src/communityai_desktop/gate13_playthrough.py @@ -0,0 +1,959 @@ +"""Automate the real packaged Gate 13 desktop playthrough. + +This module is deliberately part of the frozen desktop rather than an external UI +mock. Qualification invocations reproduce the platform-specific sequence from the +accepted manual run using the normal window, real sharing-policy dialog, literal +Start/Pause buttons, and bounded localhost inference with an ephemeral client key. +It retains only bounded acceptance facts. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import stat +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +from communityai_desktop.client import normalize_loopback_url + +SCHEMA_VERSION = 2 +SCOPE = "gate13-packaged-desktop-playthrough" +MAX_CONFIG_BYTES = 65_536 +MAX_RESPONSE_BYTES = 1_048_576 +QUALIFICATION_KEY_LABEL = "Gate 13 automated qualification" +MANUAL_ROUTE_WAIT_SECONDS = 450.0 +MANUAL_ROUTE_POLL_SECONDS = 5.0 +MAX_PROGRESS_BYTES = 16_384 + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_MODEL_RE = re.compile(r"[ -~]{1,128}") +_INFERENCE_DETAILS = { + "inference_failed", + "inference_rejected", + "inference_transport_failed", + "inference_timed_out", + "inference_response_invalid", + "inference_selection_changed", + "inference_key_baseline_missing", + "inference_key_baseline_dirty", + "inference_key_response_invalid", + "inference_model_mismatch", + "inference_token_count_invalid", + "inference_unexpected_error", + "inference_key_cleanup_failed", +} +_POLICY_FIELDS = { + "sharing_enabled", + "allowed_models", + "preferred_models", + "denied_models", + "max_disk_space", + "max_vram", + "max_bandwidth_mbps", + "max_power_watts", + "pause_timeout", + "schedule", +} + + +def _manual_schedule() -> dict[str, Any]: + return { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + } + + +_CONFIG_FIELDS = { + "schema_version", + "run_id", + "platform", + "stage", + "model_id", + "manifest_digest", + "total_blocks", + "policy", + "timeout_seconds", + "inference_timeout_seconds", +} + + +class PlaythroughError(ValueError): + """A qualification plan or observed desktop state failed closed.""" + + +def _safe_inference_detail(message: str) -> str: + if message in _INFERENCE_DETAILS or re.fullmatch(r"inference_http_[1-5][0-9]{2}", message): + return message + return "inference_failed" + + +def _reject_constant(_value: str) -> None: + raise PlaythroughError("configuration contains a non-finite value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise PlaythroughError("configuration contains a duplicate field") + value[key] = item + return value + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise PlaythroughError("configuration is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise PlaythroughError("configuration is not a bounded regular file") + try: + return path.read_bytes() + except OSError as exc: + raise PlaythroughError("configuration is unreadable") from exc + + +def _bounded_number(value: Any, label: str, *, minimum: float, maximum: float) -> float: + if type(value) not in (int, float): + raise PlaythroughError(f"{label} is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise PlaythroughError(f"{label} is invalid") + return rendered + + +def _selectors(value: Any, label: str) -> tuple[str, ...]: + if not isinstance(value, list) or len(value) > 8: + raise PlaythroughError(f"{label} is invalid") + clean: list[str] = [] + folded: set[str] = set() + for item in value: + if not isinstance(item, str) or _MODEL_RE.fullmatch(item) is None or item != item.strip(): + raise PlaythroughError(f"{label} is invalid") + canonical = item.casefold() + if canonical in folded: + raise PlaythroughError(f"{label} contains a duplicate selector") + folded.add(canonical) + clean.append(item) + return tuple(clean) + + +def _policy(value: Any, model_id: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _POLICY_FIELDS: + raise PlaythroughError("sharing policy schema is invalid") + allowed = _selectors(value["allowed_models"], "allowed models") + preferred = _selectors(value["preferred_models"], "preferred models") + denied = _selectors(value["denied_models"], "denied models") + if value["sharing_enabled"] is not True: + raise PlaythroughError("sharing must be enabled for the start stage") + if model_id not in allowed or model_id not in preferred or denied: + raise PlaythroughError("sharing policy does not select the qualification model") + if value["max_disk_space"] != "32GB": + raise PlaythroughError("storage ceiling does not match the proven manual replay") + if value["max_vram"] != "20GB": + raise PlaythroughError("memory ceiling does not match the proven manual replay") + bandwidth = _bounded_number(value["max_bandwidth_mbps"], "bandwidth ceiling", minimum=0.001, maximum=1_000_000) + if bandwidth != 100.0: + raise PlaythroughError("bandwidth ceiling does not match the proven manual replay") + if value["max_power_watts"] is not None: + raise PlaythroughError("the manual CPU-host replay requires an unset power ceiling") + pause = _bounded_number(value["pause_timeout"], "pause timeout", minimum=1, maximum=300) + if pause != 120.0: + raise PlaythroughError("pause timeout does not match the proven manual replay") + if value["schedule"] != _manual_schedule(): + raise PlaythroughError("sharing schedule does not match the proven manual replay") + return { + "sharing_enabled": True, + "allowed_models": list(allowed), + "preferred_models": list(preferred), + "denied_models": list(denied), + "max_disk_space": value["max_disk_space"], + "max_vram": value["max_vram"], + "max_bandwidth_mbps": bandwidth, + "max_power_watts": None, + "pause_timeout": pause, + "schedule": _manual_schedule(), + } + + +@dataclass(frozen=True) +class PlaythroughPlan: + run_id: str + platform: str + stage: str + model_id: str + manifest_digest: str + total_blocks: int + policy: Mapping[str, Any] + timeout_seconds: float + inference_timeout_seconds: float + + @classmethod + def load(cls, path: Path) -> "PlaythroughPlan": + payload = _regular_bytes(path, MAX_CONFIG_BYTES) + try: + raw = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PlaythroughError("configuration is invalid JSON") from exc + if not isinstance(raw, dict) or set(raw) != _CONFIG_FIELDS or raw.get("schema_version") != SCHEMA_VERSION: + raise PlaythroughError("configuration schema is invalid") + run_id = raw["run_id"] + platform = raw["platform"] + stage = raw["stage"] + model_id = raw["model_id"] + digest = raw["manifest_digest"] + total_blocks = raw["total_blocks"] + if not isinstance(run_id, str) or _RUN_RE.fullmatch(run_id) is None: + raise PlaythroughError("run id is invalid") + if platform not in ("windows", "linux"): + raise PlaythroughError("playthrough platform is invalid") + if stage not in ("initial", "restart"): + raise PlaythroughError("playthrough stage is invalid") + if not isinstance(model_id, str) or _MODEL_RE.fullmatch(model_id) is None or model_id != model_id.strip(): + raise PlaythroughError("model id is invalid") + if not isinstance(digest, str) or _DIGEST_RE.fullmatch(digest) is None: + raise PlaythroughError("manifest digest is invalid") + if type(total_blocks) is not int or not 1 <= total_blocks <= 512: + raise PlaythroughError("block count is invalid") + timeout = _bounded_number(raw["timeout_seconds"], "playthrough timeout", minimum=30, maximum=3_600) + inference_timeout = _bounded_number( + raw["inference_timeout_seconds"], "inference timeout", minimum=10, maximum=600 + ) + return cls( + run_id=run_id, + platform=platform, + stage=stage, + model_id=model_id, + manifest_digest=digest, + total_blocks=total_blocks, + policy=_policy(raw["policy"], model_id), + timeout_seconds=timeout, + inference_timeout_seconds=inference_timeout, + ) + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ARG002 + return None + + +def _completion_request(url: str, secret: str, timeout: float) -> Mapping[str, Any]: + body = json.dumps( + { + "model": "auto", + "messages": [{"role": "user", "content": "Reply with one word."}], + "max_tokens": 1, + "stream": False, + }, + separators=(",", ":"), + ).encode("utf-8") + request = Request( + url, + data=body, + method="POST", + headers={ + "Authorization": f"Bearer {secret}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + opener = build_opener(ProxyHandler({}), _RejectRedirects()) + try: + with opener.open(request, timeout=timeout) as response: + if response.status != 200 or response.headers.get_content_type() != "application/json": + raise PlaythroughError("inference_rejected") + payload = response.read(MAX_RESPONSE_BYTES + 1) + except HTTPError as exc: + raise PlaythroughError(_safe_inference_detail(f"inference_http_{exc.code}")) from exc + except (URLError, OSError, TimeoutError) as exc: + timed_out = isinstance(exc, TimeoutError) or isinstance(getattr(exc, "reason", None), TimeoutError) + raise PlaythroughError("inference_timed_out" if timed_out else "inference_transport_failed") from exc + if not 1 <= len(payload) <= MAX_RESPONSE_BYTES: + raise PlaythroughError("inference_response_invalid") + try: + value = json.loads(payload.decode("utf-8"), parse_constant=_reject_constant) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PlaythroughError("inference_response_invalid") from exc + if not isinstance(value, dict): + raise PlaythroughError("inference_response_invalid") + return value + + +def _manual_route_ready(status: Mapping[str, Any], plan: PlaythroughPlan) -> bool: + selection = status.get("auto_selection") + models = status.get("models") + if not isinstance(selection, dict) or not isinstance(models, list): + return False + selected = next( + ( + item + for item in models + if isinstance(item, dict) + and item.get("id") == plan.model_id + and item.get("manifest_digest") == plan.manifest_digest + ), + None, + ) + route = selected.get("route") if selected is not None else None + return bool( + selection.get("status") == "selected" + and selection.get("model") == plan.model_id + and selection.get("manifest_digest") == plan.manifest_digest + and selection.get("covered_blocks") == plan.total_blocks + and selection.get("total_blocks") == plan.total_blocks + and isinstance(selection.get("peer_count"), int) + and selection["peer_count"] > 0 + and isinstance(route, dict) + and route.get("status") == "complete" + and route.get("covered_blocks") == plan.total_blocks + and route.get("total_blocks") == plan.total_blocks + ) + + +def _completion_after_manual_readiness_wait( + controller: Any, + plan: PlaythroughPlan, + url: str, + secret: str, +) -> Mapping[str, Any]: + """Replay the manual Model-unavailable -> wait-for-complete -> retry sequence.""" + + try: + return _completion_request(url, secret, plan.inference_timeout_seconds) + except PlaythroughError as first_error: + deadline = time.monotonic() + min(MANUAL_ROUTE_WAIT_SECONDS, plan.inference_timeout_seconds) + while time.monotonic() < deadline: + time.sleep(min(MANUAL_ROUTE_POLL_SECONDS, max(0.0, deadline - time.monotonic()))) + try: + status = controller.client.status() + except BaseException: + continue + if _manual_route_ready(status, plan): + try: + return _completion_request(url, secret, plan.inference_timeout_seconds) + except PlaythroughError as retry_error: + raise retry_error from first_error + raise first_error + + +def qualify_localhost_inference(controller: Any, plan: PlaythroughPlan) -> dict[str, Any]: + """Run one response-content-free localhost inference and restore the API-key baseline.""" + + baseline_items = [item for item in controller.client.list_keys() if item.get("revoked_at") is None] + baseline = {item["id"] for item in baseline_items} + if not baseline: + raise PlaythroughError("inference_key_baseline_missing") + if any(item.get("label") == QUALIFICATION_KEY_LABEL for item in baseline_items): + raise PlaythroughError("inference_key_baseline_dirty") + created_id = "" + secret = "" + failure_detail = None + cleanup_failed = False + completion_count = 0 + generated_token_count = 0 + try: + status = controller.client.status() + selection = status["auto_selection"] + if ( + selection.get("status") != "selected" + or selection.get("model") != plan.model_id + or selection.get("manifest_digest") != plan.manifest_digest + ): + raise PlaythroughError("inference_selection_changed") + created = controller.client.create_key(QUALIFICATION_KEY_LABEL) + created_id = created.get("key", {}).get("id", "") + secret = created.get("secret", "") + if ( + not isinstance(created_id, str) + or not created_id + or not isinstance(secret, str) + or not 1 <= len(secret) <= 512 + ): + raise PlaythroughError("inference_key_response_invalid") + base = normalize_loopback_url(status["openai_base_url"]) + completion = _completion_after_manual_readiness_wait( + controller, + plan, + f"{base}/v1/chat/completions", + secret, + ) + if completion.get("model") != plan.model_id: + raise PlaythroughError("inference_model_mismatch") + usage = completion.get("usage") + generated = usage.get("completion_tokens") if isinstance(usage, dict) else None + if type(generated) is not int or generated != 1: + raise PlaythroughError("inference_token_count_invalid") + completion_count = 1 + generated_token_count = generated + except PlaythroughError as exc: + failure_detail = _safe_inference_detail(str(exc)) + except BaseException: + failure_detail = "inference_unexpected_error" + finally: + secret = "" + try: + active = {item["id"] for item in controller.client.list_keys() if item.get("revoked_at") is None} + candidates = active - baseline + if created_id and created_id in active: + candidates.add(created_id) + if len(candidates) != 1: + raise PlaythroughError("temporary client key identity is ambiguous") + controller.client.revoke_key(next(iter(candidates))) + after = {item["id"] for item in controller.client.list_keys() if item.get("revoked_at") is None} + if after != baseline: + raise PlaythroughError("temporary client key cleanup failed") + except BaseException: + cleanup_failed = True + if cleanup_failed: + raise PlaythroughError("inference_key_cleanup_failed") + if failure_detail is not None: + raise PlaythroughError(failure_detail) + return { + "passed": True, + "model_id": plan.model_id, + "manifest_digest": plan.manifest_digest, + "completion_count": completion_count, + "generated_token_count": generated_token_count, + "response_content_retained": False, + "token_identifiers_retained": False, + "temporary_key_removed": True, + } + + +def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: + destination = Path(path).absolute() + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and (destination.is_symlink() or not destination.is_file()): + raise PlaythroughError("evidence destination is unsafe") + payload = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + temporary_name = "" + try: + with tempfile.NamedTemporaryFile( + prefix=".gate13-playthrough-", suffix=".tmp", dir=destination.parent, delete=False + ) as out: + temporary_name = out.name + out.write(payload) + out.flush() + os.fsync(out.fileno()) + os.replace(temporary_name, destination) + temporary_name = "" + except OSError as exc: + raise PlaythroughError("evidence could not be persisted") from exc + finally: + if temporary_name: + try: + Path(temporary_name).unlink() + except OSError: + pass + + +class Gate13Playthrough: + """A bounded Qt state machine that drives the real packaged controls.""" + + def __init__( + self, + plan: PlaythroughPlan, + evidence_path: Path, + *, + screenshot_path: Path | None = None, + inference_runner: Callable[[Any, PlaythroughPlan], Mapping[str, Any]] = qualify_localhost_inference, + clock: Callable[[], float] = time.monotonic, + start_observation_seconds: float | None = None, + restart_observation_seconds: float | None = None, + ): + self.plan = plan + self.evidence_path = Path(evidence_path) + self.screenshot_path = None if screenshot_path is None else Path(screenshot_path) + self._inference_runner = inference_runner + self._clock = clock + default_start_observation = 25.0 if plan.platform == "windows" else 20.0 + self._start_observation_seconds = ( + default_start_observation + if start_observation_seconds is None + else _bounded_number(start_observation_seconds, "start observation", minimum=0.05, maximum=60) + ) + self._restart_observation_seconds = ( + 15.0 + if restart_observation_seconds is None + else _bounded_number(restart_observation_seconds, "restart observation", minimum=0.05, maximum=60) + ) + self._started = clock() + self._state = "wait_ready" + self._logged_phase: str | None = None + self._bootstrap_failure = "" + self._done = False + self._inference: Mapping[str, Any] | None = None + self._window = None + self._application = None + self._qt: Mapping[str, Any] = {} + self._timer = None + self._observation_deadline: float | None = None + self._ui = { + "real_window_opened": True, + "policy_dialog_saved": False, + "start_clicked": False, + "pause_control_observed": False, + "pause_clicked": False, + "restart_resume_observed": False, + "sharing_intent_enabled_observed": False, + "sharing_intent_disabled_observed": False, + } + + def _log_progress(self, message: str) -> None: + """Keep readable diagnostics even when a windowed bundle has no stderr.""" + path = self.evidence_path.with_suffix(".log") + payload = ("gate13-playthrough: " + " ".join(message.split())[:700] + "\n").encode("utf-8") + try: + if path.is_symlink(): + return + with path.open("ab") as stream: + if stream.tell() + len(payload) <= MAX_PROGRESS_BYTES: + stream.write(payload) + stream.flush() + except OSError: + # Logging must never change the replay's acceptance result. + pass + + def install(self, window: Any, application: Any, qt: Mapping[str, Any]) -> None: + self._window = window + self._application = application + self._qt = qt + timer_type = qt["QTimer"] + self._timer = timer_type(window) + self._timer.setInterval(200) + self._timer.timeout.connect(self._tick) + self._timer.start() + timer_type.singleShot(max(1, int(self.plan.timeout_seconds * 1_000)), self._timeout) + + def _timeout(self) -> None: + if not self._done: + self._fail("playthrough_timed_out") + + def _ready(self) -> bool: + window = self._window + if window is None or window._controller is None or window._busy: + return False + snapshot = window._snapshot + selection = snapshot.get("auto_selection", {}) + models = snapshot.get("models", []) + selected = next((item for item in models if item.get("id") == self.plan.model_id), None) + return bool( + selection.get("status") == "selected" + and selection.get("model") == self.plan.model_id + and selection.get("manifest_digest") == self.plan.manifest_digest + and selection.get("covered_blocks") == self.plan.total_blocks + and selection.get("total_blocks") == self.plan.total_blocks + and isinstance(selection.get("peer_count"), int) + and selection["peer_count"] > 0 + and selected is not None + and selected.get("route_complete") is True + and selected.get("covered_blocks") == self.plan.total_blocks + and selected.get("total_blocks") == self.plan.total_blocks + ) + + def _tick(self) -> None: + if self._done or self._window is None: + return + try: + if self._state != self._logged_phase: + self._log_progress(f"stage={self.plan.stage} phase={self._state}") + self._logged_phase = self._state + if self._state == "wait_ready": + detail_label = getattr(self._window, "connection_detail", None) + detail = detail_label.text() if detail_label is not None else "" + bootstrap_failure = ( + detail[:600] + if isinstance(detail, str) + and detail.startswith( + ( + "The signed model catalog could not be installed:", + "The signed model catalog installation timed out after ", + ) + ) + else "" + ) + if bootstrap_failure != self._bootstrap_failure: + self._bootstrap_failure = bootstrap_failure + if bootstrap_failure: + self._log_progress(f"stage={self.plan.stage} phase=wait_ready {bootstrap_failure}") + if not self._ready(): + return + if self.plan.stage == "initial": + self._begin_inference("after_initial_inference") + elif self.plan.platform == "windows": + self._begin_policy_edit() + else: + if not self._show_sharing_page(): + self._fail() + return + self._state = "wait_resumed" + elif self._state == "wait_policy": + contribution = self._window._snapshot.get("contribution", {}) + observed_policy = dict(contribution.get("policy") or {}) + if observed_policy.get("max_processing_percent") == 100: + observed_policy.pop("max_processing_percent") + if not self._window._busy and observed_policy == self.plan.policy: + self._ui["policy_dialog_saved"] = True + # The manual Windows run toggled the selected model before + # using the master Start control. Automatic placement can + # race the policy refresh and start that model first; in + # that state the master control already says Pause and the + # old replay waited forever for a Start button. Restore a + # paused baseline through the literal per-model control, + # then replay the literal master Start action. + if contribution.get("intent_enabled"): + desired = any( + worker.get("model") == self.plan.model_id and worker.get("desired_running") + for worker in self._window._snapshot.get("workers", []) + ) + if desired: + self._click_model_toggle_to_pause() + else: + self._pause_before_policy_action("wait_prestart_paused") + else: + self._click_start() + elif self._state == "wait_policy_paused": + contribution = self._window._snapshot.get("contribution", {}) + if not self._window._busy and not contribution.get("intent_enabled"): + self._begin_policy_edit() + elif self._state == "wait_prestart_paused": + contribution = self._window._snapshot.get("contribution", {}) + if not self._window._busy and contribution.get("intent_enabled"): + desired = any(worker.get("desired_running") for worker in self._window._snapshot.get("workers", [])) + if not desired: + self._pause_before_policy_action("wait_prestart_paused") + if ( + not self._window._busy + and not contribution.get("intent_enabled") + and self._start_control_available() + ): + self._click_start() + elif self._state == "wait_started_intent": + contribution = self._window._snapshot.get("contribution", {}) + if contribution.get("intent_enabled") and self._pause_control_available(): + self._ui["sharing_intent_enabled_observed"] = True + self._ui["pause_control_observed"] = True + if self._observation_deadline is None: + self._observation_deadline = self._clock() + self._start_observation_seconds + if self._clock() >= self._observation_deadline: + if self.plan.platform == "windows": + self._click_pause() + else: + self._pass() + elif self._state == "wait_resumed": + contribution = self._window._snapshot.get("contribution", {}) + workers = self._window._snapshot.get("workers", []) + desired = any( + item.get("model") == self.plan.model_id and item.get("desired_running") for item in workers + ) + if contribution.get("intent_enabled") and desired and self._pause_control_available(): + self._ui["sharing_intent_enabled_observed"] = True + self._ui["pause_control_observed"] = True + self._ui["restart_resume_observed"] = True + if self._observation_deadline is None: + self._observation_deadline = self._clock() + self._restart_observation_seconds + if self._clock() >= self._observation_deadline: + self._click_pause() + elif self._state == "wait_paused_intent": + contribution = self._window._snapshot.get("contribution", {}) + if ( + not self._window._busy + and not contribution.get("intent_enabled") + and self._start_control_available() + ): + self._ui["sharing_intent_disabled_observed"] = True + if self.plan.platform == "linux": + self._begin_inference("after_restart_inference") + else: + self._pass() + except BaseException: + self._fail() + + def _begin_inference(self, waiting_state: str) -> None: + self._state = waiting_state + controller = self._window._controller + + def finished(result: Mapping[str, Any]) -> None: + self._inference = dict(result) + if waiting_state == "after_initial_inference" and self.plan.platform == "linux": + self._begin_policy_edit() + else: + self._pass() + + self._window._submit( + lambda: self._inference_runner(controller, self.plan), + finished, + lambda message: self._fail("inference_failed", _safe_inference_detail(message)), + ) + + def _begin_policy_edit(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + if self._window._snapshot.get("contribution", {}).get("intent_enabled"): + self._pause_before_policy_action("wait_policy_paused") + return + if not self._show_more_settings(): + self._fail() + return + if self._window.edit_policy_button.isEnabled() is False: + self._fail() + return + self._window.pages.currentWidget().ensureWidgetVisible(self._window.edit_policy_button) + if not self._window.edit_policy_button.isVisible(): + self._fail() + return + self._state = "editing_policy" + self._qt["QTimer"].singleShot(100, self._fill_policy_dialog) + self._window.edit_policy_button.click() + + def _pause_before_policy_action(self, state: str) -> None: + if not self._show_sharing_page(): + self._fail() + return + button = self._window.master_share_button + if button.text() != "Pause sharing" or not button.isEnabled(): + return + self._state = state + button.click() + + def _show_more_settings(self) -> bool: + window = self._window + matches = [ + button + for button in window.findChildren(type(window.master_share_button)) + if button.accessibleName() == "More sharing settings" + ] + # Older qualified shells presented these controls directly on Sharing. + if not matches: + edit_button = getattr(window, "edit_policy_button", None) + return edit_button is None or not hasattr(edit_button, "isVisible") or edit_button.isVisible() + if len(matches) != 1 or not matches[0].isEnabled(): + return False + button = matches[0] + window.pages.currentWidget().ensureWidgetVisible(button) + if not button.isChecked(): + button.click() + return bool(button.isChecked()) + + def _fill_policy_dialog(self) -> None: + try: + dialog = self._window.findChild(self._qt["QDialog"], "sharingPolicyDialog") + if dialog is None: + raise PlaythroughError("sharing policy dialog did not open") + checkbox = dialog.findChild(self._qt["QCheckBox"], "policy_sharing_enabled") + checkbox.setChecked(True) + for field in ("allowed_models", "preferred_models", "denied_models"): + editor = dialog.findChild(self._qt["QPlainTextEdit"], f"policy_{field}") + editor.setPlainText("\n".join(self.plan.policy[field])) + for field in ( + "max_disk_space", + "max_vram", + "max_bandwidth_mbps", + "max_power_watts", + "pause_timeout", + ): + editor = dialog.findChild(self._qt["QLineEdit"], f"policy_{field}") + value = self.plan.policy[field] + editor.setText("" if value is None else f"{value:g}" if isinstance(value, float) else str(value)) + schedule = dialog.findChild(self._qt["QPlainTextEdit"], "policy_schedule") + schedule.setPlainText(json.dumps(self.plan.policy["schedule"], separators=(",", ":"))) + buttons = dialog.findChild(self._qt["QDialogButtonBox"], "sharingPolicyButtons") + save = buttons.button(self._qt["QDialogButtonBox"].StandardButton.Save) + self._state = "wait_policy" + save.click() + except BaseException: + self._fail() + + def _show_sharing_page(self) -> bool: + window = self._window + buttons = None if window is None else getattr(window, "_page_buttons", None) + if not isinstance(buttons, list) or len(buttons) != 4: + return False + button = buttons[2] + if button.text() != "Sharing" or not button.isEnabled(): + return False + if not button.isChecked(): + button.click() + return bool(button.isChecked()) + + def _click_start(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + button = self._window.master_share_button + if button.text() != "Start sharing" or not button.isEnabled(): + return + self._state = "wait_started_intent" + self._observation_deadline = None + self._ui["start_clicked"] = True + button.click() + + def _click_model_toggle_to_pause(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + if not self._show_more_settings(): + self._fail() + return + checkbox_type = self._qt.get("QCheckBox") + if checkbox_type is None: + self._fail() + return + expected_name = f"Share compute with {self.plan.model_id}" + matches = [ + checkbox + for checkbox in self._window.findChildren(checkbox_type) + if checkbox.accessibleName() == expected_name + ] + desired = any( + worker.get("model") == self.plan.model_id and worker.get("desired_running") + for worker in self._window._snapshot.get("workers", []) + ) + if len(matches) != 1 or not desired or not matches[0].isChecked(): + self._fail() + return + if not matches[0].isEnabled(): + return + self._state = "wait_prestart_paused" + matches[0].click() + + def _click_pause(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + button = self._window.master_share_button + if button.text() != "Pause sharing" or not button.isEnabled(): + return + self._state = "wait_paused_intent" + self._observation_deadline = None + self._ui["pause_clicked"] = True + button.click() + + def _pause_control_available(self) -> bool: + button = self._window.master_share_button + return bool(button.text() == "Pause sharing" and button.isEnabled()) + + def _start_control_available(self) -> bool: + button = self._window.master_share_button + # A just-paused worker may remain temporarily resource-suspended while its + # process exits, which legitimately leaves Start disabled. The manual run + # accepted the intent transition and literal control text at this boundary. + return bool(button.text() == "Start sharing") + + def _base_result(self, result: str) -> dict[str, Any]: + duration = max(0.0, self._clock() - self._started) + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "run_id": self.plan.run_id, + "platform": self.plan.platform, + "stage": self.plan.stage, + "result": result, + "model_id": self.plan.model_id, + "manifest_digest": self.plan.manifest_digest, + "duration_seconds": round(duration, 6), + } + + def _pass(self) -> None: + inference_required = (self.plan.platform, self.plan.stage) in { + ("windows", "initial"), + ("linux", "initial"), + ("linux", "restart"), + } + if self._done or (inference_required and self._inference is None): + self._fail() + return + value = self._base_result("passed") + value.update( + { + "route": { + "rendered_in_real_window": True, + "complete": True, + "covered_blocks": self.plan.total_blocks, + "total_blocks": self.plan.total_blocks, + }, + "inference": None if self._inference is None else dict(self._inference), + "ui": dict(self._ui), + "limits": { + "storage": self._ui["policy_dialog_saved"], + "memory_or_vram": self._ui["policy_dialog_saved"], + "bandwidth": self._ui["policy_dialog_saved"], + "power": False, + "pause_timeout": self._ui["policy_dialog_saved"], + "schedule": self._ui["policy_dialog_saved"], + }, + "timing": { + "start_observation_seconds": ( + self._start_observation_seconds if self._ui["start_clicked"] else 0.0 + ), + "restart_observation_seconds": ( + self._restart_observation_seconds if self._ui["restart_resume_observed"] else 0.0 + ), + }, + "privacy": { + "prompt_retained": False, + "response_content_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + }, + } + ) + self._finish(value) + + def _fail(self, failure_code: str = "playthrough_failed", failure_detail: str | None = None) -> None: + if self._done: + return + value = self._base_result("failed") + value["failure_code"] = failure_code + value["failure_phase"] = self._state + if failure_detail is None and self._state == "wait_ready" and self._bootstrap_failure: + failure_detail = "bootstrap_failed" + if failure_detail is not None: + value["failure_detail"] = failure_detail + self._finish(value) + + def _finish(self, value: Mapping[str, Any]) -> None: + self._done = True + if self._timer is not None: + self._timer.stop() + try: + if self.screenshot_path is not None and self._window is not None: + self.screenshot_path.parent.mkdir(parents=True, exist_ok=True) + if not self._window.grab().save(str(self.screenshot_path)): + raise PlaythroughError("playthrough screenshot failed") + _atomic_json(self.evidence_path, value) + except BaseException: + fallback = self._base_result("failed") + fallback["failure_code"] = "evidence_write_failed" + fallback["failure_phase"] = self._state + try: + _atomic_json(self.evidence_path, fallback) + except BaseException: + pass + finally: + if self._application is not None: + self._application.quit() diff --git a/desktop/src/communityai_desktop/lifecycle.py b/desktop/src/communityai_desktop/lifecycle.py index ce8fde2c7..842986419 100644 --- a/desktop/src/communityai_desktop/lifecycle.py +++ b/desktop/src/communityai_desktop/lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import os import socket import subprocess @@ -67,6 +68,15 @@ def _port_is_open(node_url: str, timeout: float) -> bool: return False +def _bootstrap_output_detail(stdout: str | bytes | None, stderr: str | bytes | None) -> str: + for output in (stderr, stdout): + if isinstance(output, bytes): + output = output.decode("utf-8", errors="replace") + if output and output.strip(): + return output.strip().splitlines()[-1][:600] + return "" + + class NodeLifecycleSupervisor: """Start one native-key node sidecar and stop only the process it owns.""" @@ -80,7 +90,7 @@ def __init__( node_command: Optional[Sequence[str]] = None, bootstrap_config_path: Optional[Path | str] = None, bootstrap_command: Optional[Sequence[str]] = None, - bootstrap_timeout: float = 60.0, + bootstrap_timeout: float = 300.0, startup_timeout: float = 45.0, poll_interval: float = 0.2, client_timeout: float = 2.0, @@ -168,7 +178,8 @@ def _record_failure(self) -> None: def _ensure_config(self) -> None: if self.config_path.is_symlink(): raise NodeLifecycleError(f"CommunityAI refuses the unsafe configuration link {self.config_path.name}") - if self.config_path.is_file(): + existing = self.config_path.is_file() + if existing and (self.bootstrap_config_path is None or not self.bootstrap_command): return if self.bootstrap_config_path is None: raise NodeLifecycleError( @@ -190,12 +201,13 @@ def _ensure_config(self) -> None: str(self.data_dir), "--node_config", str(self.config_path), + *(("--refresh_if_needed",) if existing else ()), ) kwargs = { "stdin": subprocess.DEVNULL, "capture_output": True, "text": True, - "timeout": self.bootstrap_timeout, + "timeout": min(self.bootstrap_timeout, 30.0) if existing else self.bootstrap_timeout, "cwd": str(self.data_dir), "close_fds": True, } @@ -204,12 +216,30 @@ def _ensure_config(self) -> None: try: result = self._bootstrap_runner(command, **kwargs) except subprocess.TimeoutExpired as exc: - raise NodeLifecycleError("The signed model catalog installation timed out") from exc + if existing: + logging.getLogger(__name__).warning( + "Catalog migration timed out; starting the saved node configuration" + ) + return + detail = _bootstrap_output_detail(exc.stdout, exc.stderr) + suffix = f": {detail}" if detail else "" + raise NodeLifecycleError( + f"The signed model catalog installation timed out after {self.bootstrap_timeout:g} seconds{suffix}" + ) from exc except OSError as exc: + if existing: + logging.getLogger(__name__).warning( + "Catalog migration unavailable; starting the saved node configuration" + ) + return raise NodeLifecycleError(f"Could not run the bundled catalog installer: {exc}") from exc if result.returncode: - output = (result.stderr or result.stdout or "").strip().splitlines() - detail = output[-1][:600] if output else f"exit code {result.returncode}" + if existing: + logging.getLogger(__name__).warning( + "Catalog migration failed verification; starting the saved node configuration" + ) + return + detail = _bootstrap_output_detail(result.stdout, result.stderr) or f"exit code {result.returncode}" raise NodeLifecycleError(f"The signed model catalog could not be installed: {detail}") if not self.config_path.is_file() or self.config_path.is_symlink(): raise NodeLifecycleError("The signed model catalog installer did not create a safe node configuration") @@ -243,8 +273,9 @@ def _start(self) -> None: raise NodeLifecycleError(f"Could not start the bundled local node: {exc}") from exc def _stop_owned_process(self) -> None: - process, self._process = self._process, None + process = self._process if process is None or process.poll() is not None: + self._process = None return try: process.terminate() @@ -253,10 +284,13 @@ def _stop_owned_process(self) -> None: try: process.kill() process.wait(timeout=5) - except OSError: + except (OSError, subprocess.TimeoutExpired): pass except OSError: pass + if process.poll() is None: + raise NodeLifecycleError("The owned node did not stop; application files must not be replaced") + self._process = None def _wait_until_ready(self, client: NodeClient) -> None: deadline = self._clock() + self.startup_timeout diff --git a/desktop/src/communityai_desktop/maintenance.py b/desktop/src/communityai_desktop/maintenance.py new file mode 100644 index 000000000..28e4a603a --- /dev/null +++ b/desktop/src/communityai_desktop/maintenance.py @@ -0,0 +1,56 @@ +"""Same-user shutdown handshake used before replacing installed application files.""" + +import hashlib +import time +from pathlib import Path + + +def prepare_update(*, timeout=45.0, instance_name=None): + from communityai_desktop.pyside_shell import _single_instance_server_name + from communityai_desktop.startup import SingleInstanceError + from PySide6.QtCore import QCoreApplication, QLockFile, QStandardPaths + from PySide6.QtNetwork import QLocalSocket + + application = QCoreApplication.instance() or QCoreApplication([]) + application.setApplicationName("CommunityAI") + application.setOrganizationName("CommunityAI") + root = QStandardPaths.writableLocation(QStandardPaths.AppLocalDataLocation) + if not root: + raise SingleInstanceError("The application-data location is unavailable") + name = instance_name or _single_instance_server_name(root) + lock_digest = hashlib.sha256(name.encode("utf-8")).hexdigest()[:20] + lock = QLockFile(str(Path(root) / f"instance-{lock_digest}.lock")) + lock.setStaleLockTime(0) + if not Path(root).exists(): + return 0 + if lock.tryLock(0): + lock.unlock() + return 0 + socket = QLocalSocket(application) + socket.connectToServer(name) + deadline = time.monotonic() + timeout + while socket.state() != QLocalSocket.ConnectedState and time.monotonic() < deadline: + application.processEvents() + time.sleep(0.01) + if socket.state() != QLocalSocket.ConnectedState: + socket.abort() + raise SingleInstanceError("CommunityAI is starting or cannot acknowledge shutdown; retry the installer") + socket.write(b"shutdown\n") + response = bytearray() + while time.monotonic() < deadline: + socket.flush() + application.processEvents() + response.extend(bytes(socket.read(64 - len(response)))) + if b"\n" in response or len(response) >= 64: + break + time.sleep(0.01) + socket.abort() + if bytes(response) != b"stopped\n": + raise SingleInstanceError("CommunityAI could not finish shutting down; installation was not started") + while time.monotonic() < deadline: + if lock.tryLock(0): + lock.unlock() + return 0 + application.processEvents() + time.sleep(0.01) + raise SingleInstanceError("CommunityAI has not released its instance lock; retry the installer") diff --git a/desktop/src/communityai_desktop/model_health.py b/desktop/src/communityai_desktop/model_health.py new file mode 100644 index 000000000..3341bd71a --- /dev/null +++ b/desktop/src/communityai_desktop/model_health.py @@ -0,0 +1,448 @@ +"""Live block coverage, observed peers, and this computer's artifact transfers.""" + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QFrame, + QGridLayout, + QHBoxLayout, + QHeaderView, + QLabel, + QProgressBar, + QPushButton, + QTableWidget, + QTableWidgetItem, + QVBoxLayout, + QWidget, +) + +from communityai_desktop.presentation import model_name + +COLORS = { + "covered": "#237851", + "replicated": "#35b779", + "joining": "#367ed6", + "reserved": "#8060dc", + "missing": "#303b4d", + "offline": "#954354", + "unknown": "#242b38", +} + + +def caption(value, style="bodyMuted"): + widget = QLabel(value) + widget.setTextFormat(Qt.PlainText) + widget.setObjectName(style) + widget.setWordWrap(True) + return widget + + +def byte_text(value): + if value is None: + return "unknown" + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024 or unit == "TiB": + return f"{value:.1f} {unit}" if unit != "B" else f"{value:.0f} B" + value /= 1024 + + +def block_ranges(indices): + spans = [] + for value in sorted(set(indices)): + if spans and value == spans[-1][1] + 1: + spans[-1][1] = value + else: + spans.append([value, value]) + return ", ".join(str(start) if start == end else f"{start}–{end}" for start, end in spans) or "—" + + +class DownloadCard(QFrame): + def __init__(self): + super().__init__() + self.setObjectName("listRow") + layout = QVBoxLayout(self) + self.title = caption("Your download", "bodyStrong") + self.detail = caption("") + self.bar = QProgressBar() + self.bar.setRange(0, 1000) + self.bar.setTextVisible(False) + self.bar.setAccessibleName("Current file download progress") + self.totals = caption("") + for widget in (self.title, self.detail, self.bar, self.totals): + layout.addWidget(widget) + + def set_state(self, name, progress, model_state=None): + name = model_name(name) + if progress is None: + self.title.setText(f"{name} · Not downloaded") + self.detail.setText("Downloads when needed.") + self.bar.hide() + self.totals.setText("") + self.totals.hide() + return + state = progress["state"] + labels = { + "waiting": "Waiting", + "checking": "Checking downloaded files", + "downloading": "Downloading", + "retrying": "Retrying download", + "verifying": "Verifying files", + "loading": "Loading model", + "ready": "Ready", + "failed": "Couldn’t prepare this model", + "paused": "Paused", + } + if state == "ready" and model_state == "known": + labels["ready"] = "Downloaded" + self.title.setText(f"{name} · {labels[state]}") + size, received = progress["artifact_bytes"], progress["artifact_received_bytes"] + artifact = progress.get("artifact") or "Preparing download" + speed = progress.get("bytes_per_second") or 0 + detail = f"{byte_text(received or 0)} / {byte_text(size)}" if size else "Preparing download" + if state == "downloading" and speed: + detail += f" · {byte_text(speed)}/s" + self.detail.setText(detail) + self.detail.setToolTip(artifact) + self.bar.setVisible(bool(size)) + self.bar.setValue(min(1000, int(1000 * (received or 0) / size)) if size else 0) + self.bar.setAccessibleDescription(f"{artifact} · {detail}") + diagnostic = ( + f"{byte_text(progress['verified_bytes'])} verified across {progress['verified_files'] or 0} files" + f" · {byte_text(progress['received_bytes'])} received or cached" + + (f" · Resumed {byte_text(progress['resumed_bytes'])}" if progress["resumed_bytes"] else "") + + (f" · {progress['retries']} retries" if progress["retries"] else "") + ) + files, total_files = progress.get("verified_files"), progress.get("selected_files") + self.totals.setText(f"{files or 0} of {total_files} files checked" if total_files else "") + self.totals.setVisible(bool(total_files)) + self.totals.setToolTip(diagnostic) + self.setToolTip(diagnostic) + + +class DownloadsPanel(QFrame): + def __init__(self): + super().__init__() + self.setObjectName("card") + self.layout = QVBoxLayout(self) + self.layout.addWidget(caption("Your downloads", "sectionTitle")) + self.empty = caption("No model downloads have started on this computer.") + self.layout.addWidget(self.empty) + self.cards = {} + + def set_state(self, snapshot): + entries = [(f"model:{m['id']}", m["id"], m.get("download_progress"), m["state"]) for m in snapshot["models"]] + entries += [ + (f"worker:{w['id']}", f"Sharing · {w['model']}", w.get("download_progress"), w["state"]) + for w in snapshot["workers"] + ] + entries = [entry for entry in entries if entry[2] is not None][:64] + keys = {entry[0] for entry in entries} + for key in list(self.cards): + if key not in keys: + self.layout.removeWidget(self.cards[key]) + self.cards.pop(key).deleteLater() + for key, name, progress, state in entries: + if key not in self.cards: + self.cards[key] = DownloadCard() + self.layout.addWidget(self.cards[key]) + self.cards[key].set_state(name, progress, state) + self.empty.setVisible(not entries) + + +class _ModelDisclosureButton(QPushButton): + def sizeHint(self): + return self.layout().totalSizeHint() if self.layout() else super().sizeHint() + + def minimumSizeHint(self): + return self.layout().totalMinimumSize() if self.layout() else super().minimumSizeHint() + + +class ModelHealthCard(QFrame): + def __init__(self): + super().__init__() + self.setObjectName("card") + self.layout = QVBoxLayout(self) + self.layout.setContentsMargins(8, 8, 8, 8) + self.expand_button = _ModelDisclosureButton() + self.expand_button.setObjectName("modelDisclosure") + self.expand_button.setCheckable(True) + self.expand_button.setStyleSheet( + "QPushButton#modelDisclosure { background: transparent; border: 0; padding: 0; text-align: left; }" + "QPushButton#modelDisclosure:hover { background: #192131; }" + "QPushButton#modelDisclosure:focus { border: 1px solid #826BFF; }" + ) + header = QHBoxLayout(self.expand_button) + header.setContentsMargins(12, 10, 12, 10) + copy = QVBoxLayout() + copy.setSpacing(4) + self.title = caption("", "sectionTitle") + self.summary = caption("") + self.summary.setStyleSheet("font-size: 13px;") + self.disclosure = caption("Show details", "bodyStrong") + for widget in (self.title, self.summary, self.disclosure): + widget.setAttribute(Qt.WA_TransparentForMouseEvents) + copy.addWidget(self.title) + copy.addWidget(self.summary) + header.addLayout(copy, 1) + header.addWidget(self.disclosure) + self.layout.addWidget(self.expand_button) + self.details = QWidget() + details_layout = QVBoxLayout(self.details) + details_layout.setContentsMargins(12, 4, 12, 12) + details_layout.setSpacing(10) + self.layout.addWidget(self.details) + self.details.hide() + self.expand_button.toggled.connect(self._toggle_details) + self.legend = caption("Green: available · Blue: joining · Purple: reserved · Red: offline · Gray: missing") + self.legend.setToolTip("Brighter green means a block has more than one copy. Select a block for details.") + self.grid = QGridLayout() + self.grid.setSpacing(5) + self.grid.setAlignment(Qt.AlignLeft) + self.cells = [] + self.selected = 0 + self.block_detail = caption("Select a block for details.") + self.local_info = caption("") + self.download = DownloadCard() + self.worker_downloads = {} + self.worker_downloads_layout = QVBoxLayout() + self.peer_button = QPushButton("Contributors") + self.peer_button.setCheckable(True) + self.peer_button.toggled.connect(self._toggle_peers) + self.peer_table = QTableWidget(0, 3) + self.peer_table.setStyleSheet( + "QTableWidget { background: #10151f; color: #e7eaf0; gridline-color: #293344; border: 1px solid #293344; }" + "QHeaderView::section { background: #192231; color: #aab8cc; border: 0; padding: 7px; }" + "QTableWidget::item:selected { background: #334861; }" + ) + self.peer_table.setHorizontalHeaderLabels(["Contributor", "Status", "Blocks"]) + self.peer_table.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch) + self.peer_table.verticalHeader().hide() + self.peer_table.setEditTriggers(QTableWidget.NoEditTriggers) + self.peer_table.setAccessibleName("Model contributors") + self.peer_table.hide() + self.peer_note = caption("No contributors connected yet.") + self.peer_note.hide() + details_layout.addWidget(self.legend) + details_layout.addLayout(self.grid) + for widget in (self.block_detail, self.local_info, self.download): + details_layout.addWidget(widget) + details_layout.addLayout(self.worker_downloads_layout) + for widget in (self.peer_button, self.peer_table, self.peer_note): + details_layout.addWidget(widget) + + def _toggle_details(self, checked): + self.details.setVisible(checked) + self.disclosure.setText("Hide details" if checked else "Show details") + self._update_accessible_header() + + def _update_accessible_header(self): + self.expand_button.setAccessibleName(f"{self.title.text()}. {self.summary.text()}. {self.disclosure.text()}") + self.expand_button.setAccessibleDescription("Expanded" if self.expand_button.isChecked() else "Collapsed") + + def _toggle_peers(self, checked): + self.peer_table.setVisible(checked and self.peer_table.rowCount() > 0) + self.peer_note.setVisible(checked and self.peer_table.rowCount() == 0) + self.peer_button.setText(f"{'Hide contributors' if checked else 'Contributors'} ({self.peer_table.rowCount()})") + + def _select(self, index): + self.selected = index + self.block_detail.setText(self.cells[index].toolTip()) + + def set_state(self, model, workers=()): + health = model["health"] + local = model["execution"] == "local" + total = min(512, health["total_blocks"]) if not local else 0 + while len(self.cells) != total: + if len(self.cells) > total: + cell = self.cells.pop() + self.grid.removeWidget(cell) + cell.deleteLater() + else: + index = len(self.cells) + cell = QPushButton(str(index)) + cell.setFixedSize(34, 28) + cell.clicked.connect(lambda checked=False, index=index: self._select(index)) + self.grid.addWidget(cell, index // 16, index % 16) + self.cells.append(cell) + self.title.setText(model_name(model["id"])) + age = health["last_updated_age"] + stale = health["status"] not in ("complete", "incomplete") or (age is not None and age > 120) + if local: + summary = ( + "On this computer · Ready" if model["state"] == "ready" else "On this computer · Downloads when needed" + ) + elif stale: + summary = "Checking availability" + elif model.get("route_complete", health["status"] == "complete"): + count = model.get("peer_count") or 0 + summary = f"Available · {count} {'contributor' if count == 1 else 'contributors'}" + elif model.get("chat_ready") is False and health["status"] == "complete": + summary = "Model blocks available · Waiting for a peer to handle chat" + else: + summary = f"Waiting for contributors · {model['coverage']} blocks available" + progress = model.get("download_progress") + if local and progress and progress["state"] == "ready" and model["state"] != "ready": + summary = "On this computer · Downloaded" + if progress and progress["state"] in ("downloading", "retrying", "verifying", "loading", "failed", "paused"): + activity = { + "downloading": "Downloading", + "retrying": "Retrying download", + "verifying": "Checking downloaded files", + "loading": "Loading", + "failed": "Couldn’t prepare this model", + "paused": "Download paused", + }[progress["state"]] + summary = f"{'On this computer' if local else 'Your download'} · {activity}" + self.summary.setText(summary) + self.summary.setToolTip(f"Updated {int(age)} seconds ago" if age is not None and not local else "") + self._update_accessible_header() + self.legend.setVisible(not local) + self.block_detail.setVisible(not local) + size = model.get("selected_whole_shard_bytes") + self.local_info.setText(f"Download size: {byte_text(size)}" if size else "Uses this computer for answers.") + self.local_info.setVisible(local) + for index, cell in enumerate(self.cells): + replicas = health["replica_counts"][index] if health["replica_counts"] is not None else None + joining = health["joining_counts"][index] if health["joining_counts"] is not None else 0 + offline = health["offline_counts"][index] if health["offline_counts"] is not None else 0 + reservations = [r for r in health["reservations"] if r["start_block"] <= index < r["end_block"]] + failures = [] + for worker in workers: + span = worker.get("placement", {}).get("block_indices") or "" + parts = span.split(":") + if ( + worker["model"] == model["id"] + and worker["state"] == "crashed" + and len(parts) == 2 + and all(p.isdigit() for p in parts) + ): + if int(parts[0]) <= index < int(parts[1]): + failures.append(worker["id"]) + state = ( + "unknown" + if stale or replicas is None + else "replicated" + if replicas > 1 + else "covered" + if replicas + else "joining" + if joining + else "reserved" + if reservations + else "offline" + if offline or failures + else "missing" + ) + detail = f"Block {index} · {state.capitalize()} · {replicas if replicas is not None else 'Unknown'} copies" + if joining: + detail += f" · {joining} joining" + if reservations: + detail += f" · {len(reservations)} reservations" + owners = [p["public_name"] or p["peer_id"][:12] for p in health["peers"] if index in p["online_blocks"]] + if owners: + detail += " · Peers: " + ", ".join(owners[:8]) + if failures: + detail += " · Sharing failed on this computer" + cell.setToolTip(detail) + cell.setAccessibleName(detail) + cell.setStyleSheet( + f"QPushButton {{ background: {COLORS[state]}; color: #edf6ff; padding: 0; border: 1px solid #526174; border-radius: 4px; font-size: 10px; }} QPushButton:focus {{ border: 2px solid white; }}" + ) + if self.cells: + self._select(min(self.selected, len(self.cells) - 1)) + self.download.set_state("Your download", model.get("download_progress"), model["state"]) + self.download.setVisible(model.get("download_progress") is not None) + downloading_workers = [ + worker + for worker in workers + if worker["model"] == model["id"] and worker.get("download_progress") is not None + ] + worker_ids = {worker["id"] for worker in downloading_workers} + for worker_id in list(self.worker_downloads): + if worker_id not in worker_ids: + card = self.worker_downloads.pop(worker_id) + self.worker_downloads_layout.removeWidget(card) + card.deleteLater() + for worker in downloading_workers: + if worker["id"] not in self.worker_downloads: + card = self.worker_downloads[worker["id"]] = DownloadCard() + self.worker_downloads_layout.addWidget(card) + self.worker_downloads[worker["id"]].set_state( + "Sharing download", worker["download_progress"], worker["state"] + ) + peer_rows = [] + by_id = {p["peer_id"]: p for p in health["peers"]} + for reservation in health["reservations"]: + by_id.setdefault( + reservation["peer_id"], + { + "peer_id": reservation["peer_id"], + "public_name": None, + "online_blocks": [], + "joining_blocks": [], + "offline_blocks": [], + }, + ) + for peer in by_id.values(): + reserved = sorted( + { + i + for r in health["reservations"] + if r["peer_id"] == peer["peer_id"] + for i in range(r["start_block"], r["end_block"]) + } + ) + states = [] + if peer["online_blocks"]: + states.append(f"Serving {len(peer['online_blocks'])}") + if peer["joining_blocks"]: + states.append(f"Preparing {len(peer['joining_blocks'])}") + if reserved: + states.append(f"Planning to share {len(reserved)}") + if peer["offline_blocks"]: + states.append("Offline") + blocks = block_ranges(peer["online_blocks"] + peer["joining_blocks"] + peer["offline_blocks"] + reserved) + runtime = ( + " · ".join( + filter( + None, + [ + peer.get("version"), + peer.get("torch_dtype"), + peer.get("quant_type"), + "Relay" if peer.get("using_relay") else None, + ], + ) + ) + or "Not reported" + ) + peer_rows.append( + ( + peer["public_name"] or (peer["peer_id"] or "Unknown")[:16], + " · ".join(states) + (" · Stale" if stale else ""), + blocks, + runtime, + peer["peer_id"], + ) + ) + for worker in workers: + if worker["model"] == model["id"]: + progress = worker.get("download_progress") + state = progress["state"] if progress is not None else worker["display_status"] + peer_rows.append( + ( + "This computer", + state.capitalize(), + worker.get("placement", {}).get("block_indices") or "Unassigned", + "Local contribution", + worker["id"], + ) + ) + self.peer_table.setRowCount(len(peer_rows)) + self.peer_table.setFixedHeight(min(300, 55 + 30 * len(peer_rows))) + for row, values in enumerate(peer_rows): + for column, value in enumerate(values[:3]): + item = QTableWidgetItem(value) + item.setToolTip(f"{values[4]}\n{values[3]}" if column == 0 else value) + self.peer_table.setItem(row, column, item) + self.peer_button.setVisible(not local) + self._toggle_peers(not local and self.peer_button.isChecked()) diff --git a/desktop/src/communityai_desktop/presentation.py b/desktop/src/communityai_desktop/presentation.py new file mode 100644 index 000000000..345b9e0cc --- /dev/null +++ b/desktop/src/communityai_desktop/presentation.py @@ -0,0 +1,105 @@ +"""Short, human-readable desktop summaries of observed node state.""" + +from __future__ import annotations + +import re +from typing import Any + + +def model_name(value: str | None) -> str: + if not value: + return "No model selected" + value = re.sub(r"[- ](?:Local|FP8[- ]Dequant)$", "", value, flags=re.IGNORECASE) + return re.sub(r"(?<=\d)-(?=\d)|(?<=[A-Za-z])-?(?=\d+B\b)", " ", value) + + +def memory_text(value: int | float | None) -> str: + if value is None: + return "Not available" + return f"{value / 1024**3:.1f} GB" + + +def model_summary(snapshot: dict[str, Any]) -> tuple[str, str, str]: + selection = snapshot.get("auto_selection", {}) + selected = selection.get("model") + if not selected: + return "No model selected", "Open Models to check what is available.", "" + model = next((item for item in snapshot.get("models", []) if item["id"] == selected), {}) + local = selection.get("source") == "local" or model.get("execution") == "local" + if snapshot.get("inference_mode") == "local_only": + reason = "You chose to use only this computer." + elif local: + reason = "The community cannot answer right now. Using this computer until it is available." + else: + reason = "The community model is ready to answer your messages." + return model_name(selected), reason, "On this computer" if local else "With the community" + + +def sharing_reason(reason: str | None) -> str: + """Translate operational reasons without dumping internal policy text into the UI.""" + text = (reason or "").casefold() + if any(word in text for word in ("vram", "gpu memory", "accelerator", "cuda", "memory budget")): + if any(word in text for word in ("unavailable", "not available", "no cuda", "not detected")): + return "Your graphics card is not available for sharing. Check its driver." + return "Not enough GPU memory. Increase the memory limit or close another app." + if any(word in text for word in ("disk", "storage", "artifact set")): + return "Not enough storage. Free some space or increase the storage limit." + if "schedule" in text: + return "Sharing will start during the hours you chose." + if any(word in text for word in ("power", "battery")): + return "Sharing is paused to stay within your power settings." + if "bandwidth" in text: + return "Sharing is waiting for your download limit to allow it." + if any(word in text for word in ("coverage", "discovery", "peer", "bootstrap", "connect", "network")): + return "Connecting to the community. Sharing will start when connected." + if any(word in text for word in ("placement", "candidate", "no eligible", "no community model")): + return "Finding a model your computer can help with." + if any(word in text for word in ("denied", "allowed", "disabled", "policy")): + return "Your sharing settings are preventing this model from starting." + if "download" in text: + return "Downloading the files needed for sharing." + if "changed elsewhere" in text: + return "Your settings changed. Try again." + return "Sharing could not start. Try again or check your settings." + + +def sharing_summary(snapshot: dict[str, Any]) -> tuple[str, str, str]: + contribution = snapshot.get("contribution", {}) + workers = snapshot.get("workers", []) + wanted = contribution.get("intent_enabled", False) + active = [worker for worker in workers if worker.get("sharing_active", worker.get("state") == "running")] + if active: + names = ", ".join( + dict.fromkeys(model_name(worker.get("model")) for worker in active if worker.get("model") != "auto") + ) + return "Sharing is on", f"Helping with {names}." if names else "Helping the community.", "running" + if not wanted: + if contribution.get("editable") is False: + return "Sharing is unavailable", "Restart CommunityAI to try again.", "waiting" + return "Sharing is off", "", "off" + selected = [ + worker + for worker in workers + if not worker.get("operator_paused") + and (worker.get("desired_running") or (worker.get("placement") or {}).get("automatic")) + ] + if not selected: + return "Sharing is paused", "", "paused" + reasons = contribution.get("selected_blocked_reasons") or [] + if reasons: + return "Sharing is waiting", sharing_reason(reasons[0]), "waiting" + for worker in selected: + progress = worker.get("download_progress") or {} + if progress.get("state") in ("downloading", "verifying", "retrying"): + return "Downloading for sharing", "You can keep using your computer.", "starting" + if worker.get("state") == "crashed": + return "Sharing stopped", "The model could not start. Try again.", "error" + placement = worker.get("placement") or {} + if placement.get("automatic") and not placement.get("block_indices"): + reason = placement.get("reason") + return ( + "Preparing to share", + sharing_reason(reason) if reason else "Finding a part of the model for your computer.", + "starting", + ) + return "Starting sharing", "Preparing the model. You can keep using your computer.", "starting" diff --git a/desktop/src/communityai_desktop/pyside_shell.py b/desktop/src/communityai_desktop/pyside_shell.py index d354e704b..50ecf37b7 100644 --- a/desktop/src/communityai_desktop/pyside_shell.py +++ b/desktop/src/communityai_desktop/pyside_shell.py @@ -12,9 +12,11 @@ from pathlib import Path from typing import Any, Callable, Dict +from communityai_desktop.presentation import memory_text, model_name, model_summary, sharing_reason, sharing_summary from communityai_desktop.startup import LoginStartupError, SingleInstanceError, login_startup_enabled, set_login_startup APP_STYLESHEET = """ +QLabel { color: #E7EAF0; font-family: "Segoe UI"; font-size: 14px; } QMainWindow, QWidget#appShell, QScrollArea, QScrollArea > QWidget > QWidget { background: #090C12; color: #F4F6FA; @@ -48,7 +50,7 @@ QLabel#metricLabel { color: #8C96AA; font-size: 12px; } QLabel#metricNote { color: #667187; font-size: 11px; } QLabel#bodyStrong { color: #EDEFF5; font-size: 14px; font-weight: 650; } -QLabel#bodyMuted { color: #7F899D; font-size: 12px; } +QLabel#bodyMuted { color: #A0AABC; font-size: 13px; } QLabel#endpointText { color: #D9DDFE; background: #111625; border: 1px solid #29304A; border-radius: 9px; padding: 11px 13px; font-family: "Consolas"; font-size: 12px; @@ -188,10 +190,14 @@ def run( activate_existing_instance: bool = True, instance_name: str | None = None, before_termination_restore: Callable[[], None] | None = None, + qualification_automation=None, # noqa: ANN001 + updater=None, # noqa: ANN001 ) -> int: if controller is None and connect is None: raise ValueError("the desktop requires an initial controller or connector") + from communityai_desktop.model_health import DownloadCard, ModelHealthCard + from communityai_desktop.resource_controls import ResourceControls from PySide6.QtCore import QLockFile, QObject, QRunnable, QStandardPaths, Qt, QThreadPool, QTimer, Signal, Slot from PySide6.QtGui import QFont, QGuiApplication, QIcon from PySide6.QtNetwork import QLocalServer, QLocalSocket @@ -221,6 +227,7 @@ def run( def label(text: str = "", name: str | None = None) -> QLabel: item = QLabel(text) + item.setTextFormat(Qt.PlainText) if name: item.setObjectName(name) return item @@ -262,9 +269,17 @@ def __init__(self, operation: Callable[[], Any]): @Slot() def run(self): try: - self.signals.result.emit(self.operation()) + result = self.operation() except Exception as exc: # GUI boundary: show a friendly state and remain responsive. - self.signals.error.emit(str(exc)) + signal, result = self.signals.error, str(exc) + else: + signal = self.signals.result + try: + signal.emit(result) + except RuntimeError as exc: + # A bounded node request may finish after the user has closed Qt. + if "deleted" not in str(exc): + raise class MainWindow(QMainWindow): def __init__(self): @@ -275,6 +290,15 @@ def __init__(self): self._pool = QThreadPool.globalInstance() self._tasks = set() self._busy = 0 + self._refreshing = False + self._change_version = 0 + self._sharing_pending = None + self._awaiting_sharing_snapshot = False + self._sharing_error = None + self._mode_pending = False + self._awaiting_mode_snapshot = False + self._closing = False + self._update_notice = "" self._controller = controller self._snapshot: Dict[str, Any] = { "models": [], @@ -312,6 +336,16 @@ def __init__(self): self._timer.timeout.connect(self.refresh) self._timer.start() self.refresh() + if updater is not None: + self._update_timer = QTimer(self) + self._update_timer.setInterval(500) + self._update_timer.timeout.connect(self._render_update) + self._update_timer.start() + self._update_check_timer = QTimer(self) + self._update_check_timer.setInterval(6 * 60 * 60 * 1000) + self._update_check_timer.timeout.connect(updater.check) + self._update_check_timer.start() + QTimer.singleShot(10_000, updater.check) def _build_sidebar(self) -> QFrame: sidebar = QFrame() @@ -349,17 +383,15 @@ def _build_sidebar(self) -> QFrame: layout.addWidget(button) layout.addStretch(1) - privacy_card, privacy_layout = card() - privacy_card.setStyleSheet("QFrame#card { background: #0B1018; }") - privacy_layout.setContentsMargins(13, 12, 13, 12) - privacy_layout.setSpacing(4) - privacy_layout.addWidget(label("A note on privacy", "bodyStrong")) - note = label("Computers helping with a request may be able to see what was sent.", "privacySmall") - note.setWordWrap(True) - privacy_layout.addWidget(note) - layout.addWidget(privacy_card) - layout.addSpacing(12) - + if updater is not None: + self.update_detail = label("", "bodyMuted") + self.update_detail.setWordWrap(True) + self.update_detail.hide() + self.update_button = QPushButton("Check for updates") + self.update_button.setObjectName("textButton") + self.update_button.clicked.connect(self._update_clicked) + layout.addWidget(self.update_detail) + layout.addWidget(self.update_button) status_row = QHBoxLayout() self.sidebar_dot = label("●", "sidebarDot") self.sidebar_status = label("Connecting", "sidebarStatus") @@ -369,6 +401,30 @@ def _build_sidebar(self) -> QFrame: layout.addLayout(status_row) return sidebar + def _render_update(self): + state = updater.snapshot() + busy = state["status"] in ("checking", "downloading", "installing") + self.update_button.setEnabled(not busy) + self.update_button.setText(state["message"] if state["status"] != "error" else "Retry update") + detail = self._update_notice or (state["message"] if state["status"] == "error" else "") + self.update_detail.setText(detail) + self.update_detail.setVisible(bool(detail)) + + def _update_clicked(self): + self._update_notice = "" + if updater.snapshot()["status"] == "ready": + if any(model.get("active_requests", 0) for model in self._snapshot.get("models", [])): + self._update_notice = "Wait for the current answer to finish, then try again." + self._render_update() + return + try: + updater.install() + except ValueError as exc: + self._update_notice = str(exc) + else: + updater.check() + self._render_update() + def _scroll_page(self, title: str, subtitle: str) -> tuple[QScrollArea, QVBoxLayout]: scroll = QScrollArea() scroll.setFrameShape(QFrame.NoFrame) @@ -380,10 +436,10 @@ def _scroll_page(self, title: str, subtitle: str) -> tuple[QScrollArea, QVBoxLay layout.setContentsMargins(28, 27, 28, 32) layout.setSpacing(20) layout.addWidget(label(title, "pageTitle")) - subtitle_label = label(subtitle, "pageSubtitle") - subtitle_label.setWordWrap(True) - layout.addWidget(subtitle_label) - layout.addSpacing(3) + if subtitle: + subtitle_label = label(subtitle, "pageSubtitle") + subtitle_label.setWordWrap(True) + layout.addWidget(subtitle_label) scroll.setWidget(content) return scroll, layout @@ -406,123 +462,123 @@ def _metric_card(self, title: str, note: str) -> tuple[QFrame, QLabel]: return frame, value def _build_home_page(self) -> QScrollArea: - page, layout = self._scroll_page("Welcome home", "Everything you need, without the network homework.") - - self.connection_banner = QFrame() - self.connection_banner.setObjectName("connectionBanner") - banner_layout = QHBoxLayout(self.connection_banner) - banner_layout.setContentsMargins(16, 13, 14, 13) - banner_copy = QVBoxLayout() - banner_copy.setSpacing(2) - self.connection_title = label("Getting things ready…", "bodyStrong") - self.connection_detail = label("CommunityAI connects automatically.", "bodyMuted") - banner_copy.addWidget(self.connection_title) - banner_copy.addWidget(self.connection_detail) - banner_layout.addLayout(banner_copy, 1) + page, layout = self._scroll_page("Home", "") + self.connection_banner, banner_layout = card("connectionBanner") + self.connection_title = label("Connecting…", "bodyStrong") + self.connection_detail = label("Starting CommunityAI.", "bodyMuted") + self.connection_detail.setWordWrap(True) + banner_layout.addWidget(self.connection_title) + banner_layout.addWidget(self.connection_detail) self.retry_button = QPushButton("Try again") - self.retry_button.setObjectName("ghostButton") self.retry_button.clicked.connect(self._reset_connection) - self.retry_button.hide() banner_layout.addWidget(self.retry_button) layout.addWidget(self.connection_banner) hero, hero_layout = card("heroCard") - hero_row = QHBoxLayout() - hero_copy = QVBoxLayout() - hero_copy.setSpacing(5) - hero_copy.addWidget(label("LOCAL AI", "eyebrow")) - self.hero_title = label("Your AI is getting ready", "pageTitle") - self.hero_title.setStyleSheet("font-size: 23px;") - hero_copy.addWidget(self.hero_title) - self.hero_subtitle = label("Your apps will connect here automatically.", "pageSubtitle") - hero_copy.addWidget(self.hero_subtitle) - hero_row.addLayout(hero_copy, 1) - endpoint_box = QVBoxLayout() - endpoint_box.setSpacing(7) - endpoint_box.addWidget(label("ENDPOINT URL", "eyebrow")) - endpoint_row = QHBoxLayout() - self.endpoint = label("http://127.0.0.1:8080/v1", "endpointText") - self.endpoint.setMinimumWidth(0) - self.endpoint.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Fixed) - self.endpoint.setTextInteractionFlags(Qt.TextSelectableByMouse | Qt.TextSelectableByKeyboard) - self.endpoint.setAccessibleName("Local API endpoint URL") - endpoint_row.addWidget(self.endpoint, 1) - copy_button = QPushButton("Copy") - copy_button.clicked.connect(self._copy_endpoint) - endpoint_row.addWidget(copy_button) - endpoint_box.addLayout(endpoint_row) - hero_row.addLayout(endpoint_box, 1) - hero_layout.addLayout(hero_row) + model_header = QHBoxLayout() + model_header.addWidget(label("YOUR MODEL", "eyebrow"), 1) + self.home_model_location = label("", "bodyMuted") + model_header.addWidget(self.home_model_location) + hero_layout.addLayout(model_header) + self.hero_title = label("Checking your model…", "pageTitle") + self.hero_title.setAccessibleName("Selected model") + self.hero_subtitle = label("", "pageSubtitle") + self.hero_subtitle.setWordWrap(True) + hero_layout.addWidget(self.hero_title) + hero_layout.addWidget(self.hero_subtitle) layout.addWidget(hero) - metrics = QHBoxLayout() - model_metric, self.models_metric = self._metric_card("Models ready", "Available to your apps") - peer_metric, self.peers_metric = self._metric_card("Peers online", "Across the community") - region_metric, self.regions_metric = self._metric_card("World regions", "Community around the world") - metrics.addWidget(model_metric) - metrics.addWidget(peer_metric) - metrics.addWidget(region_metric) - layout.addLayout(metrics) - - lower = QHBoxLayout() - models_card, models_layout = card() - models_header = QHBoxLayout() - models_header.addLayout(self._section_header("Ready to use", "Available models"), 1) - view_models = QPushButton("View all") - view_models.setObjectName("textButton") - view_models.clicked.connect(lambda: self._show_page(1)) - models_header.addWidget(view_models) - models_layout.addLayout(models_header) - self.home_models_layout = QVBoxLayout() - self.home_models_layout.setSpacing(8) - models_layout.addLayout(self.home_models_layout) - models_layout.addStretch(1) - - regions_card, regions_layout = card() - regions_layout.addLayout(self._section_header("Around the world", "Peers by region")) - self.region_layout = QVBoxLayout() - self.region_layout.setSpacing(6) - regions_layout.addLayout(self.region_layout) - regions_layout.addStretch(1) - lower.addWidget(models_card, 3) - lower.addWidget(regions_card, 2) - layout.addLayout(lower) + hardware, hardware_layout = card() + hardware_layout.addWidget(label("Your hardware", "sectionTitle")) + hardware_grid = QFormLayout() + hardware_grid.setVerticalSpacing(16) + hardware_grid.setHorizontalSpacing(26) + self.home_gpu = label("Checking…", "bodyStrong") + self.home_cpu = label("Checking…", "bodyStrong") + self.home_gpu.setWordWrap(True) + self.home_cpu.setWordWrap(True) + self.home_gpu.setAccessibleName("Graphics card") + self.home_cpu.setAccessibleName("Processor") + hardware_grid.addRow(label("Graphics card", "bodyMuted"), self.home_gpu) + hardware_grid.addRow(label("Processor", "bodyMuted"), self.home_cpu) + hardware_layout.addLayout(hardware_grid) + self.home_hardware_detail = label("", "bodyMuted") + self.home_hardware_detail.hide() + hardware_layout.addWidget(self.home_hardware_detail) + layout.addWidget(hardware) + + sharing, sharing_layout = card() + row = QHBoxLayout() + self.home_sharing_title = label("Sharing is off", "sectionTitle") + row.addWidget(self.home_sharing_title, 1) + self.home_share_button = QPushButton("Start sharing") + self.home_share_button.setObjectName("primaryButton") + self.home_share_button.setAccessibleName("Home sharing control") + self.home_share_button.clicked.connect(self._toggle_all_sharing) + row.addWidget(self.home_share_button) + sharing_layout.addLayout(row) + self.home_sharing_detail = label("", "bodyMuted") + self.home_sharing_detail.setWordWrap(True) + self.home_sharing_detail.hide() + sharing_layout.addWidget(self.home_sharing_detail) + limits = QHBoxLayout() + vram = QVBoxLayout() + vram.addWidget(label("GPU memory for sharing", "bodyMuted")) + self.home_vram = label("Checking…", "sectionTitle") + self.home_vram.setAccessibleName("GPU memory limit") + vram.addWidget(self.home_vram) + processing = QVBoxLayout() + processing.addWidget(label("Computing power", "bodyMuted")) + self.home_processing = label("100%", "sectionTitle") + self.home_processing.setAccessibleName("Computing power limit") + processing.addWidget(self.home_processing) + limits.addLayout(vram, 1) + limits.addLayout(processing, 1) + settings = QPushButton("Change limits") + settings.setObjectName("textButton") + settings.clicked.connect(lambda: self._show_page(2)) + limits.addWidget(settings, 0, Qt.AlignBottom) + sharing_layout.addLayout(limits) + layout.addWidget(sharing) layout.addStretch(1) return page def _build_models_page(self) -> QScrollArea: - page, layout = self._scroll_page( - "Models", "Pick a model in your AI app. CommunityAI connects you automatically." - ) - info, info_layout = card("heroCard") - info_layout.addWidget(label("COMMUNITY LIBRARY", "eyebrow")) - info_layout.addWidget(label("One place. Every available model.", "sectionTitle")) - info_layout.addWidget( - label("Models appear here when enough community computers are online to run them.", "sectionSubtitle") - ) - info_layout.addSpacing(8) - self.auto_selection_title = label("auto is waiting for a complete route", "bodyStrong") + page, layout = self._scroll_page("Models", "") + selection, selection_layout = card() + selection_row = QHBoxLayout() + summary = QVBoxLayout() + self.auto_selection_title = label("Checking your model…", "sectionTitle") self.auto_selection_title.setAccessibleName("Automatic model selection") - self.auto_selection_detail = label("CommunityAI checks live route coverage before choosing.", "bodyMuted") + self.auto_selection_detail = label("", "bodyMuted") self.auto_selection_detail.setWordWrap(True) - info_layout.addWidget(self.auto_selection_title) - info_layout.addWidget(self.auto_selection_detail) - layout.addWidget(info) + summary.addWidget(self.auto_selection_title) + summary.addWidget(self.auto_selection_detail) + selection_row.addLayout(summary, 1) + self.inference_mode_button = QPushButton("Use only this computer") + self.inference_mode_button.setAccessibleName("Switch local-only inference") + self.inference_mode_button.clicked.connect(self._toggle_inference_mode) + selection_row.addWidget(self.inference_mode_button) + selection_layout.addLayout(selection_row) + layout.addWidget(selection) self.models_list_layout = QVBoxLayout() + self.model_health_cards = {} self.models_list_layout.setSpacing(10) layout.addLayout(self.models_list_layout) + privacy = label("Community members helping with your messages may see their contents.", "bodyMuted") + privacy.setWordWrap(True) + layout.addWidget(privacy) layout.addStretch(1) return page def _build_sharing_page(self) -> QScrollArea: - page, layout = self._scroll_page("Sharing", "Help the community when it suits you. You stay in control.") + page, layout = self._scroll_page("Sharing", "") hero, hero_layout = card("heroCard") top = QHBoxLayout() copy = QVBoxLayout() - copy.setSpacing(4) - copy.addWidget(label("SHARING", "eyebrow")) - self.sharing_title = label("Your GPU is not sharing right now", "sectionTitle") - self.sharing_detail = label("Choose models below, then start whenever you're ready.", "sectionSubtitle") + self.sharing_title = label("Sharing is off", "sectionTitle") + self.sharing_detail = label("", "bodyMuted") + self.sharing_detail.setWordWrap(True) copy.addWidget(self.sharing_title) copy.addWidget(self.sharing_detail) top.addLayout(copy, 1) @@ -532,104 +588,61 @@ def _build_sharing_page(self) -> QScrollArea: top.addWidget(self.master_share_button) hero_layout.addLayout(top) layout.addWidget(hero) + self.sharing_downloads_layout = QVBoxLayout() + self.sharing_download_cards = {} + layout.addLayout(self.sharing_downloads_layout) memory_card, memory_layout = card() - memory_header = QHBoxLayout() - memory_header.addLayout( - self._section_header( - "Configured GPU memory budget", - "Read from the node's enforced policy; unavailable data never becomes an invented default.", - ), - 1, - ) - self.memory_value = label("Unavailable", "metricValue") - self.memory_value.setStyleSheet("font-size: 22px;") - memory_header.addWidget(self.memory_value) - memory_layout.addLayout(memory_header) - self.memory_detail = label("No accelerator budget is reported.", "bodyMuted") + memory_layout.addWidget(label("How much to share", "sectionTitle")) + self.memory_value = label("", "bodyStrong") + self.memory_detail = label("", "bodyMuted") + self.memory_detail.setWordWrap(True) + memory_layout.addWidget(self.memory_value) memory_layout.addWidget(self.memory_detail) - self.memory_bar = QProgressBar() - self.memory_bar.setRange(0, 100) - self.memory_bar.setValue(0) - self.memory_bar.setTextVisible(False) - self.memory_bar.setAccessibleName("Configured GPU memory budget") - memory_layout.addWidget(self.memory_bar) + self.resource_controls = ResourceControls() + self.resource_controls.apply_requested.connect(self._apply_resource_limits) + memory_layout.addWidget(self.resource_controls) layout.addWidget(memory_card) - policy_card, policy_layout = card() - policy_header = QHBoxLayout() - policy_header.addLayout( - self._section_header( - "Node-enforced sharing limits", - "These values and admission decisions come from the authenticated local node.", - ), - 1, - ) - self.edit_policy_button = QPushButton("Edit sharing limits") - self.edit_policy_button.setObjectName("ghostButton") - self.edit_policy_button.clicked.connect(self._edit_contribution_policy) - policy_header.addWidget(self.edit_policy_button) - policy_layout.addLayout(policy_header) - self.policy_status = label("Contribution policy is unavailable.", "bodyStrong") - self.policy_status.setWordWrap(True) - policy_layout.addWidget(self.policy_status) - self.disk_policy = label("Storage: unavailable", "bodyMuted") - self.bandwidth_policy = label("Bandwidth: unavailable", "bodyMuted") - self.power_policy = label("Power: unavailable", "bodyMuted") - self.schedule_policy = label("Schedule: unavailable", "bodyMuted") - for policy_line in ( - self.disk_policy, - self.bandwidth_policy, - self.power_policy, - self.schedule_policy, - ): - policy_line.setWordWrap(True) - policy_layout.addWidget(policy_line) - layout.addWidget(policy_card) - - startup_card, startup_layout = card() - startup_header = QHBoxLayout() - startup_copy = self._section_header( - "Start after sign-in", - "CommunityAI can reconnect your local service automatically when you sign in.", - ) - startup_header.addLayout(startup_copy, 1) - self.login_startup_toggle = QCheckBox("Start CommunityAI when I sign in") + advanced, advanced_layout = card() + more = QPushButton("More settings") + more.setCheckable(True) + more.setObjectName("textButton") + more.setAccessibleName("More sharing settings") + advanced_layout.addWidget(more) + body = QWidget() + body_layout = QVBoxLayout(body) + body_layout.setContentsMargins(0, 8, 0, 0) + body_layout.setSpacing(14) + self.login_startup_toggle = QCheckBox("Open CommunityAI when I sign in") self.login_startup_toggle.setAccessibleName("Start CommunityAI when I sign in") try: startup_enabled = login_startup_enabled() - startup_detail = "Enabled for this user" if startup_enabled else "Off" - except LoginStartupError as exc: + startup_detail = "" + except LoginStartupError: startup_enabled = False - startup_detail = f"Unavailable: {str(exc)[:180]}" + startup_detail = "Sign-in settings could not be read." self.login_startup_toggle.setDisabled(True) self.login_startup_toggle.setChecked(startup_enabled) - startup_header.addWidget(self.login_startup_toggle) - startup_layout.addLayout(startup_header) self.login_startup_detail = label(startup_detail, "bodyMuted") - startup_layout.addWidget(self.login_startup_detail) + self.login_startup_detail.setVisible(bool(startup_detail)) self.login_startup_toggle.toggled.connect(self._set_login_startup) - layout.addWidget(startup_card) - - selection_card, selection_layout = card() - selection_layout.addLayout( - self._section_header("Models you want to help", "Turn models on or off. CommunityAI handles the rest.") - ) + body_layout.addWidget(self.login_startup_toggle) + body_layout.addWidget(self.login_startup_detail) + self.edit_policy_button = QPushButton("Storage, internet and schedule…") + self.edit_policy_button.clicked.connect(self._edit_contribution_policy) + body_layout.addWidget(self.edit_policy_button) self.contribution_models_layout = QVBoxLayout() - self.contribution_models_layout.setSpacing(9) - selection_layout.addLayout(self.contribution_models_layout) - layout.addWidget(selection_card) - - privacy, privacy_layout = card() - privacy_layout.addWidget(label("A quick privacy note", "bodyStrong")) - privacy_text = label( - "When sharing is on, your computer helps process requests. Their content may be visible to you or " - "software running on your computer.", - "bodyMuted", - ) - privacy_text.setWordWrap(True) - privacy_layout.addWidget(privacy_text) - layout.addWidget(privacy) + body_layout.addLayout(self.contribution_models_layout) + advanced_layout.addWidget(body) + body.hide() + + def expand_settings(expanded): + body.setVisible(expanded) + more.setText("Hide settings" if expanded else "More settings") + + more.toggled.connect(expand_settings) + layout.addWidget(advanced) layout.addStretch(1) return page @@ -646,10 +659,10 @@ def _build_api_page(self) -> QScrollArea: self.api_endpoint.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Fixed) self.api_endpoint.setTextInteractionFlags(Qt.TextSelectableByMouse | Qt.TextSelectableByKeyboard) endpoint_row.addWidget(self.api_endpoint, 1) - copy = QPushButton("Copy URL") - copy.setObjectName("primaryButton") - copy.clicked.connect(self._copy_endpoint) - endpoint_row.addWidget(copy) + self.copy_endpoint_button = QPushButton("Copy URL") + self.copy_endpoint_button.setObjectName("primaryButton") + self.copy_endpoint_button.clicked.connect(self._copy_endpoint) + endpoint_row.addWidget(self.copy_endpoint_button) endpoint_layout.addLayout(endpoint_row) layout.addWidget(endpoint_card) @@ -676,59 +689,80 @@ def _show_page(self, index: int) -> None: button.setChecked(button_index == index) def _set_connection_state(self, connected: bool) -> None: - self.connection_banner.setProperty("connectionState", "online" if connected else "offline") - self.connection_banner.style().unpolish(self.connection_banner) - self.connection_banner.style().polish(self.connection_banner) - if connected: - self.connection_title.setText("Everything is connected") - self.connection_detail.setText("Your local AI is ready for apps and sharing.") - self.retry_button.hide() - self.sidebar_dot.setStyleSheet("color: #5EE1A2;") - self.sidebar_status.setText("Online") - else: - self.connection_title.setText("CommunityAI is still getting ready") - self.connection_detail.setText("It isn't ready yet. We'll keep trying automatically.") - self.retry_button.show() - self.sidebar_dot.setStyleSheet("color: #F3B76A;") - self.sidebar_status.setText("Getting ready") + self.connection_banner.setVisible(not connected) + self.retry_button.setVisible(not connected) + self.sidebar_dot.setStyleSheet("color: #5EE1A2;" if connected else "color: #F3B76A;") + self.sidebar_status.setText("Connected" if connected else "Connecting…") def _set_busy(self, change: int) -> None: - self._busy += change + self._busy = max(0, self._busy + change) busy = self._busy > 0 self.retry_button.setDisabled(busy) self.create_key_button.setDisabled(busy or self._controller is None) + self.inference_mode_button.setDisabled( + busy + or self._mode_pending + or self._controller is None + or not self._snapshot.get("inference_mode_editable", False) + ) contribution = self._snapshot.get("contribution", {}) + self.resource_controls.set_state( + contribution, busy=busy or self._controller is None or self._sharing_pending is not None + ) self.edit_policy_button.setDisabled( busy or self._controller is None or not contribution.get("editable", False) or contribution.get("intent_enabled", False) ) - action_available = ( - contribution.get("can_pause") if contribution.get("intent_enabled") else contribution.get("can_start") - ) - self.master_share_button.setDisabled( - busy or self._controller is None or not self._snapshot.get("workers") or not action_available - ) + for button in (self.master_share_button, self.home_share_button): + button.setDisabled( + busy + or self._sharing_pending is not None + or self._controller is None + or not contribution.get("editable", False) + ) + for index in range(self.contribution_models_layout.count()): + widget = self.contribution_models_layout.itemAt(index).widget() + if widget is not None: + widget.setDisabled(busy or self._sharing_pending is not None or self._controller is None) def _submit( self, operation: Callable[[], Any], on_result: Callable[[Any], None], on_error: Callable[[str], None] | None = None, + *, + background: bool = False, ) -> None: - self._set_busy(1) + if self._closing: + return + if background: + self._refreshing = True + else: + self._change_version += 1 + self._set_busy(1) task = Task(operation) self._tasks.add(task) def finish(result: Any) -> None: self._tasks.discard(task) - self._set_busy(-1) + if self._closing: + return + if background: + self._refreshing = False + else: + self._set_busy(-1) on_result(result) def fail(message: str) -> None: self._tasks.discard(task) - self._set_busy(-1) + if self._closing: + return + if background: + self._refreshing = False + else: + self._set_busy(-1) (on_error or self._connection_failed)(message) task.signals.result.connect(finish) @@ -736,13 +770,27 @@ def fail(message: str) -> None: self._pool.start(task) def refresh(self) -> None: - if self._busy: + if self._busy or self._refreshing: return if self._controller is None: self.sidebar_status.setText("Connecting") self._submit(connect, self._connected) return - self._submit(self._controller.snapshot, self._render, self._snapshot_failed) + version = self._change_version + + def refreshed(snapshot): + if version == self._change_version: + self._render(snapshot) + else: + QTimer.singleShot(0, self.refresh) + + def refresh_failed(message): + if version == self._change_version: + self._snapshot_failed(message) + else: + QTimer.singleShot(0, self.refresh) + + self._submit(self._controller.snapshot, refreshed, refresh_failed, background=True) def _connected(self, connected_controller) -> None: # noqa: ANN001 self._controller = connected_controller @@ -750,9 +798,18 @@ def _connected(self, connected_controller) -> None: # noqa: ANN001 def _connection_failed(self, message: str) -> None: self._set_connection_state(False) - self.connection_detail.setText(str(message)[:300]) - self.hero_title.setText("Your AI will appear here") - self.hero_subtitle.setText("CommunityAI connects automatically as soon as it is ready.") + self.connection_title.setText("Could not connect to CommunityAI") + self.connection_detail.setText("Try again. If this keeps happening, restart CommunityAI.") + self.connection_detail.setToolTip(str(message)[:300]) + self.hero_title.setText("Model unavailable") + self.hero_subtitle.setText("Waiting for CommunityAI to reconnect.") + for widget in (self.sharing_title, self.home_sharing_title): + widget.setText("Checking sharing…") + widget.setStyleSheet("color: #F3C46C;") + for widget in (self.sharing_detail, self.home_sharing_detail): + widget.setText("Waiting for CommunityAI to reconnect.") + widget.show() + self._set_busy(0) def _snapshot_failed(self, message: str) -> None: self._controller = None @@ -764,244 +821,152 @@ def _reset_connection(self) -> None: def _render(self, snapshot: Dict[str, Any]) -> None: self._snapshot = snapshot + if self._awaiting_sharing_snapshot: + self._awaiting_sharing_snapshot = False + self._sharing_pending = None + self._sharing_error = None + if self._awaiting_mode_snapshot: + self._awaiting_mode_snapshot = False + self._mode_pending = False self._set_busy(0) self._set_connection_state(True) - self.hero_title.setText("Your local AI is ready") - self.hero_subtitle.setText("Use community models from any compatible app on this computer.") - endpoint = snapshot["openai_base_url"] - self.endpoint.setText(endpoint) - self.api_endpoint.setText(endpoint) - - auto_selection = snapshot["auto_selection"] - self.auto_selection_title.setText(auto_selection["title"]) - self.auto_selection_detail.setText(auto_selection["reason"]) - ready_models = [model for model in snapshot["models"] if model["route_complete"]] - self.models_metric.setText(str(len(ready_models))) - network = snapshot["network"] - self.peers_metric.setText(str(network["peer_count"])) - self.regions_metric.setText(str(len(network["regions"]))) - self._render_home_models(snapshot["models"]) - self._render_regions(network["regions"]) + name, reason, location = model_summary(snapshot) + self.hero_title.setText(name) + self.hero_subtitle.setText(reason) + self.home_model_location.setText(location) + self.auto_selection_title.setText(name) + self.auto_selection_detail.setText(reason) + self.api_endpoint.setText(snapshot["openai_base_url"]) + hardware = snapshot.get("hardware", {}) + self.home_gpu.setText(hardware.get("gpu_name") or "No supported graphics card detected") + self.home_cpu.setText(hardware.get("cpu_name") or "Processor name unavailable") + device = hardware.get("inference_device") or "" + device_text = ( + ( + "Your model runs on the graphics card." + if device.startswith("cuda") + else "Your model runs on the processor." + if device == "cpu" + else "" + ) + if location == "On this computer" + else "" + ) + self.home_hardware_detail.setText(device_text) + self.home_hardware_detail.setVisible(bool(device_text)) + self.inference_mode_button.setText( + "Changing…" + if self._mode_pending + else "Use community models too" + if snapshot.get("inference_mode") == "local_only" + else "Use only this computer" + ) self._render_models(snapshot["models"]) self._render_sharing(snapshot) self._render_keys(snapshot["keys"]) - def _model_row(self, model: Dict[str, Any]) -> QFrame: - row = QFrame() - row.setObjectName("listRow") - layout = QHBoxLayout(row) - layout.setContentsMargins(12, 10, 12, 10) - layout.setSpacing(12) - avatar = label(model["id"][:1].upper(), "avatar") - avatar.setFixedSize(38, 38) - layout.addWidget(avatar) - copy = QVBoxLayout() - copy.setSpacing(2) - copy.addWidget(label(model["id"], "bodyStrong")) - peers = model.get("peer_count") - detail = f"{model['coverage']} blocks" - availability = "Available now" if model["route_complete"] else "Incomplete route" - if isinstance(peers, int): - peer_label = "peer" if peers == 1 else "peers" - detail = f"{detail} • {peers} {peer_label} • {availability}" - else: - detail = f"{detail} • {availability}" - copy.addWidget(label(detail, "bodyMuted")) - copy.addWidget( - label( - f"First-use download/storage: {model['download_storage_estimate']}", - "bodyMuted", - ) - ) - layout.addLayout(copy, 1) - tone = "good" if model["route_complete"] else "warn" - badge = "Auto choice" if model.get("auto_selected") else "Ready" if tone == "good" else "Limited" - layout.addWidget(pill(badge, tone)) - return row - - def _render_home_models(self, models: list[Dict[str, Any]]) -> None: - clear_layout(self.home_models_layout) - if not models: - self.home_models_layout.addWidget(label("Models will appear when the network is ready.", "bodyMuted")) - return - for model in models[:3]: - self.home_models_layout.addWidget(self._model_row(model)) - def _render_models(self, models: list[Dict[str, Any]]) -> None: - clear_layout(self.models_list_layout) - if not models: - self.models_list_layout.addWidget(label("No models are available yet.", "bodyMuted")) - return + keys = {model["id"] for model in models} + for key in list(self.model_health_cards): + if key not in keys: + self.models_list_layout.removeWidget(self.model_health_cards[key]) + self.model_health_cards.pop(key).deleteLater() for model in models: - self.models_list_layout.addWidget(self._model_row(model)) - - def _render_regions(self, regions: list[Dict[str, Any]]) -> None: - clear_layout(self.region_layout) - if not regions: - self.region_layout.addWidget(label("Region view is warming up.", "bodyMuted")) - return - maximum = max((region["peers"] for region in regions), default=1) - for region in regions: - line = QVBoxLayout() - header = QHBoxLayout() - header.addWidget(label(region["name"], "bodyMuted")) - header.addStretch(1) - header.addWidget(label(str(region["peers"]), "bodyStrong")) - line.addLayout(header) - bar = QProgressBar() - bar.setRange(0, maximum) - bar.setValue(region["peers"]) - bar.setTextVisible(False) - line.addWidget(bar) - self.region_layout.addLayout(line) + if model["id"] not in self.model_health_cards: + self.model_health_cards[model["id"]] = ModelHealthCard() + self.models_list_layout.addWidget(self.model_health_cards[model["id"]]) + self.model_health_cards[model["id"]].set_state(model, self._snapshot["workers"]) def _render_sharing(self, snapshot: Dict[str, Any]) -> None: contribution = snapshot["contribution"] - workers = snapshot["workers"] - enabled = contribution["enabled"] - intent_enabled = contribution["intent_enabled"] - active_models = contribution["active_models"] - if enabled: - self.sharing_title.setText(f"You're helping with {', '.join(active_models)}") - self.sharing_detail.setText("The node is enforcing every configured sharing limit.") - elif intent_enabled: - self.sharing_title.setText("Sharing is waiting on the node policy") - self.sharing_detail.setText( - contribution["selected_blocked_reasons"][0] - if contribution["selected_blocked_reasons"] - else "The selected worker is paused or stopping." + title, detail, state = sharing_summary(snapshot) + if self._sharing_pending is not None: + title = "Starting sharing…" if self._sharing_pending else "Stopping sharing…" + detail = "" + elif self._sharing_error: + title, detail = "Sharing could not change", self._sharing_error + state = "error" + if self._sharing_pending is not None: + state = "starting" + for widget in (self.sharing_title, self.home_sharing_title): + widget.setText(title) + widget.setStyleSheet( + "color: " + + { + "running": "#72E7AE", + "starting": "#B6A5FF", + "waiting": "#F3C46C", + "error": "#EF8E9D", + "off": "#F4F6FA", + "paused": "#F4F6FA", + }[state] + + ";" ) - else: - self.sharing_title.setText("Your computer is not sharing right now") - self.sharing_detail.setText( - "Choose an admitted model below, then start whenever you're ready." - if contribution["can_start"] - else ( - contribution["blocked_reasons"][0] - if contribution["blocked_reasons"] - else "No contribution worker is available." + for widget in (self.sharing_detail, self.home_sharing_detail): + widget.setText(detail) + widget.setVisible(bool(detail)) + for button in (self.master_share_button, self.home_share_button): + if self._sharing_pending is not None: + button.setText("Starting…" if self._sharing_pending else "Stopping…") + else: + button.setText( + "Pause sharing" if contribution.get("intent_enabled") and state != "paused" else "Start sharing" ) + button.setObjectName( + "ghostButton" if contribution.get("intent_enabled") and state != "paused" else "primaryButton" ) - if intent_enabled: - self.master_share_button.setText("Pause sharing") - self.master_share_button.setObjectName("ghostButton") - else: - self.master_share_button.setText("Start sharing") - self.master_share_button.setObjectName("primaryButton") - self.master_share_button.style().unpolish(self.master_share_button) - self.master_share_button.style().polish(self.master_share_button) - - vram_status = contribution["vram_status"] - if vram_status == "configured": - percent = contribution["vram_percent"] - shared = _gib_text(contribution["vram_bytes"]) - pool = _gib_text(contribution["vram_pool_bytes"]) - self.memory_value.setText(f"{percent}%") - self.memory_detail.setText(f"{shared} of {pool} is reserved per configured worker.") - self.memory_bar.setValue(percent) - elif vram_status == "varies": - self.memory_value.setText("Varies") - self.memory_detail.setText("Configured accelerator limits differ between workers.") - self.memory_bar.setValue(0) - else: - self.memory_value.setText("Unavailable") - self.memory_detail.setText("No accelerator budget is reported; no default is assumed.") - self.memory_bar.setValue(0) - - def limit_summary(key: str, unit: str, *, byte_size: bool = False) -> str: - values = [worker["limits"][key] for worker in workers] - configured = {value for value in values if value is not None} - if not configured: - return "not configured" - if len(configured) != 1 or len(configured) != len(values) and any(value is None for value in values): - return "varies by worker" - value = next(iter(configured)) - return _gib_text(value) if byte_size else f"{value:g} {unit}" - - def measurement_summary(key: str, unit: str) -> str: - values = [worker["measurements"][key] for worker in workers] - present = {value for value in values if value is not None} - if not present: - return "telemetry unavailable" - if len(present) != 1 or any(value is None for value in values): - return "telemetry varies or is unavailable" - return f"{next(iter(present)):g} {unit} measured" - - admitted_models = sum(worker["policy_admitted"] for worker in workers) - policy_text = ( - f"Model policy admits {admitted_models} of {len(workers)} configured workers." - if workers - else "No contribution workers are configured." - ) - if contribution["blocked_reasons"]: - policy_text += f" {contribution['blocked_reasons'][0]}" - self.policy_status.setText(policy_text) - self.disk_policy.setText(f"Storage ceiling: {limit_summary('disk_bytes', '', byte_size=True)}") - self.bandwidth_policy.setText( - "Bandwidth ceiling: " - f"{limit_summary('bandwidth_mbps', 'Mbps')} · " - f"{measurement_summary('bandwidth_mbps', 'Mbps')}" + button.style().unpolish(button) + button.style().polish(button) + self.resource_controls.set_state( + contribution, busy=self._busy > 0 or self._controller is None or self._sharing_pending is not None ) - self.power_policy.setText( - "Power ceiling: " f"{limit_summary('power_watts', 'W')} · " f"{measurement_summary('power_watts', 'W')}" - ) - closed_reasons = [] - for worker in workers: - if not worker["schedule_admitted"] and worker["schedule_reason"] not in closed_reasons: - closed_reasons.append(worker["schedule_reason"]) - self.schedule_policy.setText( - "Schedule: unavailable" - if not workers - else (f"Schedule: {closed_reasons[0]}" if closed_reasons else "Schedule: open now") + hardware = snapshot.get("hardware", {}) + total = hardware.get("gpu_total_bytes") or contribution.get("vram_pool_bytes") + allowed = contribution.get("vram_bytes") + if allowed is not None and total: + self.home_vram.setText(f"{memory_text(allowed)} of {memory_text(total)}") + elif not total: + self.home_vram.setText("No GPU memory available") + else: + self.home_vram.setText("Checking memory limit…") + percent = contribution.get( + "processing_percent", (contribution.get("policy") or {}).get("max_processing_percent", 100) ) - + self.home_processing.setText(f"{percent:g}%") + self.memory_value.setText(hardware.get("gpu_name") or "") + self.memory_value.setVisible(bool(hardware.get("gpu_name"))) + self.memory_detail.setText("") + self.memory_detail.hide() + downloads = { + worker["id"]: worker + for worker in snapshot.get("workers", []) + if worker.get("download_progress") + and worker["download_progress"].get("state") not in ("ready", "paused") + } + for key in list(self.sharing_download_cards): + if key not in downloads: + self.sharing_downloads_layout.removeWidget(self.sharing_download_cards[key]) + self.sharing_download_cards.pop(key).deleteLater() + for key, worker in downloads.items(): + if key not in self.sharing_download_cards: + self.sharing_download_cards[key] = DownloadCard() + self.sharing_downloads_layout.addWidget(self.sharing_download_cards[key]) + name = "Sharing download" if worker["model"] == "auto" else model_name(worker["model"]) + self.sharing_download_cards[key].set_state(name, worker["download_progress"], worker["state"]) clear_layout(self.contribution_models_layout) - workers_by_model: Dict[str, list[Dict[str, Any]]] = {} - for worker in workers: - workers_by_model.setdefault(worker["model"], []).append(worker) - for model in snapshot["models"]: - model_workers = workers_by_model.get(model["id"], []) - row = QFrame() - row.setObjectName("listRow") - row_layout = QHBoxLayout(row) - row_layout.setContentsMargins(14, 12, 14, 12) - avatar = label(model["id"][:1].upper(), "avatar") - avatar.setFixedSize(38, 38) - row_layout.addWidget(avatar) - copy = QVBoxLayout() - copy.setSpacing(2) - copy.addWidget(label(model["id"], "bodyStrong")) - selected = any(worker["desired_running"] for worker in model_workers) - statuses = list(dict.fromkeys(worker["display_status"] for worker in model_workers)) - detail = "; ".join(statuses) if statuses else "Available after sharing setup" - status_label = label(detail, "bodyMuted") - status_label.setWordWrap(True) - copy.addWidget(status_label) - copy.addWidget( - label( - f"First-use download/storage: {model['download_storage_estimate']}", - "bodyMuted", - ) - ) - row_layout.addLayout(copy, 1) - toggle = QCheckBox() - toggle.setChecked(selected) - toggle.setEnabled( - bool(model_workers) - and self._busy == 0 - and (selected or any(worker["can_start"] for worker in model_workers)) - ) - toggle.setAccessibleName(f"Share compute with {model['id']}") - worker_ids = [worker["id"] for worker in model_workers] - startable_ids = [worker["id"] for worker in model_workers if worker["can_start"]] - toggle.stateChanged.connect( - lambda state, all_ids=worker_ids, start_ids=startable_ids: self._set_model_sharing( - start_ids if state == Qt.Checked.value else all_ids, - state == Qt.Checked.value, - ) + # Explicit per-model overrides remain in More settings; automatic + # contribution needs only the main sharing switch. + for worker in snapshot.get("workers", []): + if (worker.get("placement") or {}).get("automatic"): + continue + toggle = QCheckBox(model_name(worker.get("model"))) + toggle.setChecked(worker.get("desired_running", False)) + toggle.setEnabled(not self._busy and self._sharing_pending is None and self._controller is not None) + toggle.setAccessibleName(f"Share compute with {worker['model']}") + toggle.toggled.connect( + lambda enabled, worker_id=worker["id"]: self._set_model_sharing([worker_id], enabled) ) - row_layout.addWidget(toggle) - self.contribution_models_layout.addWidget(row) + self.contribution_models_layout.addWidget(toggle) def _render_keys(self, keys: list[Dict[str, Any]]) -> None: clear_layout(self.keys_layout) @@ -1038,10 +1003,34 @@ def _set_login_startup(self, enabled: bool) -> None: self.login_startup_toggle.blockSignals(True) self.login_startup_toggle.setChecked(not enabled) self.login_startup_toggle.blockSignals(False) - self.login_startup_detail.setText(f"Could not change login startup: {str(exc)[:180]}") + self.login_startup_detail.setText("Could not save this setting. Try again.") + self.login_startup_detail.setToolTip(str(exc)[:180]) + self.login_startup_detail.show() QMessageBox.warning(self, "Login startup", str(exc)[:300]) return - self.login_startup_detail.setText("Enabled for this user" if enabled else "Off") + self.login_startup_detail.setText( + "CommunityAI will open when you sign in." if enabled else "Automatic opening is off." + ) + self.login_startup_detail.show() + + def _apply_resource_limits(self, changes, revision) -> None: + if self._controller is None or self._busy: + return + + def applied(result): + self.resource_controls.applied(result) + self.refresh() + + def failed(message): + self.resource_controls.failed(sharing_reason(message)) + self.resource_controls.message.setToolTip(str(message)[:300]) + self.refresh() + + self._submit( + lambda: self._controller.update_resource_limits(changes, expected_revision=revision), + applied, + failed, + ) def _edit_contribution_policy(self) -> None: contribution = self._snapshot.get("contribution", {}) @@ -1060,6 +1049,7 @@ def _edit_contribution_policy(self) -> None: return dialog = QDialog(self) + dialog.setObjectName("sharingPolicyDialog") dialog.setWindowTitle("Edit sharing limits") dialog.setMinimumWidth(620) layout = QVBoxLayout(dialog) @@ -1072,6 +1062,7 @@ def _edit_contribution_policy(self) -> None: form = QFormLayout() sharing_enabled = QCheckBox("Allow this node to share compute") + sharing_enabled.setObjectName("policy_sharing_enabled") sharing_enabled.setChecked(policy["sharing_enabled"]) form.addRow("Sharing", sharing_enabled) @@ -1082,6 +1073,7 @@ def _edit_contribution_policy(self) -> None: ("denied_models", "Denied models"), ): editor = QPlainTextEdit() + editor.setObjectName(f"policy_{field}") editor.setPlainText("\n".join(policy[field])) editor.setPlaceholderText("One exact model selector per line") editor.setFixedHeight(64) @@ -1098,6 +1090,7 @@ def _edit_contribution_policy(self) -> None: ("pause_timeout", "Pause timeout (seconds)", "10"), ): editor = QLineEdit() + editor.setObjectName(f"policy_{field}") value = policy[field] editor.setText("" if value is None else f"{value:g}" if isinstance(value, float) else str(value)) editor.setPlaceholderText(placeholder) @@ -1106,6 +1099,7 @@ def _edit_contribution_policy(self) -> None: form.addRow(title, editor) schedule = QPlainTextEdit() + schedule.setObjectName("policy_schedule") schedule.setPlainText("" if policy["schedule"] is None else json.dumps(policy["schedule"], indent=2)) schedule.setPlaceholderText( '{"timezone":"local","windows":[{"days":["mon"],"start":"22:00","end":"06:00"}]}' @@ -1116,6 +1110,7 @@ def _edit_contribution_policy(self) -> None: layout.addLayout(form) buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Save | QDialogButtonBox.StandardButton.Cancel) + buttons.setObjectName("sharingPolicyButtons") buttons.accepted.connect(dialog.accept) buttons.rejected.connect(dialog.reject) layout.addWidget(buttons) @@ -1128,6 +1123,7 @@ def optional_text(field: str): try: updated = { + **policy, "sharing_enabled": sharing_enabled.isChecked(), **{ field: [line for line in editor.toPlainText().splitlines() if line.strip()] @@ -1158,38 +1154,77 @@ def optional_text(field: str): ) def _sharing_action_failed(self, message: str) -> None: - self.sharing_title.setText("The node rejected the sharing change") - self.sharing_detail.setText(str(message)[:300]) + self._sharing_pending = None + self._awaiting_sharing_snapshot = False + self._sharing_error = sharing_reason(message) + self._render_sharing(self._snapshot) + self.sharing_detail.setToolTip(str(message)[:300]) + self.home_sharing_detail.setToolTip(str(message)[:300]) + self._set_busy(0) + self.refresh() + + def _sharing_changed(self, result) -> None: + self._awaiting_sharing_snapshot = True + self.refresh() def _set_model_sharing(self, worker_ids: list[str], enabled: bool) -> None: if not worker_ids or self._controller is None or self._busy: return + self._sharing_pending = enabled + self._sharing_error = None + self._render_sharing(self._snapshot) + controller = self._controller self._submit( - lambda: self._controller.set_workers_enabled(worker_ids, enabled), - lambda result: self.refresh(), + lambda: controller.set_workers_enabled(worker_ids, enabled), + self._sharing_changed, self._sharing_action_failed, ) - def _toggle_all_sharing(self) -> None: - workers = self._snapshot.get("workers", []) - contribution = self._snapshot.get("contribution", {}) - if not workers or self._controller is None: + def _toggle_inference_mode(self) -> None: + if self._controller is None or self._busy or self._mode_pending: return - enable = not contribution.get("intent_enabled", False) - worker_ids = [ - worker["id"] for worker in workers if (worker["can_start"] if enable else worker["desired_running"]) - ] - if not worker_ids: + mode = "auto" if self._snapshot.get("inference_mode") == "local_only" else "local_only" + controller = self._controller + self._mode_pending = True + self.inference_mode_button.setText("Changing…") + + def changed(_): + self._awaiting_mode_snapshot = True + self.refresh() + + def failed(message): + self._mode_pending = False + self._render(self._snapshot) + self.auto_selection_detail.setText("Could not change this setting. Try again.") + self.auto_selection_detail.setToolTip(str(message)[:300]) + + self._submit( + lambda: controller.client.set_inference_mode(mode), + changed, + failed, + ) + + def _toggle_all_sharing(self) -> None: + if self._controller is None or self._busy or self._sharing_pending is not None: return + enable = ( + not self._snapshot.get("contribution", {}).get("intent_enabled", False) + or sharing_summary(self._snapshot)[2] == "paused" + ) + self._sharing_pending = enable + self._sharing_error = None + self._render_sharing(self._snapshot) + controller = self._controller self._submit( - lambda: self._controller.set_workers_enabled(worker_ids, enable), - lambda result: self.refresh(), + lambda: controller.set_sharing_enabled(enable), + self._sharing_changed, self._sharing_action_failed, ) def _copy_endpoint(self) -> None: - QGuiApplication.clipboard().setText(self.endpoint.text()) - self.connection_detail.setText("Endpoint URL copied") + QGuiApplication.clipboard().setText(self.api_endpoint.text()) + self.copy_endpoint_button.setText("Copied") + QTimer.singleShot(2000, lambda: self.copy_endpoint_button.setText("Copy URL")) def _create_key(self) -> None: if self._controller is None: @@ -1243,6 +1278,7 @@ def _revoke_key(self, key_id: str) -> None: instance_server = None instance_lock = None instance_server_name = None + shutdown_sockets = [] if single_instance: data_location = QStandardPaths.writableLocation(QStandardPaths.AppLocalDataLocation) if not data_location: @@ -1305,16 +1341,39 @@ def notify_existing_instance(timeout_ms: int) -> bool: raise SingleInstanceError(f"could not establish the per-user CommunityAI instance endpoint: {error}") window = MainWindow() + + def stop_window_refreshes(): + window._closing = True + window._timer.stop() + if updater is not None: + updater.close() + + application.aboutToQuit.connect(stop_window_refreshes) window._show_page(max(0, min(3, screenshot_page))) if start_minimized: window.showMinimized() else: window.show() + if qualification_automation is not None: + qualification_automation.install( + window, + application, + { + "QTimer": QTimer, + "QDialog": QDialog, + "QDialogButtonBox": QDialogButtonBox, + "QCheckBox": QCheckBox, + "QPlainTextEdit": QPlainTextEdit, + "QLineEdit": QLineEdit, + }, + ) + if instance_server is not None: def activate_window() -> None: should_activate = False + should_shutdown = False while instance_server.hasPendingConnections(): socket = instance_server.nextPendingConnection() socket.setReadBufferSize(64) @@ -1322,8 +1381,15 @@ def activate_window() -> None: raw_message = bytes(socket.read(64)) message = raw_message.strip() if len(raw_message) <= 32 and socket.bytesAvailable() == 0 else b"" should_activate = should_activate or message == b"activate" - socket.abort() - socket.deleteLater() + if message == b"shutdown": + shutdown_sockets.append(socket) + should_shutdown = True + else: + socket.abort() + socket.deleteLater() + if should_shutdown: + application.quit() + return if should_activate: window.showNormal() window.raise_() @@ -1337,7 +1403,6 @@ def close_instance_server() -> None: instance_lock.unlock() instance_server.newConnection.connect(activate_window) - application.aboutToQuit.connect(close_instance_server) if instance_server.hasPendingConnections(): QTimer.singleShot(0, activate_window) @@ -1353,8 +1418,24 @@ def capture() -> None: if auto_close_seconds is not None: QTimer.singleShot(max(1, int(float(auto_close_seconds) * 1000)), application.quit) restore_termination_handlers = _install_posix_termination_bridge(application, QTimer) + + def finish_desktop_cleanup(): + response = b"failed\n" + try: + if before_termination_restore is not None: + before_termination_restore() + response = b"stopped\n" + finally: + for socket in shutdown_sockets: + socket.write(response) + socket.flush() + socket.waitForBytesWritten(1000) + socket.disconnectFromServer() + if instance_server is not None: + close_instance_server() + return _exec_with_termination_cleanup( application, restore_termination_handlers, - before_termination_restore, + finish_desktop_cleanup, ) diff --git a/desktop/src/communityai_desktop/release.py b/desktop/src/communityai_desktop/release.py new file mode 100644 index 000000000..fe0e5fff0 --- /dev/null +++ b/desktop/src/communityai_desktop/release.py @@ -0,0 +1,3 @@ +"""Release identity compiled into the desktop executable.""" + +RELEASE_VERSION = "0.1.0-alpha.20260909.3" diff --git a/desktop/src/communityai_desktop/resource_controls.py b/desktop/src/communityai_desktop/resource_controls.py new file mode 100644 index 000000000..296d4f653 --- /dev/null +++ b/desktop/src/communityai_desktop/resource_controls.py @@ -0,0 +1,130 @@ +"""The two sharing budget sliders, backed by the node's persisted policy.""" + +import math + +from PySide6.QtCore import Qt, Signal +from PySide6.QtWidgets import QHBoxLayout, QLabel, QPushButton, QSlider, QVBoxLayout, QWidget + + +class ResourceControls(QWidget): + apply_requested = Signal(object, str) + + def __init__(self, parent=None): + super().__init__(parent) + self._draft = {} + self._revision = None + self._policy = {} + self._editable = False + self._busy = False + self._vram_total = None + self._vram_available = None + self._vram_saved = None + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + self.sliders = {} + self.values = {} + for field, title in ( + ("max_vram", "GPU memory limit"), + ("max_processing_percent", "Computing"), + ): + row = QHBoxLayout() + row.addWidget(QLabel(title), 1) + value = QLabel("100%") + value.setObjectName(f"resource_{field}_value") + row.addWidget(value) + layout.addLayout(row) + slider = QSlider(Qt.Horizontal) + slider.setObjectName(f"resource_{field}") + slider.setAccessibleName(title) + slider.setRange(1, 100) + slider.setValue(100) + slider.setPageStep(10) + slider.valueChanged.connect(lambda percent, name=field: self._changed(name, percent)) + layout.addWidget(slider) + self.sliders[field] = slider + self.values[field] = value + self.message = QLabel("") + self.message.setWordWrap(True) + self.message.setObjectName("bodyMuted") + layout.addWidget(self.message) + self.apply_button = QPushButton("Save changes") + self.apply_button.setObjectName("applyResourceLimits") + self.apply_button.clicked.connect(self._apply) + layout.addWidget(self.apply_button) + self._update_enabled() + + def _changed(self, field, percent): + self._draft[field] = f"{percent}%" if field == "max_vram" else percent + self.values[field].setText(self._vram_text(percent) if field == "max_vram" else f"{percent}%") + self.message.setText("Unsaved changes") + self._update_enabled() + + def _vram_text(self, percent=None): + if self._vram_total is not None: + amount = self._vram_saved + if percent is not None: + amount = self._vram_total * percent / 100 + if amount is not None: + return f"{amount / 1024**3:.1f} GB of {self._vram_total / 1024**3:.1f} GB" + return "No GPU memory detected" + + def set_state(self, contribution, *, busy=False): + revision = contribution.get("config_revision") + if revision != self._revision: + if self._draft: + self.message.setText("Settings updated elsewhere. Showing saved values.") + self._draft.clear() + self._revision = revision + self._policy = contribution.get("policy") or {} + self._editable = contribution.get("editable", False) and isinstance(revision, str) + self._busy = busy + self._vram_total = contribution.get("vram_pool_bytes") + self._vram_available = contribution.get("vram_available_bytes") + self._vram_saved = contribution.get("vram_bytes") + for field, slider in self.sliders.items(): + raw = self._draft.get(field, self._policy.get(field, 100)) + if field == "max_vram": + try: + percent = float(raw[:-1]) if isinstance(raw, str) and raw.endswith("%") else None + if percent is not None and (not math.isfinite(percent) or not 0 < percent <= 100): + percent = None + except ValueError: + percent = None + text = self._vram_text(percent) + if raw and percent is None: + text = str(raw) + else: + percent = raw + text = f"{percent:g}%" + slider.blockSignals(True) + slider.setMaximum(100) + slider.setValue(100 if percent is None else round(percent)) + slider.blockSignals(False) + self.values[field].setText(text) + self._update_enabled() + + def _update_enabled(self): + supported = "max_processing_percent" in self._policy + for slider in self.sliders.values(): + slider.setEnabled(self._editable and supported and not self._busy) + self.sliders["max_vram"].setEnabled( + self._editable and supported and not self._busy and bool(self._vram_total) and self._vram_available != 0 + ) + self.apply_button.setEnabled(self._editable and supported and bool(self._draft) and not self._busy) + if self._editable and not supported: + self.message.setText("Update the local node to use both resource controls.") + + def _apply(self): + if self.apply_button.isEnabled(): + self.message.setText("Saving changes…") + self.apply_button.setText("Saving…") + self.apply_requested.emit(dict(self._draft), self._revision) + + def applied(self, result): + self._draft.clear() + self.apply_button.setText("Save changes") + self.message.setText(result.get("message", "Limits saved.")) + + def failed(self, message): + self.apply_button.setText("Save changes") + self.message.setText(f"Could not apply limits: {str(message)[:240]}") diff --git a/desktop/src/communityai_desktop/resource_playthrough.py b/desktop/src/communityai_desktop/resource_playthrough.py new file mode 100644 index 000000000..e4990d058 --- /dev/null +++ b/desktop/src/communityai_desktop/resource_playthrough.py @@ -0,0 +1,185 @@ +"""Explicit, bounded qualification of the real packaged sharing controls. + +The host runner observes the actual node between steps and acknowledges each +observation through a local file. No control credentials enter the UI evidence. +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +from communityai_desktop.gate13_playthrough import PlaythroughError, _regular_bytes + + +class ResourcePlaythrough: + def __init__(self, plan_path: Path, evidence_path: Path): + plan = json.loads(_regular_bytes(plan_path, 16384)) + if not isinstance(plan, dict) or set(plan) != {"steps", "timeout_seconds", "acknowledgement"}: + raise PlaythroughError("Resource playthrough plan is invalid") + steps = plan["steps"] + if not isinstance(steps, list) or not 1 <= len(steps) <= 24: + raise PlaythroughError("Resource playthrough requires 1..24 steps") + for step in steps: + if not isinstance(step, dict) or step.get("action") not in ("observe", "limits", "start", "pause"): + raise PlaythroughError("Resource playthrough action is invalid") + fields = {"action"} + if step["action"] in ("observe", "limits"): + fields |= {"vram_percent", "processing_percent"} + for field in ("vram_percent", "processing_percent"): + if type(step.get(field)) is not int or not 1 <= step[field] <= 100: + raise PlaythroughError("Resource playthrough percentages must be 1..100") + if set(step) != fields: + raise PlaythroughError("Resource playthrough step fields are invalid") + timeout = plan["timeout_seconds"] + if type(timeout) is not int or not 30 <= timeout <= 3600: + raise PlaythroughError("Resource playthrough timeout must be 30..3600 seconds") + acknowledgement = plan["acknowledgement"] + if not isinstance(acknowledgement, str) or not Path(acknowledgement).is_absolute(): + raise PlaythroughError("Resource acknowledgement must be an absolute local path") + self.steps = steps + self.acknowledgement = Path(acknowledgement) + self.evidence_path = Path(evidence_path) + self.timeout = timeout + self.index = 0 + self.phase = "ready" + self.done = False + self.started = time.monotonic() + self.result = { + "scope": "packaged-resource-controls", + "frozen": bool(getattr(sys, "frozen", False)), + "result": "running", + "steps": [], + } + + def install(self, window, application, qt): + from PySide6.QtCore import Qt + from PySide6.QtTest import QTest + + self.window, self.application = window, application + self.types = qt + self.qt, self.test = Qt, QTest + self.timer = qt["QTimer"](window) + self.timer.setInterval(200) + self.timer.timeout.connect(self.tick) + self.timer.start() + self.write() + + def write(self): + temporary = self.evidence_path.with_suffix(".tmp") + temporary.write_text(json.dumps(self.result, indent=2) + "\n", encoding="utf-8") + temporary.replace(self.evidence_path) + + def finish(self, error=None): + self.done = True + self.timer.stop() + self.result["result"] = "passed" if error is None else "failed" + if error: + self.result["error"] = error + self.result["phase"] = self.phase + self.result["step_index"] = self.index + self.result["blocked_reasons"] = self.window._snapshot.get("contribution", {}).get("blocked_reasons", []) + self.window.grab().save(str(self.evidence_path.with_suffix(".png"))) + self.write() + self.application.exit(0 if error is None else 2) + + def click(self, button): + if not button.isEnabled(): + raise PlaythroughError("resource_control_disabled") + self.test.mouseClick(button, self.qt.LeftButton) + + def tick(self): + if self.done: + return + try: + if time.monotonic() - self.started > self.timeout: + self.finish("resource_playthrough_timed_out") + return + if self.phase == "acknowledgement": + if self.acknowledgement.exists(): + acknowledgement = json.loads(_regular_bytes(self.acknowledgement, 128)) + if type(acknowledgement) is int and acknowledgement == self.index: + self.index += 1 + self.phase = "ready" + if self.index == len(self.steps): + self.finish() + return + window = self.window + if window._controller is None or window._busy: + return + controls = window.resource_controls + contribution = window._snapshot.get("contribution", {}) + if not contribution.get("editable"): + return + step = self.steps[self.index] + if self.phase == "ready": + self.action_started = time.monotonic() + self.click(window._page_buttons[0 if step["action"] in ("start", "pause") else 2]) + if step["action"] == "limits": + for field, key in (("max_vram", "vram_percent"), ("max_processing_percent", "processing_percent")): + slider = controls.sliders[field] + if not slider.isEnabled(): + raise PlaythroughError("resource_slider_disabled") + if field == "max_vram" and step[key] != 100 and step[key] >= slider.maximum(): + raise PlaythroughError("resource_vram_target_above_available_memory") + target = min(step[key], slider.maximum()) + slider.setFocus() + self.test.keyClick(slider, self.qt.Key_Home) + for _ in range(target - 1): + self.test.keyClick(slider, self.qt.Key_Right) + if slider.value() != target: + raise PlaythroughError("resource_slider_value_mismatch") + self.click(controls.apply_button) + elif step["action"] in ("start", "pause"): + expected = "Start sharing" if step["action"] == "start" else "Pause sharing" + if window.home_share_button.text() != expected: + raise PlaythroughError("resource_sharing_button_mismatch") + self.click(window.home_share_button) + pending_text = "Starting…" if step["action"] == "start" else "Stopping…" + confirmed_text = "Pause sharing" if step["action"] == "start" else "Start sharing" + for button in (window.home_share_button, window.master_share_button): + if button.text() not in (pending_text, confirmed_text): + raise PlaythroughError("resource_sharing_immediate_feedback_missing") + if button.text() == pending_text and button.isEnabled(): + raise PlaythroughError("resource_sharing_pending_button_enabled") + self.immediate_feedback = window.home_share_button.text() + self.phase = "observe" + return + policy = contribution.get("policy") or {} + if step["action"] in ("observe", "limits"): + expected = { + "max_vram": f"{step['vram_percent']}%", + "max_processing_percent": step["processing_percent"], + } + if any(policy.get(field) != value for field, value in expected.items()): + return + if controls.sliders["max_vram"].value() != min( + step["vram_percent"], controls.sliders["max_vram"].maximum() + ): + raise PlaythroughError("resource_display_mismatch") + if controls.sliders["max_processing_percent"].value() != step["processing_percent"]: + raise PlaythroughError("processing_display_mismatch") + elif bool(contribution.get("intent_enabled")) != (step["action"] == "start"): + return + self.result["steps"].append( + { + "index": self.index, + **step, + "seconds": round(time.monotonic() - self.started, 3), + "action_seconds": round(time.monotonic() - self.action_started, 3), + "saved_vram": policy.get("max_vram"), + "saved_processing_percent": policy.get("max_processing_percent"), + "sharing_intent": bool(contribution.get("intent_enabled")), + "vram_display": controls.values["max_vram"].text(), + "processing_display": controls.values["max_processing_percent"].text(), + "message": controls.message.text()[:240], + **({"immediate_feedback": self.immediate_feedback} if step["action"] in ("start", "pause") else {}), + } + ) + window.grab().save(str(self.evidence_path.with_suffix(".png"))) + self.write() + self.phase = "acknowledgement" + except Exception as exc: + self.finish(str(exc) if isinstance(exc, PlaythroughError) else "resource_playthrough_failed") diff --git a/desktop/src/communityai_desktop/telemetry.py b/desktop/src/communityai_desktop/telemetry.py new file mode 100644 index 000000000..a464f532f --- /dev/null +++ b/desktop/src/communityai_desktop/telemetry.py @@ -0,0 +1,118 @@ +"""Bound optional display telemetry without changing routing or download decisions.""" + +import math +import time + + +def number(value, *, integer=False, maximum=64 * 1024**4): + if isinstance(value, bool) or not isinstance(value, int if integer else (int, float)): + return None + return value if math.isfinite(value) and 0 <= value <= maximum else None + + +def text(value, limit=256): + return " ".join(value.split())[:limit] if isinstance(value, str) else None + + +def download_view(value): + if not isinstance(value, dict) or value.get("schema_version") != 1: + return None + state = value.get("state") + if state not in { + "waiting", + "checking", + "downloading", + "retrying", + "verifying", + "loading", + "ready", + "failed", + "paused", + }: + return None + result = {"state": state, "artifact": text(value.get("artifact"))} + for key in ( + "artifact_bytes", + "artifact_received_bytes", + "selected_bytes", + "received_bytes", + "verified_bytes", + "verified_files", + "selected_files", + "resumed_bytes", + "retries", + ): + result[key] = number(value.get(key), integer=True) + result["bytes_per_second"] = number(value.get("bytes_per_second")) + result["updated_at"] = number(value.get("updated_at")) + if result["updated_at"] is not None and time.time() - result["updated_at"] > 10: + result["bytes_per_second"] = 0 + return result + + +def route_view(route): + total = number(route.get("total_blocks"), integer=True, maximum=4096) + total = total or 0 + + def counts(key): + values = route.get(key) + if not isinstance(values, list) or len(values) != total: + return None + return [number(value, integer=True, maximum=100000) for value in values] + + def blocks(value): + if not isinstance(value, list) or len(value) > total: + return [] + return sorted({index for index in value if type(index) is int and 0 <= index < total}) + + def records(key): + value = route.get(key) + return value[:256] if isinstance(value, list) else [] + + peers = [] + for peer in records("peers"): + if not isinstance(peer, dict) or not isinstance(peer.get("peer_id"), str): + continue + peers.append( + { + "peer_id": text(peer["peer_id"], 128), + "public_name": text(peer.get("public_name"), 128), + "online_blocks": blocks(peer.get("online_blocks")), + "joining_blocks": blocks(peer.get("joining_blocks")), + "offline_blocks": blocks(peer.get("offline_blocks")), + "version": text(peer.get("version"), 64), + "torch_dtype": text(peer.get("torch_dtype"), 32), + "quant_type": text(peer.get("quant_type"), 32), + "using_relay": peer.get("using_relay") if type(peer.get("using_relay")) is bool else None, + } + ) + reservations = [] + for reservation in records("reservations"): + if not isinstance(reservation, dict): + continue + start = number(reservation.get("start_block"), integer=True, maximum=total) + end = number(reservation.get("end_block"), integer=True, maximum=total) + expiry = number(reservation.get("expires_at")) + if start is None or end is None or start >= end or expiry is None or expiry <= time.time(): + continue + reservations.append( + { + "peer_id": text(reservation.get("peer_id"), 128), + "start_block": start, + "end_block": end, + "expires_at": expiry, + } + ) + return { + "total_blocks": total, + "replica_counts": counts("replica_counts"), + "joining_counts": counts("joining_counts"), + "offline_counts": counts("offline_counts"), + "peers": peers, + "reservations": reservations, + "reservations_known": isinstance(route.get("reservations"), list), + "status": text(route.get("status"), 32), + "last_updated_age": number(route.get("last_updated_age")), + "last_error": text(route.get("last_error")), + "truncated": route.get("peer_details_truncated") is True, + } diff --git a/desktop/src/communityai_desktop/updater.py b/desktop/src/communityai_desktop/updater.py new file mode 100644 index 000000000..8c7bcc5e2 --- /dev/null +++ b/desktop/src/communityai_desktop/updater.py @@ -0,0 +1,390 @@ +"""Authenticated application updates, resumable downloads and installer handoff.""" + +from __future__ import annotations + +import base64 +import hashlib +import http.client +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import urlsplit + +from communityai_desktop.release import RELEASE_VERSION +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +ORIGIN = "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev" +FEEDS = ( + ORIGIN + "/updates/alpha.json", + "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate14-20260902-b/public-alpha/updates/alpha.json", +) +PUBLIC_KEY = "9gREIiEXMW20DgpIFW1Hjzsb3hFSZ2Ya3Kx6oSTZuz0=" +SIGNATURE_DOMAIN = b"communityai-app-update-v1\x00" +MAX_FEED_BYTES = 65536 +MAX_PACKAGE_BYTES = 32 * 1024**3 + + +class UpdateError(ValueError): + pass + + +def require(condition, message): + if not condition: + raise UpdateError(message) + + +def version_key(value): + require(isinstance(value, str), "Invalid application version") + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:-alpha\.(\d{8})\.(\d+))?", value) + require(match is not None, "Invalid application version") + major, minor, patch, date, revision = match.groups() + return int(major), int(minor), int(patch), int(date is None), int(date or 0), int(revision or 0) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False).encode("ascii") + + +def unique_fields(pairs): + value = {} + for key, item in pairs: + require(key not in value, "Duplicate update field") + value[key] = item + return value + + +def verify_feed(raw, *, public_key=PUBLIC_KEY, now=None, minimum_sequence=0): + require(len(raw) <= MAX_FEED_BYTES, "Update feed is too large") + try: + envelope = json.loads(raw, object_pairs_hook=unique_fields) + require(isinstance(envelope, dict) and set(envelope) == {"signed", "signature"}, "Invalid signed update") + signed = envelope["signed"] + signature = base64.b64decode(envelope["signature"], validate=True) + Ed25519PublicKey.from_public_bytes(base64.b64decode(public_key, validate=True)).verify( + signature, SIGNATURE_DOMAIN + canonical(signed) + ) + except (InvalidSignature, TypeError, KeyError, ValueError) as exc: + raise UpdateError("Update signature could not be verified") from exc + require( + isinstance(signed, dict) + and set(signed) + == {"schema_version", "channel", "version", "sequence", "published_at", "expires_at", "artifacts"}, + "Invalid update fields", + ) + require(type(signed["schema_version"]) is int and signed["schema_version"] == 1, "Unsupported update format") + require(signed["channel"] == "alpha", "Wrong update channel") + version_key(signed["version"]) + sequence = signed["sequence"] + require(type(sequence) is int and sequence >= minimum_sequence and sequence > 0, "Older update feed rejected") + now = time.time() if now is None else now + issued, expires = signed["published_at"], signed["expires_at"] + require( + type(issued) is int + and type(expires) is int + and issued <= now + 300 + and now < expires + and 0 < expires - issued <= 180 * 86400, + "Update feed is expired or not yet valid", + ) + artifacts = signed["artifacts"] + require(isinstance(artifacts, dict) and 1 <= len(artifacts) <= 2, "Invalid update packages") + for target, item in artifacts.items(): + require(target in ("windows-x64", "linux-amd64") and isinstance(item, dict), "Unsupported update platform") + require(set(item) == {"filename", "url", "size_bytes", "sha256"}, "Invalid update package fields") + expected = ( + f"communityai-{signed['version']}-windows-setup.exe" + if target == "windows-x64" + else f"communityai_{signed['version'].replace('-alpha.', '~alpha.')}_amd64.deb" + ) + require(item["filename"] == expected, "Update filename does not match version") + require(type(item["size_bytes"]) is int and 0 < item["size_bytes"] <= MAX_PACKAGE_BYTES, "Invalid update size") + require( + isinstance(item["sha256"], str) and re.fullmatch(r"[0-9a-f]{64}", item["sha256"]), "Invalid update hash" + ) + parsed = urlsplit(item["url"]) + require( + parsed.scheme == "https" + and parsed.netloc == urlsplit(ORIGIN).netloc + and not parsed.query + and not parsed.fragment + and parsed.path.startswith("/alpha/") + and parsed.path.rsplit("/", 1)[-1] == expected + and "/../" not in parsed.path + and "%" not in parsed.path, + "Update package has an untrusted address", + ) + return signed + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise UpdateError("Update redirects are not allowed") + + +def open_url(url, *, offset=0): + headers = {"User-Agent": "CommunityAI-Online-Installer/1", "Accept-Encoding": "identity"} + if offset: + headers["Range"] = f"bytes={offset}-" + return urllib.request.build_opener(NoRedirect()).open(urllib.request.Request(url, headers=headers), timeout=20) + + +def file_hash(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024**2), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_bytes(canonical(value)) + os.replace(temporary, path) + + +def download(item, directory, progress, cancelled, *, opener=open_url): + directory.mkdir(parents=True, exist_ok=True) + require(not directory.is_symlink(), "Invalid update cache") + target = directory / item["filename"] + partial = directory / (item["filename"] + ".part") + require(not target.is_symlink() and not partial.is_symlink(), "Invalid cached update") + size = item["size_bytes"] + if target.is_file() and target.stat().st_size == size and file_hash(target) == item["sha256"]: + return target + if target.is_file(): + target.unlink() + if partial.exists() and partial.stat().st_size > size: + partial.unlink() + for attempt in range(4): + if cancelled.is_set(): + raise UpdateError("Update download paused") + offset = partial.stat().st_size if partial.exists() else 0 + if offset == size: + break + require( + shutil.disk_usage(directory).free >= size - offset + 256 * 1024**2, "Not enough storage for the update" + ) + try: + with opener(item["url"], offset=offset) as response: + require(response.headers.get("Content-Encoding", "identity") == "identity", "Encoded update rejected") + if response.status == 200: + offset = 0 # A server may ignore Range; restart rather than append. + mode = "wb" + else: + require(response.status == 206 and offset > 0, "Unexpected update response") + require( + response.headers.get("Content-Range") == f"bytes {offset}-{size - 1}/{size}", + "Invalid update range", + ) + mode = "ab" + require(response.headers.get("Content-Length") == str(size - offset), "Update size mismatch") + with partial.open(mode) as stream: + while offset < size: + if cancelled.is_set(): + raise UpdateError("Update download paused") + block = response.read(min(1024**2, size - offset)) + if not block: + raise OSError("Update download interrupted") + stream.write(block) + offset += len(block) + progress(offset, size) + require(not response.read(1), "Update exceeded its expected size") + break + except (OSError, urllib.error.URLError, http.client.HTTPException): + if attempt == 3: + raise UpdateError("Download interrupted. It will resume when you retry.") + if cancelled.wait(min(2**attempt, 4)): + raise UpdateError("Update download paused") + require(partial.is_file() and partial.stat().st_size == size, "Update download is incomplete") + if file_hash(partial) != item["sha256"]: + partial.unlink() + raise UpdateError("Update verification failed. Please retry.") + os.replace(partial, target) + return target + + +def installed_root(): + if not getattr(sys, "frozen", False): + return None + root = Path(sys.executable).resolve().parent + marker = root / ".communityai-installation" + if marker.is_file() and marker.read_text(encoding="utf-8").startswith("CommunityAI installer-managed"): + return root + return None + + +class UpdateManager: + def __init__(self, directory, *, root=None, current_version=RELEASE_VERSION, system=None): + self.directory = Path(directory) + self.root = root + self.current_version = current_version + self.system = system or platform.system() + self.target = "windows-x64" if self.system == "Windows" else "linux-amd64" + self.lock = threading.Lock() + self.cancelled = threading.Event() + self.state = {"status": "idle", "message": "Check for updates", "version": current_version} + self.thread = None + self.candidate = None + self.process = None + + def _set(self, **values): + with self.lock: + self.state.update(values) + + def snapshot(self): + with self.lock: + return dict(self.state) + + def check(self): + if self.thread and self.thread.is_alive() or self.snapshot()["status"] == "installing": + return + self.cancelled.clear() + self._set(status="checking", message="Checking for updates…") + self.thread = threading.Thread(target=self._check, daemon=True, name="CommunityAI updates") + self.thread.start() + + def _check(self): + try: + state_path = self.directory / "accepted.json" + minimum = 0 + if state_path.exists(): + accepted = json.loads(state_path.read_bytes()) + minimum = int(accepted.get("sequence", 0)) + errors = [] + signed = None + for url in FEEDS: + try: + with open_url(url) as response: + raw = response.read(MAX_FEED_BYTES + 1) + signed = verify_feed(raw, minimum_sequence=minimum) + break + except (OSError, ValueError, urllib.error.URLError) as exc: + errors.append(type(exc).__name__) + require(signed is not None, "Could not check for updates. Try again later.") + atomic_json(state_path, {"sequence": signed["sequence"]}) + if ( + version_key(signed["version"]) <= version_key(self.current_version) + or self.target not in signed["artifacts"] + ): + self._clear_installed_downloads() + self._set(status="current", message="You’re up to date") + return + item = signed["artifacts"][self.target] + self._set(status="downloading", message="Downloading update…", version=signed["version"]) + path = download( + item, + self.directory / item["sha256"], + lambda received, total: self._set(message=f"Downloading update… {received * 100 // total}%"), + self.cancelled, + ) + self.candidate = (signed, path) + self._set(status="ready", message="Restart to update", version=signed["version"]) + except Exception as exc: + self._set( + status="error", message=str(exc) if isinstance(exc, UpdateError) else "Could not update. Try again." + ) + + def install(self): + require(self.root is not None and self.candidate is not None, "Update is not ready") + require(not self.thread or not self.thread.is_alive(), "Update is still being checked") + self._set(status="installing", message="Installing update…") + self.thread = threading.Thread(target=self._install, daemon=True, name="CommunityAI install") + self.thread.start() + + def _clear_installed_downloads(self): + for directory in self.directory.iterdir(): + if not re.fullmatch(r"[0-9a-f]{64}", directory.name) or directory.is_symlink() or not directory.is_dir(): + continue + for file in directory.iterdir(): + match = re.fullmatch( + r"communityai-(.+)-windows-setup\.exe(?:\.part)?|communityai_(.+)_amd64\.deb(?:\.part)?", file.name + ) + if not match or file.is_symlink() or not file.is_file(): + continue + version = (match[1] or match[2]).replace("~alpha.", "-alpha.") + if version_key(version) <= version_key(self.current_version): + try: + file.unlink() + except OSError: + pass # Windows may still have its installer open during restart. + try: + directory.rmdir() + except OSError: + pass + + def _install(self): + try: + signed, path = self.candidate + require(time.time() < signed["expires_at"], "Update information expired. Check for updates again.") + item = signed["artifacts"][self.target] + require( + not path.is_symlink() + and path.stat().st_size == item["size_bytes"] + and file_hash(path) == item["sha256"], + "Cached update changed. Please download it again.", + ) + if self.system == "Windows": + arguments = [ + str(path), + "/SILENT", + "/SP-", + "/NORESTART", + "/UPDATE=1", + "/DIR=" + str(self.root), + "/LOG=" + str(self.directory / "install.log"), + ] + self.process = subprocess.Popen(arguments, creationflags=subprocess.CREATE_NO_WINDOW) + code = self.process.wait() + require(code == 0, "Update installation did not finish. Try again.") + else: + require( + self.root == Path("/opt/communityai") and Path("/usr/bin/pkexec").is_file(), + "Install policykit-1 to allow app updates.", + ) + result = self.directory / "install-result.txt" + result.unlink(missing_ok=True) + # Reparent before APT starts: package maintenance must never kill its own installer as a GUI child. + command = ( + "(sleep 1; /usr/bin/pkexec /usr/bin/python3 -I /opt/communityai/update_install.py " + '"$1" "$2" "$3" "$4" --noninteractive; code=$?; printf "%s" "$code" >"$5"; ' + 'if [ "$code" -eq 0 ]; then exec /usr/bin/communityai; fi) >"$6" 2>&1 &' + ) + launcher = subprocess.Popen( + [ + "/bin/sh", + "-c", + command, + "communityai-update", + str(path), + item["filename"], + str(item["size_bytes"]), + item["sha256"], + str(result), + str(self.directory / "install.log"), + ], + start_new_session=True, + ) + require(launcher.wait(timeout=10) == 0, "Could not start the update installer") + while not result.exists() and not self.cancelled.wait(1): + pass + if result.exists(): + require(result.read_text().strip() == "0", "Update cancelled or installation failed. Try again.") + except Exception as exc: + self._set( + status="error", + message=str(exc) if isinstance(exc, UpdateError) else "Could not install the update. Try again.", + ) + + def close(self): + self.cancelled.set() diff --git a/desktop/tests/test_build_desktop.py b/desktop/tests/test_build_desktop.py index d2b87e591..b8d056f65 100644 --- a/desktop/tests/test_build_desktop.py +++ b/desktop/tests/test_build_desktop.py @@ -13,6 +13,7 @@ import zipfile from pathlib import Path from tempfile import TemporaryDirectory +from unittest.mock import patch from drift.catalog_release import catalog_publication_bundle_index_digest, write_catalog_publication_bundle from drift.model_catalog import CATALOG_SCHEMA_VERSION, CatalogSigningKey, ModelCatalog, SignedModelCatalog @@ -107,6 +108,19 @@ def setUp(self) -> None: self.addCleanup(self._temporary_directory.cleanup) self.tmp_path = Path(self._temporary_directory.name) + def test_build_storage_combines_same_volume_staging_and_archive_before_writing(self): + usage = shutil.disk_usage(self.tmp_path) + with patch.object(build_desktop.shutil, "disk_usage", return_value=usage._replace(free=14 * 1024**3)): + with self.assertRaisesRegex(RuntimeError, "15.0 GiB is required"): + build_desktop._check_build_storage(self.tmp_path / "output", self.tmp_path / "build") + self.assertEqual(list(self.tmp_path.iterdir()), []) + + def test_build_storage_accepts_capacity_without_creating_output_directories(self): + usage = shutil.disk_usage(self.tmp_path) + with patch.object(build_desktop.shutil, "disk_usage", return_value=usage._replace(free=16 * 1024**3)): + build_desktop._check_build_storage(self.tmp_path / "output", self.tmp_path / "build") + self.assertEqual(list(self.tmp_path.iterdir()), []) + def test_release_inputs_require_complete_verified_bundle_and_record_identity(self): bootstrap, envelope, bundle_path, index = _release_bundle(self.tmp_path) diff --git a/desktop/tests/test_client.py b/desktop/tests/test_client.py index 0e3cf4db0..1f484896d 100644 --- a/desktop/tests/test_client.py +++ b/desktop/tests/test_client.py @@ -86,6 +86,8 @@ def test_model_download_estimate_is_bounded_and_fail_closed(self): "selected_whole_shard_bytes": 4_571_197_320, } self.assertEqual(_normalize_model_download(valid), valid) + no_download = {"schema_version": 1, "selected_whole_shard_bytes": 0} + self.assertEqual(_normalize_model_download(no_download), no_download) invalid_values = ( None, @@ -94,7 +96,7 @@ def test_model_download_estimate_is_bounded_and_fail_closed(self): {"schema_version": True, "selected_whole_shard_bytes": 4_571_197_320}, {"schema_version": 1.0, "selected_whole_shard_bytes": 4_571_197_320}, {"schema_version": 1, "selected_whole_shard_bytes": True}, - {"schema_version": 1, "selected_whole_shard_bytes": 0}, + {"schema_version": 1, "selected_whole_shard_bytes": -1}, {"schema_version": 1, "selected_whole_shard_bytes": 64 * 1024**4 + 1}, {**valid, "credential": "must-not-be-accepted"}, ) @@ -329,6 +331,14 @@ def test_controller_builds_shell_neutral_view(self): self.assertEqual(snapshot["workers"][0]["state"], "paused") self.assertEqual(snapshot["keys"][0]["label"], "bootstrap") + def test_multishard_model_with_pending_estimate_remains_visible(self): + from communityai_desktop.client import _normalize_model_download + + download = _normalize_model_download({"schema_version": 1, "selected_whole_shard_bytes": None}) + model = DesktopController._model_view({"id": "Qwen3.8", "download": download}) + self.assertEqual(model["download_storage_estimate"], "Pending verified shard selection") + self.assertFalse(model["route_complete"]) + if __name__ == "__main__": unittest.main() diff --git a/desktop/tests/test_gate13_playthrough.py b/desktop/tests/test_gate13_playthrough.py new file mode 100644 index 000000000..159ead20c --- /dev/null +++ b/desktop/tests/test_gate13_playthrough.py @@ -0,0 +1,675 @@ +from __future__ import annotations + +import json +import os +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from communityai_desktop.acceptance import fake_node +from communityai_desktop.app import main +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController +from communityai_desktop.gate13_playthrough import ( + Gate13Playthrough, + PlaythroughError, + PlaythroughPlan, + _manual_route_ready, + qualify_localhost_inference, +) + +MODEL_ID = "Qwen 3 8B" +MANIFEST_DIGEST = "sha256:" + "b" * 64 + + +def _config(stage: str, platform: str = "windows") -> dict: + return { + "schema_version": 2, + "run_id": "gate13-automated-test", + "platform": platform, + "stage": stage, + "model_id": MODEL_ID, + "manifest_digest": MANIFEST_DIGEST, + "total_blocks": 36, + "policy": { + "sharing_enabled": True, + "allowed_models": [MODEL_ID], + "preferred_models": [MODEL_ID], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": None, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + }, + }, + "timeout_seconds": 30.0, + "inference_timeout_seconds": 10.0, + } + + +def _write_plan(path: Path, stage: str, platform: str = "windows") -> PlaythroughPlan: + path.write_text(json.dumps(_config(stage, platform)), encoding="utf-8") + return PlaythroughPlan.load(path) + + +def _inference(_controller, plan): # noqa: ANN001 + return { + "passed": True, + "model_id": plan.model_id, + "manifest_digest": plan.manifest_digest, + "completion_count": 1, + "generated_token_count": 1, + "response_content_retained": False, + "token_identifiers_retained": False, + "temporary_key_removed": True, + } + + +class PlaythroughPlanTests(unittest.TestCase): + def test_manual_readiness_uses_the_node_client_nested_route(self): + plan = PlaythroughPlan( + run_id="gate13-automated-test", + platform="windows", + stage="initial", + model_id=MODEL_ID, + manifest_digest=MANIFEST_DIGEST, + total_blocks=36, + policy=_config("initial")["policy"], + timeout_seconds=30, + inference_timeout_seconds=10, + ) + with fake_node() as (url, token): + status = NodeClient(url, token).status() + selected = next(model for model in status["models"] if model["id"] == MODEL_ID) + # The desktop fake node omits model identity metadata; the production + # ModelSnapshot includes it alongside the nested route object. + selected["manifest_digest"] = MANIFEST_DIGEST + + self.assertTrue(_manual_route_ready(status, plan)) + selected["route"]["covered_blocks"] = 35 + self.assertFalse(_manual_route_ready(status, plan)) + selected["route"]["covered_blocks"] = 36 + selected["manifest_digest"] = "sha256:" + "c" * 64 + self.assertFalse(_manual_route_ready(status, plan)) + + def test_plan_is_strict_and_bounded(self): + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as directory: + root = Path(directory) + plan = _write_plan(root / "plan.json", "initial") + self.assertEqual(plan.model_id, MODEL_ID) + self.assertEqual(plan.policy["allowed_models"], [MODEL_ID]) + self.assertIsNone(plan.policy["max_power_watts"]) + + invalid = _config("initial") + invalid["policy"]["denied_models"] = [MODEL_ID] + (root / "invalid.json").write_text(json.dumps(invalid), encoding="utf-8") + with self.assertRaises(PlaythroughError): + PlaythroughPlan.load(root / "invalid.json") + + invalid_power = _config("initial") + invalid_power["policy"]["max_power_watts"] = 250.0 + (root / "invalid-power.json").write_text(json.dumps(invalid_power), encoding="utf-8") + with self.assertRaises(PlaythroughError): + PlaythroughPlan.load(root / "invalid-power.json") + + (root / "duplicate.json").write_text('{"schema_version":2,"schema_version":2}', encoding="utf-8") + with self.assertRaises(PlaythroughError): + PlaythroughPlan.load(root / "duplicate.json") + + def test_localhost_inference_restores_key_baseline_and_retains_only_counts(self): + plan = PlaythroughPlan( + run_id="gate13-automated-test", + platform="windows", + stage="initial", + model_id=MODEL_ID, + manifest_digest=MANIFEST_DIGEST, + total_blocks=36, + policy=_config("initial")["policy"], + timeout_seconds=30, + inference_timeout_seconds=10, + ) + + class Client: + def __init__(self): + self.active = { + "baseline": { + "id": "baseline", + "label": "baseline", + "revoked_at": None, + } + } + + def list_keys(self): + return list(self.active.values()) + + def status(self): + return { + "openai_base_url": "http://127.0.0.1:8080/v1", + "auto_selection": { + "status": "selected", + "model": MODEL_ID, + "manifest_digest": MANIFEST_DIGEST, + "covered_blocks": plan.total_blocks, + "total_blocks": plan.total_blocks, + "peer_count": 1, + }, + "models": [ + { + "id": MODEL_ID, + "manifest_digest": MANIFEST_DIGEST, + "route": { + "status": "complete", + "covered_blocks": plan.total_blocks, + "total_blocks": plan.total_blocks, + "peer_count": 1, + }, + } + ], + } + + def create_key(self, label): + self.active["temporary"] = {"id": "temporary", "label": label, "revoked_at": None} + return {"key": self.active["temporary"], "secret": "temporary-secret"} + + def revoke_key(self, key_id): + self.active[key_id]["revoked_at"] = 1 + return {"key": self.active[key_id]} + + client = Client() + completion = { + "model": MODEL_ID, + # The manual Gate 13 command deliberately retained no generated + # content. A one-token response is qualified by identity and the + # server-reported token count, not by decoded visible text. + "choices": [{"message": {"role": "assistant", "content": ""}}], + "usage": {"completion_tokens": 1}, + } + with patch("communityai_desktop.gate13_playthrough._completion_request", return_value=completion): + result = qualify_localhost_inference(SimpleNamespace(client=client), plan) + + self.assertEqual(result["completion_count"], 1) + self.assertFalse(result["response_content_retained"]) + self.assertEqual({item["id"] for item in client.list_keys() if item["revoked_at"] is None}, {"baseline"}) + + unavailable_then_ready = MagicMock(side_effect=[PlaythroughError("localhost inference failed"), completion]) + with ( + patch( + "communityai_desktop.gate13_playthrough._completion_request", + unavailable_then_ready, + ), + patch("communityai_desktop.gate13_playthrough.time.sleep") as readiness_sleep, + ): + retried = qualify_localhost_inference(SimpleNamespace(client=client), plan) + + self.assertTrue(retried["passed"]) + self.assertEqual(unavailable_then_ready.call_count, 2) + readiness_sleep.assert_called_once_with(5.0) + self.assertEqual({item["id"] for item in client.list_keys() if item["revoked_at"] is None}, {"baseline"}) + + with patch( + "communityai_desktop.gate13_playthrough._completion_after_manual_readiness_wait", + side_effect=PlaythroughError("inference_http_503"), + ): + with self.assertRaisesRegex(PlaythroughError, "^inference_http_503$"): + qualify_localhost_inference(SimpleNamespace(client=client), plan) + self.assertEqual({item["id"] for item in client.list_keys() if item["revoked_at"] is None}, {"baseline"}) + + unavailable_then_timeout = MagicMock( + side_effect=[PlaythroughError("inference_http_503"), PlaythroughError("inference_timed_out")] + ) + with ( + patch("communityai_desktop.gate13_playthrough._completion_request", unavailable_then_timeout), + patch("communityai_desktop.gate13_playthrough.time.sleep"), + ): + with self.assertRaisesRegex(PlaythroughError, "^inference_timed_out$"): + qualify_localhost_inference(SimpleNamespace(client=client), plan) + self.assertEqual(unavailable_then_timeout.call_count, 2) + self.assertEqual({item["id"] for item in client.list_keys() if item["revoked_at"] is None}, {"baseline"}) + + def test_inference_http_failure_retains_status_without_response_or_secret(self): + from urllib.error import HTTPError + + from communityai_desktop.gate13_playthrough import _completion_request + + opener = MagicMock() + opener.open.side_effect = HTTPError( + "http://127.0.0.1:8080/v1/chat/completions", 503, "private response detail", None, None + ) + with patch("communityai_desktop.gate13_playthrough.build_opener", return_value=opener): + with self.assertRaisesRegex(PlaythroughError, "^inference_http_503$"): + _completion_request("http://127.0.0.1:8080/v1/chat/completions", "private-key", 10) + + def test_localhost_inference_requests_exactly_one_token(self): + from communityai_desktop.gate13_playthrough import _completion_request + + response = MagicMock() + response.__enter__.return_value = response + response.status = 200 + response.headers.get_content_type.return_value = "application/json" + response.read.return_value = b'{"result":"bounded"}' + opener = MagicMock() + opener.open.return_value = response + + with patch("communityai_desktop.gate13_playthrough.build_opener", return_value=opener): + result = _completion_request("http://127.0.0.1:8080/v1/chat/completions", "secret", 10) + + self.assertEqual(result, {"result": "bounded"}) + request = opener.open.call_args.args[0] + self.assertEqual( + json.loads(request.data), + { + "model": "auto", + "messages": [{"role": "user", "content": "Reply with one word."}], + "max_tokens": 1, + "stream": False, + }, + ) + + def test_hidden_packaged_cli_installs_the_qualification_automation(self): + lifecycle = SimpleNamespace(close=lambda: None) + loaded_plan = SimpleNamespace(stage="initial") + automation = SimpleNamespace() + with ( + patch("communityai_desktop.app.NodeLifecycleSupervisor", return_value=lifecycle), + patch("communityai_desktop.gate13_playthrough.PlaythroughPlan.load", return_value=loaded_plan), + patch("communityai_desktop.gate13_playthrough.Gate13Playthrough", return_value=automation), + patch("communityai_desktop.pyside_shell.run", return_value=0) as run, + ): + result = main( + [ + "--gate13-ui-playthrough", + "plan.json", + "--gate13-ui-evidence", + "evidence.json", + ] + ) + + self.assertEqual(result, 0) + self.assertIs(run.call_args.kwargs["qualification_automation"], automation) + + with self.assertRaises(SystemExit): + main(["--self-test", "--gate13-ui-evidence", "orphan.json"]) + + +class PackagedUiPlaythroughTests(unittest.TestCase): + def test_wait_ready_logs_only_changed_bounded_bootstrap_errors_before_failure(self): + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as directory: + root = Path(directory) + evidence_path = root / "failure.json" + automation = Gate13Playthrough(_write_plan(root / "plan.json", "initial"), evidence_path) + messages = [ + "The signed model catalog could not be installed: drift bootstrap: error: " + "Another first-install catalog bootstrap is already in progress" + ] + automation._window = SimpleNamespace( + _controller=None, + _busy=False, + connection_detail=SimpleNamespace(text=lambda: messages[0]), + ) + log_path = evidence_path.with_suffix(".log") + for _ in range(5): + automation._tick() + logged = log_path.read_text(encoding="utf-8") + self.assertEqual(logged.count("Another first-install"), 1) + self.assertEqual(len(logged.splitlines()), 2) # phase, then error + self.assertFalse(automation._done) + self.assertFalse(evidence_path.exists()) + messages[0] = "The signed model catalog installation timed out after 300 seconds: " + "x" * 800 + automation._tick() + automation._tick() + logged = log_path.read_text(encoding="utf-8") + self.assertEqual(len(logged.splitlines()), 3) + retained_message = logged.splitlines()[-1].split("phase=wait_ready ", 1)[1] + self.assertEqual(retained_message, messages[0][:600]) + automation._timeout() + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + self.assertEqual(evidence["failure_code"], "playthrough_timed_out") + self.assertEqual(evidence["failure_phase"], "wait_ready") + self.assertEqual(evidence["failure_detail"], "bootstrap_failed") + self.assertNotIn("signed model catalog", json.dumps(evidence)) + + def test_progress_log_stays_bounded_and_ignores_unrelated_ui_text(self): + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as directory: + root = Path(directory) + automation = Gate13Playthrough(_write_plan(root / "plan.json", "initial"), root / "evidence.json") + automation._window = SimpleNamespace( + _controller=None, + _busy=False, + connection_detail=SimpleNamespace(text=lambda: "private unrelated UI content"), + ) + automation._tick() + for _ in range(100): + automation._log_progress("x" * 800) + logged = (root / "evidence.log").read_bytes() + self.assertLessEqual(len(logged), 16_384) + self.assertNotIn(b"private", logged) + + def test_failure_evidence_keeps_phase_and_only_controlled_inference_detail(self): + from tempfile import TemporaryDirectory + + def submit(operation, _finished, failed): + try: + operation() + except Exception as exc: + failed(str(exc)) + + with TemporaryDirectory() as directory: + root = Path(directory) + for index, (error_message, expected_detail) in enumerate( + (("inference_http_503", "inference_http_503"), ("private-key response content", "inference_failed")) + ): + with self.subTest(error_message=error_message): + evidence_path = root / f"failure-{index}.json" + automation = Gate13Playthrough( + _write_plan(root / f"plan-{index}.json", "initial"), + evidence_path, + inference_runner=MagicMock(side_effect=PlaythroughError(error_message)), + ) + automation._window = SimpleNamespace(_controller=object(), _submit=submit) + automation._begin_inference("after_initial_inference") + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + self.assertEqual(evidence["failure_phase"], "after_initial_inference") + self.assertEqual(evidence["failure_code"], "inference_failed") + self.assertEqual(evidence["failure_detail"], expected_detail) + self.assertNotIn("private-key", json.dumps(evidence)) + + evidence_path = root / "timeout.json" + automation = Gate13Playthrough(_write_plan(root / "timeout-plan.json", "initial"), evidence_path) + automation._timeout() + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + self.assertEqual(evidence["failure_phase"], "wait_ready") + self.assertEqual(evidence["failure_code"], "playthrough_timed_out") + + def test_policy_auto_start_is_normalized_through_model_toggle_before_master_start(self): + from tempfile import TemporaryDirectory + + class PageButton: + def __init__(self, label): + self.label = label + self.enabled = True + self.checked = label == "Home" + + def text(self): + return self.label + + def isEnabled(self): + return self.enabled + + def isChecked(self): + return self.checked + + def click(self): + self.checked = True + + class Window: + def __init__(self, policy): + self._busy = 0 + self._page_buttons = [PageButton(label) for label in ("Home", "Models", "Sharing", "API access")] + self._snapshot = { + "contribution": {"intent_enabled": True, "enabled": True, "policy": policy}, + "workers": [{"model": MODEL_ID, "desired_running": True}], + } + self.master_share_button = MasterButton(self) + self.model_toggle = ModelToggle(self) + + def findChildren(self, _kind): + return [self.model_toggle] + + class ModelToggle: + def __init__(self, window): + self.window = window + self.checked = True + self.clicks = 0 + + def accessibleName(self): + return f"Share compute with {MODEL_ID}" + + def isChecked(self): + return self.checked + + def isEnabled(self): + return True + + def click(self): + self.clicks += 1 + self.checked = False + self.window._snapshot["contribution"]["intent_enabled"] = False + self.window._snapshot["contribution"]["enabled"] = False + self.window._snapshot["workers"][0]["desired_running"] = False + self.window.master_share_button.label = "Start sharing" + + class MasterButton: + def __init__(self, window): + self.window = window + self.label = "Pause sharing" + self.clicks = [] + + def text(self): + return self.label + + def isEnabled(self): + return True + + def click(self): + if not self.window._page_buttons[2].isChecked(): + raise AssertionError("sharing action was invoked off the Sharing page") + self.clicks.append(self.label) + enabled = self.label == "Start sharing" + self.window._snapshot["contribution"]["intent_enabled"] = enabled + self.window._snapshot["contribution"]["enabled"] = enabled + self.window._snapshot["workers"][0]["desired_running"] = enabled + self.label = "Pause sharing" if enabled else "Start sharing" + + with TemporaryDirectory() as directory: + root = Path(directory) + now = [10.0] + plan = _write_plan(root / "windows-restart-plan.json", "restart", "windows") + window = Window(plan.policy) + application = SimpleNamespace(quit=MagicMock()) + automation = Gate13Playthrough( + plan, + root / "windows-restart-evidence.json", + clock=lambda: now[0], + start_observation_seconds=0.05, + ) + automation._application = application + automation._window = window + automation._qt = {"QCheckBox": object} + automation._state = "wait_policy" + + automation._tick() + self.assertEqual(automation._state, "wait_prestart_paused") + self.assertEqual(window.model_toggle.clicks, 1) + self.assertEqual(window.master_share_button.clicks, []) + + automation._tick() + self.assertEqual(window.master_share_button.clicks, ["Start sharing"]) + automation._tick() + now[0] += 0.1 + automation._tick() + self.assertEqual(window.master_share_button.clicks, ["Start sharing", "Pause sharing"]) + automation._tick() + + evidence = json.loads((root / "windows-restart-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(evidence["result"], "passed") + self.assertTrue(evidence["ui"]["start_clicked"]) + self.assertTrue(evidence["ui"]["pause_clicked"]) + application.quit.assert_called_once() + + def test_bandwidth_suspension_and_async_worker_exit_do_not_reintroduce_manual_false_failure(self): + from tempfile import TemporaryDirectory + + class Button: + def __init__(self, navigation): + self.label = "Pause sharing" + self.enabled = True + self.clicks = 0 + self.navigation = navigation + + def text(self): + return self.label + + def isEnabled(self): + return self.enabled + + def click(self): + if not self.navigation.isChecked(): + raise AssertionError("sharing action was invoked off the Sharing page") + self.clicks += 1 + + class PageButton: + def __init__(self, label): + self.label = label + self.enabled = True + self.checked = label == "Home" + self.clicks = 0 + + def text(self): + return self.label + + def isEnabled(self): + return self.enabled + + def isChecked(self): + return self.checked + + def click(self): + self.clicks += 1 + self.checked = True + + with TemporaryDirectory() as directory: + root = Path(directory) + now = [10.0] + pages = [PageButton(label) for label in ("Home", "Models", "Sharing", "API access")] + button = Button(pages[2]) + application = SimpleNamespace(quit=MagicMock()) + automation = Gate13Playthrough( + _write_plan(root / "windows-restart-plan.json", "restart", "windows"), + root / "windows-restart-evidence.json", + clock=lambda: now[0], + start_observation_seconds=0.05, + restart_observation_seconds=0.05, + ) + automation._application = application + automation._window = SimpleNamespace( + _busy=0, + _page_buttons=pages, + master_share_button=button, + _snapshot={ + "contribution": {"intent_enabled": True, "enabled": False}, + "workers": [ + { + "model": MODEL_ID, + "desired_running": True, + "sharing_active": False, + "resource_admitted": False, + "resource_reason": "bandwidth usage exceeds contribution budget", + } + ], + }, + ) + automation._state = "wait_started_intent" + + automation._tick() + now[0] += 0.1 + automation._tick() + + self.assertEqual(button.clicks, 1) + self.assertEqual(pages[2].clicks, 1) + self.assertEqual(automation._state, "wait_paused_intent") + button.label = "Start sharing" + button.enabled = False + automation._window._snapshot = { + "contribution": {"intent_enabled": False, "enabled": False}, + # The manual trace still counted workers after Pause. Their + # asynchronous exit is deliberately not this UI gate's boundary. + "workers": [{"model": MODEL_ID, "desired_running": False, "sharing_active": True}], + } + automation._tick() + + evidence = json.loads((root / "windows-restart-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(evidence["result"], "passed") + self.assertTrue(evidence["ui"]["sharing_intent_disabled_observed"]) + application.quit.assert_called_once() + + def test_real_window_replays_manual_platform_sequences(self): + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + try: + from PySide6.QtWidgets import QApplication + except ModuleNotFoundError as exc: + self.skipTest(f"PySide6 is unavailable: {exc}") + + from tempfile import TemporaryDirectory + + from communityai_desktop.pyside_shell import run + + QApplication.instance() or QApplication([]) + with TemporaryDirectory() as directory: + root = Path(directory) + for platform in ("windows", "linux"): + with self.subTest(platform=platform), fake_node(all_workers_paused=True) as (url, token): + controller = DesktopController(NodeClient(url, token)) + initial = Gate13Playthrough( + _write_plan(root / f"{platform}-initial-plan.json", "initial", platform), + root / f"{platform}-initial-evidence.json", + inference_runner=_inference, + start_observation_seconds=0.05, + restart_observation_seconds=0.05, + ) + with patch("communityai_desktop.pyside_shell.login_startup_enabled", return_value=False): + self.assertEqual( + run(controller, single_instance=False, qualification_automation=initial), + 0, + ) + initial_evidence = json.loads( + (root / f"{platform}-initial-evidence.json").read_text(encoding="utf-8") + ) + self.assertEqual(initial_evidence["result"], "passed") + self.assertEqual(initial_evidence["platform"], platform) + self.assertEqual(initial_evidence["ui"]["policy_dialog_saved"], platform == "linux") + self.assertEqual(initial_evidence["ui"]["start_clicked"], platform == "linux") + + restart = Gate13Playthrough( + _write_plan(root / f"{platform}-restart-plan.json", "restart", platform), + root / f"{platform}-restart-evidence.json", + inference_runner=_inference, + start_observation_seconds=0.05, + restart_observation_seconds=0.05, + ) + with patch("communityai_desktop.pyside_shell.login_startup_enabled", return_value=False): + self.assertEqual( + run(controller, single_instance=False, qualification_automation=restart), + 0, + ) + restart_evidence = json.loads( + (root / f"{platform}-restart-evidence.json").read_text(encoding="utf-8") + ) + self.assertEqual(restart_evidence["result"], "passed") + self.assertTrue(restart_evidence["ui"]["pause_control_observed"]) + self.assertTrue(restart_evidence["ui"]["pause_clicked"]) + self.assertTrue(restart_evidence["ui"]["sharing_intent_disabled_observed"]) + self.assertEqual( + restart_evidence["ui"]["restart_resume_observed"], + platform == "linux", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_installers.py b/desktop/tests/test_installers.py new file mode 100644 index 000000000..5e86fb456 --- /dev/null +++ b/desktop/tests/test_installers.py @@ -0,0 +1,137 @@ +import importlib.util +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + + +@unittest.skipUnless(sys.platform.startswith("linux"), "Linux installer process ownership") +class LinuxInstallerTests(unittest.TestCase): + @staticmethod + def maintenance(): + path = Path(__file__).resolve().parents[1] / "installers/linux_maintenance.py" + spec = importlib.util.spec_from_file_location("linux_installer_maintenance", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + @unittest.skipUnless(hasattr(os, "geteuid") and os.geteuid() == 0, "dpkg maintenance runs as root") + def test_stop_owned_tree_preserves_unrelated_process_and_user_data(self): + maintenance = self.maintenance() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + installation = root / "installation" + installation.mkdir() + (installation / ".communityai-installation").write_text("CommunityAI installer-managed fixture\n") + executable = installation / "CommunityAI" + shutil.copy2("/bin/sh", executable) + child_pid = root / "child-pid" + state = root / "user-cache" + state.write_bytes(b"retained verified data") + process = subprocess.Popen( + [str(executable), "-c", 'sleep 60 & echo $! > "$1"; wait', "probe", str(child_pid)] + ) + unrelated = subprocess.Popen(["sleep", "60"]) + child_descriptor = None + try: + deadline = time.monotonic() + 5 + while not child_pid.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(child_pid.exists()) + descendant = int(child_pid.read_text()) + child_descriptor = os.pidfd_open(descendant) + maintenance.stop_installation(installation, timeout=2) + self.assertIsNotNone(process.poll()) + self.assertNotIn(descendant, maintenance.process_snapshot()) + self.assertIsNone(unrelated.poll()) + self.assertEqual(state.read_bytes(), b"retained verified data") + finally: + if child_descriptor is not None: + try: + signal.pidfd_send_signal(child_descriptor, signal.SIGKILL) + except ProcessLookupError: + pass + finally: + os.close(child_descriptor) + for candidate in (process, unrelated): + if candidate.poll() is None: + candidate.kill() + candidate.wait(timeout=5) + + def test_unmarked_installation_is_not_touched(self): + maintenance = self.maintenance() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + preserved = root / "unrelated.txt" + preserved.write_text("preserved") + with self.assertRaises(RuntimeError): + maintenance.stop_installation(root) + self.assertEqual(preserved.read_text(), "preserved") + + @unittest.skipUnless(hasattr(os, "geteuid") and os.geteuid() == 0, "dpkg maintenance runs as root") + def test_helper_started_during_shutdown_is_stopped_after_parent_exits(self): + maintenance = self.maintenance() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / ".communityai-installation").write_text("CommunityAI installer-managed fixture\n") + executable, helper = root / "CommunityAI", root / "helper" + shutil.copy2("/bin/sh", executable) + shutil.copy2("/bin/sleep", helper) + child_pid, ready = root / "child-pid", root / "ready" + process = subprocess.Popen( + [ + str(executable), + "-c", + 'trap \'"$1" 60 & echo $! > "$2"; exit 0\' TERM; echo ready > "$3"; while :; do sleep 0.05; done', + "probe", + str(helper), + str(child_pid), + str(ready), + ] + ) + try: + deadline = time.monotonic() + 5 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(ready.exists()) + maintenance.stop_installation(root, timeout=2) + process.wait(timeout=5) + self.assertTrue(child_pid.exists()) + self.assertNotIn(int(child_pid.read_text()), maintenance.process_snapshot()) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + if child_pid.exists(): + pid = int(child_pid.read_text()) + snapshot = maintenance.process_snapshot() + if pid in snapshot and snapshot[pid][2] == helper: + descriptor = os.pidfd_open(pid) + try: + if maintenance.process_snapshot().get(pid) == snapshot[pid]: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + except ProcessLookupError: + pass + finally: + os.close(descriptor) + + def test_inaccessible_process_refuses_replacement_without_sending_signals(self): + maintenance = self.maintenance() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / ".communityai-installation").write_text("CommunityAI installer-managed fixture\n") + with patch.object(maintenance.os, "readlink", side_effect=PermissionError("restricted procfs")): + with patch.object(maintenance.signal, "pidfd_send_signal") as send: + with self.assertRaisesRegex(RuntimeError, "Cannot inspect process ownership"): + maintenance.stop_installation(root) + send.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_lifecycle.py b/desktop/tests/test_lifecycle.py index 09db4001f..48f518f13 100644 --- a/desktop/tests/test_lifecycle.py +++ b/desktop/tests/test_lifecycle.py @@ -94,6 +94,18 @@ def __call__(self, node_url, timeout): class NodeLifecycleTests(unittest.TestCase): + def test_failed_owned_shutdown_prevents_installation_acknowledgement(self): + with TemporaryDirectory() as directory: + supervisor = self._supervisor(directory, FakeStore()) + process = mock.Mock() + process.poll.return_value = None + process.wait.side_effect = lifecycle.subprocess.TimeoutExpired("owned node", 1) + supervisor._process = process + with self.assertRaises(NodeLifecycleError): + supervisor.close() + self.assertIs(supervisor._process, process) + process.kill.assert_called_once() + def test_frozen_desktop_resolves_nested_node_sidecar(self): executable = Path.cwd() / "product" / ("CommunityAI.exe" if lifecycle.os.name == "nt" else "CommunityAI") suffix = ".exe" if lifecycle.os.name == "nt" else "" @@ -306,6 +318,8 @@ def process_factory(command, **kwargs): supervisor.close() self.assertEqual(len(bootstrap_calls), 1) + self.assertEqual(bootstrap_calls[0][1]["timeout"], 300.0) + self.assertEqual(supervisor.startup_timeout, 45.0) bootstrap_command = bootstrap_calls[0][0] self.assertEqual(bootstrap_command[:2], ("python", "fake-bootstrap.py")) self.assertEqual(bootstrap_command[2], os.path.abspath(bootstrap_path)) @@ -341,6 +355,46 @@ def test_failed_catalog_bootstrap_does_not_start_the_node(self): self.assertEqual(processes, []) + def test_catalog_bootstrap_timeout_preserves_bounded_output_without_starting_node(self): + cases = ( + (b"ignored stdout", b"older line\nbootstrap: HTTPS fetch stalled\n", "bootstrap: HTTPS fetch stalled"), + ("bootstrap stdout detail\n", b"", "bootstrap stdout detail"), + (None, b"x" * 700, "x" * 600), + (None, None, ""), + ) + for stdout, stderr, expected_detail in cases: + with self.subTest(expected_detail=expected_detail), TemporaryDirectory() as directory: + root = Path(directory) + bootstrap_path = root / "catalog-bootstrap.json" + bootstrap_path.write_text("{}\n", encoding="utf-8") + processes = [] + runner = mock.Mock( + side_effect=lifecycle.subprocess.TimeoutExpired("bootstrap", 300, output=stdout, stderr=stderr) + ) + supervisor = NodeLifecycleSupervisor( + "http://127.0.0.1:8080", + FakeStore(), + config_path=root / "node-config.json", + data_dir=root / "data", + node_command=("python", "fake-node.py"), + bootstrap_command=("python", "fake-bootstrap.py"), + bootstrap_config_path=bootstrap_path, + bootstrap_runner=runner, + process_factory=lambda *args, **kwargs: processes.append((args, kwargs)), + client_factory=FakeClient, + port_probe=PortSequence(False), + ) + with self.assertRaises(NodeLifecycleError) as caught: + supervisor.ensure_client() + supervisor.close() + + expected = "The signed model catalog installation timed out after 300 seconds" + if expected_detail: + expected += f": {expected_detail}" + self.assertEqual(str(caught.exception), expected) + self.assertEqual(runner.call_args.kwargs["timeout"], 300.0) + self.assertEqual(processes, []) + if __name__ == "__main__": unittest.main() diff --git a/desktop/tests/test_login_startup_ui.py b/desktop/tests/test_login_startup_ui.py new file mode 100644 index 000000000..930b5d52f --- /dev/null +++ b/desktop/tests/test_login_startup_ui.py @@ -0,0 +1,66 @@ +import importlib.util +import os +import unittest +from pathlib import Path + +os.environ["QT_QPA_PLATFORM"] = "offscreen" +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "qualify_login_startup_source.py" +spec = importlib.util.spec_from_file_location("qualify_login_startup_source", SCRIPT) +replay = importlib.util.module_from_spec(spec) +spec.loader.exec_module(replay) + +from communityai_desktop.startup import LoginStartupError + + +class LoginStartupUiTests(unittest.TestCase): + def test_literal_checkbox_reflects_saved_state_after_window_recreation(self): + state = {"enabled": False, "writes": []} + + def save(enabled): + state["writes"].append(enabled) + state["enabled"] = enabled + + first = replay.checkbox_session(lambda: state["enabled"], save, click=True) + second = replay.checkbox_session(lambda: state["enabled"], save, click=True) + self.assertFalse(first["initial_checked"]) + self.assertTrue(first["final_checked"]) + self.assertTrue(second["initial_checked"]) + self.assertFalse(second["final_checked"]) + self.assertEqual(second["initial_detail"], "") + self.assertEqual(first["final_detail"], "CommunityAI will open when you sign in.") + self.assertEqual(second["final_detail"], "Automatic opening is off.") + self.assertEqual(state["writes"], [True, False]) + self.assertEqual(first["qt_platform"], "offscreen") + self.assertEqual(second["qt_platform"], "offscreen") + self.assertTrue(first["checkbox_visible"]) + self.assertTrue(second["checkbox_visible"]) + self.assertTrue(first["settings_initially_collapsed"]) + self.assertTrue(second["settings_initially_collapsed"]) + self.assertTrue(first["final_detail_visible"]) + self.assertTrue(second["final_detail_visible"]) + + def test_failed_native_write_reverts_checkbox_and_reports_failure(self): + def denied(enabled): + raise LoginStartupError("test registry access denied") + + result = replay.checkbox_session(lambda: False, denied, click=True) + self.assertFalse(result["initial_checked"]) + self.assertFalse(result["final_checked"]) + self.assertEqual(result["warning_count"], 1) + self.assertEqual(result["final_detail"], "Could not save this setting. Try again.") + self.assertTrue(result["checkbox_visible"]) + self.assertTrue(result["final_detail_visible"]) + + def test_unreadable_startup_registration_disables_the_checkbox(self): + def denied(): + raise LoginStartupError("test registration unreadable") + + def never_write(enabled): + raise AssertionError("A disabled control must not write") + + result = replay.checkbox_session(denied, never_write) + self.assertFalse(result["initial_enabled"]) + self.assertFalse(result["final_checked"]) + self.assertEqual(result["initial_detail"], "Sign-in settings could not be read.") + self.assertEqual(result["warning_count"], 0) + self.assertTrue(result["checkbox_visible"]) diff --git a/desktop/tests/test_maintenance.py b/desktop/tests/test_maintenance.py new file mode 100644 index 000000000..b9f56389d --- /dev/null +++ b/desktop/tests/test_maintenance.py @@ -0,0 +1,83 @@ +import os +import subprocess +import sys +import tempfile +import time +import unittest +import uuid +from pathlib import Path +from unittest import mock + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +class MaintenanceTests(unittest.TestCase): + def test_broken_installed_runtime_returns_failure_without_traceback_dialog(self): + from communityai_desktop.app import main + + with mock.patch("communityai_desktop.maintenance.prepare_update", side_effect=ImportError("Qt unavailable")): + with self.assertRaises(SystemExit) as caught: + main(["--prepare-update"]) + self.assertEqual(caught.exception.code, 2) + + def test_shutdown_acknowledges_only_after_owned_cleanup(self): + name = "communityai-maintenance-test-" + uuid.uuid4().hex + + def request_shutdown(): + return subprocess.run( + [ + sys.executable, + "-c", + "import sys; from communityai_desktop.maintenance import prepare_update; " + "sys.exit(prepare_update(instance_name=sys.argv[1], timeout=10))", + name, + ], + capture_output=True, + timeout=15, + ).returncode + + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "stopped.txt" + ready = Path(directory) / "ready.txt" + script = Path(directory) / "desktop.py" + script.write_text( + """ +import sys +from pathlib import Path +from PySide6.QtCore import QTimer +from communityai_desktop.acceptance import fake_node +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController +from communityai_desktop.pyside_shell import run + +class Automation: + def install(self, window, application, types): + QTimer.singleShot(100, lambda: Path(sys.argv[2]).write_text('ready')) + +with fake_node() as (url, token): + run(DesktopController(NodeClient(url, token)), instance_name=sys.argv[1], + auto_close_seconds=20, qualification_automation=Automation(), + before_termination_restore=lambda: Path(sys.argv[3]).write_text('owned cleanup complete')) +""", + encoding="utf-8", + ) + process = subprocess.Popen( + [sys.executable, str(script), name, str(ready), str(marker)], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + try: + deadline = time.monotonic() + 15 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + self.assertTrue(ready.exists()) + self.assertEqual(request_shutdown(), 0) + self.assertEqual(marker.read_text(), "owned cleanup complete") + self.assertEqual(process.wait(timeout=10), 0) + self.assertEqual(request_shutdown(), 0) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + process.stderr.close() diff --git a/desktop/tests/test_model_health.py b/desktop/tests/test_model_health.py new file mode 100644 index 000000000..0729e3982 --- /dev/null +++ b/desktop/tests/test_model_health.py @@ -0,0 +1,202 @@ +import os +import time +import unittest + +os.environ["QT_QPA_PLATFORM"] = "offscreen" + +from communityai_desktop.model_health import DownloadCard, ModelHealthCard +from communityai_desktop.telemetry import download_view, route_view +from PySide6.QtCore import Qt +from PySide6.QtTest import QTest +from PySide6.QtWidgets import QApplication + + +class ModelHealthTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.application = QApplication.instance() or QApplication([]) + + def test_grid_separates_serving_joining_reservations_and_unknown(self): + route = { + "total_blocks": 5, + "status": "incomplete", + "last_updated_age": 1, + "replica_counts": [2, 1, 0, 0, 0], + "joining_counts": [0, 0, 1, 0, 0], + "offline_counts": [0, 0, 0, 0, 1], + "reservations": [{"peer_id": "reserved", "start_block": 3, "end_block": 4, "expires_at": time.time() + 60}], + "peers": [ + {"peer_id": "one", "public_name": "literal name", "online_blocks": [0, 1]}, + {"peer_id": "two", "online_blocks": [0], "joining_blocks": [2]}, + ], + } + model = { + "id": "Test model", + "execution": "distributed", + "coverage": "2/5", + "peer_count": 2, + "state": "known", + "health": route_view(route), + } + widget = ModelHealthCard() + widget.set_state(model) + widget.resize(900, 600) + widget.show() + self.application.processEvents() + self.assertFalse(widget.expand_button.isChecked()) + self.assertFalse(widget.details.isVisible()) + self.assertFalse(widget.cells[0].isVisible()) + self.assertTrue(widget.title.isVisible()) + self.assertEqual(widget.title.text(), "Test model") + self.assertEqual(widget.summary.text(), "Waiting for contributors · 2/5 blocks available") + self.assertLess(widget.sizeHint().height(), 150) + QTest.mouseClick(widget.expand_button, Qt.LeftButton) + self.application.processEvents() + self.assertTrue(widget.details.isVisible()) + self.assertTrue(widget.cells[0].isVisible()) + self.assertFalse(widget.download.isVisible()) + for index, state in enumerate(("Replicated", "Covered", "Joining", "Reserved", "Offline")): + self.assertIn(state, widget.cells[index].toolTip()) + QTest.mouseClick(widget.cells[3], Qt.LeftButton) + self.assertIn("1 reservations", widget.block_detail.text()) + QTest.mouseClick(widget.peer_button, Qt.LeftButton) + self.assertEqual(widget.peer_table.rowCount(), 3) + self.assertEqual(widget.peer_table.columnCount(), 3) + self.assertTrue(widget.peer_table.isVisible()) + self.assertEqual(widget.peer_table.item(0, 0).text(), "literal name") + model["health"]["status"] = "unknown" + widget.set_state(model) + self.assertIn("Unknown", widget.cells[0].toolTip()) + self.assertEqual(widget.summary.text(), "Checking availability") + self.assertTrue(widget.expand_button.isChecked()) + self.assertTrue(widget.details.isVisible()) + self.assertTrue(widget.peer_button.isChecked()) + self.assertEqual(widget.selected, 3) + QTest.mouseClick(widget.expand_button, Qt.LeftButton) + self.assertFalse(widget.peer_table.isVisible()) + widget.set_state(model) + self.assertFalse(widget.expand_button.isChecked()) + self.assertFalse(widget.details.isVisible()) + widget.expand_button.setFocus() + QTest.keyClick(widget.expand_button, Qt.Key_Space) + self.assertTrue(widget.details.isVisible()) + self.assertTrue(widget.peer_table.isVisible()) + self.assertEqual(widget.selected, 3) + widget.close() + + def test_download_bytes_do_not_imply_verification_and_stale_speed_is_zero(self): + progress = download_view( + { + "schema_version": 1, + "state": "verifying", + "artifact": "weights", + "artifact_bytes": 100, + "artifact_received_bytes": 100, + "verified_bytes": 0, + "received_bytes": 100, + "verified_files": 0, + "selected_files": 2, + "resumed_bytes": 50, + "retries": 2, + "updated_at": time.time() - 20, + "bytes_per_second": 100, + } + ) + self.assertEqual(progress["bytes_per_second"], 0) + widget = DownloadCard() + widget.set_state("Test", progress) + self.assertEqual(widget.bar.value(), 1000) + self.assertIn("Verifying", widget.title.text()) + self.assertEqual(widget.detail.text(), "100 B / 100 B") + self.assertEqual(widget.totals.text(), "0 of 2 files checked") + self.assertIn("0 B verified", widget.toolTip()) + self.assertIn("2 retries", widget.toolTip()) + widget.close() + + def test_local_model_hides_network_details_and_keeps_download_inside_disclosure(self): + model = { + "id": "Qwen3.5-0.8B-Local", + "execution": "local", + "coverage": "0/0", + "state": "known", + "health": route_view({}), + } + widget = ModelHealthCard() + widget.set_state(model) + widget.show() + self.application.processEvents() + self.assertEqual(widget.title.text(), "Qwen3.5 0.8B") + self.assertEqual(model["id"], "Qwen3.5-0.8B-Local") + self.assertEqual(widget.summary.text(), "On this computer · Downloads when needed") + QTest.mouseClick(widget.expand_button, Qt.LeftButton) + self.assertFalse(widget.legend.isVisible()) + self.assertFalse(widget.block_detail.isVisible()) + self.assertFalse(widget.peer_button.isVisible()) + self.assertFalse(widget.peer_note.isVisible()) + self.assertFalse(widget.download.isVisible()) + model["download_progress"] = download_view( + { + "schema_version": 1, + "state": "downloading", + "artifact_bytes": 1000, + "artifact_received_bytes": 250, + "bytes_per_second": 50, + } + ) + widget.set_state(model) + self.assertTrue(widget.expand_button.isChecked()) + self.assertTrue(widget.download.isVisible()) + self.assertEqual(widget.download.bar.value(), 250) + self.assertEqual(widget.summary.text(), "On this computer · Downloading") + self.assertIn("250 B / 1000 B", widget.download.detail.text()) + QTest.mouseClick(widget.expand_button, Qt.LeftButton) + model["download_progress"]["artifact_received_bytes"] = 500 + widget.set_state(model) + self.assertFalse(widget.download.isVisible()) + self.assertEqual(widget.download.bar.value(), 500) + widget.close() + + def test_malformed_optional_peer_data_is_ignored(self): + view = route_view({"total_blocks": 64, "peers": True, "reservations": 42}) + self.assertEqual(view["peers"], []) + self.assertEqual(view["reservations"], []) + + def test_sharing_download_is_available_inside_model_details(self): + model = { + "id": "Community model", + "execution": "distributed", + "coverage": "0/1", + "state": "known", + "health": route_view({"total_blocks": 1, "status": "incomplete"}), + } + worker = { + "id": "one", + "model": model["id"], + "state": "running", + "display_status": "Loading", + "download_progress": download_view( + { + "schema_version": 1, + "state": "downloading", + "artifact_bytes": 1000, + "artifact_received_bytes": 250, + } + ), + } + widget = ModelHealthCard() + widget.set_state(model, [worker]) + widget.show() + self.application.processEvents() + download = widget.worker_downloads["one"] + self.assertFalse(download.isVisible()) + QTest.mouseClick(widget.expand_button, Qt.LeftButton) + self.assertTrue(download.isVisible()) + self.assertEqual(download.title.text(), "Sharing download · Downloading") + worker["download_progress"]["artifact_received_bytes"] = 500 + widget.set_state(model, [worker]) + self.assertIs(widget.worker_downloads["one"], download) + self.assertEqual(download.bar.value(), 500) + widget.set_state(model, []) + self.assertEqual(widget.worker_downloads, {}) + self.assertTrue(widget.expand_button.isChecked()) + widget.close() diff --git a/desktop/tests/test_native_runtime_self_test.py b/desktop/tests/test_native_runtime_self_test.py new file mode 100644 index 000000000..fcd5c20bf --- /dev/null +++ b/desktop/tests/test_native_runtime_self_test.py @@ -0,0 +1,145 @@ +"""Control-flow and failure regressions; these tests never load a native runtime.""" + +import json +import os +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from desktop import launch_node + + +class NativeRuntimeSelfTestTests(unittest.TestCase): + def setUp(self): + self.torch = MagicMock() + self.torch.__version__ = "2.6.0+cu124" + self.torch.version.cuda = "12.4" + self.torch.allclose.return_value = True + self.torch.cuda.is_available.return_value = True + self.torch.linalg.svd.return_value = (MagicMock(), MagicMock(), MagicMock()) + self.torch.linspace.return_value.shape = (64,) + self.torch.linspace.return_value.device = "cuda:0" + self.restored = MagicMock() + self.restored.shape = (64,) + self.restored.device = "cuda:0" + self.restored.__sub__.return_value.abs.return_value.max.return_value.item.return_value = 0.1 + self.functional = MagicMock() + self.functional.quantize_4bit.return_value = (MagicMock(), MagicMock()) + self.functional.dequantize_4bit.return_value = self.restored + self.bnb = SimpleNamespace(cextension=SimpleNamespace(), functional=self.functional) + modules = patch.dict(sys.modules, {"torch": self.torch, "bitsandbytes": self.bnb}) + modules.start() + self.addCleanup(modules.stop) + frozen = patch.object(sys, "frozen", False, create=True) + frozen.start() + self.addCleanup(frozen.stop) + + def gpu_run(self): + with patch.object(launch_node, "_bitsandbytes_native_path", return_value="libbitsandbytes_cuda124.so"): + return launch_node._native_runtime_contract(require_cuda=True) + + def test_cpu_mode_requires_correct_math_and_does_not_initialize_cuda(self): + result = launch_node._native_runtime_contract() + self.assertTrue(result["cpu_matmul_passed"]) + self.assertFalse(result["cuda_test_performed"]) + self.torch.cuda.is_available.assert_not_called() + self.torch.cuda.synchronize.assert_not_called() + self.functional.quantize_4bit.assert_not_called() + self.torch.allclose.return_value = False + with self.assertRaisesRegex(RuntimeError, "CPU matrix"): + launch_node._native_runtime_contract() + + def test_required_cuda_cannot_silently_fall_back_to_cpu(self): + self.torch.cuda.is_available.return_value = False + with self.assertRaisesRegex(RuntimeError, "requires an available CUDA GPU"): + self.gpu_run() + self.functional.quantize_4bit.assert_not_called() + + def test_runtime_build_must_match_the_pruned_cuda_profile(self): + self.torch.version.cuda = "12.6" + with self.assertRaisesRegex(RuntimeError, "pinned torch"): + launch_node._native_runtime_contract() + self.torch.tensor.assert_not_called() + + def test_cuda_mode_checks_linalg_nf4_and_synchronizes_before_success(self): + result = self.gpu_run() + self.assertTrue(result["cpu_matmul_passed"]) + self.assertTrue(result["cuda_matmul_passed"]) + self.assertTrue(result["cuda_linalg_passed"]) + self.assertTrue(result["bitsandbytes_nf4_roundtrip_passed"]) + self.assertEqual(result["bitsandbytes_nf4_maximum_absolute_error"], 0.1) + self.torch.cuda.synchronize.assert_called_once_with() + + def test_incorrect_cuda_math_or_linalg_fails(self): + for checks, message in (([True, False], "CUDA matrix"), ([True, True, False], "linalg reconstruction")): + with self.subTest(message=message): + self.torch.allclose.side_effect = checks + with self.assertRaisesRegex(RuntimeError, message): + self.gpu_run() + + def test_nf4_rejects_inaccurate_nonfinite_and_misplaced_results(self): + for error in (0.21, float("inf"), float("nan")): + with self.subTest(error=error): + self.restored.__sub__.return_value.abs.return_value.max.return_value.item.return_value = error + with self.assertRaisesRegex(RuntimeError, "NF4 roundtrip returned"): + self.gpu_run() + self.restored.device = "cpu" + with self.assertRaisesRegex(RuntimeError, "shape or device"): + self.gpu_run() + self.restored.device = "cuda:0" + self.restored.shape = (32,) + with self.assertRaisesRegex(RuntimeError, "shape or device"): + self.gpu_run() + + def test_dispatch_keeps_cuda_explicit_and_rejects_extra_flags(self): + expected = {"cpu_matmul_passed": True} + for required in (False, True): + args = ["CommunityAI-Node", "--native-self-test"] + (["--require-cuda"] if required else []) + with ( + patch.object(sys, "argv", args), + patch.object(launch_node, "_native_runtime_contract", return_value=expected) as native, + patch.object(launch_node.multiprocessing, "freeze_support"), + patch("builtins.print") as output, + ): + self.assertEqual(launch_node.main(), 0) + native.assert_called_once_with(require_cuda=required) + output.assert_called_once_with(json.dumps(expected, sort_keys=True)) + for args in (["--require-cuda"], ["--native-self-test", "--download-model"]): + with patch.object(sys, "argv", ["CommunityAI-Node", *args]): + with self.assertRaisesRegex(RuntimeError, "only the optional"): + launch_node.main() + + def test_native_backend_must_be_exact_package_local_cuda124_and_inside_frozen_root(self): + with TemporaryDirectory() as temporary: + root = Path(temporary) + package = root / "bitsandbytes" + package.mkdir() + native_name = "libbitsandbytes_cuda124" + (".dll" if os.name == "nt" else ".so") + native = package / native_name + native.write_bytes(b"test placeholder, never loaded") + extension = SimpleNamespace( + __file__=str(package / "cextension.py"), + lib=SimpleNamespace(compiled_with_cuda=True, _lib=SimpleNamespace(_name=str(native))), + ) + with patch.object(sys, "frozen", True), patch.object(sys, "_MEIPASS", str(root), create=True): + self.assertEqual(launch_node._bitsandbytes_native_path(extension), native_name) + wrong = package / native_name.replace("124", "126") + wrong.write_bytes(b"test placeholder, never loaded") + extension.lib._lib._name = str(wrong) + with self.assertRaisesRegex(RuntimeError, "unexpected native"): + launch_node._bitsandbytes_native_path(extension) + extension.lib._lib._name = str(native) + extension.lib.compiled_with_cuda = False + with self.assertRaisesRegex(RuntimeError, "did not load a native CUDA"): + launch_node._bitsandbytes_native_path(extension) + extension.lib.compiled_with_cuda = True + extension.__file__ = str(root.parent / "outside" / "cextension.py") + with self.assertRaisesRegex(RuntimeError, "outside the frozen runtime"): + launch_node._bitsandbytes_native_path(extension) + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_release_downloads.py b/desktop/tests/test_release_downloads.py new file mode 100644 index 000000000..a24e803b6 --- /dev/null +++ b/desktop/tests/test_release_downloads.py @@ -0,0 +1,131 @@ +import hashlib +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from installers import release_downloads + + +def artifact(platform="windows-x64"): + windows = platform == "windows-x64" + version = "0.1.0-alpha.1" if windows else "0.1.0~alpha.1" + filename = f"communityai-{version}-windows-setup.exe" if windows else f"communityai_{version}_amd64.deb" + return { + "platform": platform, + "kind": "offline-installer", + "format": "exe" if windows else "deb", + "version": version, + "filename": filename, + "url": f"https://downloads.example.invalid/alpha/{filename}", + "sha256": hashlib.sha256(b"fixture offline installer").hexdigest(), + "size_bytes": len(b"fixture offline installer"), + "publisher": "CommunityAI engineering", + } + + +class ReleaseDownloadsTests(unittest.TestCase): + def test_one_or_both_pinned_platforms_and_sizes_above_two_gib(self): + entries = {platform: artifact(platform) for platform in release_downloads.PLATFORMS} + entries["windows-x64"]["size_bytes"] = 2_519_046_440 + entries["linux-amd64"]["size_bytes"] = 3_781_591_484 + manifest = {"schema_version": 1, "artifacts": entries} + self.assertEqual(release_downloads.select_artifact(manifest, "linux-amd64"), entries["linux-amd64"]) + one = {"schema_version": 1, "artifacts": {"windows-x64": entries["windows-x64"]}} + self.assertEqual(release_downloads.validate_release_manifest(one), one) + + def test_invalid_pins_and_metadata_are_rejected(self): + mutations = ( + {"kind": "pip-wheel"}, + {"platform": "linux-amd64"}, + {"format": "zip"}, + {"filename": "../setup.exe"}, + {"filename": "other.exe"}, + {"version": "latest"}, + {"version": "0.1.0\nmalicious"}, + {"sha256": "0" * 63}, + {"sha256": "F" * 64}, + {"size_bytes": True}, + {"size_bytes": 0}, + {"size_bytes": 4.0}, + {"size_bytes": release_downloads.MAX_PACKAGE_BYTES + 1}, + {"publisher": 'bad"publisher'}, + {"publisher": "bad\npublisher"}, + {"command": "arbitrary command"}, + ) + for mutation in mutations: + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + release_downloads.validate_artifact(dict(artifact(), **mutation), "windows-x64") + + def test_network_and_path_confusion_is_rejected(self): + good = artifact() + filename = good["filename"] + urls = ( + f"http://downloads.example.invalid/{filename}", + f"https://user:secret@downloads.example.invalid/{filename}", + f"https://downloads.example.invalid:8443/{filename}", + good["url"] + "?token=secret", + good["url"] + "#fragment", + f"https://downloads.example.invalid/../{filename}", + f"https://downloads.example.invalid/%2e%2e/{filename}", + f"https://downloads.example.invalid/%2f/{filename}", + f"https://downloads.example.invalid/%5c/{filename}", + f"https://downloads.example.invalid/%00/{filename}", + f"https://downloads.example.invalid/%ZZ/{filename}", + "https://downloads.example.invalid/different.exe", + "https://downloads.example.invalid/\n" + filename, + ) + for url in urls: + with self.subTest(url=url), self.assertRaises(ValueError): + release_downloads.validate_artifact(dict(good, url=url)) + + def test_manifest_shape_cannot_silently_select_wrong_platform(self): + for manifest in ( + {"schema_version": True, "artifacts": {"windows-x64": artifact()}}, + {"schema_version": 2, "artifacts": {"windows-x64": artifact()}}, + {"schema_version": 1, "artifacts": {}}, + {"schema_version": 1, "artifacts": {"linux-amd64": artifact()}}, + {"schema_version": 1, "artifacts": {"macos": artifact()}}, + {"schema_version": 1, "artifacts": {"windows-x64": artifact()}, "latest": True}, + ): + with self.subTest(manifest=manifest), self.assertRaises(ValueError): + release_downloads.validate_release_manifest(manifest) + with self.assertRaises(ValueError): + release_downloads.select_artifact( + {"schema_version": 1, "artifacts": {"windows-x64": artifact()}}, "linux-amd64" + ) + + def test_file_hash_and_size_are_derived_from_the_exact_payload(self): + with tempfile.TemporaryDirectory() as directory: + expected = artifact("linux-amd64") + payload = Path(directory) / expected["filename"] + payload.write_bytes(b"fixture offline installer") + result = release_downloads.artifact_from_file( + "linux-amd64", payload, expected["version"], "https://downloads.example.invalid/alpha" + ) + expected.pop("publisher") + self.assertEqual(result, expected) + payload.write_bytes(b"different package") + updated = release_downloads.artifact_from_file( + "linux-amd64", payload, expected["version"], "https://downloads.example.invalid/alpha" + ) + self.assertNotEqual(updated["sha256"], result["sha256"]) + self.assertEqual(updated["size_bytes"], len(b"different package")) + + def test_duplicate_json_fields_and_oversized_manifests_fail(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "release.json" + raw = json.dumps({"schema_version": 1, "artifacts": {"windows-x64": artifact()}}) + path.write_text(raw.replace('"schema_version": 1', '"schema_version": 1, "schema_version": 1')) + with self.assertRaises(ValueError): + release_downloads.load_release_manifest(path) + path.write_text(" " * (release_downloads.MAX_MANIFEST_BYTES + 1)) + with self.assertRaises(ValueError): + release_downloads.load_release_manifest(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_resource_controls.py b/desktop/tests/test_resource_controls.py new file mode 100644 index 000000000..67bce7062 --- /dev/null +++ b/desktop/tests/test_resource_controls.py @@ -0,0 +1,310 @@ +import copy +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import QApplication + +from communityai_desktop.acceptance import fake_node +from communityai_desktop.client import NodeClient, NodeClientError +from communityai_desktop.controller import DesktopController +from communityai_desktop.resource_controls import ResourceControls + + +class ResourceControlsTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.application = QApplication.instance() or QApplication([]) + + def test_two_default_sliders_preserve_draft_until_apply(self): + widget = ResourceControls() + saved = { + "editable": True, + "config_revision": "revision", + "policy": {"max_vram": "100%", "max_processing_percent": 100}, + } + widget.set_state(saved) + self.assertEqual([slider.value() for slider in widget.sliders.values()], [100, 100]) + changes = [] + widget.apply_requested.connect(lambda fields, revision: changes.append((fields, revision))) + widget.sliders["max_processing_percent"].setValue(25) + widget.set_state(saved) + self.assertEqual(widget.sliders["max_processing_percent"].value(), 25) + self.assertEqual(changes, []) + widget.apply_button.click() + self.assertEqual(changes, [({"max_processing_percent": 25}, "revision")]) + widget.close() + + def test_preserves_custom_vram_and_does_not_enable_old_nodes(self): + widget = ResourceControls() + saved = { + "editable": True, + "config_revision": "revision", + "policy": {"max_vram": "2GiB", "max_processing_percent": 100}, + } + widget.set_state(saved) + self.assertEqual(widget.values["max_vram"].text(), "2GiB") + widget.sliders["max_processing_percent"].setValue(50) + self.assertEqual(widget._draft, {"max_processing_percent": 50}) + widget.set_state({**saved, "config_revision": "other"}) + self.assertFalse(widget.apply_button.isEnabled()) + widget.set_state({**saved, "policy": {"max_vram": "50%"}}) + self.assertFalse(widget.sliders["max_processing_percent"].isEnabled()) + widget.close() + + def test_full_memory_limit_is_not_reduced_for_local_fallback(self): + widget = ResourceControls() + saved = { + "editable": True, + "config_revision": "revision", + "policy": {"max_vram": "100%", "max_processing_percent": 100}, + "vram_bytes": 8 * 1024**3, + "vram_pool_bytes": 8 * 1024**3, + "vram_available_bytes": 8 * 1024**3, + } + widget.set_state(saved) + self.assertEqual(widget.values["max_vram"].text(), "8.0 GB of 8.0 GB") + self.assertEqual(widget.sliders["max_vram"].maximum(), 100) + self.assertEqual(widget.sliders["max_vram"].value(), 100) + self.assertEqual(widget._draft, {}) + # Saving only computing must retain the original 100% memory policy. + widget.sliders["max_processing_percent"].setValue(50) + widget.set_state(saved) + changes = [] + widget.apply_requested.connect(lambda fields, revision: changes.append(fields)) + widget.apply_button.click() + self.assertEqual(changes, [{"max_processing_percent": 50}]) + self.assertEqual(widget._policy["max_vram"], "100%") + widget.sliders["max_vram"].setValue(25) + self.assertEqual(widget.values["max_vram"].text(), "2.0 GB of 8.0 GB") + self.assertEqual(widget._draft["max_vram"], "25%") + # Every slider position expresses the configured fraction of the card. + widget.sliders["max_vram"].setValue(widget.sliders["max_vram"].maximum()) + self.assertEqual(widget._draft["max_vram"], "100%") + self.assertEqual(widget.values["max_vram"].text(), "8.0 GB of 8.0 GB") + widget.set_state(saved) + self.assertEqual(widget.values["max_vram"].text(), "8.0 GB of 8.0 GB") + self.assertEqual(widget.sliders["max_vram"].value(), 100) + self.assertEqual(widget.values["max_processing_percent"].text(), "50%") + widget.close() + + +class ResourceControllerTests(unittest.TestCase): + def setUp(self): + with fake_node() as (url, token): + self.status = NodeClient(url, token).status() + self.status["contribution"]["policy"]["policy"]["max_processing_percent"] = 100 + self.revision = self.status["contribution"]["policy"]["config_revision"] + + def client(self, *, failure=None): + outer = self + + class Client: + def __init__(self): + self.actions = [] + + def status(self): + return copy.deepcopy(outer.status) + + def worker_action(self, worker, action): + self.actions.append((action, worker)) + if failure == action: + raise NodeClientError("injected " + action) + + def update_contribution_policy(self, policy, *, expected_revision): + self.actions.append(("save", policy)) + if failure == "save": + raise NodeClientError("disk full") + return {"policy": policy} + + return Client() + + def test_pauses_before_save_preserves_policy_and_only_resumes_selected_workers(self): + client = self.client() + result = DesktopController(client).update_resource_limits( + {"max_processing_percent": 25}, expected_revision=self.revision + ) + self.assertEqual([action for action, _ in client.actions], ["pause", "pause", "pause", "save", "start"]) + self.assertEqual(client.actions[-1], ("start", "worker-b")) + saved = {**self.status["contribution"]["policy"]["policy"], "max_processing_percent": 25} + self.assertEqual(result["policy"], saved) + + def test_failed_pause_or_save_never_restarts_workers(self): + for failure in ("pause", "save"): + client = self.client(failure=failure) + with self.assertRaises(NodeClientError): + DesktopController(client).update_resource_limits({"max_vram": "25%"}, expected_revision=self.revision) + self.assertNotIn("start", [action for action, _ in client.actions]) + + def test_stale_revision_does_not_stop_workers(self): + client = self.client() + with self.assertRaises(NodeClientError): + DesktopController(client).update_resource_limits({"max_vram": "25%"}, expected_revision="stale") + self.assertEqual(client.actions, []) + + def test_saving_limits_keeps_automatic_sharing_requested_while_placement_is_pending(self): + for worker in self.status["contribution"]["workers"]: + worker["desired_running"] = False + self.status["contribution"]["policy"]["policy"]["sharing_enabled"] = True + client = self.client() + DesktopController(client).update_resource_limits({"max_vram": "50%"}, expected_revision=self.revision) + self.assertIn(("start", "worker-b"), client.actions) + + def test_pending_automatic_reason_is_visible_but_individual_pause_is_preserved(self): + contribution = self.status["contribution"] + contribution["policy"]["policy"]["sharing_enabled"] = True + worker = next(worker for worker in contribution["workers"] if worker["id"] == "worker-b") + worker["desired_running"] = False + worker["state"] = "paused" + worker["policy"].update(admitted=False, reason="automatic placement is waiting for fresh eligible coverage") + viewed = DesktopController._worker_view(worker) + result = DesktopController._contribution_view(contribution, [viewed]) + self.assertEqual(result["selected_blocked_reasons"], [worker["policy"]["reason"]]) + self.assertTrue(result["intent_enabled"]) + self.assertFalse(result["enabled"]) + worker["operator_paused"] = True + result = DesktopController._contribution_view(contribution, [DesktopController._worker_view(worker)]) + self.assertEqual(result["selected_blocked_reasons"], []) + client = self.client() + DesktopController(client).update_resource_limits({"max_vram": "50%"}, expected_revision=self.revision) + self.assertNotIn(("start", "worker-b"), client.actions) + + def test_first_start_saves_explicit_opt_in_and_defaults_before_worker_start(self): + self.status["contribution"]["policy"]["policy"].update( + sharing_enabled=False, max_vram=None, max_disk_space=None + ) + client = self.client() + DesktopController(client).set_sharing_enabled(True) + actions = [action for action, _ in client.actions] + self.assertEqual(actions, ["pause", "pause", "pause", "save", "start", "start", "start"]) + policy = client.actions[3][1] + self.assertTrue(policy["sharing_enabled"]) + self.assertEqual(policy["max_vram"], "100%") + self.assertEqual(policy["max_processing_percent"], 100) + self.assertEqual(policy["max_disk_space"], "20GiB") + + def test_pause_is_persisted_and_failed_opt_in_never_starts_workers(self): + client = self.client() + DesktopController(client).set_sharing_enabled(False) + self.assertEqual([action for action, _ in client.actions], ["pause", "pause", "pause", "save"]) + self.assertFalse(client.actions[-1][1]["sharing_enabled"]) + client = self.client(failure="save") + with self.assertRaises(NodeClientError): + DesktopController(client).set_sharing_enabled(True) + self.assertNotIn("start", [action for action, _ in client.actions]) + + def test_pending_worker_reports_persisted_intent_and_physical_memory_without_worker_budget(self): + current = self.status["contribution"] + current["workers"] = [] + current["policy"]["policy"]["sharing_enabled"] = True + result = DesktopController._contribution_view( + current, + [], + { + "gpu_total_bytes": 8_000_000_000, + "sharing_vram_bytes": 4_500_000_000, + "sharing_vram_available_bytes": 4_500_000_000, + }, + ) + self.assertTrue(result["intent_enabled"]) + self.assertTrue(result["can_pause"]) + self.assertFalse(result["enabled"]) + self.assertEqual(result["vram_bytes"], 4_500_000_000) + self.assertEqual(result["vram_pool_bytes"], 8_000_000_000) + self.assertEqual(result["processing_percent"], 100) + + def test_real_policy_api_stops_old_process_and_preserves_limits_on_reload(self): + from fastapi.testclient import TestClient + + from drift.node.config import NodeConfig + from drift.node.model_manager import ModelManager + from drift.node.policy_store import ContributionPolicyStore + from drift.node.server import create_node_app + from drift.node.worker_supervisor import WorkerLaunch, WorkerSupervisor, WorkerSupervisorSettings + + with tempfile.TemporaryDirectory() as directory: + config_path = Path(directory) / "node.json" + document = { + "schema_version": 1, + "models": [{"manifest": "manifest.json", "initial_peers": ["peer-one"]}], + "workers": [{"id": "worker", "model": "model", "identity_path": "worker.key", "num_blocks": 1}], + } + config_path.write_text(json.dumps(document), encoding="utf-8") + + def settings(config): + percent = config.contribution_policy.max_processing_percent + return WorkerSupervisorSettings( + launches=( + WorkerLaunch( + "worker", + "model", + (sys.executable, "-c", "import time; time.sleep(60)", str(percent)), + policy_admitted=config.contribution_policy.sharing_enabled, + policy_reason=None if config.contribution_policy.sharing_enabled else "sharing is disabled", + ), + ), + stop_timeout=2, + ) + + supervisor = WorkerSupervisor(settings(NodeConfig.load(config_path)).launches) + store = ContributionPolicyStore(config_path, supervisor, settings) + manager = ModelManager() + app = create_node_app( + manager, + worker_supervisor=supervisor, + contribution_policy_store=store, + control_keys=["test-control"], + api_keys=["test-inference"], + ) + try: + with TestClient(app) as api: + + class Client(NodeClient): + def _request(self, method, path, *, payload=None): + response = api.request( + method, path, json=payload, headers={"Authorization": "Bearer test-control"} + ) + if response.status_code >= 400: + raise NodeClientError(str(response.json())) + return response.json() + + controller = DesktopController(Client("http://127.0.0.1:8080", "test-control")) + self.assertFalse(NodeConfig.load(config_path).contribution_policy.sharing_enabled) + self.assertIsNone(NodeConfig.load(config_path).contribution_policy.max_disk_space) + self.assertIsNone(NodeConfig.load(config_path).contribution_policy.max_vram) + controller.set_sharing_enabled(True) + started_policy = NodeConfig.load(config_path).contribution_policy + self.assertTrue(started_policy.sharing_enabled) + self.assertEqual(started_policy.max_disk_space, "20GiB") + self.assertEqual(started_policy.max_vram, "100%") + old_process = supervisor._record("worker").process + self.assertIsNotNone(old_process) + controller.update_resource_limits( + {"max_processing_percent": 25, "max_vram": "50%"}, + expected_revision=store.snapshot()["config_revision"], + ) + new_process = supervisor._record("worker").process + self.assertIsNotNone(old_process.poll()) + self.assertIsNone(new_process.poll()) + self.assertNotEqual(old_process.pid, new_process.pid) + self.assertEqual(supervisor.launches[0].command[-1], "25.0") + reloaded = NodeConfig.load(config_path) + self.assertEqual(reloaded.contribution_policy.max_processing_percent, 25) + self.assertEqual(reloaded.contribution_policy.max_vram, "50%") + self.assertEqual(json.loads(config_path.read_text())["workers"], document["workers"]) + supervisor.pause_worker("worker") + controller.update_resource_limits( + {"max_processing_percent": 100}, expected_revision=store.snapshot()["config_revision"] + ) + self.assertIsNotNone(new_process.poll()) + self.assertIsNone(supervisor._record("worker").process) + controller.set_sharing_enabled(False) + self.assertFalse(NodeConfig.load(config_path).contribution_policy.sharing_enabled) + finally: + supervisor.shutdown() + manager.shutdown() diff --git a/desktop/tests/test_resource_playthrough.py b/desktop/tests/test_resource_playthrough.py new file mode 100644 index 000000000..6c5783fe6 --- /dev/null +++ b/desktop/tests/test_resource_playthrough.py @@ -0,0 +1,131 @@ +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import QApplication, QPushButton, QWidget + +from communityai_desktop.app import main +from communityai_desktop.resource_controls import ResourceControls +from communityai_desktop.resource_playthrough import ResourcePlaythrough + + +class ResourcePlaythroughTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.application = QApplication.instance() or QApplication([]) + + def test_home_start_observes_immediate_feedback_without_legacy_opt_in_dialog(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + plan = root / "plan.json" + plan.write_text( + json.dumps( + {"steps": [{"action": "start"}], "timeout_seconds": 30, "acknowledgement": str(root / "ack.json")} + ), + encoding="utf-8", + ) + playback = ResourcePlaythrough(plan, root / "evidence.json") + window = QWidget() + window._controller = object() + window._busy = 0 + window._page_buttons = [QPushButton(str(index), window) for index in range(3)] + window.home_share_button = QPushButton("Start sharing", window) + window.master_share_button = QPushButton("Start sharing", window) + window.resource_controls = ResourceControls(window) + contribution = {"editable": True, "intent_enabled": False, "policy": {"sharing_enabled": False}} + window._snapshot = {"contribution": contribution} + home_clicks = [] + window._page_buttons[0].clicked.connect(lambda: home_clicks.append(True)) + + def start(): + for button in (window.home_share_button, window.master_share_button): + button.setText("Starting…") + button.setEnabled(False) + + window.home_share_button.clicked.connect(start) + playback.install(window, self.application, {"QTimer": QTimer}) + playback.timer.stop() + playback.tick() + self.assertEqual(playback.phase, "observe") + self.assertEqual(playback.immediate_feedback, "Starting…") + self.assertEqual(home_clicks, [True]) + contribution.update(intent_enabled=True, policy={"sharing_enabled": True}) + playback.tick() + self.assertEqual(playback.phase, "acknowledgement") + self.assertEqual(playback.result["steps"][0]["immediate_feedback"], "Starting…") + window.close() + + def test_saved_full_memory_uses_full_slider_position(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + plan = root / "plan.json" + plan.write_text( + json.dumps( + { + "steps": [{"action": "observe", "vram_percent": 100, "processing_percent": 100}], + "timeout_seconds": 30, + "acknowledgement": str(root / "ack.json"), + } + ), + encoding="utf-8", + ) + playback = ResourcePlaythrough(plan, root / "evidence.json") + window = QWidget() + window._controller = object() + window._busy = 0 + window._page_buttons = [QPushButton(str(index), window) for index in range(3)] + window.resource_controls = ResourceControls(window) + contribution = { + "editable": True, + "config_revision": "test", + "intent_enabled": False, + "policy": {"sharing_enabled": False, "max_vram": "100%", "max_processing_percent": 100}, + "vram_pool_bytes": 8 * 1024**3, + "vram_bytes": 8 * 1024**3, + "vram_available_bytes": 8 * 1024**3, + } + window._snapshot = {"contribution": contribution} + window.resource_controls.set_state(contribution) + playback.install(window, self.application, {"QTimer": QTimer}) + playback.timer.stop() + playback.tick() + playback.tick() + self.assertEqual(playback.phase, "acknowledgement") + self.assertEqual(window.resource_controls.sliders["max_vram"].value(), 100) + self.assertEqual(playback.result["steps"][0]["saved_vram"], "100%") + window.close() + + def test_explicit_qualification_cannot_activate_users_running_desktop(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + plan = root / "plan.json" + plan.write_text( + json.dumps( + {"steps": [{"action": "start"}], "timeout_seconds": 30, "acknowledgement": str(root / "ack.json")} + ), + encoding="utf-8", + ) + with patch("communityai_desktop.pyside_shell.run", return_value=0) as run: + self.assertEqual( + main( + [ + "--no-manage-node", + "--resource-ui-playthrough", + str(plan), + "--resource-ui-evidence", + str(root / "evidence.json"), + ] + ), + 0, + ) + self.assertFalse(run.call_args.kwargs["single_instance"]) + self.assertIsInstance(run.call_args.kwargs["qualification_automation"], ResourcePlaythrough) + with patch("communityai_desktop.pyside_shell.run", return_value=0) as run: + self.assertEqual(main(["--no-manage-node"]), 0) + self.assertTrue(run.call_args.kwargs["single_instance"]) diff --git a/desktop/tests/test_runtime_packaging.py b/desktop/tests/test_runtime_packaging.py new file mode 100644 index 000000000..cf42327e1 --- /dev/null +++ b/desktop/tests/test_runtime_packaging.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import errno +import os +import stat +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from desktop import runtime_packaging as packaging +from desktop.installers import build_deb + + +class RuntimePackagingTests(unittest.TestCase): + def setUp(self): + self.temporary = TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) / "node" + self.root.mkdir() + + def write(self, name, payload=b"native-library"): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return path + + def normalize(self, platform="Linux"): + with patch.dict(os.environ, {"BNB_CUDA_VERSION": ""}): + return packaging.normalize_runtime( + self.root, target_platform=platform, torch_version=packaging.TORCH_PROFILE + ) + + def bnb(self, suffix="so"): + return self.write(f"_internal/bitsandbytes/libbitsandbytes_cuda124.{suffix}", b"cu124") + + def test_prunes_only_other_cuda_versions_and_keeps_cpu_without_gpu_detection(self): + retained = self.bnb() + cpu = self.write("_internal/bitsandbytes/libbitsandbytes_cpu.so", b"cpu") + optional = self.write("_internal/bitsandbytes/libbitsandbytes_cuda124_nocublaslt.so", b"124-alt") + self.write("_internal/bitsandbytes/libbitsandbytes_cuda118.so", b"118") + self.write("_internal/libbitsandbytes_cuda118.so", b"118") + innocent = self.write("_internal/bitsandbytes/cuda118.py", b"python-source") + result = self.normalize() + self.assertEqual(len(result["removed_bitsandbytes_variants"]), 2) + self.assertTrue(all(path.is_file() for path in (retained, cpu, optional, innocent))) + self.assertEqual(result["before"]["logical_file_bytes"] - result["after"]["logical_file_bytes"], 6) + + def test_hardlinks_identical_libraries_and_preserves_every_path_and_digest(self): + self.bnb() + first = self.write("_internal/libtorch_cuda.so", b"torch-data") + second = self.write("_internal/torch/lib/libtorch_cuda.so", b"torch-data") + different = self.write("_internal/torch/lib/libdifferent.so", b"other-data") + innocent = self.write("_internal/metadata.txt", b"torch-data") + result = self.normalize() + self.assertFalse(second.is_symlink()) + self.assertTrue(os.path.samefile(first, second)) + self.assertFalse(os.path.samefile(first, innocent)) + self.assertEqual(second.read_bytes(), b"torch-data") + self.assertEqual(different.read_bytes(), b"other-data") + self.assertEqual(result["before"]["logical_file_bytes"], result["after"]["logical_file_bytes"]) + self.assertEqual(result["before"]["unique_file_bytes"] - result["after"]["unique_file_bytes"], 10) + self.assertEqual(len(result["hardlinked_native_libraries"]), 1) + self.assertEqual(self.normalize()["hardlinked_native_libraries"], []) + + def test_windows_prunes_profile_variants_without_merging_libraries(self): + self.bnb("dll") + first = self.write("_internal/a.dll") + second = self.write("_internal/b.dll") + self.write("_internal/bitsandbytes/libbitsandbytes_cuda126.dll") + result = self.normalize("Windows") + self.assertEqual(result["hardlinked_native_libraries"], []) + self.assertEqual(len(result["removed_bitsandbytes_variants"]), 1) + self.assertFalse(os.path.samefile(first, second)) + + def test_profile_mismatch_and_missing_library_fail_before_mutation(self): + removed = self.write("libbitsandbytes_cuda118.so") + for version in ("2.6.0+cu126", "2.6.0", "2.7.0+cu124"): + with self.assertRaisesRegex(RuntimeError, "pinned torch"): + packaging.normalize_runtime(self.root, target_platform="Linux", torch_version=version) + with self.assertRaisesRegex(RuntimeError, "missing its CUDA"): + self.normalize() + self.assertTrue(removed.is_file()) + self.bnb() + with patch.dict(os.environ, {"BNB_CUDA_VERSION": "118"}): + with self.assertRaisesRegex(RuntimeError, "conflicts"): + packaging.normalize_runtime(self.root, target_platform="Linux", torch_version=packaging.TORCH_PROFILE) + self.assertTrue(removed.is_file()) + + def test_different_modes_are_never_merged(self): + self.bnb() + first = self.write("_internal/a.so") + second = self.write("_internal/b.so") + second.chmod(stat.S_IREAD) + self.addCleanup(second.chmod, stat.S_IREAD | stat.S_IWRITE) + self.normalize() + self.assertFalse(os.path.samefile(first, second)) + + def test_changed_source_fails_before_link_replacement(self): + self.bnb() + first = self.write("_internal/a.so") + second = self.write("_internal/b.so") + real_hash = packaging._sha256 + + def hash_and_change(path): + digest = real_hash(path) + if path == first: + first.write_bytes(b"changed-longer") + return digest + + with patch.object(packaging, "_sha256", side_effect=hash_and_change): + with self.assertRaisesRegex(RuntimeError, "changed during"): + self.normalize() + self.assertFalse(os.path.samefile(first, second)) + + def test_deb_staging_preserves_hardlinks_even_across_device_copy_fallback(self): + first = self.write("a.so", b"a" * 1025) + os.link(first, self.root / "b.so") + real_link = os.link + destination = self.root.parent / "stage" + + def cross_device_link(source, target, **kwargs): + if Path(source).parent == self.root: + raise OSError(errno.EXDEV, "fixture cross-device link") + return real_link(source, target, **kwargs) + + with patch.object(build_deb.os, "link", side_effect=cross_device_link): + build_deb.copy_bundle(self.root, destination) + self.assertTrue(os.path.samefile(destination / "a.so", destination / "b.so")) + self.assertEqual(build_deb.installed_size_kib(destination), 2) + self.assertEqual(packaging.storage_metrics(destination)["logical_file_bytes"], 2050) + self.assertEqual(packaging.storage_metrics(destination)["unique_file_bytes"], 1025) + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_simple_desktop.py b/desktop/tests/test_simple_desktop.py new file mode 100644 index 000000000..75162d1f9 --- /dev/null +++ b/desktop/tests/test_simple_desktop.py @@ -0,0 +1,202 @@ +import copy +import os +import threading +import time +import unittest +from unittest.mock import patch + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from communityai_desktop.acceptance import fake_node +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController +from communityai_desktop.presentation import model_summary, sharing_summary + + +class SimpleSummaryTests(unittest.TestCase): + def test_local_choice_explains_fallback_and_respects_explicit_local_mode(self): + snapshot = {"auto_selection": {"model": "Qwen3.5-0.8B-Local", "source": "local"}} + name, reason, location = model_summary(snapshot) + self.assertEqual(name, "Qwen3.5 0.8B") + self.assertIn("community cannot answer right now", reason) + self.assertEqual(location, "On this computer") + snapshot["inference_mode"] = "local_only" + self.assertEqual(model_summary(snapshot)[1], "You chose to use only this computer.") + + def test_live_process_downloading_is_not_reported_as_sharing(self): + snapshot = { + "contribution": {"intent_enabled": True}, + "workers": [ + { + "model": "Qwen", + "state": "running", + "sharing_active": False, + "desired_running": True, + "download_progress": {"state": "downloading"}, + } + ], + } + self.assertEqual(sharing_summary(snapshot)[0], "Downloading for sharing") + + def test_paused_worker_can_resume_and_automatic_model_is_not_shown_as_a_name(self): + snapshot = { + "contribution": {"intent_enabled": True}, + "workers": [ + {"model": "auto", "operator_paused": True, "state": "paused", "placement": {"automatic": True}} + ], + } + self.assertEqual(sharing_summary(snapshot), ("Sharing is paused", "", "paused")) + snapshot["workers"][0].update(operator_paused=False, sharing_active=True) + self.assertEqual(sharing_summary(snapshot), ("Sharing is on", "Helping the community.", "running")) + + +class SimpleDesktopInteractionTests(unittest.TestCase): + def test_start_has_immediate_feedback_and_polling_does_not_disable_controls(self): + self._exercise_poll_race(False) + + def test_stale_poll_error_does_not_cancel_a_successful_start(self): + self._exercise_poll_race(True) + + def _exercise_poll_race(self, fail_poll): + from PySide6.QtCore import QTimer + from PySide6.QtWidgets import QLabel + + from communityai_desktop.pyside_shell import run + + with fake_node(all_workers_paused=True) as (url, token): + state = DesktopController(NodeClient(url, token)).snapshot() + state["hardware"] = { + "cpu_name": "Test processor", + "gpu_name": "Test graphics card", + "gpu_total_bytes": 8 * 1024**3, + "inference_device": "cuda:0", + } + state["contribution"].update( + intent_enabled=False, + enabled=False, + vram_bytes=4 * 1024**3, + vram_pool_bytes=8 * 1024**3, + vram_available_bytes=4 * 1024**3, + processing_percent=100, + ) + state["contribution"]["policy"].update(sharing_enabled=False, max_processing_percent=100) + action_started = threading.Event() + allow_action = threading.Event() + poll_started = threading.Event() + allow_poll = threading.Event() + confirmation_started = threading.Event() + allow_confirmation = threading.Event() + observed = [] + errors = [] + + class Controller: + block_poll = False + + def snapshot(self): + captured = copy.deepcopy(state) + if self.block_poll: + self.block_poll = False + poll_started.set() + allow_poll.wait(5) + if fail_poll: + raise RuntimeError("Old request lost its connection") + elif action_started.is_set(): + confirmation_started.set() + allow_confirmation.wait(5) + return captured + + def set_sharing_enabled(self, enabled): + action_started.set() + allow_action.wait(5) + state["contribution"]["intent_enabled"] = enabled + state["contribution"]["policy"]["sharing_enabled"] = enabled + for worker in state["workers"]: + worker["desired_running"] = enabled + worker["state"] = "paused" + worker["operator_paused"] = False + return {"message": "Sharing enabled."} + + controller = Controller() + + class Automation: + phase = 0 + + def install(self, window, application, types): + self.started = time.monotonic() + self.timer = QTimer(window) + self.timer.setInterval(15) + + def tick(): + try: + if time.monotonic() - self.started > 8: + raise AssertionError("UI interaction did not finish") + if self.phase == 0: + if not window._snapshot.get("models"): + return + visible = [ + item.text() for item in window.pages.widget(0).findChildren(QLabel) if item.isVisible() + ] + joined = "\n".join(visible) + for removed in ( + "Ready to use", + "World regions", + "Your local AI is ready", + "Your downloads", + "Everything is connected", + "ENDPOINT URL", + ): + assert removed not in joined, removed + assert window.home_vram.text() == "4.0 GB of 8.0 GB" + assert window.home_gpu.text() == "Test graphics card" + controller.block_poll = True + window.refresh() + self.phase = 1 + elif self.phase == 1 and poll_started.is_set(): + assert window.home_share_button.isEnabled(), "Background polling disabled Start" + window.home_share_button.click() + assert window.home_share_button.text() == "Starting…" + assert window.master_share_button.text() == "Starting…" + assert not window.home_share_button.isEnabled() + observed.append("immediate-feedback") + self.phase = 2 + elif self.phase == 2 and action_started.is_set(): + allow_action.set() + allow_poll.set() + self.phase = 3 + elif self.phase == 3 and confirmation_started.is_set(): + assert window.home_share_button.text() == "Starting…" + assert not window.home_share_button.isEnabled(), "Start unlocked before state confirmation" + assert window._controller is controller, "Stale error disconnected the current controller" + observed.append("waiting-for-confirmation") + allow_confirmation.set() + self.phase = 4 + elif self.phase == 4 and window.home_share_button.text() == "Pause sharing": + assert window._snapshot["contribution"]["intent_enabled"], "Stale poll overwrote Start" + assert window.master_share_button.text() == "Pause sharing" + observed.append("confirmed-intent") + window._snapshot_failed("Connection interrupted") + assert window.home_sharing_title.text() == "Checking sharing…" + assert not window.home_share_button.isEnabled() + assert not window.resource_controls.sliders["max_processing_percent"].isEnabled() + self.timer.stop() + application.quit() + except BaseException as exc: + errors.append(exc) + allow_action.set() + allow_poll.set() + allow_confirmation.set() + self.timer.stop() + application.quit() + + self.timer.timeout.connect(tick) + self.timer.start() + + with patch("communityai_desktop.pyside_shell.login_startup_enabled", return_value=False): + run(controller, single_instance=False, qualification_automation=Automation(), auto_close_seconds=10) + if errors: + raise errors[0] + self.assertEqual(observed, ["immediate-feedback", "waiting-for-confirmation", "confirmed-intent"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_update_installer.py b/desktop/tests/test_update_installer.py new file mode 100644 index 000000000..00d26cffc --- /dev/null +++ b/desktop/tests/test_update_installer.py @@ -0,0 +1,112 @@ +"""Actual tiny Inno upgrade through UpdateManager; never touches the real installation.""" + +import hashlib +import os +import subprocess +import sys +import tempfile +import time +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch + +from communityai_desktop.updater import UpdateManager + + +@unittest.skipUnless( + sys.platform == "win32" and os.environ.get("COMMUNITYAI_INNO_COMPILER"), "Requires Windows and Inno" +) +class UpdateInstallerTests(unittest.TestCase): + def test_verified_update_stops_old_installation_replaces_payload_and_reopens(self): + compiler = Path(os.environ["COMMUNITYAI_INNO_COMPILER"]) + csc = Path(os.environ["SystemRoot"]) / "Microsoft.NET/Framework64/v4.0.30319/csc.exe" + installer_source = Path(__file__).resolve().parents[1] / "installers/communityai.iss" + with tempfile.TemporaryDirectory(prefix="communityai-update-test-") as directory: + root = Path(directory) + bundle, install, output = (root / name for name in ("bundle", "installed", "output")) + for path in (bundle, install, output): + path.mkdir() + source = root / "App.cs" + source.write_text( + r""" +using System; using System.IO; using System.Diagnostics; +class App { static int Main(string[] args) { + string root = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName); + File.AppendAllText(Path.Combine(root,"lifecycle.txt"), args.Length > 0 && args[0] == "--prepare-update" ? "stop\n" : "open\n"); + return 0; +} } +""" + ) + real_popen = subprocess.Popen + flags = subprocess.CREATE_NO_WINDOW + subprocess.run( + [str(csc), "/nologo", "/target:winexe", "/out:" + str(bundle / "CommunityAI.exe"), str(source)], + check=True, + capture_output=True, + creationflags=flags, + ) + import shutil + + shutil.copy2(bundle / "CommunityAI.exe", install / "CommunityAI.exe") + (install / ".communityai-installation").write_text("CommunityAI installer-managed fixture") + (install / "_internal").mkdir() + (install / "_internal/obsolete.dll").write_bytes(b"old") + (install / "settings.json").write_text("keep settings") + (bundle / "_internal").mkdir() + (bundle / "_internal/current.dll").write_bytes(b"new") + identifier = "CommunityAI.UpdateFixture." + uuid.uuid4().hex + result = subprocess.run( + [ + str(compiler), + "/Qp", + "/DBundleDir=" + str(bundle), + "/DOutputPath=" + str(output), + "/DAppVersion=0.1.0-alpha.20260909.3", + "/DAppIdentifier=" + identifier, + str(installer_source), + ], + capture_output=True, + creationflags=flags, + ) + self.assertEqual(result.returncode, 0, result.stdout[-3000:] + result.stderr[-1000:]) + package = next(output.glob("*.exe")) + item = {"size_bytes": package.stat().st_size, "sha256": hashlib.sha256(package.read_bytes()).hexdigest()} + manager = UpdateManager(root / "cache", root=install, system="Windows") + manager.directory.mkdir() + manager.candidate = {"artifacts": {"windows-x64": item}, "expires_at": int(time.time()) + 60}, package + arguments = [] + + def hidden_setup(argv, **kwargs): + arguments.append(list(argv)) + # Production /SILENT displays progress; this fixture must never display windows. + argv = ["/VERYSILENT" if arg == "/SILENT" else arg for arg in argv] + ["/NOICONS"] + return real_popen(argv, **kwargs) + + try: + with patch("communityai_desktop.updater.subprocess.Popen", side_effect=hidden_setup): + manager.install() + manager.thread.join(60) + self.assertFalse(manager.thread.is_alive()) + self.assertEqual(manager.process.returncode, 0, manager.snapshot()) + deadline = time.monotonic() + 10 + while "open" not in (install / "lifecycle.txt").read_text() and time.monotonic() < deadline: + time.sleep(0.1) + self.assertEqual((install / "lifecycle.txt").read_text(), "stop\nopen\n") + self.assertEqual((install / "settings.json").read_text(), "keep settings") + self.assertFalse((install / "_internal/obsolete.dll").exists()) + self.assertEqual((install / "_internal/current.dll").read_bytes(), b"new") + self.assertIn("/UPDATE=1", arguments[0]) + finally: + uninstaller = install / "unins000.exe" + if uninstaller.exists(): + subprocess.run( + [str(uninstaller), "/VERYSILENT", "/SUPPRESSMSGBOXES", "/NORESTART"], + creationflags=flags, + timeout=60, + check=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_updater.py b/desktop/tests/test_updater.py new file mode 100644 index 000000000..4e7203769 --- /dev/null +++ b/desktop/tests/test_updater.py @@ -0,0 +1,340 @@ +import base64 +import hashlib +import io +import json +import os +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from communityai_desktop import updater +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat + + +class Response(io.BytesIO): + def __init__(self, body, status=200, headers=None): + super().__init__(body) + self.status = status + self.headers = headers or {"Content-Length": str(len(body))} + + +class UpdateTests(unittest.TestCase): + def setUp(self): + self.key = Ed25519PrivateKey.generate() + self.public = base64.b64encode(self.key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)).decode() + self.payload = b"verified fixture installer" + self.item = { + "filename": "communityai-0.1.0-alpha.20260909.3-windows-setup.exe", + "url": updater.ORIGIN + "/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe", + "sha256": hashlib.sha256(self.payload).hexdigest(), + "size_bytes": len(self.payload), + } + self.signed = { + "schema_version": 1, + "channel": "alpha", + "version": "0.1.0-alpha.20260909.3", + "sequence": 2026090903, + "published_at": 1000, + "expires_at": 3000, + "artifacts": {"windows-x64": self.item}, + } + + def envelope(self): + return updater.canonical( + { + "signed": self.signed, + "signature": base64.b64encode( + self.key.sign(updater.SIGNATURE_DOMAIN + updater.canonical(self.signed)) + ).decode(), + } + ) + + def verify(self, raw=None, **kwargs): + return updater.verify_feed(raw or self.envelope(), public_key=self.public, now=2000, **kwargs) + + def test_signed_feed_rejects_tampering_and_wrong_key(self): + self.assertEqual(self.verify()["version"], self.signed["version"]) + forged = json.loads(self.envelope()) + forged["signed"]["artifacts"]["windows-x64"]["sha256"] = "a" * 64 + with self.assertRaises(updater.UpdateError): + self.verify(updater.canonical(forged)) + with self.assertRaises(updater.UpdateError): + updater.verify_feed(self.envelope(), now=2000) + + def test_expiry_rollback_unknown_platform_and_external_url_rejected(self): + for change in ( + {"expires_at": 1999}, + {"published_at": 2500}, + {"channel": "stable"}, + {"schema_version": True}, + {"sequence": 0}, + ): + original = dict(self.signed) + self.signed.update(change) + with self.assertRaises(updater.UpdateError): + self.verify() + self.signed = original + with self.assertRaises(updater.UpdateError): + self.verify(minimum_sequence=2026090904) + self.item["url"] = self.item["url"].replace(updater.ORIGIN, "https://example.com") + with self.assertRaises(updater.UpdateError): + self.verify() + + def test_duplicate_fields_and_oversized_feed_rejected(self): + with self.assertRaises(updater.UpdateError): + self.verify(b'{"signed":{},"signed":{},"signature":""}') + with self.assertRaises(updater.UpdateError): + self.verify(b" " * (updater.MAX_FEED_BYTES + 1)) + + def test_versions_compare_numerically_and_stable_is_newer(self): + self.assertLess(updater.version_key("0.1.0-alpha.20260909.2"), updater.version_key("0.1.0-alpha.20260909.10")) + self.assertLess(updater.version_key("0.1.0-alpha.20260909.10"), updater.version_key("0.1.0")) + with self.assertRaises(updater.UpdateError): + updater.version_key("../../installer") + + def test_download_resumes_verified_prefix_and_reuses_complete_file(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / (self.item["filename"] + ".part")).write_bytes(self.payload[:8]) + offsets = [] + + def open_fixture(url, offset=0): + offsets.append(offset) + return Response( + self.payload[offset:], + 206, + { + "Content-Length": str(len(self.payload) - offset), + "Content-Range": f"bytes {offset}-{len(self.payload) - 1}/{len(self.payload)}", + }, + ) + + path = updater.download(self.item, root, lambda *args: None, threading.Event(), opener=open_fixture) + self.assertEqual(path.read_bytes(), self.payload) + updater.download(self.item, root, lambda *args: None, threading.Event(), opener=open_fixture) + self.assertEqual(offsets, [8]) + + def test_ignored_range_restarts_download(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / (self.item["filename"] + ".part")).write_bytes(b"old bytes") + path = updater.download( + self.item, + root, + lambda *args: None, + threading.Event(), + opener=lambda *args, **kwargs: Response(self.payload), + ) + self.assertEqual(path.read_bytes(), self.payload) + + def test_wrong_range_and_corrupt_download_never_become_ready(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + partial = root / (self.item["filename"] + ".part") + partial.write_bytes(self.payload[:2]) + with self.assertRaises(updater.UpdateError): + updater.download( + self.item, + root, + lambda *args: None, + threading.Event(), + opener=lambda *args, **kwargs: Response( + self.payload[2:], + 206, + {"Content-Length": str(len(self.payload) - 2), "Content-Range": "bytes 0-1/2"}, + ), + ) + partial.unlink() + with self.assertRaises(updater.UpdateError): + updater.download( + self.item, + root, + lambda *args: None, + threading.Event(), + opener=lambda *args, **kwargs: Response(b"x" * len(self.payload)), + ) + self.assertFalse((root / self.item["filename"]).exists()) + self.assertFalse(partial.exists()) + + def test_cancellation_and_low_disk_leave_existing_data_intact(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + sentinel = root / "user-settings" + sentinel.write_text("preserve") + cancel = threading.Event() + cancel.set() + with self.assertRaises(updater.UpdateError): + updater.download(self.item, root, lambda *args: None, cancel) + cancel.clear() + with patch.object(updater.shutil, "disk_usage", return_value=type("Disk", (), {"free": 0})()): + with self.assertRaises(updater.UpdateError): + updater.download(self.item, root, lambda *args: None, cancel) + self.assertEqual(sentinel.read_text(), "preserve") + + def test_manager_downloads_without_installing_and_rechecks_cached_hash(self): + with tempfile.TemporaryDirectory() as directory: + manager = updater.UpdateManager(directory, root=Path(directory), system="Windows") + self.signed.update(published_at=int(time.time()) - 10, expires_at=int(time.time()) + 1000) + actual_verify = updater.verify_feed + + def verified(raw, **kwargs): + return actual_verify(raw, public_key=self.public, **kwargs) + + with ( + patch.object(updater, "verify_feed", side_effect=verified), + patch.object(updater, "open_url", return_value=Response(self.envelope())), + patch.object(updater, "download", return_value=Path(directory) / "setup.exe"), + patch.object(updater.subprocess, "Popen") as launch, + ): + manager.check() + manager.thread.join(5) + self.assertEqual(manager.snapshot()["status"], "ready") + launch.assert_not_called() + manager.candidate[1].write_bytes(b"tampered") + manager.install() + manager.thread.join(5) + self.assertEqual(manager.snapshot()["status"], "error") + launch.assert_not_called() + + def test_windows_handoff_uses_verified_setup_and_existing_install_directory(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / self.item["filename"] + package.write_bytes(self.payload) + self.signed["expires_at"] = int(time.time()) + 1000 + manager = updater.UpdateManager(root, root=root / "installed app", system="Windows") + manager.candidate = self.signed, package + with patch.object(updater.subprocess, "Popen") as launch, patch.object( + updater.subprocess, "CREATE_NO_WINDOW", 0, create=True + ): + launch.return_value.wait.return_value = 1 + manager.install() + manager.thread.join(5) + argv = launch.call_args.args[0] + self.assertEqual(argv[0], str(package)) + self.assertIn("/UPDATE=1", argv) + self.assertIn("/DIR=" + str(root / "installed app"), argv) + self.assertEqual(manager.snapshot()["status"], "error") + + def test_real_http_interruption_resumes_without_redownloading_prefix(self): + import http.server + import urllib.request + + payload = b"a" * (1024**2) + b"b" * 2048 + requests = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + offset = int(self.headers.get("Range", "bytes=0-")[6:-1]) + requests.append(offset) + self.send_response(206 if offset else 200) + self.send_header("Content-Length", str(len(payload) - offset)) + if offset: + self.send_header("Content-Range", f"bytes {offset}-{len(payload)-1}/{len(payload)}") + self.end_headers() + self.wfile.write(payload[offset:] if offset else payload[: 1024**2]) + self.close_connection = True + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + item = {**self.item, "sha256": hashlib.sha256(payload).hexdigest(), "size_bytes": len(payload)} + + def opener(url, offset=0): + headers = {"Range": f"bytes={offset}-"} if offset else {} + return urllib.request.urlopen( + urllib.request.Request(f"http://127.0.0.1:{server.server_port}/setup", headers=headers) + ) + + with tempfile.TemporaryDirectory() as directory: + result = updater.download(item, Path(directory), lambda *args: None, threading.Event(), opener=opener) + self.assertEqual(result.read_bytes(), payload) + self.assertEqual(requests, [0, 1024**2]) + finally: + server.shutdown() + server.server_close() + thread.join(5) + + +class UpdateUiTests(unittest.TestCase): + def test_progress_restart_and_active_answer_feedback_are_visible(self): + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + from communityai_desktop.acceptance import fake_node + from communityai_desktop.client import NodeClient + from communityai_desktop.controller import DesktopController + from communityai_desktop.pyside_shell import run + from PySide6.QtCore import QTimer + + errors = [] + + class Updates: + state = {"status": "idle", "message": "Check for updates"} + installed = False + + def snapshot(self): + return self.state + + def check(self): + self.state = {"status": "checking", "message": "Checking for updates…"} + + def install(self): + self.installed = True + self.state = {"status": "installing", "message": "Installing update…"} + + def close(self): + pass + + updates = Updates() + + class Automation: + def install(self, window, application, types): + def exercise(): + try: + assert window.update_button.isVisible() + window.update_button.click() + assert window.update_button.text() == "Checking for updates…" + assert not window.update_button.isEnabled() + updates.state = {"status": "downloading", "message": "Downloading update… 50%"} + window._render_update() + assert "50%" in window.update_button.text() + updates.state = {"status": "ready", "message": "Restart to update"} + window._render_update() + window._snapshot["models"] = [{"active_requests": 1}] + window.update_button.click() + window._render_update() + assert window.update_detail.isVisible() and "answer" in window.update_detail.text() + assert not updates.installed + window._snapshot["models"] = [] + window.update_button.click() + assert updates.installed and window.update_button.text() == "Installing update…" + except BaseException as exc: + errors.append(exc) + finally: + application.quit() + + QTimer.singleShot(500, exercise) + + with fake_node() as (url, token), patch( + "communityai_desktop.pyside_shell.login_startup_enabled", return_value=False + ): + run( + DesktopController(NodeClient(url, token)), + updater=updates, + single_instance=False, + qualification_automation=Automation(), + auto_close_seconds=5, + ) + if errors: + raise errors[0] + + +if __name__ == "__main__": + unittest.main() diff --git a/desktop/tests/test_windows_online_installer.py b/desktop/tests/test_windows_online_installer.py new file mode 100644 index 000000000..00d2ce648 --- /dev/null +++ b/desktop/tests/test_windows_online_installer.py @@ -0,0 +1,192 @@ +"""Exercise the PowerShell build boundary without downloading or running setup.""" + +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +BUILDER = ROOT / "desktop/installers/build_windows_online_installer.ps1" +POWERSHELL = shutil.which("pwsh") or shutil.which("powershell") + + +class WindowsOnlineSilentFailureContracts(unittest.TestCase): + """Static dialog-boundary guard; native download acceptance remains separate.""" + + def test_script_error_dialogs_are_confined_to_the_interactive_reporter(self): + source = (BUILDER.parent / "communityai-online.iss").read_text(encoding="utf-8") + reporter = re.search(r"procedure ReportFailure\([^\n]+\);\s*begin\s*(.*?)\s*end;", source, re.S) + self.assertIsNotNone(reporter) + self.assertRegex(reporter.group(1), r"Log\(Message\);") + self.assertRegex( + reporter.group(1), + r"if not WizardSilent then\s+SuppressibleMsgBox\(Message, mbError, MB_OK, IDOK\);", + ) + remainder = source[: reporter.start()] + source[reporter.end() :] + self.assertNotRegex( + remainder, r"\b(?:SuppressibleMsgBox|MsgBox|TaskDialogMsgBox|SuppressibleTaskDialogMsgBox)\s*\(" + ) + + def test_missing_verification_exits_before_execution_with_a_failure_code(self): + source = (BUILDER.parent / "communityai-online.iss").read_text(encoding="utf-8") + handoff = source.split("procedure CurStepChanged", 1)[1] + guard = re.search(r"if not DownloadVerified then begin\s*(.*?)\s*end;", handoff, re.S) + self.assertIsNotNone(guard) + self.assertRegex(guard.group(1), r"ChildExitCode := 1;") + self.assertRegex(guard.group(1), r"ReportFailure\([\s\S]+\);\s*Exit;") + self.assertLess(guard.end(), handoff.index("ChildStarted := Exec(")) + self.assertNotIn("RaiseException(", handoff) + self.assertRegex(handoff, r"function GetCustomSetupExitCode: Integer;\s*begin\s*Result := ChildExitCode;") + + +@unittest.skipUnless(os.name == "nt" and POWERSHELL, "Windows PowerShell builder") +class WindowsOnlineInstallerTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="communityai online test ") + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.output = self.directory / "output with spaces" + self.artifact = { + "platform": "windows-x64", + "kind": "offline-installer", + "format": "exe", + "version": "0.0.0-test", + "filename": "communityai-0.0.0-test-windows-setup.exe", + "url": "https://example.invalid/releases/0.0.0-test/communityai-0.0.0-test-windows-setup.exe", + "sha256": "a" * 64, + "size_bytes": 2519046440, + "publisher": "CommunityAI test fixture", + } + + def invoke(self, *extra): + manifest = self.directory / "release fixture.json" + manifest.write_text(json.dumps({"schema_version": 1, "artifacts": {"windows-x64": self.artifact}})) + return subprocess.run( + [ + POWERSHELL, + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(BUILDER), + "-Manifest", + str(manifest), + "-OutputDirectory", + str(self.output), + "-PythonCommand", + sys.executable, + *extra, + ], + capture_output=True, + text=True, + timeout=20, + creationflags=subprocess.CREATE_NO_WINDOW, + ) + + def compiler(self, *, fails=False): + compiler = self.directory / "record compiler.ps1" + compiler.write_text( + "param([Parameter(ValueFromRemainingArguments=$true)][string[]]$CompilerArguments)\n" + "$definitions = @{}\n" + "foreach ($argument in $CompilerArguments) {\n" + " if ($argument.StartsWith('/D')) {\n" + " $pieces = $argument.Substring(2).Split('=',2)\n" + " $definitions[$pieces[0]] = $pieces[1]\n" + " }\n" + "}\n" + "$destination = $definitions.OutputPath\n" + "$CompilerArguments | ConvertTo-Json | Set-Content -LiteralPath " + "(Join-Path $destination 'compiler-arguments.json') -Encoding utf8\n" + + ( + "exit 7\n" + if fails + else "$fixture = Join-Path $destination ('communityai-' + $definitions.AppVersion + " + "'-windows-online-setup.exe')\n" + "[IO.File]::WriteAllBytes($fixture, [Text.Encoding]::UTF8.GetBytes('not executable - build fixture'))\n" + "exit 0\n" + ) + ) + return compiler + + def test_validation_preserves_exact_large_size_without_building(self): + result = self.invoke("-UnsignedAlpha", "-ValidateOnly") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), self.artifact) + self.assertFalse(self.output.exists()) + + def test_explicit_signing_choice_is_required_before_output(self): + result = self.invoke("-ValidateOnly") + self.assertNotEqual(result.returncode, 0) + self.assertIn("UnsignedAlpha", result.stderr) + self.assertFalse(self.output.exists()) + + def test_preprocessor_delimiters_in_publisher_are_rejected(self): + for publisher in ("owner's label", "{unsafe}"): + with self.subTest(publisher=publisher): + self.artifact["publisher"] = publisher + result = self.invoke("-UnsignedAlpha", "-ValidateOnly") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.output.exists()) + + def test_mutable_or_untrusted_url_is_rejected_before_compiler(self): + for url in ( + self.artifact["url"].replace("https:", "http:"), + self.artifact["url"] + "?latest=true", + self.artifact["url"] + "#fragment", + ): + with self.subTest(url=url): + self.artifact["url"] = url + result = self.invoke("-UnsignedAlpha", "-ValidateOnly") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.output.exists()) + + def test_metadata_and_compiler_arguments_bind_the_pinned_offline_setup(self): + result = self.invoke("-UnsignedAlpha", "-Compiler", str(self.compiler())) + self.assertEqual(result.returncode, 0, result.stderr) + arguments = json.loads((self.output / "compiler-arguments.json").read_text(encoding="utf-8-sig")) + for key, value in { + "InstallerUrl": self.artifact["url"], + "InstallerFilename": self.artifact["filename"], + "InstallerSha256": self.artifact["sha256"], + "InstallerSize": str(self.artifact["size_bytes"]), + "OutputPath": str(self.output), + }.items(): + self.assertIn(f"/D{key}={value}", arguments) + installer = self.output / "communityai-0.0.0-test-windows-online-setup.exe" + metadata = json.loads(installer.with_suffix(".exe.json").read_text(encoding="utf-8-sig")) + self.assertEqual(metadata["offline_installer"], self.artifact) + self.assertEqual(metadata["sha256"], hashlib.sha256(installer.read_bytes()).hexdigest()) + helper = self.output / "downloader-build/WindowsDownload.exe" + self.assertIn(f"/DDownloadHelper={helper}", arguments) + self.assertEqual(metadata["download_helper_sha256"], hashlib.sha256(helper.read_bytes()).hexdigest()) + self.assertEqual( + metadata["download_helper_source_sha256"], + hashlib.sha256((BUILDER.parent / "WindowsDownload.cs").read_bytes()).hexdigest(), + ) + self.assertFalse(metadata["live_download_verified"]) + + def test_failed_compiler_cannot_create_success_metadata(self): + result = self.invoke("-UnsignedAlpha", "-Compiler", str(self.compiler(fails=True))) + self.assertNotEqual(result.returncode, 0) + self.assertIn("compiler failed: 7", result.stderr) + self.assertEqual(list(self.output.glob("*.exe.json")), []) + + def test_existing_online_installer_is_not_overwritten(self): + self.output.mkdir() + original = self.output / "communityai-0.0.0-test-windows-online-setup.exe" + original.write_bytes(b"preserved fixture") + result = self.invoke("-UnsignedAlpha", "-Compiler", str(self.compiler())) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(original.read_bytes(), b"preserved fixture") + self.assertFalse((self.output / "compiler-arguments.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ALPHA_INSTALL.md b/docs/ALPHA_INSTALL.md new file mode 100644 index 000000000..477dc7fab --- /dev/null +++ b/docs/ALPHA_INSTALL.md @@ -0,0 +1,96 @@ +# Install CommunityAI + +[Download the latest alpha](https://github.com/flujo-app/CommunityAI/releases/tag/v0.1.0-alpha.20260909.3) +for Windows or Ubuntu/Debian. Community inference now needs no local community +model downloads. Sharing uses the full configured GPU memory budget. + +**Already using the September 9 updater release? Check for updates in the sidebar.** +September 8 installations need one manual installer upgrade. Your settings and +downloaded models are preserved. Updates download in the app and show +**Restart to update** when ready. You choose when to restart. + +| Platform | Small online installer | Complete offline installer | +| --- | --- | --- | +| Windows | [Online setup](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-online-setup.exe) | [Offline setup](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe) | +| Ubuntu/Debian | [Online installer](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-linux-online.py) | [Offline .deb](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai_0.1.0~alpha.20260909.3_amd64.deb) | + +The online installer downloads and verifies the complete offline package. It makes +the initial download smaller; the total runtime download is the same. Local +fallback and sharing roles download model files when needed. Community inference +sends text to peers and requires no community weight downloads. Both platforms +include the required runtime libraries; GPU use still requires a compatible +NVIDIA driver. + +[Installer SHA-256 checksums](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/INSTALLER-SHA256SUMS) +are available for all four downloads. + +## Windows + +Open the downloaded setup and follow the prompts. It installs for your current +Windows user without administrator elevation. The alpha has no Authenticode +publisher signature, so Windows may show an unknown-publisher prompt. +Launch **CommunityAI** from the Start menu. + +## Ubuntu/Debian + +Run the downloaded online installer as your normal user: + +```sh +python3 communityai-0.1.0-alpha.20260909.3-linux-online.py +``` + +Or install the downloaded offline package: + +```sh +sudo apt install './communityai_0.1.0~alpha.20260909.3_amd64.deb' +communityai +``` + +The package targets amd64 Debian 12+/Ubuntu 22.04+. Run CommunityAI as your normal +desktop user with an unlocked credential store. Installation and in-app updates +ask for administrator authentication. + +## Using and updating the app + +Home shows the selected model and why, your hardware, GPU memory budget in GB, +and computing limit. Use **Start sharing** to contribute. The resource controls +default to 100%; you can reduce them before starting. A 100% GPU memory limit +allows sharing to use the full capacity, with no permanent fallback reservation. +Models expands to show block health, contributors and your downloads. + +The app checks for updates shortly after opening and every six hours. New releases +download automatically with visible progress. **Restart to update** installs the +update after the current answer finishes. Settings, credentials and model caches +are preserved. See [how updates work](AUTOMATIC_UPDATES.md). + +Community availability depends on contributors, including a peer serving the +input/output stages. The app can use its eligible local fallback when the mesh +cannot answer. Text-peer roles currently require operator setup through the +source CLI (`drift text-peer`), rather than the desktop sharing controls. +Computers helping answer a request may be able to see its contents. + +Normal uninstall preserves settings and model downloads. Before uninstalling, +turn off **Start CommunityAI when I sign in**. See +[removal and optional data cleanup](DESKTOP_UNINSTALL.md). + +## Release records + +The immutable release is `alpha/20260909.3/` on Cloudflare R2. +[Release metadata](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-release-metadata.zip) +binds the packages to source commit `d7f4333cc8ff74ce060076c1d6effb67fb4c6d2c`. +The R2 development hostname is rate limited. + +Focused source checks and real text-only public-mesh completion/chat passed. +See [consumer evidence](evidence/text-only-mesh-consumer-20260909.md) and +[release evidence](evidence/text-mesh-release-20260909.json). The Windows updater +handoff fixture belongs to the earlier September 9 updater release. Packaging uses +the normal build checks, uploaded size/checksum metadata and public download samples. +Small public files receive complete hash checks. This release does +not claim a new full desktop, GPU, cloud or installed Linux updater test run. +Earlier installer/native-runtime checks remain recorded in +[release readiness](RELEASE_READINESS.md). + +Maintainers must retain the update mirror branch and renew its signed feed before +expiry as described in [automatic updates](AUTOMATIC_UPDATES.md). The separately +signed model catalog expires September 28, 2026 at 19:35 UTC; its existing renewal +and branch requirements are recorded in [hosting notes](CLOUDFLARE_RELEASE.md). diff --git a/docs/AUTOMATIC_UPDATES.md b/docs/AUTOMATIC_UPDATES.md new file mode 100644 index 000000000..55dca23d0 --- /dev/null +++ b/docs/AUTOMATIC_UPDATES.md @@ -0,0 +1,60 @@ +# Application updates + +Installer-managed CommunityAI checks for updates shortly after opening and every +six hours while running. It downloads a newer release automatically. The sidebar +shows download progress and then **Restart to update**. Clicking that button runs +the normal installer and reopens CommunityAI. A current answer must finish first. +Settings, credentials and model caches stay outside the installed program files. + +The September 8 release did not contain an updater. Users of that release must +install an updater-enabled release once; subsequent releases use this flow. +Source previews and portable build folders do not update the installed application. + +Windows uses the ordinary-user Inno installer and its existing shutdown handshake. +Linux uses the installed, root-owned Python package helper through PolicyKit; the +desktop asks for administrator authentication before APT changes the installation. +The Linux installer is detached from the desktop process tree before APT starts, +so package maintenance cannot terminate its own installer. A cancelled or failed +handoff leaves a retry message when the old desktop remains running. + +## Release authentication and recovery + +- The executable pins an Ed25519 public key. The update document cannot introduce + a new signing key or redirect downloads to another host. +- The signed document binds channel, version, monotonically increasing sequence, + publication/expiry times and each exact package URL, size and SHA-256. +- Expired documents, older sequences, wrong-platform filenames, duplicates and + unsupported fields are rejected. A lower application version is never installed. +- Interrupted downloads retain their prefix and resume with an exact HTTP range. + Complete files are checked before becoming ready and rechecked before execution. +- Downloaded installers for the running or older version are removed after a + successful update check. Cache removal is restricted to updater-owned files. +- The app never closes just because an update becomes available. The user chooses + when to restart. Errors remain visible with a retry control. + +## Publishing + +1. Set `desktop/src/communityai_desktop/release.py` to the new release version. + Build both runtimes and installers from the reviewed commit. Installer versions + must match the compiled version (`-alpha.` becomes `~alpha.` for Debian). +2. Qualify the packages and upload immutable files to the versioned R2 directory. + Verify the complete public bytes. Build the online installers from the same + exact offline package manifest. +3. Run `desktop/installers/sign_app_update.py` with that manifest, release version, + a strictly increasing sequence and an output path. It reads a base64 Ed25519 + private key from standard input, never from a command-line argument. The local + release key is protected with Windows DPAPI outside the repository. +4. Publish the signed document **last**, to R2 `updates/alpha.json` with + `Cache-Control: no-store`. Commit the identical document at + `public-alpha/updates/alpha.json` on `codex/gate14-20260902-b` for the GitHub + fallback. Keep that branch available until clients ship a replacement mirror. +5. Verify both public documents with the pinned client key. Publish the GitHub + prerelease with the download links, small online installers and release hashes. + +Feed signatures are valid for 90 days. Renew the signed feed before expiry even +when the package version is unchanged; increment its sequence. Retain the same +key securely. Key rotation requires a release that trusts the replacement key. +This feed signature is separate from Windows Authenticode signing. + +The Windows handoff uses documented [Inno command-line options](https://jrsoftware.org/ishelp/topic_setupcmdline.htm) +and [post-install launch behavior](https://jrsoftware.org/ishelp/topic_runsection.htm). diff --git a/docs/CATALOG_BOOTSTRAP_V1.md b/docs/CATALOG_BOOTSTRAP_V1.md index dbc3d49c4..8fd18a123 100644 --- a/docs/CATALOG_BOOTSTRAP_V1.md +++ b/docs/CATALOG_BOOTSTRAP_V1.md @@ -1,5 +1,7 @@ # Catalog bootstrap v1 +Publisher key location and emergency recovery: [CATALOG_SIGNING_KEY.md](CATALOG_SIGNING_KEY.md). + Status: the strict sidecar consumer, desktop lifecycle integration, last-known-good cache, best-effort-alpha publication-bundle contract, and fail-closed packaging handoff are implemented. The threshold-one `public-alpha/catalog-v1` bundle publishes the diff --git a/docs/CATALOG_SIGNING_KEY.md b/docs/CATALOG_SIGNING_KEY.md new file mode 100644 index 000000000..e27c912a5 --- /dev/null +++ b/docs/CATALOG_SIGNING_KEY.md @@ -0,0 +1,62 @@ +# Catalog publisher key and emergency recovery + +Updated 2026-09-06. This is the permanent locator for the **catalog signing key**. +It is separate from worker identities, desktop API credentials and installer signing. +Never put its contents into Git, release archives, logs, conversations or worker VMs. + +## Current key and three backups + +Public key ID: `sha256:9505d3ac8ec996d4b794bd43d09dd84447acc5da8a0bb1eac1be4e9c9a34b14f`. + +| Copy | Location | +| --- | --- | +| Working publisher key | `%LOCALAPPDATA%\CommunityAI\publisher-keys\catalog-20260906\catalog-private.pem` | +| **Emergency online backup** | [Google Secret Manager: communityai-catalog-signer-20260906](https://console.cloud.google.com/security/secret-manager/secret/communityai-catalog-signer-20260906/versions?project=community-ai-506321), **version 1**, project `community-ai-506321` | +| Second backup | `G:\CommunityAI-Publisher-Backup\catalog-20260906\catalog-private.pem` | +| Third backup | Repository-local `.publisher-secrets/catalog-20260906/catalog-private.pem`, explicitly gitignored | + +The online resource is +`projects/1091886728019/secrets/communityai-catalog-signer-20260906/versions/1`. +It has automatic replication, no expiry and no public IAM grant. The current project +IAM policy has one owner; workers were not granted access. Local directories have +Windows ACLs restricted to the publisher's Windows identity. G: and repository +copies stay in place by explicit owner request. The Google copy survives losing this PC. + +The local non-secret registry is +`%LOCALAPPDATA%\CommunityAI\publisher-keys\active-catalog.json`; copies accompany +the G: and repository backups. It records the private paths, public identity and +pinned Google backup version. This document remains available if that registry is lost. + +## Verify and recover + +Authenticate Google Cloud CLI as the CommunityAI project owner. With the project +Python environment installed, run `python scripts/catalog_key_backup.py`. This +retrieves version 1 into memory, checks its public identity, signs and verifies a +challenge, and prints only the verification result. It never prints the key. + +For emergency recovery, open the Google link above, select **version 1**, and save +the value as `catalog-private.pem` in an owner-restricted publisher directory. +Alternatively use `gcloud secrets versions access 1 --secret=communityai-catalog-signer-20260906 +--project=community-ai-506321 --out-file=` as a single command. +Use `--out-file`; omitting it displays the private key in the terminal. +Verify the recovered key's public ID before signing. Do not generate another key +merely because the local working copy is missing. + +`scripts/sign_catalog_candidate.py` loads the registered working key and verifies +the Google emergency copy before sealing a reviewed candidate. Sealing does not +publish it. `scripts/provision_catalog_signer.ps1` is for deliberate creation of a +new identity, not routine recovery. + +## September 2026 replacement + +The previous public key ID ended in `5ac8435`. Its original generation command was +found in the Gate 12 execution history: `.gatev-runtime/gate12-release/communityai-public-alpha-v1.pem`. +That directory no longer exists; no private material was recovered. Commit +`26be579` contains only its public root and signed catalog. + +The replacement Qwen bundle uses catalog sequence 2 and a separate publication +directory, `public-alpha/catalog-qwen-v2`. Existing installations need the new +desktop build, whose bundled bootstrap explicitly authorizes replacing the exact +previous root digest. Network catalogs cannot change trust roots. The old +`catalog-v1` publication stays available for older clients; rollback watermarks and +user configuration survive the application migration. diff --git a/docs/CLOUDFLARE_RELEASE.md b/docs/CLOUDFLARE_RELEASE.md new file mode 100644 index 000000000..a36f155e9 --- /dev/null +++ b/docs/CLOUDFLARE_RELEASE.md @@ -0,0 +1,115 @@ +# Cloudflare alpha downloads + +The owner activated R2 on September 8, 2026. The `communityai-releases` bucket +uses Standard storage. Its initial test origin is +`https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev`. +The fresh account has no custom domain. Cloudflare rate-limits `r2.dev` and +intends it for development; use a custom domain for wider distribution. +[Public bucket guidance](https://developers.cloudflare.com/r2/buckets/public-buckets/). + +The owner authorized a personal API token named **CommunityAI release storage +CLI**, restricted to **Workers R2 Storage: Edit** on this account. It is stored +outside the repository using Windows DPAPI for the current user. No credential +is embedded in an installer, release manifest, GitHub secret, or this document. + +Wrangler **4.130.0** created the bucket and enabled its public URL. AWS CLI +**1.46.1** uses the same authorized token through R2's S3-compatible interface. +Both clients uploaded harmless test objects, and anonymous HTTPS requests +returned their exact content with HTTP 200. This establishes hosting access; +it does not establish installer download or installation acceptance. + +Wrangler's object uploader limits individual uploads to 300 MiB. Use the S3 CLI +for the multi-gigabyte installers. The local upload profile limits concurrency +to two requests and uses 16 MiB multipart chunks above a 64 MiB threshold. +Cloudflare documents deriving the S3 access key ID from the token ID and its +secret from SHA-256 of the token value; keep both values private. +[R2 authentication](https://developers.cloudflare.com/r2/api/tokens/), +[large-object uploads](https://developers.cloudflare.com/r2/objects/upload-objects/). + +## Publication sequence + +1. Build and qualify the normalized offline EXE and Debian package. Preserve the + previously qualified files and record each replacement's version, exact byte + count, SHA-256 and source provenance. +2. Choose a new immutable release prefix, such as `alpha/20260908.1`. Check that + its objects are absent; do not overwrite a published version. +3. Upload only the named release artifacts with the S3 CLI. Check each object's + byte count and anonymously download/hash the resulting HTTPS object. + Quote the PowerShell cache argument as + `--cache-control 'public,max-age=31536000,immutable'`. Finalize object metadata + before starting qualification downloads, then leave both payload and metadata + unchanged while downloads are active. +4. Create `release-downloads.json` with + `desktop/installers/release_downloads.py create`, supplying the real HTTPS + base URL and each exact offline file/version. Build the Windows and Linux + online installers from that pinned manifest. +5. Qualify actual downloader-to-installer handoff, then upload the small online + files, checksums and release metadata. Publish their verified links in the + release notes and installation guide. + +The versioned release prefix is `alpha/20260908.1/`. Both offline installers are +uploaded and passed complete hosted download/hash verification. Windows's +2,462,345,104-byte setup passed actual ordinary-user handoff through the updated +online wrapper, installed CPU diagnostics and removal. The retained offline-child +handle returned exit 0; temporary files were removed and the saved user-state, +menu and native-registration baselines matched after uninstall. +[Windows hosted acceptance](evidence/normalized-online-windows-installer-20260908.md). +Prior installed CUDA/NF4 checks cover this exact offline package. + +The published [Windows online setup](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe) +is 2,107,751 bytes, SHA-256 +`8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861`. +Its complete anonymous public body matched that checksum. The +original downloader's two connection failures and the first resumable helper's +fatal progress update remain separate evidence. The current helper treats +progress records as optional while retaining independent complete-file checks; +an observed skipped progress frame did not interrupt the successful handoff. + +Linux's 2,302,428,788-byte hosted package passed full download/hash, +protected-copy/APT installation and removal, using the +[scoped log/state audit](evidence/alpha-online-linux-hosted-20260908.md). +Its 13,662-byte online script and nine curated Linux metadata files are published. +Every public small-file body was downloaded anonymously and matched to its +local SHA-256. Prior native CPU/CUDA checks cover the exact offline package. +Keep each platform's `provenance.json`, `desktop-metrics.json`, `SHA256SUMS` and +companions under `metadata/windows/` or `metadata/linux/`. Those generic names +must not overwrite the other platform's records. The Windows online builder +used the exact `metadata/windows/windows-release-downloads.json`; preserve its +bytes because the executable's companion binds the whole manifest hash. The +combined `release-downloads.json` lists both exact offline artifacts, while +`INSTALLER-SHA256SUMS` covers both offline and both online files. Build companions +record compilation with `live_download_verified=false`; the separate acceptance +records establish later actual download and installation results. + +At **2026-09-09 02:02 UTC**, the final +[publication audit](evidence/alpha-cloudflare-publication-20260909.json) matched +all 24 small public object bodies: both online installers, 19 curated platform +records, the combined manifest, installer checksums and metadata ZIP. The +[installer checksums](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/INSTALLER-SHA256SUMS), +[combined manifest](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/release-downloads.json) +and [metadata ZIP](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-release-metadata.zip) +are available. The ZIP is 1,009,867 bytes, SHA-256 +`9e3424eb8587b09b02b464dc2456aecd1150f154c4fe719069e39374129bb60b`. +Both offline objects also returned HEAD 200 with exact lengths; their complete +byte identities come from the actual online acceptance records. This publishes +the qualified candidate downloads; it does not claim Gate 16's combined canary +has run or establish broad service availability. + +Both offline runtimes identify source +`84205f93fc73d3babd39e238944b97fab0d11b3e`. The Windows online helper, Inno script +and builder were built from working-tree files and subsequently matched to all +three Git blobs at `b6c8aad9cea208630785d890cfb966093f809e7e`. +`metadata/windows/online-source-commit.json` records that separate binding; +neither source identity substitutes for the other. + +Harmless probes under `_checks/` are not release installers. The complete new +offline installers total 4,764,773,892 bytes, down from 6,300,637,924 bytes. Avoid +retaining multiple full releases without checking account storage usage. +R2's included usage is an allowance, not a spending cap. +[Pricing](https://developers.cloudflare.com/r2/pricing/). + +The original and revised Windows downloaders passed real HTTPS negative checks: it rejected a +harmless 52-byte object with a deliberately wrong embedded hash and exited with +code 1, without launching a child installer. Its temporary directory was removed. +[Original check](evidence/alpha-online-windows-https-rejection-20260908.json), +[current progress-fix check](evidence/alpha-online-windows-progress-fix-https-rejection-20260909.json). diff --git a/docs/COMMUNITY_AI_MODEL_LADDER.md b/docs/COMMUNITY_AI_MODEL_LADDER.md new file mode 100644 index 000000000..0ee4507d1 --- /dev/null +++ b/docs/COMMUNITY_AI_MODEL_LADDER.md @@ -0,0 +1,147 @@ +# CommunityAI model ladder + +Reviewed: 2026-09-07 against this working tree, the publisher configurations, and +the owner's [model-ladder discussion](codex://threads/01a06441-735b-74d2-a2c8-273dad789e6e). +This is the product plan and implementation audit, not a signed model approval. + +## Intended progression + +**Local Qwen3.5 → community Qwen3.8-27B → DeepSeek-V4-Flash → GLM-5.3-Flash.** + +Local Qwen remains available when the community route is incomplete, unreachable, +or too slow. Larger models become candidates only after their exact runtime +profiles pass qualification and a signed catalog approves them. Increasing the +number of connected PCs cannot enable an unsupported model. + +| Stage | Role | Actual status on 2026-09-07 | +| --- | --- | --- | +| Qwen3.5 local | An answer even when the user is alone | Exact 0.8B BF16/eager profile passed real offline inference through source and packaged nodes on an 8 GB RTX 2070 SUPER. Automatic local selection, budgets, local-only preference and cancellation are implemented. Larger local profiles and target RTX 30/40/50 hardware are unqualified. | +| Qwen3.8-27B FP8 | First community model | Complete CPU/mixed inference, reference comparison, packaged completion/chat and cache restart passed. The actual one-click CPU test also passed automatic 64-block formation, six-client promotion/inference, five-client local fallback, and unattended recovery using real source Qt windows. [Evidence](evidence/qwen-formation-passed-20260907.json). Final packages and broader consumer GPU envelopes remain open. | +| DeepSeek-V4-Flash | Next larger community target | No `deepseek_v4` DRIFT adapter or pinned candidate manifest in this checkout. Existing `deepseek_v3` support does not establish V4 support. | +| GLM-5.3-Flash | Larger frontier target | No `glm5_next`/`glm5_next_text` DRIFT adapter or pinned candidate manifest in this checkout. | + +The published [sequence 1](../public-alpha/catalog-v1/catalog.signed.json) still +contains Qwen3.5 2B and Gemma 4 E2B. A separate, signed +[sequence 2 candidate](../public-alpha/catalog-qwen-v2/catalog.signed.json) contains +local Qwen3.5-0.8B and distributed Qwen3.8-27B. It is **published at a separate +qualification path**. The current Windows bootstrap passed clean HTTPS catalog +installation and migration of an online sequence-1 fixture while preserving +preferences and cache data. [Online evidence](evidence/qwen-catalog-online-20260906.json). +The former signer could not be recovered; the replacement has three verified +backups and an explicit application-delivered trust-root migration. See +[CATALOG_SIGNING_KEY.md](CATALOG_SIGNING_KEY.md). Publication alone cannot update +old software that trusts only the former key. + +Consumer NVIDIA RTX 30/40/50-series PCs are the target population. L4 and T4 are +test hardware. GPU generation alone does not establish support for a particular +quantized kernel, driver, memory budget, or runtime profile. + +## What actually exists in the code + +| Mechanism | Implemented behavior | Remaining product work | +| --- | --- | --- | +| Desktop `auto` | [ModelManager](../src/drift/node/model_manager.py) requires catalog eligibility for community selection and falls back to verified standalone Qwen. Exact selectors and active requests remain pinned; local-only mode blocks new community requests. Assigned mixed-route packaged tests and the bounded autonomous CPU desktop test passed. | Carry fixes into final packages and broaden hardware/conversation qualification. | +| Strict eligibility | [model_selection.py](../src/drift/node/model_selection.py) enforces freshness, soak, replicas, independent routes, surviving coverage, latency and throughput using real probes. All six clients promoted under unchanged signed sequence 2 in the passing N2 CPU formation test. | Broader performance qualification. Earlier failed CPU measurements remain historical failures; no thresholds were weakened. | +| Automatic contribution | [contribution_planner.py](../src/drift/node/contribution_planner.py) chooses a configured model and a contiguous under-covered span with residency/cooldown, dispersion, demand bounds and artifact budgets. Real desktops formed all 64 blocks in the passing staggered CPU test; unique spans stayed in place during slow growth. | Qualify consumer GPU contributors and simultaneous joins. Add staged growth that preserves the working lower route before activating future model families. | +| Catalog installation | [catalog_bootstrap.py](../src/drift/node/catalog_bootstrap.py) verifies signatures, compatibility, expiry and rollback state; periodic refresh stages immutable files and activates after active requests drain. Packaged HTTPS bootstrap and explicit application root migration passed a bounded live test. | Finish the ordinary-user desktop lifecycle around migration and canary disable/recovery. Network payloads cannot rotate trust roots. | +| Local fallback | [local_inference.py](../src/drift/node/local_inference.py) loads the exact verified standalone checkpoint, enforces context/token/time and CUDA allocation limits, and supports cancellation. The 0.8B GPU test stayed below its 3 GiB allocation budget. Packaged local inference alongside one automatically placed Qwen3.8 block also passed on the 8 GB card. The v9 package includes the cache-accounting fixes and passed bounded resource checks. | Broaden resource and platform observations. CPU admission estimates are not an OS memory cap. | +| Recovery | The Qwen3.8 cloud experiment preserved the original client session when one worker was replaced. | That proves recovery within one exact model. Switching model families requires new model state and a new tokenization/prefill; it cannot reuse another model's KV cache. | + +The placement simulations exercise useful anti-herding behavior; they are not a +real consumer swarm or a complete model-migration acceptance test. + +## The first 8 GB user, then a growing network + +This is the acceptance scenario to implement and demonstrate: + +1. **One person opens the app.** Select a qualified local Qwen profile that fits + the *available user budget*, including context/cache, RAM, and desktop overhead. + Do not promise that Qwen3.5 9B fits every 8 GB card. A smaller qualified profile + is preferable to exhausting memory. Checkpoint download size and resident + quantized memory are different measurements. +2. **People join and opt into sharing.** Publish bounded, authenticated capability + and placement information. Stage only the missing Qwen3.8 spans within each + person's download/storage/compute limits. While the route forms, local chat + remains usable. Local inference and contribution share one memory budget; + they need scheduling or eviction if both cannot remain resident. +3. **A complete route becomes useful.** Probe all 64 blocks and measure latency, + throughput, stability, and the declared availability policy. A PC count or a + sum of advertised VRAM is not a readiness signal. The CPU smoke test was + functional but is not a conversational performance qualification. +4. **Eligible clients move up.** New `auto` requests can choose Qwen3.8 after the + readiness window. Clients make local decisions, so there is no requirement for + a synchronized global switch. Explicit model choices and active generations + stay pinned. At a chat-turn boundary, a model change needs the retained text + history formatted and tokenized for the new model, followed by fresh prefill. + Keep the selected model visible and honor local-only/privacy preferences. +5. **More capacity arrives.** After DeepSeek support and qualification, use spare + capacity to stage it while retaining a useful Qwen route. Soak/probe the new + route, then prefer it for eligible new requests. Apply the same process to GLM. +6. **Capacity leaves.** Stop selecting an unhealthy upper route, recover an active + same-model request when possible, and offer the healthy lower/local route for + subsequent requests. Exercise downgrade, rejoin, cache reuse, and repeated + threshold crossings without repeated mass downloads. + +For the first best-effort alpha, one declared complete Qwen route plus local +fallback can be an honest availability policy. Independent spare coverage and +retained lower-route capacity become increasingly important for unattended +promotion. Do not silently lower an already signed policy to force eligibility. + +## Work to support the two larger models + +DeepSeek's official [configuration](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/raw/main/config.json) +declares `deepseek_v4`, FP4 expert weights, FP8 settings with `ue8m0` scales, +hyper-connection parameters, and compressed-attention settings. Work includes +the block/config adapter, exact tensor/shard ownership, the mixed-format loader, +residual and attention-state handling, and replay/recovery. The V3 adapter and +generic FP8 conversion are reusable foundations, not a complete V4 implementation. + +GLM's official [configuration](https://huggingface.co/zai-org/GLM-5.3-Flash/raw/main/config.json) +and [model card](https://huggingface.co/zai-org/GLM-5.3-Flash) describe a +`glm5_next` wrapper, `glm5_next_text` text tower, FP8 weights, hyper-connections, +and alternating sparse and linear attention. It needs its own block/loader +integration, recurrent and attention cache management, and worker-loss replay. +Start with text inference; image/video input is a separate scope. + +For each model: pin an exact revision and manifest, prove one real block against +the publisher/reference implementation, measure the smallest assigned unit on a +consumer GPU, then prove a full route, recovery, and packaged acquisition before +catalog activation. Reuse the successful Qwen runners rather than building a +second cloud-control framework. + +Dequantizing to BF16 can provide an initial correctness path, but can make a +single large block too large for an 8 GB contributor. Compact downloads do not +guarantee compact execution. Efficient FP4/FP8 execution, finer splitting, or +larger-memory contributors may be required for practical later rungs. Likewise, +the client-side embeddings/head and download need their own envelope for each +model; fitting worker blocks alone is insufficient. + +Treat this as a sensible target order, not a permanent ranking by model size. +Promote only when the measured quality and latency improve the actual product. +DeepSeek and GLM implementation remain after the Qwen public-alpha path. + +## Credits after the inference alpha + +The current desktop release metadata has `credits_enabled: false`; no +contribution-credit protocol or settlement ledger exists in the inspected product +code. Node identities, route metrics, and signatures provide foundations. + +The smallest useful next stage is **shadow credits**: count accepted block-token +work, issue bounded signed receipts without prompt/output content, deduplicate +retries and recovery replay, and display estimated/pending contributions. Compare +against independent work observations; do not yet grant or deny service. + +Spendable credits then need an explicit unit/pricing policy, an authoritative +auditable ledger with earning/reservation/spending/refund rules, replay and +double-spend protection, collusion/Sybil checks, account/key recovery, and bounded +outage/reconciliation behavior. A signature does not prove useful work, and the +DHT does not provide ordered balance settlement. The existing roadmap prefers +testing federated settlement; a temporary operator ledger is a separate product +decision, not an implicit architecture change. + +Buying credits or paying contributors is a further marketplace project involving +payment integration, separate buyer balances/provider earnings/promotions, +fraud handling, reconciliation, and jurisdiction/provider review. See the existing +[credit design](REVIVAL.md#identity-keys-accounting-and-credits) and +[marketplace design](REVIVAL.md#compute-marketplace-and-provider-payouts). +None of these features is established by the Qwen inference test. diff --git a/docs/DESKTOP_UNINSTALL.md b/docs/DESKTOP_UNINSTALL.md new file mode 100644 index 000000000..12def1a76 --- /dev/null +++ b/docs/DESKTOP_UNINSTALL.md @@ -0,0 +1,94 @@ +# Remove CommunityAI or reclaim its downloaded models + +The alpha installers preserve your settings, credentials and downloaded models. +Reinstalling the application can reuse them. Deleting downloads is an explicit +manual choice in this alpha; the uninstaller does not offer an automatic data +deletion checkbox. + +## Before removing the application + +1. Open **Sharing** and turn off **Start CommunityAI when I sign in**. Confirm + that the status below it says **Off**. The alpha uninstaller does not remove + an existing per-user login entry. +2. Pause sharing, then close the CommunityAI window. The installer also performs + a shutdown check and refuses maintenance if shutdown fails. +3. Choose which data to keep below. Perform any credential reset while the + installed application is still available, then uninstall. + +On Windows, use **Settings → Apps → CommunityAI → Uninstall** or its Start-menu +uninstall shortcut. On Ubuntu/Debian, run `sudo apt remove communityai`. +Package removal, including `apt purge`, does not delete files in your home +directory. + +## Keep everything for a later reinstall + +Uninstall after the steps above. Your resource preferences, model policy, API +keys, peer identity and verified downloads remain. Install the next setup or +`.deb` to use the retained state. Updates also preserve these files. + +## Delete downloaded models but keep settings + +After CommunityAI has stopped, open your home directory and review the exact +managed cache folder: + +| Platform | Default managed model cache | +| --- | --- | +| Windows | `%USERPROFILE%\.drift\node\model-cache` | +| Ubuntu/Debian | `~/.drift/node/model-cache` | + +Delete only that **model-cache** folder if you want to reclaim its disk space. +Keep its parent **node** folder and `node-config.json` to retain your settings. +This removes local model and sharing-worker downloads; using those models again +requires downloading and verifying the missing artifacts. + +If you have changed cache locations or imported an existing node configuration, +review the `cache_dir` values in `node-config.json` first. Each model or worker +may use another directory. Delete a custom directory only after checking that it +contains solely model data you want removed. The legacy `DRIFT_CACHE` location +and `~/.cache/drift` can be shared with other Drift installations. Removing the +managed folder does not clear those other locations. + +## Reset this user's CommunityAI settings and credentials + +This is a separate choice from freeing downloaded models. It discards resource +preferences, API keys, peer identity, catalog state and downloads stored under +the default node directory. Existing local API integrations will need new keys +after setup. + +First stop CommunityAI and disable login startup as described above. While it +is still installed, remove its native control credential: + +Windows PowerShell, for the default installation location: + +```powershell +& "$env:LOCALAPPDATA\Programs\CommunityAI\CommunityAI.exe" --delete-control-key +``` + +Ubuntu/Debian, as your ordinary desktop user: + +```sh +communityai --delete-control-key +``` + +Then delete the exact `%USERPROFILE%\.drift\node` folder on Windows or +`~/.drift/node` folder on Linux, after reviewing its contents. Do not delete the +whole `.drift` directory if another Drift installation uses it. If you imported +headless state, used custom paths or intentionally share this node with another +client, review that setup before deleting its state or credential. + +Uninstall the application when finished. Reinstallation starts fresh and needs +new API keys and new downloads. Normal filesystem deletion is not a secure erase +of recoverable disk blocks or backups. + +## If you already uninstalled with login startup enabled + +Reinstall in the same location, open Sharing and disable the sign-in toggle, +then uninstall again. This is the supported manual cleanup path for the alpha. +An advanced user can instead remove only the corresponding **CommunityAI** +entry from their Windows per-user Run key or +`~/.config/autostart/communityai.desktop` on Linux after checking its command. +On Linux, `XDG_CONFIG_HOME` can place the autostart file elsewhere. + +The [Gate 15 choice evidence](evidence/gate15-20260908-windows-data-login-choices.json) +records the tested scope. The separate installer lifecycle evidence proves +settings/cache retention during actual replacement and removal. diff --git a/docs/GATE13_ONE_CLICK_GCP.md b/docs/GATE13_ONE_CLICK_GCP.md new file mode 100644 index 000000000..08ad06db1 --- /dev/null +++ b/docs/GATE13_ONE_CLICK_GCP.md @@ -0,0 +1,106 @@ +# Gate 13 one-click GCP replay + +Run Gate 13 GCP.cmd is the human entry point for the complete GCP replay. +It accepts no arguments and asks no questions while it is running. + +Double-click Run Gate 13 GCP.cmd in Explorer, or run this one command from +the repository root: + + & '.\Run Gate 13 GCP.cmd' + +The command does all of the following: + +1. fails closed before provisioning unless GitHub, GCP, the network, images, + protected bootstrap, L4 quota, and exact target absence pass; +2. resolves or builds the exact Windows and Linux production artifacts from + the pushed branch HEAD, downloads their audit provenance, and binds both + the GitHub wrapper and inner archive hashes and byte counts; +3. creates the run-scoped GCP L4 route plus the DHT, IAP, and private relay + firewall rules used by the successful `gate13-20260901-a` run; +4. installs the exact retained route wheel, signed catalog, setup script, and + helper commits used by that run—no route wheel is rebuilt—and executes the + separately staged final route fence; +5. repeats the successful artifact path for each platform: the route downloads + the multi-gigabyte GitHub wrapper with `curl`, verifies wrapper and inner + archive hash and size, and exposes it on the private network at port 38081; +6. creates the clean client with the original startup script, which downloads + that wrapper from the private route relay, verifies the inner archive, and + installs the product; the relay is removed once the client is ready; +7. runs the Windows/Qwen qualification from an ordinary interactive user and + deletes its VM and disk; +8. repeats the same fence, relay, client, qualification, and deletion sequence + for Linux/Gemma from an ordinary desktop user; +9. deletes and proves absence of every run-scoped VM, disk, and all three + firewalls; and +10. writes one terminal `result.json`. + +The human does not copy URLs, issue SSH commands, click through the desktop +flow, or repair a run in flight. A failure still drives exact cleanup. If the +launcher process or computer is interrupted, the next double-click recovers +the prior run ID using that run's immutable provider-config snapshot, cleans +and verifies it, and only then begins a new run. +Provider maximum lifetimes remain a final backstop: six hours for a client and +sixteen hours for the route. + +## One-time machine prerequisites + +The Windows machine needs Python 3, Git, GitHub CLI, and Google Cloud CLI. The +exact successful-run route wheel must remain at the path pinned in +`config/gate13_gcp.json`; the launcher verifies its 389,107-byte size and +`7a42803811289e14f69835331e0fbab69dd353c70c835131c10bdfa96ca5f111` +hash before provisioning anything. +The flujo-app/CommunityAI GitHub account and the GCP account must already be +authenticated, and the current named branch HEAD must be pushed to origin. +Authentication is deliberately outside the replay because browser login is +interactive; the replay never prompts for or stores credentials. + +If GCP authentication has expired, refresh it once before double-clicking: + + gcloud auth login + +GitHub authentication, reusable GCP authentication, and the pushed-HEAD check +currently pass. The preflight stops before cloud mutation if any of those +checks later fail. + +## Expected duration + +The last successful automated host jobs took: + +- Windows: 344 seconds (5 minutes 44 seconds) +- Linux: 294 seconds (4 minutes 54 seconds) +- Both qualification jobs: 638 seconds (10 minutes 38 seconds) + +Those are the jobs themselves, after their clean VMs and route were ready. +The last successful final cloud window—from starting the Windows job through +Linux completion and exact cleanup—was about 4,081 seconds (1 hour 8 minutes). + +For this full one-click command, reserve about 90 minutes when matching +production artifacts already exist. If it must build fresh GitHub artifacts, +reserve about 2.5 hours. Network/model download variance can extend either +estimate; the command prints ongoing phase updates. + +## Result and privacy boundary + +Each run writes ignored local state under: + + .gate13-runs/gcp// + +result.json is the single pass/fail record. The two bounded client evidence +files are retained beside it. The command journal retains action names, +durations, and exit codes only. It does not retain command arguments, command +output, GitHub tokens, signed URLs, prompts, or model responses. Exactly as in +the successful run, a signed package URL is temporarily placed in the route +VM's `artifact-probe-url` metadata so the route can perform the download. It is +removed immediately after that download attempt, including on failure, and is +not retained in the result or command journal. + +The implementation is separated into: + +- gate13_cloud_orchestrator.py: provider-neutral lifecycle and cleanup order; +- gate13_gcp_provider.py: GCP provisioning plus GitHub artifact adapter; +- run_gate13_gcp.py: zero-input launcher, lock, recovery, and result display. + +That boundary is the seam for the Azure adapter and, later, the deployment +tool. Route creation, client creation, client preparation, job execution, +resource deletion, and cleanup verification are already separate provider +operations. diff --git a/docs/GATE16_CANARY.md b/docs/GATE16_CANARY.md new file mode 100644 index 000000000..8cda3d764 --- /dev/null +++ b/docs/GATE16_CANARY.md @@ -0,0 +1,253 @@ +# Gate 16: bounded alpha canary + +Gate 16 is **open**. The September 7 local preflight establishes useful safety +prerequisites; it does not establish a monitored public canary. Use the qualified +installers and exact manifest/catalog identities from +[release readiness](RELEASE_READINESS.md). Signing is owner-deferred after alpha. + +September 8 scope update: the owner deferred additional conversation/performance +measurements and frozen periodic catalog-update qualification after alpha. The +catalog withdrawal/restore phases below are retained as beta procedures, not +current alpha blockers. Existing real worker-loss, fallback, rejoin, formation +and shutdown evidence must be reused before planning further work. The owner has +asked what a new integrated run adds; Gate 16's final alpha scope is under review. +No combined public canary is claimed to have passed. This scope update takes +precedence over the original full-run acceptance wording below. + +## Reproducible local prerequisites + +Run these from the repository root in the maintained Python environment. They do +not require model downloads or a GPU. The source suite includes a real loopback +TLS p2pd rejection test and signed DHT announcement round-trip, alongside isolated +admission, identity, health, selection, catalog and privacy tests. + +```powershell +$env:PYTHONPATH = "$PWD\src;$PWD\desktop\src;$PWD\tests" +python -m pytest tests/test_server_admission.py tests/test_protocol_identity.py tests/test_protocol_identity_network.py tests/test_public_worker_health.py tests/test_discovery.py tests/test_route_health.py tests/test_measured_model_selection.py tests/test_catalog_refresh.py tests/test_route_metrics.py tests/test_automatic_placement_privacy.py tests/test_gate16_catalog_drill.py -q +``` + +The frozen node probe uses a **new** output directory, empty local cache, CPU-only +local configuration, no configured peers and no contribution worker. It tests HTTP +authentication separation, malformed input, unknown-model rejection, policy +revision conflicts, mode persistence and disposable-key revocation. It sends no +valid inference request. Its five-second HTTP deadlines are probe limits, not +evidence for stalled generation deadlines. + +```powershell +python scripts/gate16_local_preflight.py --node .gate13-runs/gate14-release-windows-v2-output/CommunityAI/node/CommunityAI-Node.exe --expected-node-sha256 158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a --manifest public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json --output .gate13-runs/gate16-local-new-run --port 18116 +``` + +The node digest above identifies the already-qualified Windows runtime from +`76b6d84fc52342af4fd2315926b187aaa36b1378`. Replacing that artifact requires a fresh +verified digest; do not copy a digest from an unverified replacement. The output +directory retains private host-local logs and a sanitized `result.json`; only the +latter is suitable for evidence export. The runner stops only its owned process +and verifies descendant exit, then removes its three generated credential files. +Native desktop credentials are outside this headless probe. + +The signed catalog drill uses the real Qwen catalog/manifests with an ephemeral +test signing key and in-memory fetcher. It withdraws the community model, rejects +a lower-sequence rollback, and restores known-good content at a higher sequence. +It checks the custom local cache/device/timeout and resource preferences survive. +No production signing key or mirror is involved. + +## Bounded live RPC driver + +`scripts/gate16_live_rpc.py` now connects the malformed/admission cases to the +real worker RPC protocol. It defaults to **local preflight with zero network +connections**. Run it beside an already-owned worker's current public-health +file, in its maintained Python environment. Record that worker's exact direct +TCP multiaddress, manifest, served block and effective launch settings in the +private route inventory first. The health file does not identify its peer, so +that pairing is an operator prerequisite, not something the driver can prove. + +Copy `docs/gate16-rpc-policy.example.json` to the private run directory and adjust +it to the actual effective settings. The example is the maintained public-route +launcher's policy; it is not evidence of any desktop worker's configuration. +The probe requires global active capacity greater than one, a per-peer active +limit of one, training disabled, finite timeouts no greater than 60 seconds, +and enough idle time for token-bucket refill before the second stream. + +Use these variables from the recorded owned-route inventory; do not substitute +a shared bootstrap address or an arbitrary worker. Each output directory must +be new. The second command is the explicit network action. + +```powershell +$env:PYTHONPATH = "$PWD\src;$PWD\tests" +$rpcArguments = @( + 'scripts/gate16_live_rpc.py', + '--manifest', $ownedWorkerManifest, + '--expected-manifest-digest', $ownedManifestDigest, + '--worker-multiaddr', $ownedWorkerMultiaddr, + '--worker-label', 'canary-worker-a', + '--block', $ownedServedBlock, + '--health', $ownedWorkerHealth, + '--policy', $verifiedEffectivePolicy +) +python @rpcArguments --output .gate13-runs/gate16-rpc-preflight-new +python @rpcArguments --execute --output .gate13-runs/gate16-rpc-live-new +``` + +The hard envelope is one ephemeral TLS client, 20 RPC calls, 128 KiB of total +protobuf request payload, 600 seconds of operations and up to 80 seconds of +cleanup. It sends no valid inference tensor and loads no model. Relays, automatic +NAT discovery, port mapping and IPFS bootstrap are disabled. The runner checks +TLS and the exact responding peer, waits for token refill while the first idle +lease remains active, checks the second same-peer stream rejects, and observes +worker-side lease release before closing its own input producer. Each malformed +case must match its own rejection category; overload cannot count as success. +Cache-token capacity and bounded public-health counters must recover. + +The runner terminates only its own descendant client processes and separately +checks worker sessions/pushes released. Cleanup failures produce a failed +`result.json` and a nonzero exit. Raw errors, peer addresses and identities stay +out of exported JSON; retain any console logs privately. A passed RPC result is +partial evidence: valid post-probe inference, global saturation, identity churn, +GUI disclosure, full route cleanup and the other phases below remain required. + +## Isolated live catalog channel + +`scripts/gate16_catalog_channel.py` prepares an isolated channel from the real +release manifests and observes its ordinary packaged consumer refresh. It never +publishes or provisions anything. Preparation generates a distinct ephemeral +trust root, pre-signs baseline/withdrawal/restore at sequences 1/2/3, and retains +no private signing key. All phases expire after two hours. It prepares new +private node state offline with local-only mode and sharing paused; it downloads +no model and starts no process. + +```powershell +python scripts/gate16_catalog_channel.py prepare --release public-alpha/catalog-qwen-v2 --base-url $ownedCanaryHttpsBase --initial-peer $ownedBootstrapMultiaddr --run-id gate16-new --output .gate13-runs/gate16-channel-new +``` + +The HTTPS base must be an already-authorized public HTTPS path ending in `/`. +Publish **only** the generated `channel/` contents to that path using its existing +operator workflow. Keep `private-node/` local; it contains host-local configuration. +Start the qualified installed desktop against the prepared `private-node` state, +under its normal owning lifecycle and a unique native credential service/account. +Keep source or automated Qt work offscreen. The prepared custom trust root is +deliberately separate from the bundled production root; this tests subsequent +ordinary refresh, not first-install trust bootstrap. The desktop retains its +saved custom-root state when bundled-root migration is not authorized. + +The observer only reads authenticated loopback status and local signed state. +It neither starts the desktop nor creates/deletes its native credential. Bind +`$canaryConfig`, `$canaryNodeUrl` and `$canaryCredentialService` to that exact +private desktop lifecycle. The default observation deadline is 420 seconds +(maximum 900), allowing the ordinary 300-second refresh interval. + +```powershell +$channel = '.gate13-runs/gate16-channel-new' +$observeArguments = @( + 'scripts/gate16_catalog_channel.py', 'observe', '--bundle', $channel, + '--node-config', $canaryConfig, '--node-url', $canaryNodeUrl, + '--credential-service', $canaryCredentialService, + '--credential-account', 'control', '--timeout', '420' +) +python @observeArguments --phase baseline --output .gate13-runs/gate16-catalog-baseline-new +python scripts/gate16_catalog_channel.py advance-local --bundle $channel --phase withdrawal +# Publish the updated channel/catalog.signed.json through the owned HTTPS workflow. +python @observeArguments --phase withdrawal --previous .gate13-runs/gate16-catalog-baseline-new/result.json --output .gate13-runs/gate16-catalog-withdrawal-new +python scripts/gate16_catalog_channel.py advance-local --bundle $channel --phase restore +# Publish the updated channel/catalog.signed.json through the same owned workflow. +python @observeArguments --phase restore --previous .gate13-runs/gate16-catalog-withdrawal-new/result.json --output .gate13-runs/gate16-catalog-restore-new +``` + +Local phase activation requires exactly the next sequence and verifies all signed +phase hashes. Observation requires the expected installed signed catalog, a +**running** node with a later start identity after each transition, and an +authenticated in-memory `config_revision` matching the exact saved config bytes. +Updating files alone, an unrelated restart serving the older configuration, or +a stopping node cannot pass. The existing API exposes no active catalog digest; +the evidence binds saved signed policy to the node's active configuration revision. +It checks removal/restoration of automatic community priority, preserved local +preferences, no loaded model and paused workers. It does not prove active-generation +drain, inference/cache retention, GUI health or route shutdown. The lifecycle +owner must stop the private desktop and remove its disposable native credential. + +Both drivers have focused local tests, including the real handler over loopback +TLS without model allocation, false-pass regressions and cleanup-failure evidence: + +```powershell +python -m pytest tests/test_gate16_live_rpc.py tests/test_gate16_catalog_channel.py -q +``` + +Actual live prerequisites still include an owned formed route and live health +access, the recorded effective worker policy, an owned HTTPS channel with its +publication workflow, and qualified installed consumers. None is created by these +drivers, and no public Gate 16 pass is claimed by the local tests. + +## Public run envelope + +Use an explicitly identified canary route and an allowlist of its exact worker +identities. Record the installer checksums, runtime commits, manifest digest, +signed catalog digest/sequence, client OS/hardware class, and route inventory before +starting. Keep public identities/addresses in a private operational record; export +opaque roles and aggregate counts. Existing historical cloud budgets do not +authorize a new paid route. Use already-authorized resources, or obtain a fresh +bounded budget before provisioning. + +Allow 30 minutes after route formation, at most two ordinary desktop clients, +and at most two concurrent ordinary generations. Use fixed synthetic prompts, +at most 32 generated tokens per normal request, and at most 12 ordinary requests. +Reserve the last five minutes for shutdown and independent cleanup. Every operator +action must name an owned route/process/resource; do not stop a shared bootstrap. + +Record the **effective** admission and timeout values on every worker. The +maintained public-route launcher currently fixes 8 active sessions, 1 per peer, +2 new sessions/second globally with burst 4, 0.25 per peer/second with burst 1, +512 tracked peers with 300-second expiry, and 4 pending pushes. It fixes request +and session timeouts at 60 seconds, step timeout at 30 seconds, batch size 1, +512 cache tokens and 16 MiB chunks. Desktop contribution launchers may use their +own finite settings; report those actual values rather than attributing the +public-route launcher's settings to them. + +An explicit `request_timeout` is not an end-to-end generation deadline. Record +client retry count, backoff and local `local_max_seconds` separately. Cap each +canary action with an operator deadline and terminate only the exact test client +on expiry. A client-side timeout is a failed observation until worker counters +and owned process/cache release are independently checked. + +## Required observations + +| Phase | Bounded action | Required evidence | +| --- | --- | --- | +| Baseline | Open both actual installed clients. Make one local and one community request with the fixed synthetic prompt. | Local inference works; community route identity and complete coverage match the allowlist; generated-token count and latency are recorded without text. Sharing starts only after opt-in. | +| Disclosure | Read the Home and Sharing pages before enabling contribution. | Home states that computers helping may see submitted content; Sharing states that request content may be visible to the contributor or software on that machine. Do not present transport encryption as end-to-end inference confidentiality. | +| Admission | Use the live RPC driver to hold one first-message-idle inference stream, wait for token refill while its lease remains active, then attempt one additional stream from the same peer. | The second stream is rejected before cache allocation, the first expires within effective step/session timeout plus 5 seconds and active-session counters return to baseline. Separately prove one valid subsequent request succeeds. | +| Malformed input | Use the live RPC driver for one request per invalid case to one explicitly allowlisted worker: oversized inference metadata, mismatched manifest, non-dictionary metadata, non-finite allocation timeout, invalid maximum length, and disabled training RPC. Wait at least the per-peer refill period between attempts. | All invalid requests receive their specific rejection within the deadline; no worker restart or cache-capacity loss; counters remain bounded. Separately inspect bounded rejection logging for absence of raw payloads. The driver is implemented; an actual owned-route run is still required. | +| Health reconstruction | Pause one owned contributor through the real desktop, wait past its signed announcement/intent validity, then restart it. | Grid distinguishes serving coverage from joining/reserved blocks and local failures. When a unique span disappears, status becomes incomplete/unavailable and automatic selection falls back. On return, coverage reconstructs from fresh signed records; stale reservations do not count as service. Record time to disappearance and recovery. | +| Catalog withdrawal | In an isolated canary catalog channel, publish a correctly signed higher sequence retaining the approved local model/rung and withdrawing the community model/rung. Wait for an ordinary refresh and idle reconfiguration. | The consumer authenticates the new digest; automatic selection and contribution approval exclude the removed community digest. Local inference and custom settings/cache survive. Invalid signature, equivocation and lower sequence remain rejected. | +| Route disable | Pause/stop each exact canary worker, then select local-only in both canary clients. Do not rely on catalog withdrawal alone as an emergency stop. | Every owned worker tree exits; remote coverage ages out; automatic requests remain local; no local controller silently restarts the route. | +| Restore | Publish the previous known-good catalog contents under a **newer** sequence, restore only the recorded owned workers, and leave sharing disabled until deliberately resumed. | Fresh authenticated coverage is required again. No rollback guard is erased. A normal request succeeds within the stated deadline. | +| Cleanup | Exit canary clients, stop owned route processes/resources, remove disposable API/native credentials, and preserve settings/cache selected for retention. | Independent audit confirms exact resource/process absence, no remaining test credential, and retained-data sentinel/hash. Export only sanitized aggregates and evidence digests. | + +Catalog withdrawal deliberately retains older exact model selectors as manual +choices. It changes automatic selection and contribution approval; it does not +remotely revoke a user's explicit model configuration. Local-only currently +governs automatic selection as well. Emergency route disable therefore requires +stopping the **actual canary workers**, and an explicit-selector request should be +observed failing/unavailable once that exact route is gone. Independent workers +outside the owned route cannot be stopped by this drill. + +The consumer refresh interval defaults to 300 seconds and waits for active loads +and generations to finish before restarting. Record the observed delay; a public +catalog change is not an immediate kill switch. Keep the canary catalog separate +from the shared alpha channel until its withdrawal/restore sequence passes. + +## Stop conditions and evidence + +Stop the canary on any leaked credential/content, accepted malformed signed +record, unbounded queue/cache/session growth, repeated worker restarts, stuck +cleanup, unintended peer/route use, or an operator deadline. Preserve the failing +phase and sanitized reason; do not rerun over the same record and label it passed. +If an ordinary request is rejected by expected admission, record that as an +overload result and retry only once after the configured refill interval. + +The final Gate 16 record must bind each phase to the qualified installer/runtime, +give wall-clock start/end and monotonic durations, record aggregate health before +and after, include outcome and cleanup for failed attempts, and list any omitted +observation. Gate 16 passes only after every required live phase and independent +cleanup pass. Local test success or a generated protocol cannot close it. + +September 7 local evidence is in +[`gate16-20260907-local-preflight.json`](evidence/gate16-20260907-local-preflight.json). diff --git a/docs/MODEL_CATALOG_V1.md b/docs/MODEL_CATALOG_V1.md index 7b67ca96c..0f3b6291f 100644 --- a/docs/MODEL_CATALOG_V1.md +++ b/docs/MODEL_CATALOG_V1.md @@ -1,107 +1,74 @@ # Signed model catalog and elastic capacity ladder v1 -Status: strict schema, independent Ed25519 signing keys, threshold verification, -expiry, persistent rollback protection, local rung selection, bounded HTTPS fetching, -exact manifest installation, first-install node configuration, and desktop-sidecar -consumption are implemented. The model-agnostic qualification runner and an exact -bootstrap evidence pin are also implemented; Qwen3 1.7B passed full-artifact audit, -local Windows CPU parity, and selected-worker recovery. That 2025-generation checkpoint -proves the harness but is not a production-ladder candidate. The production backlog was -refreshed against official publisher releases on 2026-08-23. The dense Qwen3.5 text -adapter now has exact synthetic block, cached-decode, nested-wrapper loading, and real -local Hivemind RPC parity. The exact Qwen3.5 2B and Gemma 4 E2B manifests and the first -threshold-one alpha catalog/bootstrap are published. Trust-root rotation, periodic -catalog refresh, larger-rung migration, edge envelopes, and packaged inference remain -open. Gate 11 route operation passed through the generic product node with direct, -manifest-verified Hugging Face artifact delivery. - -`ModelManifest v1` identifies one exact checkpoint and execution profile. A model -catalog answers a separate question: which immutable manifests does one community -approve, and when is each capacity rung healthy enough to become the default for a -new request? - -The catalog is advisory and forkable. It cannot change a manifest digest, allocate a -user's GPU, move an in-flight request to another model, or prevent an installation -from subscribing to another root or selecting an exact manifest. - -## Elastic ladder - -The small-model rungs exist to bootstrap and test the network. They are not the -product destination. The default `auto` policy should advance toward progressively -larger current-generation models as independently measured network capacity becomes -sufficient. - -For a profile using `bytes_per_parameter` and two complete replicas, the weight-only -approximation is: +Reviewed: 2026-09-06. The product order is **local Qwen3.5 → Qwen3.8-27B → +DeepSeek-V4-Flash → GLM-5.3-Flash**. See +[COMMUNITY_AI_MODEL_LADDER.md](COMMUNITY_AI_MODEL_LADDER.md) for the current model +status, consumer hardware constraints, automatic-growth acceptance scenario, and +missing adapters. The superseded broad candidate inventory is retained in +[RELEASE_READINESS_HISTORY.md](RELEASE_READINESS_HISTORY.md#original-model-catalog-documentation-snapshot). + +The published [signed alpha catalog](../public-alpha/catalog-v1/catalog.signed.json) +is still sequence 1 with Qwen3.5 2B and Gemma 4 E2B. It is historical qualification +and bootstrap evidence, not an approval of the intended new ladder. Qwen3.8 now +has a pinned FP8 manifest and [real full-route/recovery evidence](QWEN_FULL_INFERENCE_RESULTS.md), +but still needs the remaining [product qualification](RELEASE_READINESS.md). +DeepSeek V4 and GLM 5.3 do not yet have DRIFT adapters or candidate manifests. +The signed [Qwen sequence 2](../public-alpha/catalog-qwen-v2/catalog.signed.json) +is now published at its separate qualification path. It adds local 0.8B and community +27B, with the explicit application trust-root migration documented in +[CATALOG_SIGNING_KEY.md](CATALOG_SIGNING_KEY.md). +Clean HTTPS catalog installation and explicit old-root migration passed through +the current Windows packaged bootstrap; ordinary-user release lifecycle and +remaining Qwen qualification are still open. [Online evidence](evidence/qwen-catalog-online-20260906.json). + +## Implemented boundaries + +`ModelManifest v1` identifies one exact checkpoint and execution profile. The +catalog separately authorizes immutable manifests and declares when each rung +may become eligible. Catalog schema validation, independent Ed25519 keys, +threshold signatures, expiry, persistent rollback protection, bounded HTTPS +fetching, exact manifest installation, and first-install desktop/node consumption +are implemented. Periodic authenticated refresh, runtime architecture checks, +preservation of user settings, immutable catalog/bootstrap files and activation +after active requests drain are now implemented. Windows/Linux packaged inference +was proven for the old Qwen/Gemma fixtures; the new local 0.8B profile also passed +offline packaged Windows GPU inference. Full packaged ladder qualification is open. + +The working-tree schema requires one primary per rung and permits optional +standbys, explicit local execution, and zero surviving replicas for a declared +best-effort policy. A different lower rung or local fallback need not be an alternative +model at the same rung. Preserve the existing signed sequence; changing allowed +models, profiles, or policies requires a newly signed sequence. + +The catalog is advisory and forkable. It cannot override a user's resource +limits, change an exact manifest selection, or move an active generation to a +different model. Download bytes, resident weights, client-side tensors, context +cache, and migration capacity are separate budgets. Total stored MoE parameters +determine weight capacity; active parameters describe token-time computation. + +## Promotion evidence and runtime integration + +The strict `select_highest_eligible_model` helper evaluates each exact manifest +against signed requirements for fresh observations, continuous stability, +minimum per-block replicas, independent complete routes, coverage after losing +the largest peer, p95 first-token latency, and generation throughput. It prefers +the highest eligible rung, then its primary over an optional standby. + +The node now calls that helper through `MeasuredModelSelector`. Bounded synthetic +generations measure first-token latency and throughput; fresh signed discovery +observations establish continuous coverage. A changed route invalidates its +measurements. New `auto` requests remain local until the declared policy passes. +The live CPU retry found a complete route but initially measured less than one +token per minute, so promotion remains unproved rather than bypassing the rule. + +Automatic contribution already scores configured models and under-covered spans, +checks exact selected-artifact budgets, and applies cooldown/residency and +anti-herding rules. Remaining integration must stage an upper route without +destroying a useful lower route and prove zero-to-complete desktop formation. +The lone-user local 0.8B backend now passes real GPU inference; live +promotion/downgrade and simultaneous contribution need qualification. Clients may converge at different +times; an active generation remains on its selected manifest. -```text -maximum parameters = usable contributed VRAM bytes / (2 * bytes_per_parameter) -``` - -Raw VRAM is not usable VRAM. Promotion also reserves capacity for local embeddings and -heads, KV caches, activations, framework overhead, churn, and graceful migration. MoE -rungs are placed by total stored parameters; active parameters describe token-time -compute and do not reduce the bytes required to keep two complete routes available. - -The current qualification backlog is below. Names link to the exact official repository -that a future manifest must pin. Estimates use total parameters and two unquantized BF16 -replicas; an FP8, INT8, or lower-bit artifact is a separate profile with its own manifest -and qualification evidence. - -| Rung by total parameters | Preferred candidate | Standby candidate | Approx. two-replica BF16 weights | -| --- | --- | --- | ---: | -| Edge, 2-5B | [`Qwen/Qwen3.5-2B`](https://huggingface.co/Qwen/Qwen3.5-2B) | [`google/gemma-4-E2B-it`](https://huggingface.co/google/gemma-4-E2B-it), 5.1B total / 2.3B effective | 9.1-20.5 GB | -| Compact, 4-8B | [`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B) | [`google/gemma-4-E4B-it`](https://huggingface.co/google/gemma-4-E4B-it), 8.0B total / 4.5B effective | 18.6-32.0 GB | -| Standard, 9-12B | [`Qwen/Qwen3.5-9B`](https://huggingface.co/Qwen/Qwen3.5-9B) | [`google/gemma-4-12B-it`](https://huggingface.co/google/gemma-4-12B-it) | 38.6-47.8 GB | -| Collective, 27-31B | [`Qwen/Qwen3.8-27B`](https://huggingface.co/Qwen/Qwen3.8-27B) | [`google/gemma-4-31B-it`](https://huggingface.co/google/gemma-4-31B-it) | 111-125 GB | -| Cluster MoE, 109-125B | [`Qwen/Qwen3.5-122B-A10B`](https://huggingface.co/Qwen/Qwen3.5-122B-A10B), about 125B total / 10B active | [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct), about 109B total / 17B active | 435-500 GB | -| Frontier MoE, 397-402B | [`Qwen/Qwen3.5-397B-A17B`](https://huggingface.co/Qwen/Qwen3.5-397B-A17B), about 403B total / 17B active | [`meta-llama/Llama-4-Maverick-17B-128E-Instruct`](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct), about 402B total / 17B active | about 1.61 TB | - -These are candidates, not published approvals. Each exact revision, tokenizer, -runtime profile, quantization, license, artifact inventory, distributed parity, -failure recovery, and edge envelope must pass qualification before its digest enters -a catalog. The Qwen3.5 through Qwen3.8 releases use `qwen3_5` or `qwen3_5_moe`, not the -implemented `qwen3` architecture. Llama 4 uses `llama4`, not the implemented dense -`llama` adapter. Both families need explicit DRIFT adapters. Gemma 4 and Gemma 4 Unified -have DRIFT adapters and focused stock-parity tests, but still need exact real-checkpoint -qualification. Llama 4 artifacts are manually gated on Hugging Face and require a -distribution and operator-access review before catalog use. - -[`Qwen/Qwen3.8-2.4T-A95B`](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) is the current -top Qwen release, with about 2.45T total and 95B active parameters. Two BF16 replicas -alone require roughly 9.8 TB. It remains a frontier preview rather than an activatable -rung because it has no comparable standby, uses the separate `qwen3.8-max` license, and -needs the `qwen3_5_moe_text` adapter plus qualification. Qwen3.5 0.8B may be used for -adapter bring-up but is not a selectable production rung. - -Each rung contains exactly one primary and at least one standby. The standby is an -approved replacement, not a requirement to keep both choices resident in volunteer -VRAM. Hosting two alternatives with two replicas each would double the capacity -requirement and fragment coverage. - -## Promotion evidence - -The selector uses observations for exact manifest digests. It examines the minimum -coverage across all blocks rather than summing advertised VRAM. A model is eligible -only when it simultaneously meets its signed rung policy: - -- minimum bottleneck replicas across every block; -- minimum independent complete routes; -- minimum surviving coverage after removing the largest peer; -- a continuous stability soak; -- a fresh observation window; -- maximum measured p95 time to first token; and -- minimum measured generation throughput. - -The highest eligible rung wins, with its primary preferred over its standby. If no -model in a higher rung qualifies, selection remains on the highest lower rung with -complete evidence. Missing or stale evidence never promotes a model. - -The selector only answers which exact manifest a new `auto` request should use. The -promotion controller still needs to preannounce demand, download and verify artifacts, -establish two independent routes, soak them, atomically update the default alias, and -retain the previous rung as a fallback. Explicit manifest requests and in-flight -requests remain pinned. ## Catalog trust versus artifact delivery @@ -218,20 +185,16 @@ the required number of distinct signatures is present. ## Remaining integration work -1. Publish the Qwen3.5 2B and Gemma 4 E2B Windows/Linux Gate 9 acquisition records and - steady-state edge envelopes through the direct manifested-artifact path in - [`EDGE_RESOURCE_ENVELOPE_RUNBOOK.md`](EDGE_RESOURCE_ENVELOPE_RUNBOOK.md). -2. Preserve the published threshold-one alpha catalog/bootstrap and migrate its - branch-scoped HTTPS mirror only through a newly signed sequence and packaged bootstrap - before deleting the branch. Multiple interchangeable mirrors and independently - operated seeds are post-alpha hardening. -3. Bundle that bootstrap and pass the clean-install packaged inference gate. The - implemented sidecar consumer fetches manifests, verifies their digest against the - catalog, and registers them without trusting catalog display metadata. -4. Extend placement evidence with selected-shard byte cost and verified cache affinity while - preserving user bandwidth/storage ceilings and the existing anti-herding margins. -5. Reconstruct capacity observations from authenticated DHT records and completed route - probes rather than accepting a central capacity total; then add staged promotion, - fallback/downgrade drills, and deterministic churn simulation. -6. Design and validate signed trust-root rotation before the public root has multiple - independent maintainers. +1. Qualify packaged Qwen3.8 and exact local Qwen fallback profiles, including + resource envelopes, acquisition/cache reuse, correctness, and recovery. +2. Publish a new signed sequence after acceptance; preserve the historical + Qwen/Gemma catalog and its existing mirror until migration is supported. +3. Add authenticated refresh for existing installations without overwriting + user contribution/privacy settings or disrupting active generations. +4. Feed authenticated route observations and completed probes into the strict + eligibility policy used by the actual node `auto` path. +5. Prove staged local-to-Qwen promotion, lower-route retention, downgrade, + same-model recovery, and repeated growth/churn through real desktops. +6. After the Qwen alpha, add and qualify DeepSeek-V4 and GLM-5.3 adapters before + publishing their manifests. Independent trust-root governance/rotation and + mirror/seed redundancy remain separate post-alpha hardening. diff --git a/docs/MODEL_MANIFEST_V1.md b/docs/MODEL_MANIFEST_V1.md index 21eb6ad58..618258251 100644 --- a/docs/MODEL_MANIFEST_V1.md +++ b/docs/MODEL_MANIFEST_V1.md @@ -95,8 +95,10 @@ branches and tags are deliberately invalid. Names and aliases are display and AP metadata only; they never select a swarm. V1 supports the current `hidden-states-v1` tensor boundary, protocol version 1, -`float32`/`float16`/`bfloat16`, `none`/`int8`/`nf4` quantization, and -`auto`/`eager`/`sdpa` attention selection. `adapter_profile` is either `none` or a +`float32`/`float16`/`bfloat16`, `none`/`int8`/`nf4`/`fp8_dequant` quantization, and +`auto`/`eager`/`sdpa` attention selection. `fp8_dequant` identifies an official +fine-grained FP8 source checkpoint whose selected blocks are dequantized to the declared +runtime dtype during loading; it is distinct from native FP8 execution. `adapter_profile` is either `none` or a `sha256:` reference. The parser reserves digest references now, but this release refuses to execute them until adapter manifests can pin and verify every adapter artifact. diff --git a/docs/NODE_CONFIG_V1.md b/docs/NODE_CONFIG_V1.md index 4520ece30..2fba1f68d 100644 --- a/docs/NODE_CONFIG_V1.md +++ b/docs/NODE_CONFIG_V1.md @@ -32,6 +32,7 @@ registration does not download tokenizers or client-side weights. "denied_models": [], "max_disk_space": "20GiB", "max_vram": "50%", + "max_processing_percent": 100, "max_bandwidth_mbps": 25, "max_power_watts": 180, "pause_timeout": 10, @@ -94,6 +95,21 @@ allow/prefer/deny selector must resolve to a configured exact model. A nonempty happens before worker launch, so changing between a name, alias, or manifest digest cannot bypass policy. +`max_processing_percent` is a finite value from 1 to 100, defaulting to 100 for +older configs. Below 100, contribution runtimes synchronize device work and add +cooldown between steps. Capped workers from one node share a lock across compute +and cooldown, so their budgets do not multiply with worker count. Pause/shutdown +interrupts waits. This limits compute duty cycle; it does not promise a flat +instantaneous utilization percentage, and excludes downloads, model loading, +local inference and other applications. At 100 no pacing or shared lock is added. + +Fresh desktop catalog installs set both VRAM and processing budgets to 100% and +leave sharing disabled. Existing settings are preserved during catalog refresh. +The desktop's Apply limits action pauses every worker before the revision-bound +policy transaction, then resumes only previously selected workers after success. +Failed persistence never resumes sharing with the old budget. Sliders cover +1–100%; use Pause sharing to stop completely. + For accelerator workers, an enabled policy also requires a finite `max_vram`. The value is either an absolute byte size such as `8GiB` or a percentage of the selected accelerator's usable memory such as `50%`. A worker inherits that ceiling @@ -141,6 +157,12 @@ duplicate manifest paths, empty peer sets, and invalid resource limits. Every manifest is loaded and runtime-validated at startup. Names, aliases, and manifest digests must be unique case-insensitively across the entire node. +Ordinary desktop API keys and bootstrap/control credentials are generated from +32 cryptographically random bytes. When importing an advanced headless +`--api-key`, supply an independently generated token of at least that entropy; +do not use a human password or a short memorable string. The API-key store +hashes opaque bearer tokens, and hashing cannot strengthen a weak imported key. + Provider tokens, local API keys, control credentials, and identity private material are deliberately absent from this format. A Hugging Face token may currently be supplied with the process secret mechanism or the existing `--token` compatibility diff --git a/docs/PACKAGED_ALPHA_OPERATIONS.md b/docs/PACKAGED_ALPHA_OPERATIONS.md index 3b3b84bf5..d3b3b6371 100644 --- a/docs/PACKAGED_ALPHA_OPERATIONS.md +++ b/docs/PACKAGED_ALPHA_OPERATIONS.md @@ -4,9 +4,16 @@ This runbook defines the Gate 13 clean-host lifecycle for the unsigned Community public-alpha packages. It applies to Windows and Linux. It does not apply to macOS, does not test credits, and does not authorize cloud creation. -Passing the controller tests in this repository does **not** pass Gate 13. Gate 13 -requires one complete real packaged lifecycle on each supported platform against the -published signed bootstrap and a live product route. +Passing the controller tests in this repository does **not** constitute a fresh live +qualification. A replay requires the exact packaged desktop on each supported platform +against the published signed bootstrap and a live product route. + +Gate 13 and Gate 15 are now separate release gates. Gate 13 covers verified package +startup, real-window inference, sharing-policy editing, Start, full application restart, +automatic sharing resume, Pause, and post-restart inference. Manual replacement, +retain/delete uninstall choices, and retained-data reinstall are Gate 15. The older +16-phase contract later in this document remains a useful combined Gate 13/15 release +exercise; it is not the shortest Gate 13 replay. ## Release boundary @@ -29,6 +36,16 @@ catalog, and model transport must remain separate. ## Required inputs +Build operators must budget for both the unpacked runtime and its install archive. +The current Windows bundle occupies about 4.5 GB unpacked and 2.7 GB compressed; +each retained build therefore uses about 7.2 GB before model caches. The builder +checks available space before compilation, allowing 8 GiB for outputs, 5 GiB for +staging, and 2 GiB reserve per affected volume. `--output-root` and `--build-root` +can place both on a larger disk. This is an estimate, not a filesystem reservation. +Retain qualification receipts and the packages needed for current/replacement +tests; review superseded build directories before starting another rebuild. Model +caches and publisher-key backups have separate retention requirements. + Resolve these before touching a clean host: 1. The exact Windows and Linux artifacts from one successful production desktop @@ -41,6 +58,9 @@ Resolve these before touching a clean host: 7. One privacy-safe run ID for each host. 8. A copy of [gate13_packaged_lifecycle.py](../scripts/gate13_packaged_lifecycle.py). +9. A copy of + [gate13_automated_playthrough.py](../scripts/gate13_automated_playthrough.py) + when replaying the current Gate 13 boundary. The controller is a standard-library qualification tool. It may be copied separately to the host, but it does not install or import CommunityAI source. The product runtime @@ -48,7 +68,118 @@ must consist only of the unpacked release executables. A source checkout, editab install, repository PYTHONPATH, developer virtual environment, or invocation of python -m drift invalidates the run. -## Evidence contract +## Current automated Gate 13 replay + +The production desktop contains a hidden qualification mode that drives the real Qt +window. It does not call the controller in place of UI actions. The first process opens +the normal window, verifies the exact selected route, performs one localhost inference, +opens and saves **Edit sharing limits**, clicks **Start sharing**, and observes the +selected worker running. The process then exits normally so the desktop-owned node is +stopped. A second fresh desktop process proves sharing resumed after restart, clicks +**Pause sharing**, proves the worker stopped, and performs another localhost inference. + +The replay preserves the literal manual control order. It opens **Sharing** before using +page-scoped controls. After saving policy, it checks whether automatic placement already +enabled the selected model. If so, it clicks the exact checked **Share compute with +<model>** control to restore a paused baseline before exercising literal **Start +sharing** and **Pause sharing**. This normalization prevents policy-save reconciliation +from bypassing the manual Start step. If the first inference reports `Model unavailable`, +the replay polls the exact model in `/v1/models` every five seconds for at most 90 seconds +and retries the same `model:auto`, one-token request once. + +Each inference creates one in-memory temporary client key, retains only completion and +token counts, revokes the key, and proves the active-key baseline was restored. It requests +and requires exactly one generated token, matching the manual Gate 13 procedure. Session +timeouts are bounded to one hour each. The outer runner verifies the production archive +digest and byte size, runs the four packaged self-tests, executes both window sessions, +validates their strict privacy-safe evidence, and removes its exact run-scoped temporary +root. + +Prepare one absolute-path config beside the staged runner. `work_root` must not exist and +its leaf must be exactly `.gate13-playthrough-`: + +~~~json +{ + "schema_version": 1, + "run_id": "gate13-replay-a", + "platform": "windows", + "source_commit": "<40 lowercase hex>", + "package_archive": "", + "package_sha256": "sha256:<64 lowercase hex>", + "package_bytes": 1, + "desktop_executable": "", + "work_root": "/.gate13-playthrough-gate13-replay-a", + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "total_blocks": 24, + "policy": { + "sharing_enabled": true, + "allowed_models": ["Qwen3.5 2B"], + "preferred_models": ["Qwen3.5 2B"], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": null, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [{ + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59" + }] + } + }, + "session_timeout_seconds": 3600, + "inference_timeout_seconds": 600 +} +~~~ + +Run it as the ordinary qualification user with tracing disabled: + +This policy is the exact CPU-host policy proven by the manual Gate 13 playthrough: +the power field stays blank because the e2 hosts have no power telemetry, while the +storage, memory, bandwidth, pause, and explicit UTC schedule fields are exercised. + +~~~text +python gate13_automated_playthrough.py --config gate13-windows-run.json > gate13-windows-evidence.json +~~~ + +The replay is desktop automation, not a headless smoke test. On Windows, provision with +[gate13_windows_client_startup.ps1](../scripts/gate13_windows_client_startup.ps1), wait +for the ordinary `M` account to own a real console session, and let the privileged host +adapter register the bound task with `Interactive` logon and `Limited` run level. `S4U`, +service-session, and SSH-session launches are invalid because Qt may start without an +actual user desktop or access to that user's Credential Manager. + +On Linux, provision with +[gate13_linux_client_startup.sh](../scripts/gate13_linux_client_startup.sh). It installs +the package's complete XCB runtime closure, starts a TCP-disabled Xvfb display, and +prepares the ordinary `gate13` account. The host adapter runs the replay inside a private +`dbus-run-session`, starts GNOME Keyring's Secret Service, and passes only the fixed +display, home, and runtime-directory values into the bounded service. Do not substitute +`QT_QPA_PLATFORM=offscreen`: the qualification requires two real X11 windows and the +same native credential session across restart. + +Use `platform: linux` and the exact Linux executable/archive for Linux. The durable +Gate 13 host-job adapter accepts this Python entrypoint on both platforms, binds the +config and source commit, and validates the aggregate before collection. A cloud replay +still requires a fresh cost authorization, route acceptance, exact clean clients, and +provider cleanup; prior Gate 13 reservations must not be reused. + +[Paid-cloud run `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) +proves this automation from production packages without manual UI recovery. Windows/Qwen +passed two real-window sessions in 260.828 and 66.328 seconds; Linux/Gemma passed in +229.270 and 44.956 seconds, including automatic sharing resume and the second inference. +Both formal client jobs passed as attempt ordinal 1. The route preflight now allows a +bounded 180 seconds for GPU service actions and keeps polling when a stale complete DHT +advertisement expires during restart. On a fresh Linux host, `LoadState=not-found` is +accepted with systemd's exact field set even though Ubuntu omits `ExecStart`; loaded units +still require the complete strict binding. All run resources were deleted after evidence +collection, L4 usage returned to zero, and the protected bootstrap remained running. + +## Combined 16-phase Gate 13/15 evidence contract Platform startup scripts perform product actions and write one local JSON phase result after each action. After final cleanup they place the ordered phase objects in one @@ -487,7 +618,7 @@ Always finish exact product cleanup. A failed action is not permission to leave worker, node, desktop process, persistent test data, credential, or phase temporary behind. -## Publication checklist +## Combined 16-phase publication checklist A Gate 13 evidence record is publishable only when: @@ -500,5 +631,8 @@ A Gate 13 evidence record is publishable only when: checksum. Archive the Windows and Linux records separately, then aggregate their bounded facts in -release readiness. Do not mark Gate 13 passed from controller unit tests, build-job -smokes, or only one supported platform. +release readiness. For a current-scope Gate 13 replay, the automated aggregate replaces +the combined 16-phase record only for the open/infer/share/restart/resume/pause boundary; +Gate 15 still requires separate replacement and uninstall evidence. Do not claim a fresh +live qualification from controller tests, build-job smokes, or only one supported +platform. diff --git a/docs/QUALIFICATION_RUNNER_OPERATIONS.md b/docs/QUALIFICATION_RUNNER_OPERATIONS.md index 35733d173..4176fc9d1 100644 --- a/docs/QUALIFICATION_RUNNER_OPERATIONS.md +++ b/docs/QUALIFICATION_RUNNER_OPERATIONS.md @@ -12,6 +12,16 @@ register one physical or virtual machine under multiple opaque machine identitie Deferred macOS qualification is a separate operation that requires distinct `macos-cpu` and `macos-mps` hosts and does not gate the public alpha. +## GCP login in this workspace + +The owner completes Google Cloud authentication in a specific browser profile. +**Agents must not run `gcloud auth login`, open a replacement login flow, or +restart a pending login session.** Verify access with a read-only provider query. +If reauthentication is required, let the owner handle sign-in, then recheck access. +Overlapping browser flows can produce a callback state mismatch. Existing test +controllers may resume with refreshed credentials; inspect their progress before +restarting any harness or provisioning another run. + ## Combined-cloud cost guard Run `scripts/qualification_cost_guard.py` before any new GCP or Fly resource is diff --git a/docs/QWEN_DESKTOP_PRODUCT_RESULTS.md b/docs/QWEN_DESKTOP_PRODUCT_RESULTS.md new file mode 100644 index 000000000..9ad7f9219 --- /dev/null +++ b/docs/QWEN_DESKTOP_PRODUCT_RESULTS.md @@ -0,0 +1,407 @@ +# Qwen desktop product work + +Updated 2026-09-06. This report separates implemented behavior from live acceptance. +The earlier [full Qwen cloud results](QWEN_FULL_INFERENCE_RESULTS.md) remain valid +for their recorded source/profile and assigned-span topology. + +The September 7 [frozen Windows catalog startup replay](evidence/qwen-catalog-desktop-20260907.md) +now observes automatic signed sequence-1 to sequence-2 migration through the real +ordinary-user desktop, preserved resource preferences/cache, a normal restart, +old-root rejection and native-credential/process cleanup. It covers startup +migration; periodic newer-catalog activation during an active generation and the +Linux update observation remain separate. The final +[Gate 14 resource matrix](evidence/gate14-20260907-final-resource-acceptance.md) +supersedes the earlier resource-control limitations recorded below for its exact +Windows/Linux runtime and hardware scope. + +## Implemented + +- Verified standalone Qwen3.5-0.8B BF16/eager backend, selected by `auto` when the + approved community route is unavailable or ineligible. Downloads are pinned to + the manifest, revision, byte counts and hashes. No remote model code executes. +- Local-only preference in the desktop and authenticated control API. Changing it + applies to new requests; existing generations retain their model lease. +- Local context/token/time admission and CUDA allocation limits; disconnect + cancellation holds the lease until the generation thread has stopped. +- Measured community selection: synthetic token generation, route freshness, + continuous soak, replicas, independent routes, surviving coverage and catalog + latency/throughput thresholds. Complete block coverage alone is insufficient. +- A separate readiness observer now samples discovery while a long generation or + probe is busy. Previously, the observation gap could reset continuous soak even + with fresh discovery, making eligibility depend on status polling. The regression + failed before this change; 109 focused selector/API/download/lifecycle tests + passed afterward. Actual missing or stale coverage still removes eligibility. +- Chat accepts an optional strict boolean `enable_thinking`. Passing `false` to + `/v1/chat/completions` selects the verified tokenizer's short-answer mode; omitting + it preserves that model's template default. The local source node answered + `Paris\n` in four completion tokens with this option. This is not yet evidence + of packaged community chat. Invalid boolean values return HTTP 422. +- Periodic authenticated catalog refresh with rollback/equivocation protection, + runtime architecture checks, immutable files, preserved user preferences and + activation after active requests drain. An application bundle may explicitly + authorize replacing an exact former trust root; network catalogs cannot. +- Authenticated discovery now reconciles overlapping signed-announcement renewal + generations per peer before advancing replay state. Each newest valid signed + span remains authoritative, avoiding fictitious missing blocks caused by DHT + keys refreshing at different times. Replay, expiry and equivocation checks remain. +- Discovery reconnects to its configured seeds when a live DHT loses all routing + peers. A live process with an empty routing table cannot publish a remote lease. + Model reads and worker announcements now also retry those validated seeds in + place, explicitly reconnect transport and clear only a successfully validated + seed's failed-query backoff. Concurrent model lookups are capped at four. + These measures preserve a loaded worker's RPC identity. A real public-seed test cleared + only its own test routing table and recovered a peer on the same protocol. + A short discovery interruption preserves an already admitted exact span only + while the last successful observation is recent, the remote lease remains + valid, and current policy/artifact admission still permits that same claim. +- Successful immutable artifact plans are cached across placement ticks, while + current policy and budgets remain checked. Loading a shard no longer requires + reopening the same pinned planning metadata every five seconds. +- Large Hub artifacts use bounded HTTP ranges and contiguous resumable partials. + Full size and SHA-256 verification still precede atomic cache promotion. Local + HTTP tests cover interruption/resume, ignored ranges and malformed/corrupt data. + Transient network failures, HTTP 429 and server errors have three bounded + attempts per range; other client errors and invalid content fail immediately. + A retry also recomputes its offset from the actual partial after an interrupted + full-body HTTP 200 response; switching back to HTTP 206 cannot skip bytes. +- Download-budget accounting now includes verified artifact snapshots and retained + partials alongside the legacy Hub cache. Shared files count once; Hub aliases + that still back a verified artifact are protected from eviction. A resumed + transfer reserves only its remaining bytes. Two regressions failed before the + change; 69 cache/manifest/range tests passed afterward on Windows. Existing + verified artifacts are preserved when the limit cannot accommodate a new + transfer. This change is included in the verified v9 Windows package. The + packaged resource test checks admission; cache eviction behavior is covered + by the separate source regressions, not a packaged eviction claim. +- Windows packaging isolates its build tools from ambient DLL paths. The initial + rebuild failed to load Qt because another program's ICU DLLs were on PATH; the + corrected package passed its actual UI and sidecar smoke checks. + +## Real local GPU acceptance + +Hardware: NVIDIA RTX 2070 SUPER, 8 GB VRAM. This is not evidence for untested +RTX 30/40/50 cards. Both runs used the verified cached checkpoint with Hugging Face +offline mode and no discovery peers, through the real authenticated localhost API. + +| Observation | Source node | Packaged Windows node | +| --- | --- | --- | +| Result | Passed | Passed | +| Eight generated tokens, including first load | 7.859 s | 13.579 s | +| Output | ` Paris.\nThe capital of France is` | Same | +| Peak CUDA reserved memory | 1,744,830,464 bytes | Same | +| Declared CUDA budget | 3 GiB | 3 GiB | +| Unauthenticated request rejected | Yes | Yes | +| Over-budget token request rejected | Yes | Yes | +| Local-only setting persisted | Yes | Yes | +| Actual stream disconnect stopped generation and released lease | Yes | Yes | +| Node stopped after test | Yes | Yes | + +The manifest is `qwen3.5-0.8b-local-bfloat16-eager.json`, digest +`sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0`, +upstream revision `2fc06364715b967f1860aea9cf38778875588b17`. +These short completions prove the local execution path, not broad answer quality. + +The first engineering package's ZIP SHA-256 is +`fa568d77cdb8c8a693beb33f63ee1f29508436d59bea7797aab55e540f980f45`. +It was unsigned, built from the working tree, and used an explicit local test +configuration. It does not prove clean installation of the new catalog. The new +sequence-2 package also passed the same real offline GPU tests, UI and sidecar +smokes, and independent archive verification. Its ZIP SHA-256 is +`fdf9ba76ed6a5ea8dad326c9656bb4da38cca955b6574264fc925cddf75980d0`. +[Local source and both package evidence](evidence/qwen-local-product-20260906.json). + +The current rebuilt Windows package also passed these real offline GPU checks: +eight tokens in 12.078 seconds, with the same output. Its verified ZIP SHA-256 is +`c95c1b1f94eba68a04ffc54b8dc17a49e425390eca053ef8f3996efdcc9dbf1d`. +This package includes the range retries and discovery changes above. It remains +an unsigned working-tree qualification build, not a published release. + +## Local inference alongside automatic sharing + +The clean source replay passed on the same 8 GB card. Automatic placement selected +Qwen3.8 block 6 under a 2 GiB worker budget while local Qwen had a 3 GiB budget. +The selected block loaded, became discoverable and served alongside local token +generation. Pause removed its entire process tree in 0.266 seconds; restart +required a new runtime-ready observation and discovery coverage for that exact +block. Both Pause process-tree checks passed, and the test node stopped. +[Source sharing evidence](evidence/qwen-sharing-source-20260906.json). + +Startup was slow: public bootstrap joins failed before retries succeeded, and +the selected block's download took about ten minutes. Existing cache contained +another block; the restart reused the newly verified block. This is one bounded +sharing case, not smooth-startup or complete Gate 14 acceptance. The matching +packaged replay **also passed** using automatically selected block 59 from the +verified existing cache. Both local generations alongside sharing succeeded; +Pause took 0.110 seconds, both whole-process-tree checks passed, and a new ready +runtime after restart was observed. [Packaged sharing evidence](evidence/qwen-sharing-packaged-20260906.json). + +A separate packaged control-API run also passed schedule, power, bandwidth and +artifact-storage admission checks. Each checked guard blocked a worker while the +other guards allowed it; local Qwen generated three tokens in every case. The +power sample was 55.24 W against a deliberately restrictive 1 W threshold; +bandwidth was 12.24 Mbps against a tiny test threshold. A 1 MiB disk budget +correctly rejected every one-block artifact set. [Resource-control evidence](evidence/qwen-resource-controls-20260906.json). +Power and bandwidth use sampled aggregate host telemetry to pause sharing. They +are not OS hard caps or traffic shapers. This test does not measure overshoot, +sustained load or automatic resumption. That v6 package only checked artifact-set +admission; it predates the cache-accounting fix above. Literal UI behavior and +Linux observations remain open. + +The v9 package also passed a **real power-pause and automatic-resumption** check. +An independent 25-second CUDA workload crossed a preselected 120 W threshold +while one automatically assigned Qwen3.8 block was ready. The first over-limit +sample was 129.724 W; 0.203 seconds later the worker was paused with no process, +and its complete prior process tree was verified gone. The highest sample through +pause was 134.938 W. After the unrelated load stopped, the worker resumed with a +new PID and fresh runtime-ready event, without changing policy or sending Start. +Local Qwen answered afterward; subsequent manual Pause/restart and final process +cleanup passed. This is one Windows threshold crossing, not a sustained power +cap, full-load peak measurement, or bandwidth-resumption qualification. +[Power recovery evidence](evidence/qwen-power-recovery-20260906.json). + +## Live product and numerical qualification + +The product exercises use four assigned 16-block workers and the real node, +local fallback and authenticated API. They do not establish automatic span +formation by consumer desktops. The earlier CPU and first four mixed product runs +used an ephemeral engineering catalog with a 180-second first-token ceiling, +30-second soak and at least one token per minute. Subsequent runs stage the exact +signed public sequence 2, with its 60-second first-token ceiling and 60-second +soak. Historical evidence retains its original policy; no threshold is relaxed. + +The first attempt passed local inference but did not promote. Signed-announcement +renewal exposed the discovery issue above. The warm retry installed verified +hashes of only `utils/dht.py` and `node/model_selection.py` on the coordinator and +set four OpenMP/MKL threads; its source receipt preserves this difference from +the original bundle. Complete coverage became stable. Its first three-token probe +took 189.713 seconds with a 71.997-second first token; the second took 199.981 +seconds with an 85.101-second first token. Both missed the engineering throughput +requirement, so local fallback correctly remained selected. The test was stopped +and all five VMs, disks and its firewall rules were verified absent. +[Failed promotion evidence](evidence/qwen-cpu-product-no-promotion-20260906.json). +The first mixed product run did promote after a real three-token probe (32.877 s +first token, 70.885 s total). A subsequent `auto` request returned Qwen3.8's +` Paris.\n` in 64.274 s. The later active-request transition assertion failed: +the harness observed the previous request's still-draining lease and changed mode +before submitting work had acquired its next model. That next request correctly +used local Qwen. The run is **failed overall**, and all owned resources were +verified absent. The harness now waits for the old lease to drain and observes +the new Qwen3.8 lease before changing mode. +[First mixed product attempt and cleanup](evidence/qwen-mixed-product-first-20260906.json). + +The next attempt failed during block 60 acquisition after a TCP reset; all owned +resources were verified absent. That failure led to the bounded range retries. +[Download failure and cleanup](evidence/qwen-product-download-failure-20260906.json). +The third attempt loaded all 64 blocks and promoted after a 74.983-second probe +with a 28.775-second first token. A subsequent Qwen3.8 completion took 131.359 +seconds. The following request used local fallback before the harness observed a +community lease, so that run also failed before worker loss. Its owned VMs, +disks, firewall rules and Azure resource group were verified absent. +[Third mixed attempt and cleanup](evidence/qwen-mixed-product-third-20260906.json). +The harness now waits up to ten minutes for fresh community readiness and records +intervening local fallback instead of assuming the preceding selection persists. +Neither latency nor freshness policy is weakened. + +The fourth mixed run's **source product sequence passed**. Local Qwen answered +before growth in 46.743 seconds; after measured readiness, Qwen3.8 answered in +61.912 seconds. Switching to local-only during an active community generation +preserved that answer (55.385 seconds), and the next request used local Qwen +(4.035 seconds). Killing the T4 worker process caused local fallback (3.678 seconds). +Restarting it with a new peer identity restored measured Qwen3.8 selection and an +identical answer (62.824 seconds). No intervening local fallback was needed before +the active-request assertion. This tests new-request downgrade and re-promotion; +the earlier CPU experiment separately proved same-session VM replacement. +[Fourth mixed source product evidence](evidence/qwen-mixed-product-fourth-20260906.json). +All owned VMs, disks, firewall rules and the Azure resource group were subsequently +verified absent after the packaged check could not complete acquisition. The overall +runner is failed because of that separate packaged result. This source bundle predates the separate +readiness observer and explicit chat option described above. + +The Windows v7 package passed build, UI/sidecar smokes and archive verification +(ZIP SHA-256 `cfbba0ab2d5c1fed6216a049b979d75019e521eb10bdb93b63e938a5b0a39fac`). +Its cold community check joined the private test route, but the 6,007,102,112-byte +client artifact transfer was too slow for the bounded cloud window and was stopped +with its partial preserved. Meanwhile, the v8 build containing the readiness +observer compiled successfully but ran out of disk while writing its ZIP. +It has no successful release-verification receipt. These interruptions leave +packaged community generation and direct Hub cold acquisition unqualified. + +The v8 retry **passed** UI/sidecar smokes, independent archive verification and +real offline local GPU acceptance. Its ZIP SHA-256 is +`fa4bb6aab41669bda6729041408e79d05ea17e94dee332ce1e37637bab3324c3`. +Eight local completion tokens took 21.109 seconds including load; the short chat +returned `Paris\n` with four completion tokens and a normal stop. Token admission, +local-only persistence, stream cancellation and node cleanup passed. The artifact +was moved back into the repository on C: and its archive digest rechecked. +[V8 package evidence](evidence/qwen-desktop-v8-20260906.json). + +The v9 Windows package, built entirely on C:, also **passed** independent release +verification and offline local GPU acceptance. ZIP SHA-256: +`ddc74b7aef1e29615458b930a03c8393a90dd5ec36ebed19528df2f7081d28a3`. +Eight completion tokens took 11.516 seconds including load; short chat returned +`Paris\n` in four tokens with a normal stop. Token limits, persisted local-only +preference, stream cancellation and process cleanup passed. Schedule, power, +bandwidth and declared storage admission each independently paused sharing while +local Qwen generated three tokens. These are sampled admission controls, not +sustained-load or OS hard-cap qualification. Exact build inputs are retained; +later builder/catalog-publisher changes are outside this package's source +snapshot. [V9 package evidence](evidence/qwen-desktop-v9-20260906.json). + +The Linux container build exposed missing Qt system libraries, then correctly +failed final verification because its base image had CPU-only Torch 2.6.0. +The required release runtime is `2.6.0+cu124`; the check remains enforced. +CI now installs that exact runtime on both platforms, includes the Qt backend +libraries, and builds/verifies against the signed Qwen sequence-2 bundle. The new +catalog directory is included in commit-bound source validation. The CUDA +dependency download timed out; its slow retry was stopped, with build output and +logs preserved on C:, to prioritize the Qwen client acquisition. That local +container attempt remains failed. The later Linux CUDA package on the existing +GCP coordinator **passed** archive/runtime, UI and sidecar verification. Its frozen +node then produced eight local CPU tokens in 29.398 seconds, answered the short +chat with `Paris`, rejected excessive token budgets, retained local-only mode and +released a cancelled stream's lease. The audit was downloaded to C: before cloud +cleanup. UI checks used Qt offscreen; native-user desktop and Linux GPU +qualification remain open. [Linux evidence](evidence/qwen-linux-v9-20260906.json). + +Subsequent builder checks reproduced intermittent Windows access errors during +atomic publication-directory replacement. The publisher now retries only Windows +access/sharing errors, at most six attempts over half a second, and preserves the +same atomic operation and failure rollback. The focused publication/builder suite +passed 62 tests, including retry exhaustion and immediate rejection of other +errors. This publisher change postdates the v9 package. + +The alternate coordinator transfer also averaged below 1 MB/s and was interrupted, +preserving its partial. It is not a direct-Hub cold pass. The live mixed runner +was given an explicit failed packaged receipt so its owned resources could be +released rather than kept idle during acquisition. A separate bounded v7 packaged +`edge-acquire` downloaded from the official Hub into an empty cache on C: and +**passed** verification of all eight client-selected artifacts (6,030,203,015 +bytes). The 6,007,102,112-byte client shard resumed three times, from retained +offsets 1,098,907,648, 1,098,907,648 and 4,362,076,160; its final SHA-256 is +`ddff1d6665a2b39f2612fce0ef955e2436724c565bfbcbc127c7ffd078b698ff`. +The observed acquisition took approximately two hours. It required no active +workers and does not itself prove generation. The verified cache is reusable by +the later package check. [Cold acquisition evidence](evidence/qwen-packaged-cold-acquisition-20260906.json). +Local builds and caches stay in project folders on C:, per owner preference. + +The next mixed run, `q38pm-20260906-070127-a792`, uses signed public sequence 2. +Its source client passed local inference before growth, measured promotion across +all 64 blocks, and three-token Qwen generation in 55.828 seconds. An active +community answer finished after switching to local-only; the next request used +local Qwen. GCP reauthentication temporarily stopped orchestration; access was +restored and the existing runner resumed without restart. + +The authentication delay exposed a source-test sequencing defect: a spontaneous +local fallback/recovery completed before the orchestrator killed the T4 worker. +This run's source **injected-loss/replacement claim is excluded**, despite the raw +host receipt saying passed. The prior fourth run has the correct event ordering. +The source harness now requires a unique challenge and acknowledgements after +the worker is confirmed stopped and its new peer identity is verified. A +regression reproduced the premature pass. Including the stop-state correction +below, 28 recovery/runner/action tests pass. +This correction postdates the fifth run's source bundle. It changes the test +harness; the v9 application binary is unchanged. +The independent v9 Windows client **passed** public-policy automatic selection, +three-token community completion (56.047 seconds) and a short chat (31 prompt +tokens, four output tokens, `Paris`, 285.828 seconds). Peak sampled process-tree +RSS was 4,280,983,552 bytes. It reused the separately verified cold client cache. +Catalog latency eligibility is measured with the current five-token, three-output +probe. It does not establish a 60-second chat latency bound; the longer chat above +shows why representative conversational measurements remain a release concern. +The subsequent outage check rejected `MainPID=0, ActiveState=failed`, although the +worker had stopped. Its finally block restarted the service and cleaned up the +Windows node. The overall attempt remains failed, and fallback/rejoin plus +offline-Hub restart were not completed. The check now accepts inactive or failed +units only with no main process and whole-cgroup stop semantics. The next bounded +retry reuses this verified package/cache. [Packaged community evidence](evidence/qwen-packaged-community-v9-20260906.json). + +The sixth run, `q38pm-20260906-081449-5fdb`, **passed source recovery under the +signed public policy**, with nonce-bound confirmations of the actual stop and +new-peer replacement. Three-token Qwen completion took 74.093 seconds initially +and 103.853 seconds after replacement; local fallback after confirmed loss took +7.647 seconds. The source result is independent of the packaged outcome below. +[Source public recovery evidence](evidence/qwen-source-public-recovery-20260906.json). + +Its Windows v9 node completed community generation in 78.016 seconds and the +short chat in 140.594 seconds. After fresh community selection, the harness +confirmed that CPU worker w2 had stopped; the client automatically selected local +Qwen and generated real tokens in 10.438 seconds. The worker restarted and all +64 blocks became visible again, but public-policy readiness was not satisfied +within ten minutes of restart. **The packaged attempt failed**; return-to-Qwen +inference and offline-Hub restart were not reached. The frozen node did not retain +per-probe timing, so this receipt does not isolate the cause of the rejection. +Both clouds' owned resources were verified absent. [Packaged partial results and +timeout](evidence/qwen-packaged-rejoin-timeout-v9-20260906.json). + +The seventh run, `q38pm-20260906-091609-4351`, **passed** with C3 high-memory CPU remainder workers, retaining +four vCPUs and 32 GB per worker, the same four spans, and the same public catalog +thresholds. C3 uses its own existing CPU quota, balanced disks and gVNIC. The +original E2 topology remains the default. Runner quota and provisioning checks, +recovery sequencing and owned-worker actions pass 33 focused tests; provisioning +is not itself inference or recovery evidence. Actual source-node generation took +14.250 seconds initially and 13.707 seconds after the confirmed T4 stop and +new-peer replacement. This is a separate observed hardware case, not a controlled +E2/C3 benchmark or a retroactive pass for the earlier E2 timeout. +The packaged restart check now also routes HTTP/HTTPS downloads through a local +rejecting proxy, in addition to setting Hub offline flags. This closes a test +gap: the custom range downloader uses direct HTTP and does not rely on the Hub +client's offline flag. A socket test confirmed rejection through both Requests +and HTTPX, including HTTPS tunnels. Local control and swarm RPC remained available. + +Windows v9 passed automatic community completion (12.250 seconds), short chat +(19.359 seconds), and fresh community selection before a confirmed CPU worker +stop. It answered locally after loss (21.000 seconds), then automatically returned +to Qwen after the same worker identity restarted and generated again (13.672 +seconds). Local-only mode also passed. The new process with HTTP downloads blocked +repeated community completion (12.438 seconds), chat (21.625 seconds) and local-only +inference (10.719 seconds), with **zero HTTP download attempts**. Both packaged +processes stopped cleanly. Peak sampled process-tree RSS was 4,973,985,792 bytes. +The package and independently acquired cache were reused; this run does not claim +a fresh acquisition or software upgrade. All owned GCP resources and the Azure +group were verified absent. [Complete C3 source and packaged evidence](evidence/qwen-packaged-recovery-v9-c3-20260906.json). + +The stock numerical reference harness compares three prompts, prefill and two +cached decode positions each, against Transformers' independent FP8 dequantizer. +It declares absolute tolerance 0.5 and relative tolerance 0.01 over every vocabulary +logit, plus identical greedy tokens. Four RPC workers share one 96 GiB CPU VM; +that numerical test is distinct from the earlier cross-cloud evidence. Its first +attempt failed before inference because parallel first artifact materialization +raced the verifier's snapshot-root initialization. The harness now binds verified +metadata serially before parallel downloads. The failed VM's cleanup was verified. +**The corrected reference run passed all nine positions**, with all vocabulary +logits within tolerance and identical greedy tokens. Maximum absolute logit error +was 0.28125. Its VM, disk and firewall rules were verified absent. +[Numerical reference evidence](evidence/qwen-reference-parity-20260906.json). + +## Catalog and remaining limits + +Signed sequence 2 is published at the separate `public-alpha/catalog-qwen-v2` +qualification path, in commit `2f79e6b774b599db5db1d87dbac8a25b847ab491` on the +existing publication branch. All six HTTPS files matched the validated bundle. +The current packaged bootstrap passed a clean network install, an actual online +sequence-1 fixture migration to the replacement root, preference/worker/cache-marker +preservation, repeat start without config mutation, and rejection of the former +root after migration. These CLI checks do not establish the ordinary-user +installation lifecycle. [Online catalog evidence](evidence/qwen-catalog-online-20260906.json). +The replacement signer has a verified Google Secret Manager recovery copy, +the retained G: backup, and an owner-restricted gitignored repository backup. +[Permanent key locator and recovery instructions](CATALOG_SIGNING_KEY.md). + +The 2026-09-07 [one-click formation test](QWEN_FORMATION_TEST.md) also **passed**: +four capacity-only CPU contributors formed 64 blocks through real Qt controls; +all six clients promoted and generated, five survivors answered locally after +whole-participant loss, and unattended same-identity restart restored six +community answers. Cleanup was verified and no runtime intervention was needed. +This used source Linux nodes/Qt windows and the retained Windows node with source +Qt UI; it does not qualify fresh installers, fully frozen UI, GPU contributors, +simultaneous joins or instant failover. [Evidence](evidence/qwen-formation-passed-20260907.json). + +Still required: carry the formation fixes into final packages; remaining Gate 14 resource controls +and supported-platform coverage; Gate 15 package lifecycle; +target hardware observations; complete desktop lifecycle around migration; canary. +The automatic worker default currently assigns one block. CPU memory admission is +an estimate, not an OS RSS cap. Download budgets account for owned Hub and manifest +cache files before writes; they are not an OS filesystem quota and do not delete +preexisting files when a user lowers a limit. One Windows local-plus-sharing budget case has passed; +it does not establish aggregate behavior for arbitrary worker counts or hardware. +DeepSeek/GLM adapters and credits remain post-alpha work. diff --git a/docs/QWEN_FORMATION_TEST.md b/docs/QWEN_FORMATION_TEST.md new file mode 100644 index 000000000..7ff2d82fb --- /dev/null +++ b/docs/QWEN_FORMATION_TEST.md @@ -0,0 +1,166 @@ +# Automatic Qwen formation test + +**PASSED on GCP, 2026-09-07:** the actual `Run Qwen Formation.cmd` run +`q38af-20260907-085527-0e997a` completed all acceptance steps and verified cleanup +with exit code 0. It formed 64/64 blocks automatically, promoted and generated on +all six clients, returned local answers on five survivors, then restored all six +community answers after an unattended same-identity restart. No live code, +policy, range, catalog or service intervention was needed. Total time including +setup and cleanup: 95 minutes. [Passing evidence](evidence/qwen-formation-passed-20260907.json). + +The four automatic spans were `16:32`, `48:64`, `32:48`, and `0:16`. Each real Qt +window passed Gate 13's policy Save, per-model Pause normalization and master +Start controls. Windows community requests took 30.707 seconds before loss and +29.949 seconds after recovery. The first validated local reply arrived 212.765 +seconds after the kill phase began, and the first validated recovered community +reply arrived 204.508 seconds after the restart phase began. These sequential +checkpoint timings include polling and inference; they are not per-client +detection latency. The scope limits below remain in force. + +![Windows desktop after unattended recovery](evidence/qwen-formation-desktop-recovered-20260907.png) + +**Modal currently cannot complete this test:** its sandbox filesystem rejected +the atomic settings exchange required by the production node. The real desktop's +policy Save returned HTTP 503; sharing remained disabled. All six clients had +already passed local inference and real-window selection checks. Both full Modal +attempts were cleaned. [Evidence](evidence/qwen-modal-formation-blocker-20260907.json). +The runner now checks this filesystem capability during image setup. The active +fallback is GCP N2 with actual remote Qt desktops as well. + +The retained **`Run Qwen Formation Modal.cmd`** runs from the same C: +checkout. It reuses the acceptance flow below and needs an already authenticated +Modal Python (override with `COMMUNITYAI_MODAL_PYTHON`). Four CPU contributors +have 32 GiB each; a fifth client/seed has 16 GiB. Each gets two physical cores +(four vCPU threads). All five run the production Qt desktop on Xvfb. The runner +clicks the real sharing-policy Save and Start sharing controls, observes every +remote window at each model transition, and retains screenshots. Windows uses +the retained frozen node with the source Qt UI. This is not a frozen Linux +installer qualification. Raw TCP tunnels carry libp2p's own TLS; worker +`public_port` advertises the external port while its listener remains on 31330. +Only capacity and reachable endpoints are supplied, never block ranges. + +Modal loss kills the complete contributor node and desktop process trees, verifies +they stopped, and restarts them on the same sandbox disk with the persisted +identity and policy. It does not claim destruction/replacement of that sandbox. +All five sandboxes have a six-hour maximum lifetime and are explicitly terminated +after diagnostic capture. Cleanup checks both sandbox exit and the app's stopped, +zero-task state. No named persistent volumes or deployed endpoints are created. +Evidence is under `.gate13-runs/qwen-formation-modal/q38mf-.../`. + +Run **`Run Qwen Formation.cmd`** from the C: checkout. It accepts no arguments and +uses `config/qwen_formation.json`. It follows the Gate 13 lifecycle: new run folder, +lock, preflight, source snapshot, owned cloud resources, desktop observations, +worker loss/recovery, diagnostic capture, verified cleanup, durable result. + +This test supplies **capacity, never block assignments**. Four `n2-highmem-4` +contributors each offer 16 blocks through the production node's `model: auto` +worker. An `e2-standard-4` coordinator provides the isolated discovery seed and +another client. This needs 20 GCP vCPUs; preflight checks existing quotas and does +not request increases. Hosts have an automatic deletion deadline within six hours. +The standing bootstrap VM is not a target. +The current profile uses N2 workers in `us-central1-b`, with 80 GB balanced disks. +C3 workers in `b`/`c` and an E2 coordinator in `f` encountered stockouts. All partial +attempts were cleaned up. The bounded profile also permits the original C3 workers +and zones `c`/`f` for fresh retries. Both worker types have four vCPUs and 32 GB RAM. + +Each cloud participant must first expose a fresh discovery observation within +180 seconds, then generate a real local Qwen3.5 answer. Unknown coverage cannot +pass that checkpoint. Contributors +then enable sharing one at a time, waiting for fresh coverage before the next +join. The production planner selects their exact ranges and publishes signed +intents. The runner requires successive 16/32/48/64-block coverage, automatic +Qwen3.8 selection, and real three-token answers on all five cloud nodes and the +Windows client. It kills one entire contributor process group, requires local +fallback and real answers on the survivors, restarts that participant with its +existing identity/policy, then requires full coverage and Qwen3.8 answers again. +No recovery step assigns a span. Catalog sequence 2 and its measured readiness +thresholds are unchanged. + +Startup follows Gate 13 literally: automatic worker startup stays enabled in the +saved config, while the initial sharing policy keeps contribution off. The real +desktop saves the policy. If that starts sharing before the explicit Start check, +the runner clicks the checked per-model sharing control to pause, waits for the +paused state, then clicks the actual master Start sharing button. This preserves +the saved startup behavior required for unattended whole-node recovery. + +The Windows session uses the retained hash-verified v9 node, existing verified +model caches, and the **real production Qt window run from source**. Qt automation +uses Gate 13's existing hook to observe selection and click the actual local-only +button. It records the resulting mode, displayed selection, and screenshots. +The cloud participants run production node source and the real Qt desktop on +Xvfb, with local inference on CPU. The runner clicks policy Save and Start +sharing, observes all five remote desktops, and retains their screenshots. +The Windows local model uses the configured RTX 2070 SUPER device. + +This is a **staggered formation** test. It does not certify a simultaneous cold +join burst, GPU contributors forming the whole model, a fresh installer, or the +fully frozen desktop UI. The packaged Windows participant is a client here; the +four cloud production nodes contribute. Keep those limits in any release claim. + +Evidence is under `.gate13-runs/qwen-formation/q38af-.../`: `result.json`, +`qualification/run-state.json`, source bundle/hash inventory, command journal, +per-checkpoint node/worker records, desktop screenshots and `cleanup.json`. +Each command has a fresh response ID. Status must be fresh. Diagnostics failures +cannot bypass cloud cleanup. A failed preflight creates no cloud resources. +GCP authentication must already work; this runner never invokes a login flow. + +`scripts/qualify_qwen_formation_local.py` exercises only the packaged local node +and real Qt controls against an empty local DHT. Its evidence explicitly says +`distributed_formation: false`; it cannot satisfy the cloud formation gate. + +## Current evidence + +On 2026-09-07 UTC, the first cloud attempt stopped before provisioning because native +GCP token refresh required owner reauthentication. After the owner renewed auth, +three attempts encountered regional stockouts; each verified complete cleanup. +The user requested Modal as the alternative. Its live raw-TCP nonce and cleanup +probe passed (`q38mt-20260907-053914-147072`). The first full Modal attempt +(`q38mf-20260907-054234-8995de`) was stopped after Qt reported a missing GLib +library; all five sandboxes and local processes were cleaned. The dependency and +a real-window image-build check are persisted in the runner. Its next replay, +`q38mf-20260907-054716-44636d`, passed all six local answers and UI observations, +then exposed the filesystem incompatibility described above. A new GCP N2 replay +uses the same remote desktop controls. Neither Modal attempt is a formation pass. + +The local desktop harness passed. Regression tests exposed and fixed fragmented placement, +sole-provider movement on an already complete route, and placement seeds based +on installation paths rather than persistent public identities. +See [the checkpoint](evidence/qwen-formation-checkpoint-20260907.json). +The GCP N2 run `q38af-20260907-060309-6fc196` then passed all six local-answer/UI +checks and the first 16-block automatic join. It exposed a further slow-growth +defect live: after 15 minutes, the first worker abandoned unique blocks 0–15 for +16–31 while the route was incomplete. Commit `f496b8d` requires net coverage gain +before abandoning unique blocks. The new slow-growth regression failed before +the fix; 56 related tests passed afterward. +[Slow-growth evidence](evidence/qwen-formation-slow-growth-20260907.json). + +The actual `.cmd` replay `q38af-20260907-064802-b6b85b` passed all six local +answers/UI checks and reached 32/64 blocks, automatically choosing `48:64` and +`0:16`. The first worker retained its unique span after 915 seconds, confirming +the slow-growth fix in this live case. The third participant still had unknown +discovery coverage; the runner crashed when comparing `null` with 32. Its live +parent stack showed an unbounded DHT readiness wait. Commit `17ffeb2` bounds that +parent wait, cleans unsuccessful starts for retry, handles unknown coverage, +and requires early discovery evidence. Both regressions failed before the fix; +72 related tests passed afterward. No runtime changes preceded this failure; +py-spy was installed afterward only for diagnosis. All owned resources and local +processes were verified cleaned. +[Discovery evidence](evidence/qwen-formation-discovery-startup-20260907.json). + +The next actual `.cmd` replay, `q38af-20260907-073105-de0f20`, formed all 64 blocks +automatically (`32:48`, `48:64`, `0:16`, `16:32`). All six clients promoted under +unchanged signed sequence 2 and returned real three-token Qwen3.8 answers. Windows +also passed local-only and return-to-Auto controls. After the complete contributor +loss, all five survivors returned local answers. The first validated fallback +answer arrived about five minutes after the kill phase began; sequential checks +do not establish each client's detection latency or instant fallback. + +Restart recovery did not pass: the restored node chose the missing `48:64` span +but stayed paused because the runner had saved `worker.enabled=false`. This was +a mismatch with Gate 13's policy-gated startup, corrected in `8d8fedf` using the +literal UI sequence above. Sixty related tests passed. An explicit failure marker +ended the wait; no worker was manually started and no live application code or +policy was changed. Owned resources and local processes were verified cleaned. +[Full formation/fallback and restart evidence](evidence/qwen-formation-restart-config-20260907.json). +The subsequent clean wrapper replay passed as recorded at the top of this page. +Keep this failed attempt separate from that unattended passing run. diff --git a/docs/QWEN_FULL_INFERENCE_GCP.md b/docs/QWEN_FULL_INFERENCE_GCP.md new file mode 100644 index 000000000..0f41422b3 --- /dev/null +++ b/docs/QWEN_FULL_INFERENCE_GCP.md @@ -0,0 +1,82 @@ +# Qwen full inference on a five-machine GCP CPU swarm + +Double-click `Run Qwen Full Inference GCP.cmd`, or run: + +```powershell +python scripts/run_qwen_full_inference_gcp.py +``` + +The Gate 13-derived runner uses its existing GCP command adapter, launcher lock, +atomic state writes and cleanup pattern. This is a source-runtime experiment. +It snapshots the current working source and binds every staged file and the +archive by SHA-256. It does not claim packaged-desktop or GPU qualification. + +The pinned model is `Qwen/Qwen3.8-27B-FP8`, revision +`017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`, with manifest +`sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4`. +The production manifest verifier acquires only each worker's selected shards +and converts FP8 weights to BF16 with the production block loader. + +| Role | Machine | Blocks | +| --- | --- | --- | +| Coordinator/client | e2-standard-4, 16 GiB RAM | Embeddings, final norm, language-model head | +| Worker 0 | e2-highmem-4, 32 GiB RAM | 0–15 | +| Worker 1 | e2-highmem-4, 32 GiB RAM | 16–31 | +| Worker 2 | e2-highmem-4, 32 GiB RAM | 32–47 | +| Worker 3 | e2-highmem-4, 32 GiB RAM | 48–63 | + +The configuration consumes 20 E2 vCPUs, five instances, five ephemeral external +IPs and five 80 GB standard persistent disks. External addresses provide public +checkpoint downloads; swarm connections use private IPs and run-specific +firewall tags. Administrative access uses IAP. The VMs have no service account. +No quota requests are made. Compute and disk usage are billed normally. + +The runner waits for all four workers to report healthy, then generates three +greedy tokens from the fixed synthetic prompt `The capital of France is` through +all 64 blocks. It opens another client session, generates one token, deletes the +active blocks 16–31 VM and its disk, creates a fresh worker in the same slot, +and continues the original session. Passing requires a new worker peer identity, +full route coverage, advancement of the same client session, and exact equality +with the uninterrupted three-token baseline. This tests cache reconstruction on +worker replacement; it does not establish parity with stock Transformers. + +Local state is retained under `.gate13-runs/qwen-full//`. `result.json` +is the terminal outcome. Route and token evidence, VM generation identities, +diagnostics, source inventory and `cleanup.json` are retained alongside it. +Only the synthetic test prompt is used. Diagnostic logs remain ignored. + +All cloud resources are run-scoped. The runner cleans up after success or failure, +checks exact VM/disk/firewall absence, and retries interrupted-run cleanup on the +next invocation. A six-hour maximum VM lifetime with deletion is a final compute +backstop; it does not remove firewalls if the local launcher is interrupted. +Read-only monitoring tolerates transport timeouts, and staged file copies retry +the same content before its SHA-256 is verified remotely. + +Read-only checks and source staging can be exercised without provisioning: + +```powershell +python scripts/run_qwen_full_inference_gcp.py --preflight-only +python -m unittest discover -s tests -p test_qwen_full_inference_gcp.py +``` + +Cleanup can also be resumed explicitly using the absolute retained run directory: + +```powershell +python scripts/run_qwen_full_inference_gcp.py --cleanup-run C:\path\to\run-directory +``` + +If the local runner is interrupted during replacement staging while the live +swarm is still intact, resume the original experiment with: + +```powershell +python scripts/run_qwen_full_inference_gcp.py --resume-replacement C:\path\to\run-directory +``` + +Resume verifies the retained archive, all VM ownership labels and generation +IDs, and the original client PID before proceeding. Exactly the middle worker +must have a new VM generation; the other machines and client must survive. +The original deadline remains in force. Resume cannot reconstruct a client +process that has already exited or a swarm that cleanup has already removed. + +Mixed GCP L4 + Azure T4 + CPU testing follows only after both CPU inference and +worker-loss recovery have passed. diff --git a/docs/QWEN_FULL_INFERENCE_RESULTS.md b/docs/QWEN_FULL_INFERENCE_RESULTS.md new file mode 100644 index 000000000..936893491 --- /dev/null +++ b/docs/QWEN_FULL_INFERENCE_RESULTS.md @@ -0,0 +1,129 @@ +# Qwen full-inference experiment — 2026-09-05 + +Complete 64-block CPU inference, same-session worker-loss recovery, and mixed +GCP L4 + Azure T4 + CPU inference all passed. All CPU test resources were verified +absent at 21:49:15 UTC; mixed provisioning began afterward. Mixed cleanup also +passed: all four GCP VMs, their disks and three firewall rules are absent, and the +dedicated Azure resource group and its resources are absent. + +Both experiments used `Qwen/Qwen3.8-27B-FP8` at revision +`017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`, with manifest +`sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4`. +Execution used FP8 dequantization to BF16 and eager attention throughout. + +The portable [CPU evidence record](evidence/qwen-cpu-full-inference-20260905.json) +and [mixed evidence record](evidence/qwen-mixed-full-inference-20260905.json) +include exact routes, session IDs, tokens, VM generations, source hashes, GPU +checks and cleanup results. + +## Passing CPU experiment + +Run: `q38-20260905-205155-ed78`, GCP `us-central1-b`. + +| Check | Observed result | +| --- | --- | +| Workers | Four e2-highmem-4 VMs; 16 blocks each; all 64 healthy | +| Client | One e2-standard-4 VM; original process PID 4839 | +| Prompt | `The capital of France is` | +| Uninterrupted output | ` Paris.\n` — `[11751, 13, 198]` | +| Uninterrupted generation | 148.612 seconds | +| Loss | Blocks 16–31 VM and its boot disk deleted | +| Replacement | New VM generation, new disk, fresh shard download, new peer identity | +| Recovery | Original session advanced from position 5 to 7 | +| Recovered output | ` Paris.\n` — exactly `[11751, 13, 198]` | +| Surviving spans | The other three peer IDs **and server session IDs** remained unchanged | +| Continuation time | 307.513 seconds after releasing the paused client, including its 180-second RPC timeout | + +At 21:37:01 UTC the client recorded the old worker's `TimeoutError`, selected +the replacement peer, and replayed six cached activation tokens through blocks +16–31. It completed the matching token sequence at 21:38:58 UTC. The continuation +time excludes VM provisioning, installation and shard loading. + +The replacement changed from peer +`QmemXP3tT2qbhQ9MvG8t2At6Dw6jH9ps82LvnX5jp1ojDk` to +`QmSgJ3yctUZgkp74NACwFsD2hTDnYEiTddW9kHH1oZh6jq`. +The client was observed as PID 4839 before replacement, during guarded resume, +and during actual replay. The test never restarted that client or its session. + +This run required a local controller resume: an SCP connection closed while +staging the replacement. The controller was stopped before cleanup removed any +surviving worker, then resumed after checking the original VM generations, +client PID and frozen archive. The successful baseline and live client remained +intact. The runner now retries identical file transfers and includes a guarded +`--resume-replacement` path. This is not a claim that the original invocation +completed without operator intervention. + +All five initial hosts and the replacement's installed server code were +SHA-256 verified against the frozen source inventory. Package versions were +retained: Python 3.12.3, PyTorch 2.6.0+cpu, Transformers 5.13.1 and Hivemind 1.1.12. +The model and manifest were unchanged throughout this experiment. + +## Passing mixed experiment + +Run: `q38m-20260905-215111-8174`. GCP hosts were in `us-central1-b`; the Azure +host was in `eastus`. The client was an additional GCP e2-standard-4 VM. + +| Blocks | Provider / machine | Verified execution device | +| --- | --- | --- | +| 0–15 | GCP g2-standard-8 | NVIDIA L4, CUDA, BF16 | +| 16–31 | Azure Standard_NC4as_T4_v3 | Tesla T4, CUDA, BF16 | +| 32–47 | GCP e2-highmem-4 | CPU, BF16 | +| 48–63 | GCP e2-highmem-4 | CPU, BF16 | + +All four spans became healthy. The complete route generated ` Paris.\n`, token +IDs `[11751, 13, 198]`, in **75.699 seconds**, completing at 22:33:05 UTC. +The route's peer identities and block ranges matched the inspected workers. +Its tokens exactly matched the CPU baseline. The mixed invocation reached this +result without a controller resume. + +Both GPUs passed actual BF16 matrix-multiplication and convolution operations +before loading shards. GPU hosts used PyTorch 2.6.0+cu124, CUDA 12.4 and NVIDIA +driver 580.173.02. Source hashes matched the frozen mixed bundle on all five +machines, including installed server code. T4 execution here does not imply +native BF16 tensor-core acceleration. + +No quota increases were requested. Azure's `Microsoft.DevTestLab` resource +provider was registered to support automatic VM shutdown; that registration +remains enabled. The run-specific shutdown schedule is inside the test resource +group and is removed with that group. + +These are short functional tests. The timings exclude setup and model loading; +they do not establish GPU performance qualification, long-context/concurrency +reliability, or logit parity with stock Transformers. The 13 runner/evidence tests, +15 focused Linux health tests, and formatting checks passed. + +## Earlier diagnostic attempt + +Run: `q38-20260905-194029-692c` in GCP `us-central1-b`. +Four `e2-highmem-4` workers served consecutive 16-block spans; one +`e2-standard-4` coordinator held the embeddings and language-model head. + +All four workers reported healthy. The prompt `The capital of France is` +produced ` Paris.\n`, token IDs `[11751, 13, 198]`, in 178.27 seconds. +The recorded route covered `[0,16)`, `[16,32)`, `[32,48)`, and `[48,64)` on four +distinct peers. This is evidence of complete source-runtime inference, not a +GPU performance qualification or a comparison with local Transformers logits. + +A second session generated the first token, then the blocks 16–31 VM and disk +were deleted. A fresh VM downloaded its shards and became healthy with a new +peer identity. The original client remained alive, but its old RPC had a +30-minute timeout. Before that timeout expired, local DNS/HTTPS/IAP failures +aborted the orchestration and initiated cleanup. This run does **not** prove +worker-loss recovery. + +The run exposed a production health-reporting bug: DHT metadata stores a bare +SHA-256 hash while the public health builder requires a `sha256:` identifier. +Correcting the boundary stopped healthy workers from repeatedly restarting. +The focused Linux health suite passed all 15 tests after the fix. The amended +source archive and individual files were hash-verified before inference; the +original archive and amendment records remain retained. + +The passing run used that corrected source from startup, a 180-second client RPC +timeout, and polling that tolerates temporary SSH transport failures. It retains +separate health evidence for every worker. + +Raw evidence and command journals remain in the ignored directory +`.gate13-runs/qwen-full//` and `.gate13-runs/qwen-mixed//`; +terminal results and cleanup verification are +written separately. A failed first result remains failed even if later cleanup +or a new experiment succeeds. diff --git a/docs/QWEN_MIXED_INFERENCE.md b/docs/QWEN_MIXED_INFERENCE.md new file mode 100644 index 000000000..68b61da1a --- /dev/null +++ b/docs/QWEN_MIXED_INFERENCE.md @@ -0,0 +1,76 @@ +# Qwen mixed-cloud inference + +For the combined source and packaged Windows C3 recovery/cache test, use +[Run Qwen Product Test](QWEN_PRODUCT_TEST.md). The runner below is the original +mixed full-inference proof. + +Run `Run Qwen Mixed Inference.cmd` or: + +```powershell +python scripts/run_qwen_mixed_inference.py +``` + +The runner refuses to provision a mixed swarm until a retained CPU run proves +complete inference, same-session recovery on a fresh worker identity, and +verified cleanup. It records the SHA-256 of the qualifying CPU result. + +| Role | Provider and VM | Blocks | +| --- | --- | --- | +| Client/coordinator | GCP e2-standard-4 | Embeddings, final norm and output head | +| Worker 0 | GCP g2-standard-8, L4 | 0–15 | +| Worker 1 | Azure Standard_NC4as_T4_v3, T4 | 16–31 | +| Worker 2 | GCP e2-highmem-4 | 32–47 | +| Worker 3 | GCP e2-highmem-4 | 48–63 | + +The mixed run uses the same pinned FP8 checkpoint and BF16 eager execution +profile as the CPU test. Actual GPU identity, memory, compute capability and +BF16 matrix/convolution execution are checked before loading model shards. +A successful kernel probe does not imply native BF16 acceleration on the T4. +The full inference must use the inspected L4, T4 and two CPU peer identities. + +For product recovery qualification, `scripts/run_qwen_product_mixed.py` also +accepts `--cpu-worker-machine-type c3-highmem-4`. This changes only the two CPU +remainder VMs: each still has four vCPUs and 32 GB. Preflight accounts for eight +C3 vCPUs, four E2 coordinator vCPUs and eight L4-host vCPUs (20 GCP vCPUs total). +C3 requires balanced disks and gVNIC, which the runner selects explicitly; +the supported disk/network restrictions are documented by +[Google](https://docs.cloud.google.com/compute/docs/general-purpose-machines#c3_series). +Existing quota must suffice; catalog thresholds are unchanged. The default and +original CPU proof continue to use E2. Preserve the selected provider profile +with each result rather than treating these CPU families as equivalent evidence. + +The source snapshot and package inventory are retained. GPU workers install +PyTorch 2.6.0 with CUDA 12.4; CPU hosts install its CPU build. GCP uses the pinned +Ubuntu image with a balanced boot disk. Both GPU hosts install Ubuntu's +version-pinned NVIDIA 580 server driver and verify it with `nvidia-smi`. +The model artifacts are acquired by the production manifest verifier. +G2 disk and image choices follow Google's +[G2 restrictions](https://docs.cloud.google.com/compute/docs/accelerator-optimized-machines#g2_limitations). + +Swarm peers advertise public addresses to cross the provider boundary. TCP +31330 ingress is restricted to the five run-specific public IPs. GCP SSH uses +IAP; Azure SSH is limited to the launching computer's observed public IP and +uses an ephemeral key retained only in the ignored run directory. Workers do +not receive cloud credentials or Hugging Face credentials. + +Preflight checks existing quota in both providers; it never requests more. +The runner registers `Microsoft.DevTestLab` if needed for VM auto-shutdown; +the provider registration remains enabled after resource cleanup. +Resources receive a unique run tag. GCP VMs have a maximum lifetime and Azure +has an automatic shutdown backstop. Normal cleanup deletes the exact owned +GCP resources and the dedicated Azure resource group, including its disks and +network resources, and verifies absence. The next invocation resumes incomplete +cleanup before creating anything. The shutdown backstop alone does not remove +Azure disks or public IPs. + +Evidence is retained under `.gate13-runs/qwen-mixed//`, including +`workers.json`, `client-result.json`, `result.json`, and `cleanup.json`. +This short functional experiment does not constitute GPU performance +qualification. An attempted run that fails a kernel probe, loading or routing +remains a failed result with its diagnostic evidence. + +Explicit cleanup: + +```powershell +python scripts/run_qwen_mixed_inference.py --cleanup-run C:\path\to\run-directory +``` diff --git a/docs/QWEN_PRODUCT_TEST.md b/docs/QWEN_PRODUCT_TEST.md new file mode 100644 index 000000000..9316fb20d --- /dev/null +++ b/docs/QWEN_PRODUCT_TEST.md @@ -0,0 +1,84 @@ +# One-command Qwen product replay + +For the new no-argument entry point following Gate 13's launcher and phase +recording pattern, use **[Run Qwen Qualification](QWEN_QUALIFICATION_RUNNER.md)**. +The configurable replay described below remains the lower-level tool. + +`Run Qwen Product Test.cmd` runs the maintained equivalent of the final C3 +experiment: source-node transitions, then the frozen Windows node/controller's +community completion/chat, CPU worker stop → local fallback → rejoin, local-only +preference, and a new process with HTTP downloads blocked. It automatically +records source/input provenance, writes the report, and waits for owned cloud +cleanup. It uses `MixedProductRun`; it does not introduce another cloud provider. + +The passing run `q38pm-20260906-091609-4351` used the older ignored wrapper. +The [audit](evidence/qwen-c3-run-provenance-audit-20260906.md) preserves that history. +The maintained launcher has local regression coverage and has revalidated the +old receipts; **it has not yet had an unchanged live cloud replay of its own**. +Neither this checkpoint nor the old v9 binary qualifies a new release build. + +## Run + +From the repository on C:, double-click `Run Qwen Product Test.cmd`. Its default +Python is `.gate13-runs/qwen-product-venv/Scripts/python.exe`; set +`COMMUNITYAI_TEST_PYTHON` to another prepared Python executable if necessary. +That environment needs the repository's desktop dependencies, `httpx` and +`psutil`. No build, cache relocation or authentication browser is started. + +The checked-in `config/qwen_product_test.json` selects the retained v9 package +and previously acquired caches on this machine. On another checkout, use +`--config C:\path\to\replay.json` with an existing package, its expected node +SHA-256, its `provenance.json`, both cache directories, the passed remote-cache +acquisition receipt, and `config/qwen_mixed_inference.json`. These local artifacts +are intentionally not committed. The entire package is checked against its +provenance, including dependencies; model artifacts are verified again by the +production node. Reused caches are explicitly reported as seeded. + +Useful commands (use the prepared Python above): + +```powershell +python scripts/run_qwen_product_test.py --validate-inputs +python scripts/run_qwen_product_test.py --preflight-only +python scripts/run_qwen_product_test.py +``` + +The first command checks only local inputs; the second also reads provider +authentication and quota. The full command creates billable test resources. +It requires an existing passed CPU inference/recovery/cleanup proof, choosing +the latest valid local proof or accepting `--cpu-proof C:\path\to\q38-run`. +It never loads a previous mixed attempt's configuration or reuses its output. + +## Boundaries and evidence + +The topology is GCP L4 + Azure T4 + two `c3-highmem-4` CPU workers and the GCP +coordinator. C3's existing quota, balanced disks and gVNIC requirements remain +enforced by the shared runner. Public catalog thresholds are unchanged. +Cloud runtime is bounded to three hours, with up to one hour for the Windows +phase and a cleanup reserve. Both providers retain the existing shutdown +backstops. Native cloud credentials are reused; **never run `gcloud auth login`** +from the harness. The persistent `communityai-bootstrap-1` is not a test target. + +Every invocation creates `.gate13-runs/qwen-product-mixed//`. +It retains the exact launcher/helper/runtime source in `launcher-source.tar.gz` +and hashes in `launcher-source.json`, the requested and observed provider +configurations, the actual packaged command/log, raw receipts, `launcher-result.json` +and `product-report.json`. Changes to recorded inputs during the replay prevent +a pass. Cloud-source hashes must match the launch snapshot. Package hashes are +checked before and after execution. Checkout HEAD is recorded separately from +the actual archived bytes; it is not substituted for package build provenance. + +The report checks both cache phases, real token responses, the pre-outage +community selection, actual worker-stop receipt, recovery acknowledgements, +provider identities and verified cleanup. Azure metadata is handled separately +from GCP metadata. Failed tests, missing receipts or source changes remain failed. +Old run receipts and reports are never rewritten. + +An interrupted run's explicit cleanup entry point remains: + +```powershell +python scripts/run_qwen_product_mixed.py --cleanup-run C:\path\to\q38pm-run +``` + +This exercise covers assigned cloud spans and a frozen node/controller contract. +Autonomous desktop formation, representative consumer hardware, ordinary-user +UI/install/upgrade/uninstall, and release qualification remain separate checks. diff --git a/docs/QWEN_QUALIFICATION_RUNNER.md b/docs/QWEN_QUALIFICATION_RUNNER.md new file mode 100644 index 000000000..32cc05fb3 --- /dev/null +++ b/docs/QWEN_QUALIFICATION_RUNNER.md @@ -0,0 +1,64 @@ +# Qwen qualification using Gate 13's runner pattern + +Double-click **`Run Qwen Qualification.cmd`** in the repository on C:. +It accepts no arguments and runs the audited Qwen source and Windows packaged +recovery/cache test with the configured L4/T4/C3 topology. It provisions billable +test resources, runs the checks, removes its owned resources, and prints a final +PASS/FAIL summary with the failed phase, reason and evidence paths when needed. +The window stays open until a key is pressed, as in Gate 13. + +The launcher uses the existing prepared Python environment. Its search follows +Gate 13: `.venv-cuda`, then the prepared Qwen environment, then the Windows Python +launcher or `python.exe`. `COMMUNITYAI_TEST_PYTHON` can explicitly select another +prepared interpreter. Paths containing spaces and exit codes 0/1/2 are tested. + +## What is reused + +| Gate 13 pattern | Qwen implementation | +| --- | --- | +| Small CMD wrapper, delayed exit-code handling, persistent result window | `Run Qwen Qualification.cmd` | +| No-argument Python entry point, fresh run ID, lock, fixed configuration, readable final banner | `scripts/run_qwen_qualification.py` | +| Atomic phase journal and final result | The existing Gate 13 `RunRecorder` and `_write_json`, with a Qwen-specific scope | +| Preflight before cloud creation | Existing GCP/Azure quota, identity, image and ownership checks, then package verification | +| Ordered route → client → cleanup execution | Existing `MixedProductRun`, with the packaged client executed inline inside its cleanup-protected lifecycle | +| Failures remain failed even when cleanup succeeds | Source, package, receipt, provenance and cleanup checks are all required | +| Preserve earlier results | Each click creates a fresh directory; no previous mixed-run marker or automatic recovery is consumed | + +The Qwen-specific changes are the already proven four-worker topology and test +sequence. It checks source selection/recovery first, then packaged community +completion/chat, a CPU-worker stop and rejoin, local-only preference, and an +HTTP-blocked cache restart. The packaged subprocess keeps its deadline and owned +process-tree shutdown. No separate background cloud controller is needed. + +This runner reuses the explicitly pinned Qwen engineering package and acquired +caches in `config/qwen_product_test.json`. Gate 13's fresh Windows/Linux CI package +qualification is a different scope. This runner does not claim to build or qualify +fresh Windows/Linux release packages. A passed CPU recovery proof remains a +prerequisite and is selected automatically from local evidence. + +## Evidence and boundaries + +The fresh directory is `.gate13-runs/qwen-product-mixed//`: + +- `qualification/run-state.json`: durable phases and failure information. +- `qualification/result.json`: final aggregate, using Gate 13's recorder format. +- `result.json`: the established raw cloud result, retained separately. +- `launcher-source.json` and `.tar.gz`: actual source bytes and input bindings. +- `launcher-result.json` and `product-report.json`: package/provenance checks and + validated source/client/recovery results. +- `command-journal.jsonl`, `packaged-client.log` and `packaged/result.json`: + command history and detailed client evidence. + +GCP/Azure resource operations, C3 quota/disk/gVNIC handling, source recovery, +catalog thresholds and cleanup come from the existing Qwen implementation. +The run stays on C: and reuses native cloud credentials. The harness never calls +`gcloud auth login`. The persistent `communityai-bootstrap-1` is protected. +Explicit cleanup after an externally interrupted run remains documented in +[the underlying product replay](QWEN_PRODUCT_TEST.md). + +The combined launcher/product/Gate 13 regression suite passed **74 tests**. +Local tests cover the ordered sequence, injected failures before and after +provisioning, interruption during the packaged step, cleanup/report failures, +old-run isolation, and the actual Windows CMD wrapper. **An unchanged live run of +this new entry point is still pending.** The earlier C3 pass remains the historical +result described in the [audit](evidence/qwen-c3-run-provenance-audit-20260906.md). diff --git a/docs/RELEASE_READINESS.md b/docs/RELEASE_READINESS.md index 0891abce9..d1eea4750 100644 --- a/docs/RELEASE_READINESS.md +++ b/docs/RELEASE_READINESS.md @@ -1,218 +1,440 @@ # Public inference alpha release readiness -Last verified: 2026-08-31 +Last reviewed: **2026-09-09**. This is the current release checklist. The former +checkpoint narratives, completed-gate detail, failed attempts, old model inventory, +and budget history are preserved in [RELEASE_READINESS_HISTORY.md](RELEASE_READINESS_HISTORY.md). +Implementation details belong in their linked runbooks and evidence records. -This is the live source of truth for public-alpha implementation. Update it whenever a -gate changes state. `docs/REVIVAL.md` defines the execution contract and long-term design; -`docs/REVIVAL_TEST_RESULTS.md` is the detailed evidence archive. +September 9 release `0.1.0-alpha.20260909.3` packages peer-owned input/output +processing for community inference and the full configured sharing budget, with +no permanent fallback reservation. Two real public-mesh requests completed with +an empty consumer cache and artifact downloads forbidden. The local Qwen fallback +remains when the mesh cannot answer. The desktop accepts zero-byte community +downloads and uses common discovery rather than a separate local tensor router. +Text-peer roles currently require operator setup through the source CLI; +automatic desktop placement of that role remains open. CPU-mesh answers in this proof took 107–158 +seconds. See [consumer evidence](evidence/text-only-mesh-consumer-20260909.md) and +[release records](evidence/text-mesh-release-20260909.json). The historical checks +below retain their original scope and do not qualify the new consumer path. ## Release definition -- Product: public community inference through the packaged localhost OpenAI-compatible - API, with optional bounded compute sharing. -- Label: public alpha. Do not describe it as a stable, production-SLO service. -- Supported platforms: Windows and Linux. -- Deferred platform: macOS, until later CPU/MPS and packaged-device testing passes. -- First catalog rung: Qwen3.5 2B primary, Gemma 4 E2B standby. -- Not included: credits, earnings, payments, payouts, or a compute marketplace. -- Availability promise: best effort. The alpha may initially depend on one CommunityAI - discovery seed and one complete candidate route, with a small fallback route and clear - unavailable/degraded states; it does not claim a production SLO. -- Minimum trust floor: pinned signed catalog, exact verified manifests/artifacts, - authenticated peer announcements and transport, finite public admission/time limits, - authoritative local contribution limits, prompt-visibility disclosure, and a tested - route/catalog disable procedure. -- Post-alpha hardening: independent route/seed/mirror redundancy, independent threshold - key holders, publisher-signed installers, authenticated automatic update/rollback, and - exhaustive malicious-load/Sybil/partition/long-soak programs. - -## Status vocabulary - -- `PASSED`: required real evidence exists and is linked. -- `IN PROGRESS`: implementation or a real gate run is underway. -- `READY`: prerequisites exist and the gate can be run. -- `WAITING`: a required predecessor has not passed; do not work around it. -- `PAUSED`: partial work exists, but the gate is outside the currently permitted sequence. -- `BLOCKED`: owner input or unavailable external state is required. -- `TODO`: not yet started. -- `DEFERRED`: explicitly outside the public-alpha scope. - -## Critical path - -Work from top to bottom while prerequisites are satisfied. Gate V and Gates 5–6 have passed. -The current mandatory sequence is **Gates 13–16 → Gate 17**. The visible -vertical slice proved real Qwen3.5 2B inference through a public GCP L4 worker, and the -strict four-profile Qwen and Gemma matrices now pass, and Gate 7 passed the generic -five-Machine provider recovery mechanism with TinyLlama. Per-model repetition of the -same provider recovery gate is not required. - -As of 2026-08-31, Gate 13 is `BLOCKED`: cleanup proved the Gate 11 product route, -its run-scoped firewall rules, and every Gate 13 client and disk absent, so the owner -explicitly authorized a cleanup-backed reset for the next run. The new combined -authorization epoch starts at USD 100 with no reservation recorded. The selected native -GCP account still requires interactive reauthentication, and every paid create still -requires a fresh exact source-bound conservative ledger reservation plus fail-closed -preflight. No later mandatory gate is unblocked until the live packaged route and -completed Gate 13 lifecycle evidence exist. - -Do not work on the post-alpha items in the deferred table while an alpha gate can progress. -Missing Docker, snapshots, local GPU hardware, or local host capacity is not an external -blocker: use authorized bounded infrastructure according to its role. GCP/local hosts -cover platform and CUDA qualification; Fly is CPU-only and covers the isolated -separate-machine recovery topology. A real gate failure justifies the smallest -implementation fix; speculative harness expansion does not replace the outcome. - -The former Gate 5 quota blocker is resolved. The [2026-08-27 quota/probe evidence](evidence/gcp-l4-quota-probe-20260827.json) -records `GPUS_ALL_REGIONS` limit `1`, and the completed [Gate 5 qualification](evidence/gate5-20260827-qwen3.5-2b-qualification.json) -again proves one-host-at-a-time L4 operation, zero post-run L4 usage, complete run-resource -absence, and the protected `communityai-bootstrap-1` still running. The cleaned Gate V and -Gate 5 runs remain in the historical ledger, but the owner explicitly reset the test-budget -epoch to USD 100 on 2026-08-27 after their cleanup was proved. Their unobserved maxima no -longer consume the new authorization; later billing should still be recorded for information. - -| Order | Gate | Status | Current evidence | Next action | -| ---: | --- | --- | --- | --- | -| 1 | Integrate the active revival branch and make its CI workflows dispatchable from the repository default branch | PASSED | [PR #8](https://github.com/flujo-app/CommunityAI/pull/8) integrated [commit `22b5598`](https://github.com/flujo-app/CommunityAI/commit/22b559836fa5a4c9b228d87a823d1c99dc3939a9) into `main` after [Check style](https://github.com/flujo-app/CommunityAI/actions/runs/32946456633), [Tests](https://github.com/flujo-app/CommunityAI/actions/runs/32946456596), and [Windows/Linux Production desktop](https://github.com/flujo-app/CommunityAI/actions/runs/32946456600) passed. [PR #22](https://github.com/flujo-app/CommunityAI/pull/22) later integrated the accumulated public-alpha path as merge commit `05fa84d`. Its default-branch [test run 33388263559](https://github.com/flujo-app/CommunityAI/actions/runs/33388263559) exposed one nondeterministic Ubuntu MLA paged-cache equivalence failure after the exact PR head had passed. Source `5d29416` isolates that cache contract from MoE expert routing while retaining separate dense/MoE block coverage; [PR #23 run 33388770828](https://github.com/flujo-app/CommunityAI/actions/runs/33388770828) passes the exact MLA test, 627-test Ubuntu suite, and 64-test Linux public-worker contracts. PR #23 merged as exact commit `c90625c`; its default-branch [Tests](https://github.com/flujo-app/CommunityAI/actions/runs/33389270215), [Check style](https://github.com/flujo-app/CommunityAI/actions/runs/33389270333), and [CodeQL](https://github.com/flujo-app/CommunityAI/actions/runs/33389269851) runs all pass. | Keep the same workflows green on follow-up PRs; they are now dispatchable from the default branch | -| 2 | Make Windows/Linux the strict public-alpha qualification matrix | PASSED | Default dispatch, exact-profile aggregation, fleet readiness, and the recovery controller now require Windows CPU/CUDA plus Linux CPU/CUDA; focused contract tests pass | Provision four distinct labelled runners and retain real exact-profile evidence; macOS remains a separate deferred gate | -| 3 | Prepare bounded provider automation and cost controls | PASSED | [PR #9](https://github.com/flujo-app/CommunityAI/pull/9) integrated [commit `1d4f7d4`](https://github.com/flujo-app/CommunityAI/commit/1d4f7d4453eb688994ce21c08e182c1ad8e63ae7) after [style](https://github.com/flujo-app/CommunityAI/actions/runs/32947541300), [tests](https://github.com/flujo-app/CommunityAI/actions/runs/32947541452), and [Windows/Linux production packaging](https://github.com/flujo-app/CommunityAI/actions/runs/32947541637) passed; the 29-test guard prices the serialized 13.5-hour G2/L4 fleet at USD 69 maximum (14-hour N1/T4 at USD 70), binds immutable OS images and hard deletion deadlines, supports split-region CUDA capacity, and excludes `communityai-bootstrap-1` from exact cleanup; provider automation remains passed and native `gcloud`/`flyctl` authentication was valid for the completed Gate 7 work | No further Gate 3 framework work. Revalidate native provider authentication, quota, and the exact ledger reservation immediately before every paid create | -| 4 | Build immutable Qwen3.5 2B and Gemma 4 E2B qualification images/snapshots | PASSED | [Gate 4 attempt `gate4-20260826-b`](evidence/gate4-20260826-b-qualification-image-build-attempt.json) passed both exact snapshot/in-image checks and published source `7660e33` with SLSA provenance and SPDX SBOM. [Qwen evidence](evidence/gate4-20260826-b-qwen3.5-2b-publication-evidence.json) binds `ghcr.io/flujo-app/communityai-qualification-qwen3.5-2b@sha256:129b96fd848b996a5e3a0c918c39c705d328e6e5010b3222a5c25ea10ab142ed` ([metadata](evidence/gate4-20260826-b-qwen3.5-2b-build-metadata.json)): 6,913,811,781 compressed bytes, 6,913,829,173 uncompressed, 9 GB rootfs. [Gemma evidence](evidence/gate4-20260826-b-gemma-4-e2b-publication-evidence.json) binds `ghcr.io/flujo-app/communityai-qualification-gemma-4-e2b@sha256:5f04eb8e923023ff05f64d13fde5b879e8990725518d4e81210b03b4b6047c6f` ([metadata](evidence/gate4-20260826-b-gemma-4-e2b-build-metadata.json)): 11,011,406,681 compressed bytes, 11,011,424,083 uncompressed, 13 GB rootfs. Both isolated builders and the complete retry network were deleted; the protected bootstrap remains. | Use these immutable digests and evidence-bound rootfs sizes for Gates 5 and 6 | -| V | Pass a visible public vertical slice: app observes a remote worker, `auto` selects a model, and inference succeeds | PASSED | [Run `gatev-20260827-a`](evidence/gate-v-20260827-a-public-vertical-slice.json) executed clean source `8200afc` against the immutable Qwen image and exact manifest on a public Linux G2/L4 worker. The [desktop evidence](evidence/gate-v-20260827-a-desktop-models.png) shows signed-catalog `auto` selection, 24/24 blocks, and one verified peer; the localhost OpenAI-compatible request returned one token through Qwen in 15.231 seconds. The real run exposed four bounded fixes, all focused tests and two independent reviews passed, every exact run resource is absent, global GPU usage returned to zero, and the protected bootstrap remains running. | Proceed to Gate 5 using the exact pushed source, revalidated one-L4 quota, immutable Qwen input, a new conservative reservation, sequential CUDA hosts, and complete cleanup evidence. | -| 5 | Qwen3.5 2B Windows/Linux CPU/CUDA qualification | PASSED | [Qualification and cleanup evidence](evidence/gate5-20260827-qwen3.5-2b-qualification.json) and the [strict aggregate](evidence/gate5-20260827-qwen3.5-2b-matrix.json) bind Windows CPU/CUDA and Linux CPU/CUDA passes to exact source `23a4078e17ed9d5ae6f31e7497bae69b83aecef6`, DRIFT `2.3.0.dev2`, Qwen revision `15852e8c16360a2fea060d615a32b45270f8a8fc`, and manifest `sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33`. Every profile proved exact artifacts, 24/24 manifested stock-token parity, selected-worker interruption, and recovery. All Gate 5 instances, disks, and perimeters are absent; L4 usage is zero; the protected bootstrap remains running. | Proceed to Gate 6 under the owner-reset USD 100 budget epoch. | -| 6 | Gemma 4 E2B Windows/Linux CPU/CUDA qualification | PASSED | [Qualification and cleanup evidence](evidence/gate6-20260827-gemma-4-e2b-qualification.json) and the [strict aggregate](evidence/gate6-20260827-gemma-4-e2b-matrix.json) bind Windows CPU/CUDA and Linux CPU/CUDA passes to exact source `a45025a3262a88df65217b630392488e8548aaaf`, DRIFT `2.3.0.dev2`, Gemma revision `3e22461f65e89153144f8adb70e3b8c2cc9845a7`, and manifest `sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd`. Every profile proved exact artifacts, 35/35 manifested stock-token parity, selected-worker interruption, and recovery. All Gate 6 instances, disks, firewall, NATs, routers, subnets, addresses, and VPC are absent; global GPU and regional L4 usage are zero; the protected bootstrap remains running. | Complete; Gate 7 subsequently passed. Proceed to Gate 9. | -| 7 | Provider-level real separate-machine recovery | PASSED | [Run `gate7-tiny-20260828-j`](evidence/gate7-20260828-tinyllama-recovery.json) ran one bootstrap and four CPU-only TinyLlama workers in Fly `gru` with two replicas per block. SIGKILL of `host-a` during generation caused rerouting to `host-c`, activation replay, same-session completion, and exact stock-token parity in 16.395 seconds. All five resources were destroyed and the token was revoked. The [recovery runbook](RECOVERY_TEST_RUNBOOK.md) records the artifact, control-plane, retry, and cleanup lessons. | Proceed to Gate 9. Repeat recovery only in the later clean-install automatic-placement product flow, not once per model. | -| 8 | Per-model duplicate separate-machine recovery | DEFERRED | Gates 5 and 6 already qualify Qwen and Gemma across the supported platform/device matrix; Gate 7 proves the model-independent provider recovery mechanism. Repeating the same Fly topology for each catalog model would test artifact transport rather than a new release property. | No public-alpha action. Model admission uses manifest/artifact and resource-envelope checks; product-level recovery is covered after automatic placement and catalog publication. | -| 9 | Publish edge resource envelopes for selectable profiles | PASSED | [Run `gate9-20260830-e`](evidence/gate9-20260830-e-edge-resource-envelopes.json) publishes all four privacy-safe acquisition and schema-v3 steady-state records at exact runtime source `ba410f7`. Qwen selected 4,571,197,320 bytes and Gemma 10,278,818,149 bytes from empty caches on both Windows Server 2022 and Ubuntu 24.04; every artifact SHA-256 passed with zero resumptions. Qwen measured load/first-token/decode at 25.896 s/1.785 s/1.392 tok/s on Windows and 17.115 s/2.973 s/0.800 tok/s on Linux, with process-tree RSS peaks of 1,883,205,632 and 2,832,244,736 bytes. Gemma measured 64.286 s/1.386 s/1.949 tok/s on Windows and 38.808 s/2.025 s/0.868 tok/s on Linux, with peaks of 1,728,995,328 and 2,818,523,136 bytes. Every workload generated eight tokens without retaining prompts or outputs; Windows Job Objects and Linux process groups were empty, and route/DHT, accelerator, runtime-close, and provider cleanup all passed. All four temporary instances and auto-delete disks are absent; the protected bootstrap and separately authorized Gate 11 route remain running. The Windows Gemma cache-preserving in-place memory retry created no new resource and did not raise the USD 46 Gate 9 ceiling. | Proceed immediately to Gate 13 clean packaged install/inference on Windows and Linux using these envelopes and the live bounded product route. | -| 10 | Implement automatic contributor model and block placement | PASSED | Signed bootstrap now installs one bounded `auto` worker. The local planner filters exact manifested candidates through owner policy and local resource ceilings, requires fresh authenticated replica coverage, targets the least-covered contiguous range with per-node jitter, reconciles exact-manifest launches through the existing artifact-verifying server and `WorkerSupervisor`, applies residency/cooldown/switch hysteresis, exposes placement reasons, and preserves an explicit operator pause across ineligibility or placement changes. A new or migrated worker must sign an expiring exact-manifest/range intent with fixed numeric resource claims and receive a remote DHT store acknowledgement (`exclude_self=True`) before entering the artifact path; invalid, rejected, or failed publication is fail-closed and cannot advance planner state, while a previously admitted placement is retained. Actual completed local generations feed exact-manifest demand, useful-throughput, and reliability through two bounded five-minute aggregate windows; no prompt, output, token ID, key, request ID, address, path, error, or per-request event is retained. Only a closed window with at least four completed routes may be signed by the separate router identity and published under the manifest-bound `demand-v1` DHT key with a 90-second lifetime and `exclude_self=True`. Consumers verify signature, exact schema/digest, lifetime, revocation, and replay ordering. The threshold-signed catalog may authorize 2–32 sorted RSA observer roots; missing or empty roots disable remote demand. Discovery discards unlisted identities before signature/replay work, excludes local and duplicate roots, isolates malformed records, requires two authorized roots, and medians at most 32 quantized observations. Observer keys are never generated or bundled: only a separately provisioned `route-demand.key` matching a signed root may publish, while ordinary nodes can consume without one. Any hot-edited root-list mismatch disables both publication and consumption until restart. Local utility is capped at 6 points and signed remote utility at 2, keeping the combined hint below the 10-point migration margin and 100-point replica step. Verified announcement and route-demand replay watermarks now survive restarts in one Windows-safe journal per raw manifest digest under the node data directory. Each strict journal is capped at 256 active identity scopes and 256 KiB, retains only public record kind, key ID, ordering tuple, record digest, and the bounded replay deadline, and is fsync-written through atomic replacement; malformed, duplicate, oversized, symlinked, non-regular, or unwritable state fails closed. The retained deadline prevents an older still-live record from returning after a short-lived newer record expires. The replay slice's 99-test focused protocol/discovery/planner/node-configuration matrix and 209-pass, 2-skip catalog/node/API superset pass. The Sybil slice's 122-test focused catalog/bootstrap/config/discovery matrix proves that 30 valid attacker keys plus one authorized root cannot reach threshold, two authorized roots aggregate without attacker weight, one high authorized vote cannot inflate a lower second vote, old catalogs remain signature-verifiable with remote demand disabled, and trust-epoch reload mismatches fail closed. A 190-pass, 1-skip catalog/protocol/planner/discovery/node/API superset also passes. Independent verification passed 146 focused tests and a 255-pass, 2-skip broader node/API superset, plus a native-Windows publication-boundary probe; formatting, import-order, import-smoke, and diff checks pass. The [explicit privacy review](AUTOMATIC_PLACEMENT_PRIVACY_V1.md) inventories collection, retention, public-key linkability, DHT/journal/API/log exposure, secure-deletion limits, and residual governance/host risks. Three executable privacy-contract tests fix the aggregate, intent, demand, replay, forbidden-field, and path-free warning schemas; the focused privacy/protocol/planner/discovery/node matrix passes 108 tests and the broader catalog/node/API matrix passes 258 tests with 2 skips. Independent privacy review passed 108 tests with 1 skip and a 225-pass, 2-skip broader subset; every caught observer-key exception and an unauthorized key produced no path, key ID, or exception detail, while prompt and identity-path schema injections failed closed. The [deterministic convergence and load acceptance](AUTOMATIC_PLACEMENT_ACCEPTANCE_V1.md) closes the remaining software gate: equal snapshots use node-specific 32-point model dispersion and range rendezvous ranks; a fixed 512-node cold cohort selects both models and every range below the 85% concentration boundary; two 4,096-node fresh-arrival cohorts remain below that boundary under maximum priority-aligned or standby demand; maximum demand causes zero incumbent migrations; one-replica loss migrates after residency without early reversal; rolling arrivals keep every model/block populated and repair an abrupt block loss. The alpha fails closed above 32 candidates or 512 blocks, permits one `auto` worker, clamps reconciliation to at least one second, and scans each candidate in one bounded pass. The focused planner/convergence/configuration matrix passes 78 tests and the broader catalog/protocol/discovery/node/API matrix passes 214 with 2 skips. A real Windows DHT round trip exposed and fixed a durable-replay multiprocessing regression: replay guards now omit/recreate their thread lock across serialization and reload persistent state; its 15-test protocol/network matrix passes. Independent verification reproduced the 78-test focus, passed an expanded 235-test matrix with 2 skips and the 15-test real-DHT probe, and exercised adversarial score, timing, 32-by-512 load, 1,000-case range-equivalence, and persistent replay-reload boundaries. This slice used no cloud resources and spent USD 0. | Gates 9–11 are passed. Gates 13–14 must now prove the packaged flow and real hardware ceilings using the published envelopes. | -| 11 | Operate initial public alpha routes | PASSED | [Product-node run `route-20260830-j`](evidence/gate11node-20260830-a-lifecycle.json) installed the generic CommunityAI wheel on a bounded G2/L4 VM, verified the signed catalog, downloaded both exact manifested models directly from Hugging Face into one persistent shared cache, and used the product node's automatic workers to expose complete Qwen 24/24 primary and Gemma 35/35 standby routes. No model-specific image, cache mirror, or operator-transferred model artifact was used. The privacy-safe acceptance passed one-token primary inference, deliberate primary pause, automatic Gemma selection in 58.073 seconds, standby inference, Qwen restoration in 32.042 seconds, and restored inference. Both workers were stable before the drill. After Gate 13 released the L4, the preserved route was restored without changing its model cache or source, its ephemeral endpoint was rebound, both product-node services became active, and a fresh acceptance reproved Qwen 24/24 primary inference, automatic Gemma 35/35 fallback/inference, Qwen restoration, and restored inference. The protected bootstrap remains running. A corrected 4,800-second provider DELETE backstop was set for `2026-08-31T05:28:16.516Z`, earlier than the original deadline. [Post-backstop cleanup evidence](evidence/gate11route-20260830-j-backstop-cleanup.json) and an independent recheck prove the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero remaining route availability, and the protected bootstrap still running. The same-host standby is a bounded alpha fallback, not independent infrastructure redundancy; independent redundancy remains post-alpha. | Gate 11 acceptance evidence remains complete, but no product route is live after the corrected DELETE backstop. The 2026-08-31 reset supplies a new USD 100 epoch, but any replacement route still requires refreshed native authentication, a fresh exact source-bound conservative reservation, and fail-closed preflight before provisioning. | -| 12 | Create, publish, and bundle the minimal signed alpha catalog/bootstrap | PASSED | [Run `gate12-20260829-a`](evidence/gate12-20260829-alpha-catalog-publication.json) published the deterministic [`communityai-public-alpha-v1` bundle](../public-alpha/catalog-v1/bundle.json) from source `26be579`. Its threshold-one Ed25519 root signs sequence 1 with the exact qualified Qwen primary and Gemma standby manifests, one pinned public HTTPS mirror, one public seed, a one-route best-effort policy, and no unprovisioned route-demand roots. The canonical bundle binds five members and retains `complete_release_qualification=false`. All three public objects returned HTTP 200 with exact sizes, and a fresh empty consumer fetched them remotely, verified the signature/digests, and created the two-model `auto` node configuration. The private signing key remained ignored and uncommitted. The focused publication suite passes 32 tests, the catalog/bootstrap/model/desktop superset passes 92, and the run spent USD 0. | Preserve the branch-scoped mirror until a newly signed catalog sequence and packaged bootstrap migrate it. The Gate 11 acceptance and Gate 9 envelopes exist, but no product route is currently live; Gate 13 awaits native reauthentication and fresh per-run reservations under the new epoch. Independent threshold holders and interchangeable mirror/seed governance are post-alpha. | -| 13 | Pass packaged clean-install inference on Windows and Linux | BLOCKED | [Prerequisite run `gate13-20260830-a-prerequisites`](evidence/gate13-20260830-a-prerequisites.json) established deterministic install archives, exact first-use bytes, strict provenance, and the canonical lifecycle contract. [Native-harness and production-package run `gate13-20260830-b`](evidence/gate13-20260830-b-native-harness-and-packages.json) now completes the native Windows Credential Manager/Job Object and Linux Secret Service/systemd-cgroup 16-phase adapters, exact worker and descendant cleanup proofs, 3,600-second acquisition bounds, and package/runtime/catalog cross-binding. Independent software review passed 134 focused tests plus a 113-pass broader matrix with 3 platform skips; the production-discovery correction passes 73 unittests, 4 pytest checks, self-test, formatting, and import checks. [Exact-source production run 33338872342](https://github.com/flujo-app/CommunityAI/actions/runs/33338872342) passed both jobs at source `1971f10` and published independently audited CUDA 12.4 archives: Windows `sha256:45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6` (2,695,065,068 bytes) and Linux `sha256:f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00` (3,360,717,934 bytes). Pushed source `6787272` adds the fixed stdin-only artifact downloader and exact platform configs; its 42-test adversarial suite and independent race/special-member/live-wrapper audit pass. No cloud resource was created for these prerequisites. Real completed clean-host lifecycle evidence remains absent. Provider cleanup and the temporary Gate 11 restoration are proved, but the corrected backstop has since removed that route. | [Run `gate13-20260830-c` revision 13](evidence/gate13-20260830-c-cost-authorization.json) is stopped clean. The latest Windows host passed exact package audit, clean install, four desktop self-tests, and the packaged-node self-test, then failed before model acquisition because child stderr diagnostics contaminated strict JSON captured on stdout. [Attempt, cleanup, correction, and route-restoration evidence](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves zero cache bytes, no retained credential or product process, all four exact client instances/disks absent, the bootstrap running, and the temporary restored Qwen/Gemma product route. [Post-backstop cleanup evidence](evidence/gate11route-20260830-j-backstop-cleanup.json) now proves that route, its named disk, and both exact run-scoped firewall rules absent while every Gate 13 target remains absent and the protected bootstrap remains running. Pushed source `4818da3` separates captured stdout from a dedicated NUL stderr sink and passes 15 native tests plus independent high-volume, handle-leak, descendant, timeout, and Job Object probes, but it has not completed a paid clean-host lifecycle. The cleanup-backed 2026-08-31 owner reset releases the USD 98 historical maxima and opens a new USD 100 epoch for the next run; it does not authorize any particular resource or reuse the stopped record. A read-only native-auth check on 2026-08-31 found an active account selection, but provider requests could not refresh its token without interactive reauthentication; no provider mutation or resource creation occurred. Complete both 16-phase fresh-host lifecycles only after refreshing native authentication and recording fresh exact source-bound conservative reservations for the replacement route and Gate 13 clients; do not provision, restart FLUJO, or mark Gate 13 passed before then. | -| 14 | Pass automatic-contribution and resource-control hardware checks | WAITING | [PR #11](https://github.com/flujo-app/CommunityAI/pull/11) and [PR #12](https://github.com/flujo-app/CommunityAI/pull/12) implemented the authenticated node-authoritative Sharing UI and atomic policy editing, but cross-model automatic placement and real packaged hardware evidence are absent. | After Gates 9–13, follow the [recovery runbook](RECOVERY_TEST_RUNBOOK.md) once for the clean-install product flow while validating model/block choice, exact selected-shard bytes, shared-cache affinity, download authorization, VRAM/storage/bandwidth/power limits, suspension, pause timing, cleanup, restart persistence, and unsupported telemetry on real packaged Windows/Linux hardware. | -| 15 | Complete minimal alpha release engineering | WAITING | The desktop builder now emits a stable sorted `SHA256SUMS` inventory of exact regular-file bytes and safe relative in-bundle file symlinks, source/build/catalog-bound `provenance.json`, and `release-metadata.json` with explicit unsigned public-alpha, no-publisher-signature, no-authenticated-update, Windows/Linux-only, no-credits, and incomplete-qualification claims. Structural verification binds each safe file symlink to its canonical in-bundle target, digest, and size while rejecting changed, missing, extra, absolute, external, broken, cyclic, directory-linked/junction, special, traversal, or case-colliding payloads plus unsupported or noncanonical metadata. Exact-source builds also reject dirty relevant inputs, and the expected-input fresh-process check rejects rewritten commit/tree, workflow, platform, Python, PyInstaller, or catalog evidence. Production desktop CI is configured to verify and bundle the Gate 12 inputs, bind the exact clean Git commit/tree and workflow, revalidate every expected input separately, and upload all evidence on Windows/Linux. The focused release-input/artifact suite passes 15 tests, including fresh-process CLI, dirty-source, and canonical-rewrite checks, and the broader catalog/bootstrap/model/desktop subset passes 134. Independent verification reproduced all 134, passed 58 desktop unittests with two environment skips, formatting/import-order/YAML/diff checks, an expected Gate 12/workflow fresh-process probe, and real Windows junction rejection; no cloud was used. [The first PR #22 production-desktop run](https://github.com/flujo-app/CommunityAI/actions/runs/33273518744) reached packaging on both hosts and exposed two exact cross-platform defects: PyInstaller's legitimate relative internal Qt file symlink on Ubuntu and CRLF-transformed signed Gate 12 JSON on Windows. The follow-up binds safe internal file symlinks without accepting external or directory links, forces `public-alpha/**` to LF at checkout, and includes `.gitattributes` in the clean-source boundary. [The second run](https://github.com/flujo-app/CommunityAI/actions/runs/33274432423) proved the Ubuntu package and the Windows signed-bundle/provenance path, then exposed a stale desktop contribution-status schema 2 contract when the packaged node emitted schema 3 automatic-placement evidence. Source `fcd1f41` now strictly validates schema 3 placement and rejects stale schema 2 plus missing, extra, secret-bearing, or inconsistent placement data; its 50-test node/client/lifecycle/build focus and all 59 desktop unittests passed with two environment skips. [The final run](https://github.com/flujo-app/CommunityAI/actions/runs/33275216332) bound exact source `fcd1f417d1435557addb2d6cded9dac0827c7d8c` and completed both Windows and Ubuntu package jobs, including bundle build/smoke, independent checksum/provenance verification, the Windows packaged-node/native-credential/public-seed smoke, and artifact uploads; every PR style, test, and package check is green. Source `36d85d2` makes generic release-artifact fixtures select the supported Linux archive explicitly instead of inheriting the CI host platform; the 21-test local artifact suite and [PR #22 test run 33372581439](https://github.com/flujo-app/CommunityAI/actions/runs/33372581439) pass, without expanding the supported platform matrix. Clean-install lifecycle evidence remains absent. | Retain the verified Windows/Linux artifacts as engineering evidence, then test clean install, manual upgrade/reinstall, uninstall, retained-data choice for the persistent verified model cache, and recovery instructions on both platforms against a newly authorized live product-node route and the published Gate 9 envelopes. Do not mark passed from metadata/unit tests alone. Publisher signing and automatic authenticated update/rollback are post-alpha. | -| 16 | Complete the bounded public-alpha safety canary | WAITING | [PR #13](https://github.com/flujo-app/CommunityAI/pull/13) and [PR #14](https://github.com/flujo-app/CommunityAI/pull/14) implemented bounded admission, privacy-safe aggregate health, training-off defaults, rollback procedures, and bounded routine rejection logs; no public canary has run. | After Gates 11–15, run a small monitored canary proving finite admission/timeouts, malformed-peer rejection, health reconstruction, privacy disclosure, route/catalog disable, and clean rollback. Exhaustive hostile-load, Sybil/collusion, partition, and long-soak campaigns are post-alpha. | -| 17 | Publish and observe the public alpha | TODO | Owner has authorized a public inference alpha, but preceding mandatory alpha gates are open. | After Gate V and Gates 1–16 pass, publish with explicit best-effort availability, unsigned-package, support, and prompt-privacy limitations; preserve the disable path and monitor real route/worker failures. | - -## Deferred work - -| Item | Status | Resume condition | +Ship a **best-effort Windows/Linux public inference alpha** through the packaged +desktop and localhost OpenAI-compatible API, with optional bounded compute sharing. +The intended progression is **local Qwen3.5 → community Qwen3.8-27B → +DeepSeek-V4-Flash → GLM-5.3-Flash**. The two larger community models are post-alpha +targets. Local fallback and measured selection are implemented; local offline +Windows GPU inference passed. Staggered desktop formation and recovery passed in +the bounded GCP CPU test with real source Qt windows. + +Keep exact signed catalogs/manifests, verified partial artifact downloads, +authenticated discovery/transport, finite admission/timeouts, local resource +limits, prompt-visibility disclosure, and a working route/catalog disable path. +A one-route alpha must say that availability is best effort. macOS, credits, +payments/payouts and exhaustive +hostile-network/long-soak qualification remain outside this alpha. The owner now +requires working Inno Setup and Debian installers for alpha. On September 7 the +owner explicitly deferred Windows publisher signing; unsigned alpha setup with +checksums/provenance is acceptable. Store distribution follows trusted signing; +a signed APT repository can follow the directly installable `.deb`. + +September 8 owner scope decision: the existing packaged conversation and recovery +evidence is sufficient for alpha. Additional representative conversation, +performance and hardware measurements are deferred after alpha. Frozen periodic +catalog activation and active-answer draining qualification are deferred to beta; +the owner expects only one or two more catalog changes this year. These are no +longer release blockers. This decision does not claim the omitted checks passed, +disable the existing refresh implementation, or relax catalog signature checks. + +## Qwen3.8 results: bounded alpha scope accepted + +These live Qwen3.8 tests used `Qwen/Qwen3.8-27B-FP8` revision +`017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`, manifest +`sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4`, +with FP8 weights converted to BF16 and eager attention. + +| Acceptance outcome | Result and evidence | +| --- | --- | +| Complete 64-block CPU route | **PASSED.** Four e2-highmem-4 workers, 16 blocks each, plus an e2-standard-4 client. Three generated tokens in 148.612 seconds. [CPU evidence](evidence/qwen-cpu-full-inference-20260905.json). | +| Selected-worker loss and same-session recovery | **PASSED in the tested scenario.** Deleted blocks 16–31 VM and disk; fresh replacement used a new peer identity. The original client/session continued with identical tokens, preserving the other three worker/session identities. Local orchestration needed a guarded resume after an SCP failure; the client was not restarted. | +| GCP L4 + Azure T4 + CPU remainder | **PASSED after the CPU proof.** Four 16-block spans, actual inspected devices, identical three tokens in 75.699 seconds. [Mixed evidence](evidence/qwen-mixed-full-inference-20260905.json). | +| Cleanup | **PASSED.** Both passing runs' owned cloud resources were verified absent. No quota increases requested. | +| Local fallback | **PASSED, bounded Windows GPU/Linux CPU scope.** Exact Qwen3.5-0.8B produced real tokens offline through packaged nodes; token limits, local-only persistence and stream cancellation passed. Windows used an 8 GB RTX 2070 SUPER. [Linux package evidence](evidence/qwen-linux-v9-20260906.json) and [product limits](QWEN_DESKTOP_PRODUCT_RESULTS.md). | +| Local inference plus automatic sharing | **PASSED, one Windows case.** The current package automatically selected one Qwen3.8 block under a 2 GiB worker budget alongside local Qwen's 3 GiB budget. Pause removed the entire worker process tree in 0.110 seconds; restart and concurrent local tokens passed. Public bootstrap startup retries remain a limitation. [Sharing evidence](evidence/qwen-sharing-packaged-20260906.json). | +| Resource controls and power recovery | **PASSED, bounded Windows cases.** Independent schedule/power/bandwidth/storage admission checks; a real 25-second GPU load triggered power pause and automatic resumption without policy edits. [Power evidence](evidence/qwen-power-recovery-20260906.json). The final Windows/Linux resource-control matrix also passed; see Gate 14 below. | +| Packaged cold client acquisition | **PASSED.** Eight direct-Hub artifacts, 6.03 GB, verified from an empty cache; the large shard resumed three times. Approximately two hours on the tested connection. [Acquisition evidence](evidence/qwen-packaged-cold-acquisition-20260906.json). This does not establish generation. | +| Stock/reference correctness | **PASSED, declared bounded scope.** Three prompts × prefill and two cached decode positions; all vocabulary logits within predeclared `atol=0.5`, `rtol=0.01`, and all nine greedy tokens match stock Transformers' independent FP8 dequantizer. Four RPC workers on one CPU host; separate from cross-cloud qualification. [Reference evidence](evidence/qwen-reference-parity-20260906.json). | +| Automatic promotion, preference and loss/rejoin | **PASSED through the source node under signed public sequence 2 on an assigned mixed route.** Local before growth; Qwen3.8 after measured readiness; active answer preserved when switching to local-only; local after confirmed T4 loss; Qwen3.8 after its replacement joined with a new peer identity. [Source product evidence](evidence/qwen-source-public-recovery-20260906.json). This does not prove autonomous desktop formation. | +| Packaged Qwen3.8 | **PASSED on the assigned L4/T4/C3 route under signed public sequence 2.** Windows v9 generated three tokens in 12.250 seconds; a 31-token chat prompt answered `Paris` in 19.359 seconds. Peak sampled client process-tree RSS was 4.97 GB. [Evidence](evidence/qwen-packaged-recovery-v9-c3-20260906.json). | +| Packaged worker outage and cache reuse | **PASSED on that C3 route.** Confirmed worker stop → automatic local answer → same-identity restart → Qwen answer in 13.672 seconds. A new node process repeated community completion/chat and local-only inference with HTTP downloads blocked, making zero download attempts. Owned cloud cleanup passed. The earlier [E2 rejoin timeout](evidence/qwen-packaged-rejoin-timeout-v9-20260906.json) remains a failed attempt. | +| Autonomous desktop formation and recovery | **PASSED, bounded CPU/source-UI scope.** The actual [one-click runner](QWEN_FORMATION_TEST.md) formed 64/64 blocks from four capacity-only contributors, promoted all six clients and returned real Qwen3.8 answers. Whole-participant loss produced five local answers; unattended same-identity restart restored six Qwen3.8 answers. Real Qt controls/windows passed. No runtime intervention; cleanup verified. Three-token requests took 24–34 seconds. Fallback validation took minutes. [Evidence](evidence/qwen-formation-passed-20260907.json). Fresh installers, fully frozen UI, GPU contributors and simultaneous cold joins are outside this result. | +| Consumer GPU and chat performance | **DEFERRED after alpha by the owner on September 8.** Existing conversation evidence is accepted for alpha. No RTX 30/40/50, broader conversation, context or concurrency qualification is claimed. | + +The complete [experiment report](QWEN_FULL_INFERENCE_RESULTS.md) preserves timing, +source hashes, routes, recovery limitations, and the earlier failed diagnostic +attempt. [CPU runner](QWEN_FULL_INFERENCE_GCP.md) and [mixed runner](QWEN_MIXED_INFERENCE.md) +are reusable. A production worker-health digest-format bug was fixed; the report +records the focused test results. Native FP8 remains optional for Qwen correctness. + +The [maintained product replay](QWEN_PRODUCT_TEST.md) now includes the previously +ignored wrapper/reporting steps. It has local regression coverage; its own live +replay remains open. The [run audit](evidence/qwen-c3-run-provenance-audit-20260906.md) +records the interventions in the historical passing C3 run. +The new [Gate 13-based entry point](QWEN_QUALIFICATION_RUNNER.md) runs the same +audited Qwen scope through an ordered controller and durable phase records. + +## Current gates + +`PASSED` requires the stated real evidence; `IN PROGRESS` means required outcomes +remain; `WAITING` means a dependency is open; `TODO` means not yet executed. + +| Gate | Status | What must be true before it passes | | --- | --- | --- | -| macOS CPU/MPS and packaged application support | DEFERRED | Real Apple-device hosts and testers are available | -| Credits, receipts, balances, spend authorization, earnings, and payouts | DEFERRED | Public inference alpha is live and its reliability/privacy behavior is understood | -| Compute marketplace and jurisdiction-specific payment onboarding | DEFERRED | Accounting threat model, legal review, and independent audit are complete | -| Larger model ladder rungs | DEFERRED | Once first-rung public capacity and operations are stable, test a real 27-32B split route directly and, if it passes, an exact roughly 70B candidate; intermediate sizes are not mandatory prerequisites | -| Production-SLO model-route redundancy and largest-worker-loss survival | DEFERRED | The best-effort alpha is live and its real route-loss evidence identifies the required topology | -| Independent multi-provider seeds, catalog mirrors, and outage survival | DEFERRED | The alpha seed/catalog dependency is measured and independent operators are available | -| Independent threshold catalog key holders and compromise/rotation governance | DEFERRED | The pinned single-signer alpha catalog is operating and human key holders accept responsibility | -| Publisher-signed installers plus authenticated automatic update/rollback | DEFERRED | Alpha packaging stabilizes and publisher identities/signing credentials are available | -| Exhaustive malicious-load, Sybil/collusion, partition, herd-switching, and long-soak campaigns | DEFERRED | The bounded alpha canary passes and real public telemetry supplies representative workloads | +| V and 1–13 | **PASSED, historical scope** | Integration, trust/discovery, Qwen3.5/Gemma qualification, artifact delivery, and Windows/Linux packaged inference foundations are retained. [Manual desktop evidence](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) and [automated replay](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json). These do not qualify Qwen3.8 in the current package. | +| Q3.8 | **PASSED, owner-accepted bounded alpha scope** | Runtime, packaged conversation/recovery, bounded formation and Windows/Linux startup migration passed. On September 8 the owner accepted those results for alpha and deferred additional conversation/hardware measurements and frozen periodic catalog activation/draining. Broader performance and beta update behavior remain unqualified. | +| 14 | **PASSED, bounded Windows/Linux alpha scope** | **“Sharing obeys my limits.”** Frozen packages at `76b6d84` (Windows) and `bf67f0d` (Linux packaging fixes) passed fresh 100%/100% defaults with sharing opt-in, real Qwen processing load, live VRAM changes, low-memory rejection/recovery, Pause, persistence and independent storage/bandwidth/schedule/power admission checks. Linux used ordinary-user Debian 12/Xvfb with CUDA passthrough; broader hardware/physical desktop profiles are not implied. [Final evidence](evidence/gate14-20260907-final-resource-acceptance.md). | +| 15 | **PASSED, bounded Windows/Debian/Ubuntu alpha scope** | **“Install it, replace it, remove it.”** Windows active different-version upgrade and Debian/Ubuntu active same-version replacement/removal/reinstall passed. Both frozen sign-in checkboxes passed enable/restart/disable with native registration and cleanup verified. Manual cache/reset choices passed on disposable Windows state; disable sign-in startup before uninstalling. [Combined acceptance](evidence/gate15-20260908-final-installer-acceptance.md). Unsigned alpha is owner-authorized; signing, Store and hosted signed APT follow after alpha. Automatic updates ship in the September 9 release. | +| 16 | **IN PROGRESS; existing recovery/safety evidence under release-scope review** | Real worker-loss/fallback/rejoin, formation, resource shutdown and local safety checks already passed in their recorded scopes. Credit those results before scheduling any new run. The combined public-deployment probe followed by real inference has not run; periodic live catalog withdrawal/restore qualification follows the owner's beta deferral. | +| 17 | **IN PROGRESS; candidate downloads published** | All four qualified installer options and release metadata are public and hash-verified. Prepare the draft release and observation within the declared best-effort scope; the combined Gate 16 canary remains unexecuted. | + +Gate 14 protects contributors' PCs and Gate 15 makes distribution usable; retain +both. Combine overlapping Q3.8/Gate 14/15 observations in the same bounded desktop +sessions when practical. Do not repeat old Qwen/Gemma qualification or add a new +cloud framework simply to advance gate numbers. Gate 16 provides the bounded +public safety check; exhaustive hardening is deferred. + +## Next work, in useful product order + +September 9 release `0.1.0-alpha.20260909.2` adds the repaired desktop and signed +application updates. The [installation guide](ALPHA_INSTALL.md) has current +downloads; [updater behavior and publication](AUTOMATIC_UPDATES.md) describe the +one-time manual upgrade from September 8. Existing source checks and a Windows +update-handoff fixture passed. No new full desktop, GPU, cloud or installed Linux +updater qualification is claimed. The owner explicitly requested immediate +publication using the existing checks and normal packaging/integrity checks. + +The September 8 distribution records below remain historical evidence. + +September 8 distribution refresh: the owner requested removal of duplicate +libraries and unused bitsandbytes CUDA variants, plus a small verified downloader +and the full offline installer. Both replacement runtimes identify source +`84205f93fc73d3babd39e238944b97fab0d11b3e`. The Windows +`0.1.0-alpha.20260908.1` setup is **2,462,345,104 bytes** and passed ordinary-user +installation, installed native CUDA operations and removal. +[Windows acceptance](evidence/normalized-windows-installer-20260908.md). +The Linux `0.1.0~alpha.20260908.1` package is **2,302,428,788 bytes** and passed +installation, installed CPU/CUDA/worker checks and removal on Ubuntu 22.04. +[Linux acceptance](evidence/alpha-normalized-linux-20260908.md). Measured +regular runtime payloads are 4,263,859,354 bytes on Windows and 5,161,115,250 bytes +on Linux. The [packaging evidence](evidence/runtime-packaging-reduction-20260908.md) +preserves the earlier estimates and targeted normalization rules. + +Cloudflare R2 is configured at the immutable `alpha/20260908.1/` prefix. The +actual Windows setup and Debian package are uploaded and passed complete hosted +download/hash verification. The **2,107,751-byte Windows online setup** passed +actual ordinary-user installer handoff, an installed CPU diagnostic and removal, +with exact child-process exit, temporary cleanup and persisted user-state baseline +verified. [Windows hosted acceptance](evidence/normalized-online-windows-installer-20260908.md). +The **13,662-byte Linux online installer** passed HTTPS download, protected-copy/APT +installation and removal. [Linux hosted acceptance](evidence/alpha-online-linux-hosted-20260908.md). +Earlier transport and progress-publication failures remain in those records; +these single successful handoffs establish no broad availability guarantee. + +The Windows online helper, Inno script and builder match source +`b6c8aad9cea208630785d890cfb966093f809e7e`, checked after the working-tree build; +the offline runtime source remains `84205f93`. Both online files, all 19 curated +platform records, the combined manifest, installer checksums and metadata ZIP +are published. All 24 small public object bodies matched their exact hashes. +[Publication audit](evidence/alpha-cloudflare-publication-20260909.json). +The rate-limited `r2.dev` origin serves the declared initial scope. +The [installation guide](ALPHA_INSTALL.md) carries exact hashes and availability; +[hosting records](CLOUDFLARE_RELEASE.md) keep platform provenance and the exact +embedded online manifests separate. + +Gate 14 is complete for the declared Windows/Linux alpha scope. The +[final acceptance](evidence/gate14-20260907-final-resource-acceptance.md) used complete +catalog-bearing frozen desktop/node packages at `76b6d84` (Windows) and +`bf67f0d` (Linux, with packaging fixes and unchanged application/catalog source), real Qwen block load, +literal Qt sliders, explicit opt-in, independent admission guards and native +credential stores. Both platforms passed all 11 checkpoints and cleanup. +Processing limits pace sharing compute; brief bursts, loading/downloads and +local inference remain separate. Linux used Debian 12/Xvfb with CUDA passthrough, +so this is not a physical Ubuntu/Wayland or broad GPU qualification. + +Acceptance found and fixed product defects: insufficient VRAM no longer +causes an endless worker restart loop, and migration of an identical signed +manifest into managed storage now preserves per-model cache/resource preferences. +Linux declares its missing X11 shape-library dependency and excludes optional +Triton JIT initialization that otherwise required a compiler inside the frozen app. +The block-health grid, observed peer details and local client/worker download +progress are included in both packages. The [display checkpoint](evidence/desktop-health-downloads-20260907.md) +records their state/integrity tests; remote download percentages and unreported +spare capacity are not invented. + +The final Windows setup at `0.1.0-alpha.20260907.2`, using the stable public +application ID, passed non-elevated installation, upgrade from the earlier full +setup while Qwen sharing was active, removal, reinstall and final removal. The +actual installed frozen GUI and node returned local tokens and verified the +worker cache in each launch. Complete owned trees stopped and settings/cache/ +credentials survived replacement/removal. A redundant test-driver cleanup call +failed after the final successful uninstall; the subsequent independent cleanup +audit passed. [Full evidence](evidence/gate15-20260907-frozen-windows-installer.json). + +The final Debian installer at `0.1.0~alpha.20260907.4` also passed initial +installation, active same-version replacement, removal, reinstall and final +removal with the actual ordinary-user frozen GUI/node. Local Qwen tokens and +verified sharing artifacts passed on all three launches. Settings/cache and +credentials survived maintenance, and the independent final audit found no +installed runtime/DHT processes or test credential. This is Debian 12/Xvfb with +CUDA passthrough, not a physical Ubuntu desktop or different-version upgrade. +[Final Debian evidence](evidence/gate15-20260907-frozen-debian-installer.json). + +The Debian run exposed two shutdown defects: unreadable process ownership was +silently skipped, and a fixed process snapshot missed helpers born during shutdown. +Maintenance now refuses insufficient inspection permissions and continually +discovers owned processes until repeated observations are quiet. Both failures, +targeted cleanup and the old-fails/new-passes regression are retained. Installer +scripts come from `61ab7b1`; the independently verified `bf67f0d` runtime payload +was preserved byte-for-byte while the Debian control archive was replaced. + +The subsequent Ubuntu 22.04 attempt **did not pass**: root `dpkg -i` exceeded +the 540-second harness limit during initial unpacking, with approximately 2.6 GiB +written. No installed GUI, node or inference launched. Native test credential, +DHT and display cleanup passed, and the exact disposable container was removed +with its partial installation. Caches and raw evidence were retained. The +underlying performance cause remains unconfirmed. +[Failed Ubuntu attempt](evidence/gate15-20260908-ubuntu-install-timeout.json). +The [unpack diagnosis](evidence/gate15-20260907-ubuntu-unpack-diagnosis.md) +records the 2,733-block XZ payload, relevant package-manager version differences +and a bounded profiling/repack plan; it does not claim a confirmed root cause. + +The September 8 retry **passed the complete Ubuntu 22.04 lifecycle** with the +same `.4` installer: initial installation, active replacement, removal, +reinstallation and final removal. The actual installed GUI/node produced local +Qwen tokens and verified a sharing block on all three launches. Settings/cache/ +credentials survived maintenance; independent runtime/credential cleanup passed +and the disposable container was removed. Two CPU cores and a 6 GiB memory cap +bounded local use. Initial installation took 329.971 seconds; the original timeout +was not reproduced. [Ubuntu acceptance](evidence/gate15-20260908-frozen-ubuntu-installer.md). + +The [manual uninstall choices](DESKTOP_UNINSTALL.md) now explain retaining state, +deleting only reviewed model caches, resetting the native credential and node +state, and disabling login startup before removal. Disposable Windows checks +passed, including the actual frozen credential-deletion command. The attempted +frozen sign-in-toggle test used explicitly selected mock API data and was +interrupted before any toggle succeeded; it is not a passing UI acceptance. +All its owned processes, credential and login entry were confirmed absent. +[Choice evidence](evidence/gate15-20260908-windows-data-login-choices.json). + +The unmodified frozen Linux checkbox subsequently passed enable, restart with +the setting retained, disable, and an explicit login-flag launch through AT-SPI +on a private Xvfb display. No models loaded; all three normal shutdowns and +independent credential/process cleanup passed. The initial ambiguous-action +failure and a separate virtual-display wrapper cleanup error remain recorded. +[Linux frozen control](evidence/gate15-20260908-frozen-linux-login.md). +The Windows source Qt/native-registry regression also passed while preserving +the real login entry. [Windows source evidence](evidence/gate15-20260908-source-login-checkbox.md). +The subsequent Linux CI run exposed a source-test targeting error: the helper +clicked the hidden checkbox's center outside its style-defined hit region. +The helper now exposes Sharing offscreen, clicks the actual indicator, and +cancels its session timers. All 108 Linux desktop tests completed successfully +with two existing installer-permission skips; no product change was needed. +[Portability follow-up](evidence/gate15-20260908-source-login-portability.md). + +The unmodified frozen Windows checkbox then **passed enable, normal shutdown, +restart with enabled state retained, and disable** on unswitched private desktops. +The exact qualified executable's native `REG_SZ` command was verified. Both +launches authenticated with sharing paused and no model loads; configuration and +the native credential survived restart. Both jobs were empty before closure. +An independent audit found all 18 recorded identities stopped, the test credential +absent and the original login entry state restored. Earlier reader failures are +retained as harness findings. [Windows frozen control](evidence/gate15-20260908-frozen-windows-login.md). +This completes [Gate 15's bounded installer acceptance](evidence/gate15-20260908-final-installer-acceptance.md); +actual OS sign-out/sign-in and broader physical desktop coverage are not implied. + +The normal frozen Windows and Linux desktops independently passed automatic +startup migration from real signed catalog sequence 1 to exact sequence 2. +Resource limits, local-only preference, workers, cache and native credential +survived restart; the old trust root was rejected. Linux passed on a fresh retry +after the first bootstrap child returned nonzero and the app retained sequence 1. +That failure and successful metadata-only diagnostics remain recorded; the +original child error was not retained, so its cause is unconfirmed. +[Windows acceptance](evidence/qwen-catalog-desktop-20260907.md), +[Linux acceptance and retained failure](evidence/qwen-catalog-linux-startup-20260908.md). + +Six new source integration cases connect the actual periodic refresh service, +signed installer and model manager: active leases/loading delay restart, +admission closes before restart, invalid updates preserve state, and closing the +service preserves active work. These passed without model loads. The frozen +newer-sequence/active-generation replay is deferred to beta by the September 8 +owner decision and is no longer an alpha blocker. +[Source evidence and live replay requirements](evidence/qwen-catalog-periodic-source-20260908.md). + +Gate 16 now has a [bounded canary protocol](GATE16_CANARY.md) and a passing local +preflight: 20 real frozen-node HTTP assertions and 131 source tests, including +loopback TLS/DHT and signed catalog withdrawal/forward restore. No model loaded +and no public canary ran. Catalog withdrawal removes automatic selection and +contribution approval while preserving explicit manual selectors; emergency +route disable must stop the actual owned workers. +[Local evidence](evidence/gate16-20260907-local-preflight.json). + +The prepared live RPC driver now defaults to local preflight and limits an +explicit worker probe to 20 calls/128 KiB with finite execution and cleanup. +The separate catalog driver stages an isolated signed withdrawal/restore channel +locally and observes authenticated runtime configuration after ordinary refresh. +Twenty-four focused tests passed, including the real handler over loopback TLS +with no model cache allocation. The live route, HTTPS publication and full canary +observations remain open. [Driver evidence](evidence/gate16-20260908-driver-preparation.json). +Linux CI then exposed an upstream Hivemind reset when the client finishes an +idle stream after the server timeout. The driver accepts that closure only +after observing lease release within timeout bounds and exact admission deltas; +early resets and malformed-request transport failures still fail the probe. +[Transport follow-up](evidence/gate16-20260908-linux-idle-transport-fix.md). + +At reviewed head `fdd8d0b`, all nine CI checks passed, including Linux/macOS +functional tests, style/security checks and both complete Windows/Linux production +package/installer jobs. This head adds an import-formatting fix after the +`84205f93` candidate runtime source. CI rebuilds retain their own provenance; +candidate acceptance remains bound to its exact installer/runtime identities. +The +two high-severity CodeQL findings were reviewed against their exact source-to-sink +paths and [dismissed as false positives](evidence/gate14-20260907-codeql-triage.md), +with scanning still enabled. Public-key metadata is distinct from private material, +and generated API bearer keys are distinct from human passwords; advanced imports +still require operator-supplied strong tokens. + +1. **Reuse the existing evidence when resolving Gate 16.** Worker loss, recovery, + fallback, formation, resource shutdown and local malformed/admission checks + have already passed in their recorded scopes. The owner asked which genuinely + new deployment observations remain; do not launch another full qualification + campaign merely to repeat them. The combined live canary remains unexecuted. +2. **Retain the completed distribution acceptance.** Both complete hosted + download/hash checks and actual online downloader-to-installer handoffs passed. + All four installer options and their checksums/metadata are public and verified. + Additional Q3.8 + conversations/performance measurements and periodic catalog activation/draining + are explicitly deferred and must not be reintroduced through Gate 16. +3. **Prepare the draft alpha release with the verified candidate links.** + Use the published exact checksums/provenance and declare the tested platform + limits. This can proceed while Gate 16's deployment scope is + reviewed; it does not claim the combined canary passed or current public Qwen + capacity was observed. Store, trusted Windows signing, hosted signed APT, + larger adapters and credits follow. + +The [candidate installation/download guide](ALPHA_INSTALL.md) binds the exact +installer hashes. Both new offline files exceed GitHub Releases' 2 GiB per-asset +limit and total 4,764,773,892 bytes; the owner-authorized R2 origin serves these +versioned objects. The [public metadata check](evidence/alpha-public-metadata-20260908.json) +verified the catalog, bootstrap and both manifests at 23:52 UTC on September 8. +The signed catalog expires on September 28 at 19:35 UTC. Renew it before expiry +and preserve `codex/gate-v-auto-selection` while the published URLs depend on +that branch. No peer was probed: historical successful routes do not establish +current complete community capacity. These operational facts remain distinct +from installer acceptance and the unexecuted combined Gate 16 canary. + +The [model ladder audit](COMMUNITY_AI_MODEL_LADDER.md) and +[product results](QWEN_DESKTOP_PRODUCT_RESULTS.md) separate implemented behavior +from remaining live acceptance. The corrected mixed source-node transition test +passed. The latest Windows package passed local GPU chat and independent resource +admission checks, including the new cache-accounting implementation. +[Package evidence](evidence/qwen-desktop-v9-20260906.json). Direct-Hub cold acquisition +and the Windows packaged community/recovery/cache path have passed on the assigned +C3 route. [Complete packaged result](evidence/qwen-packaged-recovery-v9-c3-20260906.json). +Bounded autonomous CPU desktop formation and recovery passed; earlier failures +are retained separately in the [formation runbook](QWEN_FORMATION_TEST.md). +The Linux CUDA package passed verification, offline local CPU chat and the +final ordinary-user frozen Qt/GPU resource-control matrix described above. +Broader hardware, physical desktop and installer qualification remain distinct. +The [catalog signer and three backups](CATALOG_SIGNING_KEY.md) are documented. +Headcount or advertised VRAM alone cannot trigger a safe upgrade. + +## Credits and deferred scope + +Credits are a separate product stage. First add measured block-token work, +content-free signed receipts, replay/double-count protection, and estimated/pending +UI in **shadow mode**. Spendable balances then need settlement, accounting units, +reserve/spend/refund rules, abuse resistance, recovery, and outage reconciliation. +Purchases/payouts add marketplace work. See the [credit audit](COMMUNITY_AI_MODEL_LADDER.md#credits-after-the-inference-alpha) +and [existing design](REVIVAL.md#identity-keys-accounting-and-credits). + +Also deferred: DeepSeek/GLM activation, independent seed/route/mirror redundancy, +independent key-holder governance, +macOS, and exhaustive malicious-load/Sybil/partition/long-soak campaigns. Preserve +their existing foundations; prioritize the usable Qwen path. ## Cloud authorization and spend ledger -Authorization applies only to CommunityAI qualification and public-alpha infrastructure. -The ceiling is USD 100 combined across new temporary GCP and Fly resources in the current -owner-authorized accounting epoch. The existing -GCP bootstrap's ordinary baseline cost is tracked separately; never delete it as test cleanup. +The compact table below preserves every legacy run ID, provider, purpose, amount, +and state because existing qualification tools parse this section. Detailed +cleanup and authorization history are in the [archive](RELEASE_READINESS_HISTORY.md#cloud-authorization-and-spend-ledger). +Historical epoch resets are not a new budget authorization; do not interpret the +old aggregate as today's available balance. September 5–6's explicitly requested +CPU/mixed experiments and verified cleanup are recorded above; their billed cost +was not reconciled in this documentation update. No cost is invented or reset here. +The protected bootstrap remains outside test cleanup. Future runs use the current +session authorization and fresh provider checks; exact cleanup remains required. -Before every paid run, add an entry with a conservative maximum. After cleanup, replace -the estimate with observed cost when available. If provider billing is delayed, retain the -maximum estimate until actual cost is known unless the owner explicitly resets the budget -after complete cleanup. On reset, keep historical rows, mark them `CLEANED-RELEASED`, and -continue recording later observed charges for information; released rows do not consume the -new epoch. +
+Legacy runner ledger (44 entries) | Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State | | --- | --- | --- | ---: | ---: | --- | --- | -| gate13-20260830-c | GCP | Gate 13 sequential clean packaged Qwen Windows and Gemma Linux lifecycles at exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7`, temporarily suspending and later restoring the Gate 11 route while reusing its sole global L4 allocation on uniquely named fresh Windows and Linux clients [plan sha256:427bc1ed8a6645ad0650d91aaba7aa753d398fa84f56d57b50aca04c4e0cc955] | USD 26.00 | — | [Cost authorization](evidence/gate13-20260830-c-cost-authorization.json) binds the passed production archives/audits, pushed download-helper/config identities, exact Actions wrapper/inner archives, exact Qwen/Gemma manifests, no service accounts/scopes, direct model transfer, native credential stores, whole-tree containment, all 16 phases, exact cleanup targets, and zero Fly/image/mirror/credits/macOS work. Revision 13 records the final Windows pre-acquisition failure, pushed correction `4818da3`, complete native cleanup, all four exact client instance/disk absences, and successful Gate 11 route restoration. [Privacy-safe final state](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves the package audit and install boundary, zero model-cache bytes, no retained credential/process/path/endpoint/provider output, protected-bootstrap health, active Qwen/Gemma route services, and fresh primary/fallback/restoration inference. The two required 16-phase lifecycles remain incomplete. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 26 maximum is historical and delayed billing remains informational. This record authorizes no later provisioning. | CLEANED-RELEASED | -| gate9-20260830-e | GCP | Gate 9 concurrent Qwen/Gemma Windows/Linux acquisition records and schema-v3 envelopes at pushed source `ba410f74f1cf625f1e1c34734b53e4514fa7c5ec`, reusing the separately authorized product route and using bounded isolated clients [plan sha256:04ba77ee68f4a895ae080a4ddcbf6805b502da6a95a4146734acbddff92de307] | USD 46.00 | — | [Passed envelopes and cleanup](evidence/gate9-20260830-e-edge-resource-envelopes.json) publish all four exact acquisition/envelope records and prove complete client cleanup; [cost authorization](evidence/gate9-20260830-e-cost-authorization.json) binds the exact wheel or exact-commit source archive, signed catalog/bootstrap, Qwen/Gemma manifests, owner-authorized parallel platform/model execution, 60-minute model windows, 90-minute client deletion backstops, exact cleanup targets, protected resources, and zero Fly/image/mirror operations. Native provider authentication was refreshed before the USD 18 Windows-client expansion and again before the zero-ceiling-increase Gemma memory retry; the exact plan permits one cache-preserving in-place resize to `e2-standard-8`. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 46 maximum is historical and delayed billing remains informational. | CLEANED-RELEASED | -| route-20260830-j | GCP | Gate 11 signed-catalog product node route [workload gcp-product-node-route] [source e1d715fd47c852fa12ca50c76e8f4c6a0831fd78] [final runtime source 4cef141746705c3ee8bc8e017693855e0bc4871e] [plan sha256:1a0927e9d83a9a409ac2ea0232c4fceb14821d3f2c5eb87def88b8e7cdcb07d8] | USD 26.00 | — | [Passed live lifecycle](evidence/gate11node-20260830-a-lifecycle.json): generic runtime, signed catalog, direct Hugging Face artifacts, shared persistent cache, complete primary/standby routes, primary/fallback/restoration inference, stable workers, no model image, and protected-bootstrap health. [Gate 13 restoration evidence](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves the route was restored, both product services became active, fresh Qwen/Gemma primary/fallback/restoration inference passed, and a corrected 4,800-second DELETE backstop ended no later than the original deadline. [Post-backstop cleanup](evidence/gate11route-20260830-j-backstop-cleanup.json) proves the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero GPU use, and the protected bootstrap running; acceptance evidence is preserved but no product route is live. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 26 maximum is historical and delayed billing remains informational. | CLEANED-RELEASED | -| cache-20260830-g | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 62be8f1c999b6ebe0ece2a660a0be4757cc83005] [plan sha256:109d2b6958ac8ced31e7202c8eb230387d29615f964d32d6726564b9366eafd7] | USD 10.00 | — | [Live lifecycle](evidence/cache-20260830-g-lifecycle.json) passed public-package/native/provider preflight, exact private repository, keyless identity, reader binding, and builder creation, then failed closed at `cache_warm` after 572.531 seconds with zero cached manifests. Cleanup passed all six exact deletes and absences, removed the ephemeral identity and repository, retained no key, public access, credential, provider output, path, identifier, or argv, and kept the protected bootstrap running. | CLEANED-RELEASED | -| cache-20260830-f | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source bff0c3203191725928246ad3e13deb01ffbab8de] [plan sha256:735cfd847291229571529c8f640fc76005e340a29680806295dad33a7e1a1fb6] | USD 10.00 | — | [Sanitized post-failure verification](evidence/cache-20260830-f-post-failure-verification.json): the private cache and keyless builder reached concurrent warm, then both exact GHCR pulls reported authentication/daemon failure because both upstream packages were still private. The failed controller was interrupted after the startup script's nonzero exit; all six exact cleanup commands and absence checks, repository deletion/absence, no public access/key/retained credential, and protected-bootstrap health passed. | CLEANED-RELEASED | -| cache-20260830-e | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source c0bd81e4e3ced3cd05a642740e343da41d05aceb] [plan sha256:fc13db74e107795c6d2896e0135c4a669a3fd7618a9ef1c4feab54f2425cf948] | USD 10.00 | — | [Live lifecycle](evidence/cache-20260830-e-lifecycle.json) passed exact private repository, ephemeral identity, reader binding, and builder creation, then failed closed at `cache_warm` after 1,653.422 seconds with no cache success claim. All six builder/perimeter/identity absences, identity removal, exact repository deletion, no public access/key/retained credential, and protected-bootstrap health passed. The [bounded acknowledgement diagnostic](evidence/cache-20260830-e-acknowledgement-diagnostic.json) proves the known exact JSON boundary while retaining no failed remote bytes. | CLEANED-RELEASED | -| cache-20260830-d | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 3ae7a094a1e4ca3865d5b6aa463816eac36318f4] [plan sha256:2387d038386ea64e6301d70133aaee4dceedb2c8279e1a341b744ffb1f9fdbc4] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-d-lifecycle.json) passed exact repository creation/configuration, then stopped before builder creation when domain-restricted sharing rejected the planned temporary `allUsers` reader binding. The [bounded policy diagnostic](evidence/cache-20260830-d-domain-policy-diagnostic.json) proves no public binding applied and exact repository deletion; [sanitized post-failure verification](evidence/cache-20260830-d-post-failure-verification.json) proves the repository and all five builder/perimeter targets absent and the protected bootstrap running. | CLEANED-RELEASED | -| cache-20260830-c | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 42241d6fb951cc6274ba991d5762558d67c376ab] [plan sha256:634c4d9db1474655065b1d4d6c2bb4066aeb6c48afa3e2eda7e85d980282104e] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-c-lifecycle.json) stopped at exact repository verification before public binding or builder creation; the [bounded provider-schema diagnostic](evidence/cache-20260830-c-repository-schema-diagnostic.json) proved GCP returns `remoteRepositoryConfig.commonRepository.uri`, deleted the exact diagnostic repository, and re-proved absence; [sanitized post-failure verification](evidence/cache-20260830-c-post-failure-verification.json) proves the API enabled, repository and all five builder/perimeter targets absent, and protected bootstrap running. | CLEANED-RELEASED | -| cache-20260830-b | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 448196300660174ae8daf5b70bb55c275dcc981d] [plan sha256:861ebeaa2af38e563bdfb736d955b23ea87bd579188636a5512577ee6b35dd52] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-b-lifecycle.json) stopped at the exact enabled-service query before repository or builder creation; [sanitized post-failure verification](evidence/cache-20260830-b-post-failure-verification.json) proves the API enabled, the exact repository and all five builder/perimeter targets absent, and the protected bootstrap running. | CLEANED-RELEASED | -| cache-20260830-a | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source a41d9ed72e333057fc017c769ed65f17c92a46e6] [plan sha256:271778431c7553f93d674dffb5131c60133449478d4103c46f366129d7eae2ab] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-a-lifecycle.json) stopped at API enablement before repository or builder creation; [sanitized post-failure verification](evidence/cache-20260830-a-post-failure-verification.json) proves the API enabled, the exact repository and all five builder/perimeter targets absent, and the protected bootstrap running. | CLEANED-RELEASED | -| route-20260830-i | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source fc4c18b045b9143ba455c38fa890eb112429ad3f] [plan sha256:c17ca0aa19f3eb79f1ae837f240b4972a17c821c5c4b8521582e2d38fbd6b99a] | USD 26.00 | — | [Concurrent-prefetch startup-health timeout and cleanup proof](evidence/gate11route-20260830-i-lifecycle.json): native/provider preflight, exact create, bootstrap, protected registry transport, authenticated concurrent prefetch, and both local digest checks passed; the direct GHCR path still exhausted startup before health, so no inference ran; five exact deletes, all six absence checks, registry removal, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-h | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source c09552e7ea0d3f0905857acb35a94affabccedbb] [plan sha256:97ce29d07b3965f8fad4272c9a7b641347622a5940b628d917f3a54fa5a17234] | USD 26.00 | — | [Startup-health timeout and cleanup proof](evidence/gate11route-20260830-h-lifecycle.json): native/provider preflight, exact create, bootstrap, protected registry transport, authenticated prefetch, and both local digest checks passed; sequential pulls exhausted the shared startup window before health, so no inference ran; five exact deletes, all six absence checks, registry removal, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-g | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 108ddbbd4a7da97a426a799e5ced71df87edad36] [plan sha256:52ff4c997508d406b71e0719e4c956829da4a24275d559428de16069c2b37fac] | USD 26.00 | — | [Registry-transport failure and cleanup proof](evidence/gate11route-20260830-g-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, registry removal, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-f | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 77eaa8ad683477ac07498d4c2420d8a959afc1e7] [plan sha256:49b182a304b1cd4dd527345cd9f64c1ec80a74dfedda740b2a198c81279e6ece] | USD 26.00 | — | [Serialized primary image-pull failure and cleanup proof](evidence/gate11route-20260830-f-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-e | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 22b468ad7901edaf85c0ff1c81594c1e90a102bd] [plan sha256:d80db65e522e6955b8d1df9853e961e0c8f0ed7e687152a26fb9d62f7dc1b016] | USD 26.00 | — | [Repeated primary image-pull failure and cleanup proof](evidence/gate11route-20260830-e-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-d | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source cc2cbb393f19e203a4c7eb5e5abfdfe772dacddc] [plan sha256:47efba5556ab8384b892d4310f3dec8760fe5642f7c20466caea77b858e5c285] | USD 26.00 | — | [Classified primary image-pull failure and cleanup proof](evidence/gate11route-20260830-d-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-c | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 47dadde939cc869f4b56ea1713127674350ece10] [plan sha256:7a535abd8b3ad6ab42a94538380897b446a280248678cef5c3cd2273020d7261] | USD 26.00 | — | [Failed start-primary and cleanup proof](evidence/gate11route-20260830-c-lifecycle.json): native/provider preflight, exact create, SSH, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | -| route-20260830-b | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 5ef5c5a389ce47080b45bebff66408174a09c4fe] [plan sha256:a87056b4659194824b1a2f0fa40d3834abc7040da78167138df217afd758be12] | USD 26.00 | USD 0 | Independent provider-free verification found a one-second float-rounding timeout overshoot after authorization. No provider call or resource creation occurred; source `47dadde` clamps the bound and uses a new run identity. | CANCELED | -| route-20260830-a | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 0ea140f3fe764a6772a3b4217ead4bcd7e93562f] [plan sha256:dc11838569220a3fd7d7afbd3e8e70f49ac9034994071252b11931dd9ad45947] | USD 26.00 | — | [Detached retry B](evidence/gate11route-20260830-a-detached-retry-b-lifecycle.json) and the concurrent [keyring failure](evidence/gate11route-20260830-a-keyring-failure-cleanup.json) both stopped before inference and proved five exact deletes, all six resource classes absent, and the protected bootstrap running. The immutable source-`0ea140f` reservation is released; the corrected source uses a new run identity. | CLEANED-RELEASED | -| gate11pub-20260829-a | GCP | Gate 11 exact Qwen/Gemma public-route image publication from source `d2ea7dea5f3541b86293279b0a650bb46ab82583`; one `e2-standard-4`, 200 GB balanced auto-delete boot disk, six-hour DELETE deadline, registry egress, and contingency | USD 10.00 | — | [Passed publications](evidence/gate11pub-20260829-a-publication-attempt.json) and the [sanitized cleanup verification](evidence/gate11pub-20260829-a-cleanup-verification-attempt.json) bind the strict [Qwen](evidence/gate11pub-20260829-a-qwen3.5-2b-publication-evidence.json) and [Gemma](evidence/gate11pub-20260829-a-gemma-4-e2b-publication-evidence.json) evidence. Registry credentials are absent; native authentication refreshed, the exact builder and auto-delete boot disk are absent, and the protected bootstrap is running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | -| gate9-20260829-d | GCP | Gate 9 sequential Qwen/Gemma edge envelopes at pushed source `480c1fa`: one G2/L4 route and one native Linux client per model, native Windows client local, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-d-edge-envelope-attempt.json): the native Windows Qwen cold cache made no progress after 22,975,832 bytes and stopped at five minutes; Linux and Gemma did not start; exact instances, disks, firewalls, model service, and benchmark process are absent; global and regional L4 usage are zero; protected bootstrap running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | -| gate9-20260829-c | GCP | Owner-authorized clean Gate 9 retry at pushed source `1e845e6`: sequential Qwen/Gemma routes and Windows/Linux cold clients, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-c-edge-envelope-attempt.json): the single Windows Qwen invocation failed before inference after MSYS converted the bootstrap multiaddr; Linux and Gemma did not start; exact instances, disks, firewalls, model service, and benchmark process are absent; GPU usage is zero; protected bootstrap running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | -| gate9-20260829-b | GCP | Owner-authorized Gate 9 attempt: sequential Qwen/Gemma routes and Windows/Linux cold clients, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-b-edge-envelope-attempt.json): Windows Qwen passed; Linux Qwen inference completed but post-close RSS failed the 16 MiB allowance; Gemma was not started; exact instances, disks, firewalls, and benchmark processes are absent; GPU usage is zero; protected bootstrap running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | -| gate9-20260829-a | GCP | Stopped Gate 9 Qwen attempt after overlapping orchestration launched two Windows cold-client processes | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-a-edge-envelope-attempt.json): both benchmark processes stopped without reports; exact instances, disks, firewalls, and model service absent; GPU usage zero; protected bootstrap running. Historical maximum released by explicit owner direction on 2026-08-29 after cleanup. | CLEANED-RELEASED | -| gatev-20260827-a | GCP | Gate V one-host Linux G2/L4 Qwen public vertical slice, 150 GB balanced disk, six-hour hard deadline, headroom, and contingency | USD 17 | — | [Passed run and cleanup proof](evidence/gate-v-20260827-a-public-vertical-slice.json): instance, disk, firewalls, subnet, network, addresses, routers, and resource policies absent at 2026-08-27T09:28:20Z; GPU usage zero; protected bootstrap running. Historical maximum released by explicit owner reset on 2026-08-27; billing remains informational. | CLEANED-RELEASED | -| gate5-20260827-a | GCP | Gate 5 Qwen3.5 2B Windows/Linux qualification and real-run source fixes | USD 69.00 | — | All four exact profile VMs/disks and both network perimeters are absent; GPU usage is zero and `communityai-bootstrap-1` remains running. Historical maximum released by explicit owner reset on 2026-08-27; billing remains informational. | CLEANED-RELEASED | -| gate5-20260827-b | GCP | Same-source `23a4078` Windows/Linux CPU retries; sequential high-memory hosts, private 150 GB disks, one-hour DELETE deadlines, 25% headroom, and fixed contingency | USD 14.00 | — | [Passed qualification and cleanup proof](evidence/gate5-20260827-qwen3.5-2b-qualification.json): Windows used N1; Linux used a lower-cost E2 fallback after N1 capacity failed in every regional zone. Both hosts/disks and the exact firewall, NAT, router, subnet, address, and network are absent; L4 usage is zero; protected bootstrap running. Historical maximum released by explicit owner reset on 2026-08-27. | CLEANED-RELEASED | -| gate6-20260827-a | GCP | Gate 6 Gemma 4 E2B four-profile qualification; serial 48 GB CUDA recovery after a native Windows failover-load crash | USD 79.00 | — | [Passed qualification and cleanup proof](evidence/gate6-20260827-gemma-4-e2b-qualification.json): all four profile hosts/disks and the exact firewall, NATs, routers, subnets, addresses, and network are absent; global GPU and regional L4 usage are zero; protected bootstrap running. Historical maximum released by explicit owner reset on 2026-08-27; billing remains informational. | CLEANED-RELEASED | -| gate7-20260827-a | FLY | Gate 7 CPU-only provider recovery mechanism | USD 30.00 | — | [Passed TinyLlama recovery and cleanup evidence](evidence/gate7-20260828-tinyllama-recovery.json): one bootstrap and four workers ran, one selected worker was killed, the route recovered with exact parity, all five Machines were destroyed, and the token was revoked. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | -| gate7pub-20260827-a | GCP | Gate 7 exact Qwen CPU image publisher after repeat 3,601.7-second Fly registry disconnects; 80 GB disk, four-hour DELETE deadline, egress, and contingency | USD 10.00 | — | [Attempt and cleanup proof](evidence/gate7-20260827-a-separate-machine-attempt.json): exact builder and boot disk absent at 2026-08-28T01:24:30Z; protected bootstrap running. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup; billing remains informational. | CLEANED-RELEASED | -| gate7pub-20260828-b | GCP | Gate 7 exact CPU-only Qwen image republish from verified source `7570d94`; `e2-standard-4`, 80 GB balanced disk, four-hour DELETE deadline, egress, and contingency | USD 10.00 | — | [Publication and cleanup evidence](evidence/gate7-20260828-b-separate-machine-attempt.json) binds the [immutable image report](evidence/gate7-20260828-b-qwen3.5-2b-publication-evidence.json); builder and disk absent, protected bootstrap running. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | -| g7mirror-20260828-c | GCP | Gate 7 immutable Qwen mirror to the isolated Fly registry; `e2-standard-2`, 30 GB disk, two-hour DELETE deadline, egress, contingency | USD 10.00 | — | [Attempt, repository initialization, credential cleanup, and builder cleanup](evidence/g7mirror-20260828-c-fly-registry-attempt.json): the first copy exposed an uninitialized Fly repository; both registry logins were removed, builder and disk are absent, the protected bootstrap remains running, and supported build-only initialization created no Machine. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | -| g7mirror-20260828-d | GCP | Final Gate 7 immutable Qwen mirror after supported Fly repository initialization; `e2-standard-2`, 30 GB disk, intended two-hour DELETE deadline, egress, contingency | USD 10.00 | — | [Failed attempt and cleanup evidence](evidence/g7mirror-20260828-d-fly-registry-attempt.json): copy did not start because GHCR authentication was rejected; the exact builder and disk are absent, Fly has zero Machines/tokens, and the protected bootstrap is running. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | -| g7mirror-20260828-e | GCP | Canceled Qwen mirror retry | USD 10.00 | USD 0 | Owner stopped the GCP-to-Fly mirror loop before provisioning; no instance or disk was created. | CANCELED | - -Owner-set accounting baseline on 2026-08-27: **USD 0 spent before `gatev-20260827-a`**. -The removed USD 99 total was a sum of worst-case reservations, not observed provider spend. -This baseline is an owner authorization decision, not a Cloud Billing reconciliation. -Read-only reconciliation at 2026-08-27T15:42:15Z confirmed billing is enabled but the -project has zero queryable BigQuery export datasets, so no observed-cost figure is available -yet. The owner reset the budget again on 2026-08-27 after Gate 6 cleanup was proved, -so its maximum remains historical evidence but no longer consumes the new epoch. The -`gate7-20260827-a` consumed a conservatively reserved USD 30 maximum and is now -cleaned. The additional -`gate7pub-20260827-a` maximum remains committed at USD 10 after its short-lived -CPU-only GCP builder published the exact image and cleanup was proved; observed billing -is still unavailable. The resulting 9 GB rootfs plan exceeded Fly's current 8 GB hard -limit before any Machine was created. Run `gate7pub-20260828-b` consumed a further -USD 10 maximum for the cleaned short-lived CPU builder that published and verified the -8 GB-compatible replacement image. Fly rejected its private external registry reference -before creating a Machine. Run `g7mirror-20260828-c` consumed USD 10 maximum for a -cleaned mirror builder; the copy exposed that the never-deployed Fly app repository first -required Fly's supported build-only initialization. That zero-byte local initialization -created no Machine. Retry `g7mirror-20260828-d` consumed its committed USD 10 maximum after creating the -bounded builder, then failed before copying because GHCR authentication was rejected. -Its exact GCP builder and disk are now proved absent and the protected bootstrap is -running. Retry `g7mirror-20260828-e` was canceled before provisioning. The four -conservatively committed GCP maxima plus the cleaned Fly reservation left USD 30 before -`gate9-20260829-a`. After that attempt's complete cleanup was proved, the owner explicitly -directed immediate continuation on 2026-08-29, resetting the combined test-budget epoch to -USD 100. The cleaned `gate9-20260829-b`, `gate9-20260829-c`, and `gate9-20260829-d` -maxima and the cleaned `gate11pub-20260829-a` publisher maximum consumed that epoch while -billing remained delayed. Native-auth verification at 2026-08-30T00:36:31Z then proved -the publisher's exact builder and disk absent and the protected bootstrap running. With -every run in that epoch cleanup-proved, the owner explicitly authorized a cleanup-backed -reset on 2026-08-30. Those four historical maxima are now `CLEANED-RELEASED`, delayed -charges remain informational, and the new combined authorization epoch starts at **USD 100**. -On 2026-08-30 the owner also designated execution speed as the operating priority: take the -shortest authorized critical path and begin bounded work as soon as its fail-closed preflight -passes. That priority does not raise the USD 100 ceiling or waive exact cleanup, protected- -resource, credential, privacy, or acceptance requirements. Fly credit is not counted as extra -authorization. - -After the Gate 9 clients, Gate 11 product route, and Gate 13 clients were all cleanup-proved, -the owner explicitly authorized another cleanup-backed reset for the next run on 2026-08-31. -Their USD 98 conservative maxima remain historical evidence but no longer consume the new -epoch; delayed observed charges remain informational. The next run starts with a new combined -authorization of **USD 100**. Every paid create still requires fresh native authentication, -an exact source-bound cost authorization, a conservative ledger reservation, and the existing -fail-closed preflight and cleanup controls. +| gate13-20260901-a | GCP | Automated Gate 13 real-window replay, finalized against production packages from `e904d36416a4f186c0bec05ff20210df9ca19848`: one bounded L4 route, then sequential ordinary-user Windows/Qwen and Linux/Gemma clients [original plan `sha256:6687b9ba098b3f6676f48f4bf03ebb92bdc6a1278bf5bc1c227819b3a3e7cbb0`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-i | GCP | Final Gate 13 manual clean-host playthrough: Gate 11 route acceptance first, then sequential ordinary-user Windows/Qwen and Linux/Gemma desktop qualification with literal UI controls and post-restart inference [plan `sha256:8525c3099f273c099aba26de57c1f610a0c74cac65ed2640589d51e874bd0c44`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-h | GCP | Final corrected Gate 13 route-first lifecycle with both four-file release-audit bundles pinned and staged, the bounded Windows user-runtime environment, exact archive preflight, and sequential ordinary-user Windows/Qwen then Linux/Gemma clients [plan `sha256:f243254cc5fb65f44d0c9e707be36feb3284fd6e15b15620882843798fb456b1`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-g | GCP | Corrected Gate 13 route-first lifecycle with a bounded standard Windows user-runtime environment, one durable foreground host-adapter execution as each ordinary OS user, exact archive preflight, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-f | GCP | Fresh Gate 13 route-first lifecycle using one durable foreground host-adapter execution over IAP SSH as each ordinary OS user, exact archive preflight, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:c9a2aafc84940df901a7db1755af2e684f845b78dcdfac04332cfed36388ba25`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-e | GCP | Fresh Gate 13 route-first lifecycle with pinned reusable route setup, corrected S4U/SID Windows host job, explicit archive download-and-hash prerequisite, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:9ca0fa516017c4a3709a467752f779bcb3bbc0a7c790f9bc61de56d385804c62`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-d | GCP | Fresh Gate 13 route-first lifecycle using the durable controller and host jobs, one bounded route and sequential clients [plan `sha256:d32050a51b8f696aa224fc7e748c9113e174e3c3069c1f8b2bc769b0c5ecea18`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-c | GCP | Gate 13 durable route-first lifecycle with the same bounded 16-hour route and sequential 6-hour clients, new exact resources, and corrected explicit IAP target-tag arguments [plan `sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-b | GCP | Gate 13 durable route-first lifecycle: one 16-hour G2/L4 product route, then sequential fresh 6-hour Windows/Qwen and Linux/Gemma CPU clients [plan `sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64`] | USD 56.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260831-a | GCP | Gate 13 replacement product-node route plus fresh CPU Windows/Linux packaged lifecycles at route source `f64a388a47b098ac7f69d2affc59816376b43bb1` and exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7` [plan sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69] | USD 52.00 | — | [Archived cleanup][ledger-history] | CLEANED-COMMITTED | +| gate13-20260830-c | GCP | Gate 13 sequential clean packaged Qwen Windows and Gemma Linux lifecycles at exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7`, temporarily suspending and later restoring the Gate 11 route while reusing its sole global L4 allocation on uniquely named fresh Windows and Linux clients [plan sha256:427bc1ed8a6645ad0650d91aaba7aa753d398fa84f56d57b50aca04c4e0cc955] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate9-20260830-e | GCP | Gate 9 concurrent Qwen/Gemma Windows/Linux acquisition records and schema-v3 envelopes at pushed source `ba410f74f1cf625f1e1c34734b53e4514fa7c5ec`, reusing the separately authorized product route and using bounded isolated clients [plan sha256:04ba77ee68f4a895ae080a4ddcbf6805b502da6a95a4146734acbddff92de307] | USD 46.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-j | GCP | Gate 11 signed-catalog product node route [workload gcp-product-node-route] [source e1d715fd47c852fa12ca50c76e8f4c6a0831fd78] [final runtime source 4cef141746705c3ee8bc8e017693855e0bc4871e] [plan sha256:1a0927e9d83a9a409ac2ea0232c4fceb14821d3f2c5eb87def88b8e7cdcb07d8] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-g | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 62be8f1c999b6ebe0ece2a660a0be4757cc83005] [plan sha256:109d2b6958ac8ced31e7202c8eb230387d29615f964d32d6726564b9366eafd7] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-f | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source bff0c3203191725928246ad3e13deb01ffbab8de] [plan sha256:735cfd847291229571529c8f640fc76005e340a29680806295dad33a7e1a1fb6] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-e | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source c0bd81e4e3ced3cd05a642740e343da41d05aceb] [plan sha256:fc13db74e107795c6d2896e0135c4a669a3fd7618a9ef1c4feab54f2425cf948] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-d | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 3ae7a094a1e4ca3865d5b6aa463816eac36318f4] [plan sha256:2387d038386ea64e6301d70133aaee4dceedb2c8279e1a341b744ffb1f9fdbc4] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-c | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 42241d6fb951cc6274ba991d5762558d67c376ab] [plan sha256:634c4d9db1474655065b1d4d6c2bb4066aeb6c48afa3e2eda7e85d980282104e] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-b | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 448196300660174ae8daf5b70bb55c275dcc981d] [plan sha256:861ebeaa2af38e563bdfb736d955b23ea87bd579188636a5512577ee6b35dd52] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| cache-20260830-a | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source a41d9ed72e333057fc017c769ed65f17c92a46e6] [plan sha256:271778431c7553f93d674dffb5131c60133449478d4103c46f366129d7eae2ab] | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-i | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source fc4c18b045b9143ba455c38fa890eb112429ad3f] [plan sha256:c17ca0aa19f3eb79f1ae837f240b4972a17c821c5c4b8521582e2d38fbd6b99a] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-h | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source c09552e7ea0d3f0905857acb35a94affabccedbb] [plan sha256:97ce29d07b3965f8fad4272c9a7b641347622a5940b628d917f3a54fa5a17234] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-g | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 108ddbbd4a7da97a426a799e5ced71df87edad36] [plan sha256:52ff4c997508d406b71e0719e4c956829da4a24275d559428de16069c2b37fac] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-f | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 77eaa8ad683477ac07498d4c2420d8a959afc1e7] [plan sha256:49b182a304b1cd4dd527345cd9f64c1ec80a74dfedda740b2a198c81279e6ece] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-e | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 22b468ad7901edaf85c0ff1c81594c1e90a102bd] [plan sha256:d80db65e522e6955b8d1df9853e961e0c8f0ed7e687152a26fb9d62f7dc1b016] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-d | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source cc2cbb393f19e203a4c7eb5e5abfdfe772dacddc] [plan sha256:47efba5556ab8384b892d4310f3dec8760fe5642f7c20466caea77b858e5c285] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-c | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 47dadde939cc869f4b56ea1713127674350ece10] [plan sha256:7a535abd8b3ad6ab42a94538380897b446a280248678cef5c3cd2273020d7261] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| route-20260830-b | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 5ef5c5a389ce47080b45bebff66408174a09c4fe] [plan sha256:a87056b4659194824b1a2f0fa40d3834abc7040da78167138df217afd758be12] | USD 26.00 | USD 0 | [Archived cleanup][ledger-history] | CANCELED | +| route-20260830-a | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 0ea140f3fe764a6772a3b4217ead4bcd7e93562f] [plan sha256:dc11838569220a3fd7d7afbd3e8e70f49ac9034994071252b11931dd9ad45947] | USD 26.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate11pub-20260829-a | GCP | Gate 11 exact Qwen/Gemma public-route image publication from source `d2ea7dea5f3541b86293279b0a650bb46ab82583`; one `e2-standard-4`, 200 GB balanced auto-delete boot disk, six-hour DELETE deadline, registry egress, and contingency | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate9-20260829-d | GCP | Gate 9 sequential Qwen/Gemma edge envelopes at pushed source `480c1fa`: one G2/L4 route and one native Linux client per model, native Windows client local, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate9-20260829-c | GCP | Owner-authorized clean Gate 9 retry at pushed source `1e845e6`: sequential Qwen/Gemma routes and Windows/Linux cold clients, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate9-20260829-b | GCP | Owner-authorized Gate 9 attempt: sequential Qwen/Gemma routes and Windows/Linux cold clients, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate9-20260829-a | GCP | Stopped Gate 9 Qwen attempt after overlapping orchestration launched two Windows cold-client processes | USD 28.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gatev-20260827-a | GCP | Gate V one-host Linux G2/L4 Qwen public vertical slice, 150 GB balanced disk, six-hour hard deadline, headroom, and contingency | USD 17 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate5-20260827-a | GCP | Gate 5 Qwen3.5 2B Windows/Linux qualification and real-run source fixes | USD 69.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate5-20260827-b | GCP | Same-source `23a4078` Windows/Linux CPU retries; sequential high-memory hosts, private 150 GB disks, one-hour DELETE deadlines, 25% headroom, and fixed contingency | USD 14.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate6-20260827-a | GCP | Gate 6 Gemma 4 E2B four-profile qualification; serial 48 GB CUDA recovery after a native Windows failover-load crash | USD 79.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate7-20260827-a | FLY | Gate 7 CPU-only provider recovery mechanism | USD 30.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate7pub-20260827-a | GCP | Gate 7 exact Qwen CPU image publisher after repeat 3,601.7-second Fly registry disconnects; 80 GB disk, four-hour DELETE deadline, egress, and contingency | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| gate7pub-20260828-b | GCP | Gate 7 exact CPU-only Qwen image republish from verified source `7570d94`; `e2-standard-4`, 80 GB balanced disk, four-hour DELETE deadline, egress, and contingency | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| g7mirror-20260828-c | GCP | Gate 7 immutable Qwen mirror to the isolated Fly registry; `e2-standard-2`, 30 GB disk, two-hour DELETE deadline, egress, contingency | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| g7mirror-20260828-d | GCP | Final Gate 7 immutable Qwen mirror after supported Fly repository initialization; `e2-standard-2`, 30 GB disk, intended two-hour DELETE deadline, egress, contingency | USD 10.00 | — | [Archived cleanup][ledger-history] | CLEANED-RELEASED | +| g7mirror-20260828-e | GCP | Canceled Qwen mirror retry | USD 10.00 | USD 0 | [Archived cleanup][ledger-history] | CANCELED | + +
+ +[ledger-history]: RELEASE_READINESS_HISTORY.md#cloud-authorization-and-spend-ledger ## Evidence update rules -- Link a passed gate to an immutable report, source commit, manifest digest, and relevant - workflow/provider run. -- Never put credentials, prompts, provider output, private paths, or private endpoints here. -- A deterministic unit/integration test may prove implementation readiness, but it cannot - pass a gate that explicitly requires external hardware, multiple hosts, public workers, - packaging, signing, or real cleanup. -- Once the required runner, adapter, or verifier exists and passes its contract tests, - additional test-harness hardening does not count as critical-path progress unless a - real gate attempt exposed the exact defect being fixed. -- When a gate fails, keep the failure evidence, use `IN PROGRESS`, `WAITING`, or `BLOCKED` - accurately, and record the concrete next action. Never lower or bypass the gate merely - to obtain a pass. +- Keep this file to current outcomes, concrete next actions, and operational inputs. + Put chronological implementation/provider detail in linked evidence or the archive. +- Record exact model/profile and source/package identity. A source-runtime test + cannot pass a packaged test; a short functional run cannot pass performance. +- Preserve failed attempts and cleanup evidence. Do not replace a failure with a + later pass or lower an acceptance requirement to obtain a green status. +- Keep private credentials, endpoints, and user content out of release records. +- New harness work must resolve a concrete implementation or observed-run gap. diff --git a/docs/RELEASE_READINESS_HISTORY.md b/docs/RELEASE_READINESS_HISTORY.md new file mode 100644 index 000000000..902518b79 --- /dev/null +++ b/docs/RELEASE_READINESS_HISTORY.md @@ -0,0 +1,739 @@ +# Release readiness history through 2026-09-05 + +Archived before the 2026-09-05 consolidation. This file preserves the previous +readiness document and superseded catalog inventory, including failed attempts, +source bindings, historical budgets, and checkpoint detail. Dates, status claims, +and next actions below describe their original snapshots; they are not current +execution instructions or new spending authorization. + +Use [RELEASE_READINESS.md](RELEASE_READINESS.md) for the current release plan, +[QWEN_FULL_INFERENCE_RESULTS.md](QWEN_FULL_INFERENCE_RESULTS.md) for the new live +results, and [COMMUNITY_AI_MODEL_LADDER.md](COMMUNITY_AI_MODEL_LADDER.md) for the +current product ladder. Relative evidence links remain valid in this directory. + +## Original release-readiness snapshot + +# Public inference alpha release readiness + +Last verified: 2026-09-03 + +This is the live source of truth for public-alpha implementation. Update it whenever a +gate changes state. `docs/REVIVAL.md` defines the execution contract and long-term design; +`docs/REVIVAL_TEST_RESULTS.md` is the detailed evidence archive. + +## Release definition + +- Product: public community inference through the packaged localhost OpenAI-compatible + API, with optional bounded compute sharing. +- Label: public alpha. Do not describe it as a stable, production-SLO service. +- Supported platforms: Windows and Linux. +- Deferred platform: macOS, until later CPU/MPS and packaged-device testing passes. +- Base CommunityAI model: Qwen3.8-27B FP8. Qwen3.5 2B and Gemma 4 E2B remain + historical qualification fixtures, while Qwen3.5 0.8B/4B/9B are local fallbacks. +- Not included: credits, earnings, payments, payouts, or a compute marketplace. +- Availability promise: best effort. The alpha may initially depend on one CommunityAI + discovery seed and one complete candidate route, with a small fallback route and clear + unavailable/degraded states; it does not claim a production SLO. +- Minimum trust floor: pinned signed catalog, exact verified manifests/artifacts, + authenticated peer announcements and transport, finite public admission/time limits, + authoritative local contribution limits, prompt-visibility disclosure, and a tested + route/catalog disable procedure. +- Post-alpha hardening: independent route/seed/mirror redundancy, independent threshold + key holders, publisher-signed installers, authenticated automatic update/rollback, and + exhaustive malicious-load/Sybil/partition/long-soak programs. + +## Status vocabulary + +- `PASSED`: required real evidence exists and is linked. +- `IN PROGRESS`: implementation or a real gate run is underway. +- `READY`: prerequisites exist and the gate can be run. +- `WAITING`: a required predecessor has not passed; do not work around it. +- `PAUSED`: partial work exists, but the gate is outside the currently permitted sequence. +- `BLOCKED`: owner input or unavailable external state is required. +- `TODO`: not yet started. +- `DEFERRED`: explicitly outside the public-alpha scope. + +## Next release gate: Qwen3.8-27B FP8 + +**Status: IN PROGRESS.** This gate takes precedence over the remaining Gate 14-17 +sequence. Existing Gate 14 implementation and evidence are preserved, but its next paid +hardware run waits until Qwen3.8 passes. + +The gate passes only when the exact pinned, anonymously downloadable 30.87 GB FP8 +checkpoint works through the product path and proves: + +- stock parity on a complete 64-block route split across independent workers; +- selected-worker interruption and same-session recovery; +- automatic model/block placement without a contributor downloading the full model; +- clean packaged direct-Hub acquisition, hash verification, restart, and cache reuse; +- bounded download, GPU memory, TTFT, and decode-throughput measurements on + representative RTX 30-, 40-, and 50-series consumer profiles; and +- publication through a new signed catalog sequence without rewriting the historical + Qwen3.5/Gemma catalog. + +Native FP8 execution is not a pass requirement. The implemented FP8-to-BF16/FP16 path may +establish correctness first; native FP8 is a later memory/throughput optimization. + +The [first source-bound loader checkpoint](evidence/gateq38-20260903-a-fp8-loader-checkpoint.md) +is pushed as `de15d9c`. It pins the official FP8 and stock-reference inventories, propagates +the source FP8 method through distributed configuration, fails closed unless source and runtime +profiles agree, dequantizes scale-grid weights into the manifested execution dtype, and carries +that profile through worker advertisement, loading, memory accounting, and the product smoke. +Independent verification passed a 158-test primary matrix and a 146-test offline subset. A clean +source checkout passed manifest structure with `artifacts_verified=false`; the exploratory +one-block product retry reached official-source acquisition but stopped at byte zero of +`tokenizer.json` on a CDN connection timeout. No official artifact inventory, real Qwen3.8 +layer, complete route, parity, recovery, package, or hardware outcome is claimed. Read-only GCP +preflight was healthy, but the current USD 100 epoch retains the prior conservative USD 56 +maximum; no new reservation or paid resource was created (USD 0). + +The [exact span-artifact planning checkpoint](evidence/gateq38-20260903-b-span-artifact-planning-checkpoint.md) +now makes the next split-worker boundary executable without acquiring model weights. It parses the +pinned checkpoint index once under exact size/digest verification, selects only startup metadata and +the union of shards required by a contiguous block span, enforces that allowlist in the actual server +loader, and uses the same exact byte count for automatic-placement admission. Cached signed-intent +reuse is bound to the current proposal, selected-set digest, normalized throughput, freshly loaded +cryptographic identity, and a finite unexpired lease. The final candidate passes 142 focused tests, +132 adjacent regressions, and 1,564 offline unit tests with 10 expected skips under independent +adversarial review. The pinned metadata declares four 16-block worker sets of approximately 6.096 GB +each while excluding 6,507,216,554 tokenizer, MTP, chat, and outside-layer bytes. These are declared +manifest/index results, not a model download, hard arbitrary-cache quota, real block execution, or +hardware outcome; no reservation or provider resource was created (USD 0). + +The [worker plan execution-binding checkpoint](evidence/gateq38-20260903-c-worker-plan-execution-binding-checkpoint.md) +now carries the exact private manifest, canonical span/cache, byte count, and artifact-set digest from +an acknowledged automatic placement into the real source or frozen server subprocess. The immutable +launch rejects a changed executable, duplicate/inline claims, `--num_blocks`, config files, custom +modules, training RPCs, and credential flags. The bound parser ignores ambient `config.yml`, and the +server independently recomputes the verified span plan before constructing its announcer or accessing +weights. Different spans sharing one physical shard set cannot substitute for each other, while the +set digest and cache path remain outside public state. The final candidate passes 147 focused tests, +259 related regressions, and 1,568 offline unit tests with 10 expected skips under independent +adversarial review. This is still USD 0 source/test evidence: no model bytes, real Qwen block, +complete route, package, cloud, or hardware outcome is claimed. The next unblocked action is a fresh +official-source single-span acquisition and real block execution through this bound command. + +The [fresh single-span execution checkpoint](evidence/gateq38-20260903-d-fresh-single-span-execution-checkpoint.md) +closes that first real-outcome boundary on a local Windows RTX 2070 SUPER. From a new isolated cache, +the bound worker anonymously acquired and rehashed the exact `config.json`, +`model.safetensors.index.json`, and 383,865,448-byte `layers-0.safetensors` selection from the +pinned official revision. Its `0:1` backend loaded in manifested BF16 and served an authenticated +exact-peer `rpc_inference`; the deterministic BF16 `[1,1,5120]` output was finite, shape +preserving, and different from the input in 0.363 seconds. That fresh process did not emit a Git +attestation at launch. A later network-disabled cache-reuse replay bound 19 production paths to +pushed commit `af7d887` and tree `4c064b2` before launch, repeated the exact input/output hashes, +and retained a zero-exit cleanup audit for worker PID, listener, and replay-tagged processes in the +[source/runtime audit](evidence/gateq38-20260903-d-source-runtime-cleanup-audit.json). This local +checkpoint used no cloud resource (USD 0). It is not complete-route parity, recovery, packaged cold +acquisition, or a required RTX 30/40/50 measurement, so Gate Q3.8 remains in progress. + +The [complete-route controller checkpoint](evidence/gateq38-20260903-e-complete-route-controller-checkpoint.md) +adds a durable USD 0 state machine for the exact four-span Qwen3.8 route. It rederives every span +from the official `model.language_model.layers` index through the source-bound production verifier, +requires protected reservation/preflight and route child evidence, writes an issuance journal before +any start decision, prevents paid replay after state loss, and binds the canonical GCP inventory, +worker plan, exact sources, pricing horizon, and ledger scope through stable plan and execution +digests. The final candidate passes 102 focused, 17 final security-regression, 423 adjacent, and +1,651 offline unit tests with 10 expected skips; independent review rejected under-priced lifetime, +bogus machine/GPU/image substitutions, and stale authorization after worker-plan or source changes. +The checked-in ledger has no exact Q3.8 reservation marker, so no paid start is authorized and this +checkpoint used USD 0. +The [GCP adapter checkpoint](evidence/gateq38-20260903-f-gcp-adapter-checkpoint.md) +adds the USD 0 provider boundary without provisioning. It source-binds the adapter, compiles but +does not execute the exact eleven-resource private GCP start specification, validates exact +run-prefixed inventory and plan-scoped network isolation, and performs retry-safe best-effort +cleanup of independently bound resources including terminal instances and disks. Paid start and +collection reject before provider access because the protected Qwen3.8 host runtime and fresh +instance-generation-bound status/evidence transport do not yet exist. The final candidate passes +137 focused, 168 adjacent, and 1,686 offline unit tests with 10 expected skips; independent +adversarial review returned PASS. The checkpoint used USD 0 and created no resource. + +The [Linux runtime-package validator checkpoint](evidence/gateq38-20260903-g-linux-runtime-package-validator-checkpoint.md) +source-binds the exact production archive, release audit, physical and semantic Qwen3.8 manifest, +and complete packaged node onedir inventory before a future privileged host stage may consume it. +The fail-closed validator rejects runtime mutation, unsafe modes and links, packaged model weights, +and archive pathname replacement, and emits one atomic canonical package record. The final local +candidate passes 26 focused tests with one Windows-only POSIX skip, 184 adjacent tests with the same +skip, and 1,712 offline unit tests with 11 expected skips under independent review. Exact-source +style and test CI pass; production package CI independently verifies the archive build but does not +invoke the new validator. This USD 0 checkpoint does not prove native Linux validator execution or +host staging. + +The [runtime-package plan-binding checkpoint](evidence/gateq38-20260903-h-runtime-package-plan-binding-checkpoint.md) +removes the circular dependency between package validation and the final route plan. A strict, +controller-protected source context now produces the complete runtime-package record, and the +controller validates and immutably carries that record through the stable plan, execution +inventory, action record, action ID, reservation, and preflight boundaries. The final candidate +passes 156 focused tests with one native-POSIX skip, all 184 Gate Q3.8 tests with the same skip, +205 adjacent tests with the same skip, and 1,733 offline unit tests with 11 expected skips; +independent adversarial and staged-snapshot reviews returned PASS. This USD 0 checkpoint does not +prove native Linux package staging, packaged execution, or status/evidence transport. + +The [Linux host-runtime preparation checkpoint](evidence/gateq38-20260903-i-linux-host-runtime-preparation-checkpoint.md) +adds a privileged, source-bound extraction and offline preflight contract for the exact packaged +node. It validates the complete package inventory and production-sized release attestations, +installs a protected root-owned runtime plus an isolated qualification-user work root, executes the +verified binary through `/proc/self/fd`, and makes cleanup and prepared-record publication +fail-closed, serialized, no-replace, and durable. The final local candidate passes 55 host-runtime +tests with three native-Linux skips, 240 Gate Q3.8 tests with four skips, 206 adjacent tests with one +skip, and 1,789 offline unit tests with 14 skips; independent adversarial and frozen-index reviews +returned PASS. This USD 0 checkpoint does not prove the native-Linux ownership/execution probes, +provider bootstrap, or status/evidence transport. + +The [instance-generation latch checkpoint](evidence/gateq38-20260903-j-instance-generation-latch-checkpoint.md) +now carries each observed GCP instance ID and creation timestamp into a project/zone/name-bound +generation digest, latches the exact five-instance set before active route phases, and forces cleanup +on deletion, recreation, or any generation drift. Non-instance resources cannot expose generation +metadata, and terminal cleanup retains the latch. The final candidate passes 170 focused tests, an +independent 28-test adversarial subset, and all 264 Gate Q3.8 tests with four native-platform skips; +formatting, import order, compilation, and whitespace checks pass. Paid start and collection remain +blocked before provider access. This checkpoint used USD 0 and created no reservation or resource. + +The [authenticated host-status envelope checkpoint](evidence/gateq38-20260903-k-authenticated-host-status-envelope-checkpoint.md) +adds a bounded canonical transport primitive for controller-issued instance contexts and HMAC-authenticated +Linux host records. The exact source, plan, execution inventory, worker plan, action IDs, provider generation, +boot UUID, monotonic revision, prepared-record digest, and strict worker/job payload are bound before a record +can be accepted. Deeply nested, oversized, noncanonical, stale, replayed, or substituted records fail closed. +The transport suite passes 41 tests, its controller/adapter matrix passes 211, and the complete Gate Q3.8 +matrix passes 305 tests with four native-platform skips. Key distribution, protected prepared-record equality, +adapter consumption, and native Linux bootstrap remain open; paid start and collection stay blocked. This +checkpoint used USD 0 and created no reservation or resource. + +The [protected host-status grounding checkpoint](evidence/gateq38-20260903-l-protected-host-status-grounding-checkpoint.md) +loads the protected controller context and key through identity-checked handles, binds exact provider generation +and boot identity into prepared state, derives status from that reopened record, and serializes preparation and +cleanup under one lifecycle lock. Cleanup publishes and fsyncs a terminal generation marker before deletion, +so interrupted cleanup blocks late preparation and remains retryable. The final candidate passes 119 focused +tests with three native skips, all 328 Gate Q3.8 tests with four skips, and 1,877 offline tests with 14 skips; +independent adversarial and staged-index reviews returned PASS. Context/key delivery, external status +publication, adapter consumption, and native Linux execution remain open; paid start and collection stay +blocked. This checkpoint used USD 0 and created no reservation or resource. + +The [protected IAP and authenticated status-consumer checkpoint](evidence/gateq38-20260903-n-protected-iap-and-status-consumer-checkpoint.md) +adds a separate exact run-scoped IAP SSH firewall to the full plan/action/inventory/cleanup contract and consumes +only canonical HMAC guest attributes through paired protected key/replay resolvers between generation-stable +complete provider inventories. Broad or substituted firewall state, malformed or ambiguous carrier data, wrong +keys, replayed revisions, instance recreation, and protected-bootstrap loss fail closed; cleanup does not depend +on status material. The firewall candidate passes 180 focused and 368 complete Gate Q3.8 tests, while the +consumer candidate passes 101 focused and 376 complete Gate Q3.8 tests, with four native skips in each complete +matrix. Independent read-only verification and compilation/whitespace checks pass. No provider request, +reservation, or resource was made (USD 0), and paid start/collection remain blocked. + +The [instance-key vault and protected-delivery checkpoint](evidence/gateq38-20260903-o-instance-key-vault-and-protected-delivery-checkpoint.md) +adds exact-generation private key records with crash-safe rotation/revocation/cleanup, a bounded authenticated +context/key bundle installed atomically as one root-private host file, and fixed IAP SSH stdin delivery bracketed +by stable provider inventories. Receipt digest and HMAC authentication precede binding and freshness policy; +key bytes and private paths never enter argv, environment, logs, receipts, or ordinary state. The final candidate +passes 147 controller tests with one native skip, 224 focused delivery tests with three native skips, and all 412 +Gate Q3.8 tests with five native skips. Black, isort, compilation, whitespace, and independent adversarial checks +pass. No provider request, reservation, or resource was made (USD 0), and paid start/collection remain blocked. + +The [native Linux protected-host probe](evidence/gateq38-20260903-p-native-linux-protected-host-probe.md) +runs the exact pushed controller, adapter, transport, privileged runtime, and staging matrix as Linux root in a +network-disabled container with read-only inputs. All 415 tests pass and only the two Windows-only checks skip; +Linux ownership, mode, nonroot traversal, symlink, atomic delivery, receipt/replay, terminal cleanup, and +fake-provider generation-bracketing paths execute. No real GCP, IAP, metadata, guest-attribute, systemd, capacity, +reservation, or paid-route result is claimed (USD 0), and paid start/collection remain blocked. + +Live provider delivery/status, complete route, parity, recovery, package, and hardware outcomes remain open. + +## Critical path + +Work from top to bottom while prerequisites are satisfied. Gate V and Gates 1–13 have +passed. The current mandatory sequence is **Gate Q3.8 → Gates 14–16 → Gate 17**. The visible +vertical slice proved real Qwen3.5 2B inference through a public GCP L4 worker, and the +strict four-profile Qwen and Gemma matrices now pass, and Gate 7 passed the generic +five-Machine provider recovery mechanism with TinyLlama. Per-model repetition of the +same provider recovery gate is not required. + +As of 2026-09-02, Gate 13 is `PASSED`. [Run `gate13-20260831-i`](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) +replaced the opaque wrapper-first approach with a literal clean-host desktop playthrough. +The route first passed Qwen primary, automatic Gemma fallback, and Qwen restoration. +Windows and Linux then ran sequentially as ordinary users from exact verified production +archives: the app opened, public inference passed, sharing was configured and started, +the app was restarted, sharing resumed, Pause sharing worked, and post-restart inference +passed on Linux. The Windows playthrough exposed and fixed the actual product blocker: +legacy MAX_PATH on a manifest-artifact lock path under the normal per-user data root. +Source `f1dc3a0` passed the regression test, rebuilt-package self-tests, and default-root +Qwen inference. Every run instance, disk, and firewall is absent, global L4 usage is zero, +and the protected bootstrap remains running. Gate 14 then became active and accumulated +partial implementation, but the owner has now made Gate Q3.8 the next release gate. Gate +15 still owns publication of the source-fixed Windows archive plus reinstall/uninstall +release work after Qwen3.8 and Gate 14 pass. + +The follow-up [automated paid-cloud replay `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) +now proves that the manual sequence is repeatable without UI assistance. Exact production +packages from source `e904d36` passed archive verification and four packaged self-tests. +Windows/Qwen passed two real-window sessions in 260.828 and 66.328 seconds; Linux/Gemma +passed in 229.270 and 44.956 seconds, including restart resume and a second inference. +Both formal client jobs passed as attempt ordinal 1. The replay now preserves the manual +Sharing-page and per-model-toggle order, handles the transient `Model unavailable` case, +fences slow GPU restarts and stale DHT advertisements, and accepts the exact fresh Ubuntu +systemd inventory. All run instances, disks, and firewalls are absent, regional L4 usage +is zero, and the protected bootstrap remains running. A future Gate 13 replay still needs +a new source-bound package, authorization, route, clean clients, and cleanup evidence. + +Do not work on the post-alpha items in the deferred table while an alpha gate can progress. +Missing Docker, snapshots, local GPU hardware, or local host capacity is not an external +blocker: use authorized bounded infrastructure according to its role. GCP/local hosts +cover platform and CUDA qualification; Fly is CPU-only and covers the isolated +separate-machine recovery topology. A real gate failure justifies the smallest +implementation fix; speculative harness expansion does not replace the outcome. + +The former Gate 5 quota blocker is resolved. The [2026-08-27 quota/probe evidence](evidence/gcp-l4-quota-probe-20260827.json) +records `GPUS_ALL_REGIONS` limit `1`, and the completed [Gate 5 qualification](evidence/gate5-20260827-qwen3.5-2b-qualification.json) +again proves one-host-at-a-time L4 operation, zero post-run L4 usage, complete run-resource +absence, and the protected `communityai-bootstrap-1` still running. The cleaned Gate V and +Gate 5 runs remain in the historical ledger, but the owner explicitly reset the test-budget +epoch to USD 100 on 2026-08-27 after their cleanup was proved. Their unobserved maxima no +longer consume the new authorization; later billing should still be recorded for information. + +| Order | Gate | Status | Current evidence | Next action | +| ---: | --- | --- | --- | --- | +| Q3.8 | Make Qwen3.8-27B FP8 the working base CommunityAI route | IN PROGRESS | [Checkpoint `de15d9c`](evidence/gateq38-20260903-a-fp8-loader-checkpoint.md) pins the exact official FP8 and stock inventories and passes bidirectional profile validation plus a synthetic FP8 checkpoint through the production config/block-load/forward path. The [span-artifact checkpoint](evidence/gateq38-20260903-b-span-artifact-planning-checkpoint.md) adds exact per-worker shard selection, admission, server allowlisting, and lease binding; 142 focused, 132 adjacent, and 1,564 offline unit tests pass. The [fresh single-span checkpoint](evidence/gateq38-20260903-d-fresh-single-span-execution-checkpoint.md) anonymously acquires and rehashes the exact 384,054,133-byte official `0:1` plan, then returns a finite changed BF16 `[1,1,5120]` result through the signed exact-peer `rpc_inference` route on local Windows/CUDA hardware. A separate network-disabled replay binds pushed source `af7d887`, repeats the deterministic result from cache, and records exact PID/listener/tagged-process cleanup. The [complete-route controller checkpoint](evidence/gateq38-20260903-e-complete-route-controller-checkpoint.md) durably binds the exact four-span artifact plan, canonical GCP launch specifications, protected USD 100 reservation/preflight evidence, exact child route evidence, and journaled cleanup/replay handling; 102 focused, 17 final security-regression, 423 adjacent, and 1,651 offline tests pass at USD 0. The [GCP adapter checkpoint](evidence/gateq38-20260903-f-gcp-adapter-checkpoint.md) source-binds and validates exact run-scoped observation/cleanup, compiles the private eleven-resource start specification without executing it, and blocks paid start/collection before provider access; 137 focused, 168 adjacent, and 1,686 offline tests pass at USD 0. The [Linux runtime-package validator checkpoint](evidence/gateq38-20260903-g-linux-runtime-package-validator-checkpoint.md) binds the production archive/audit, Qwen3.8 manifest, and complete packaged node onedir inventory; 26 focused, 184 adjacent, and 1,712 offline tests pass locally at USD 0, while native Linux validator/host execution remains open. The [runtime-package plan-binding checkpoint](evidence/gateq38-20260903-h-runtime-package-plan-binding-checkpoint.md) replaces the circular final-plan input with a strict protected source context and binds the complete immutable package record through the stable plan, execution inventory, action, authorization, and preflight identities; 156 focused, 184 Gate Q3.8, 205 adjacent, and 1,733 offline tests pass at USD 0. The [Linux host-runtime preparation checkpoint](evidence/gateq38-20260903-i-linux-host-runtime-preparation-checkpoint.md) adds strict protected extraction, an exact nonroot offline packaged preflight, all-exception process cleanup, and durable no-replace prepared state; 55 host-runtime tests with three native-Linux skips, 240 Gate Q3.8 tests with four skips, 206 adjacent tests with one skip, and 1,789 offline tests with 14 skips pass at USD 0. The [instance-generation latch checkpoint](evidence/gateq38-20260903-j-instance-generation-latch-checkpoint.md) binds exact provider instance IDs and creation timestamps into the active route state, forces cleanup on same-name recreation or generation drift, and keeps paid start/collection disabled; 170 focused and 264 complete Gate Q3.8 tests with four native skips pass at USD 0. The [authenticated host-status envelope checkpoint](evidence/gateq38-20260903-k-authenticated-host-status-envelope-checkpoint.md) adds canonical bounded controller contexts and HMAC-authenticated Linux worker/job records bound to exact source, plan, actions, provider generation, boot, revision, and prepared-record digest; 41 transport, 211 controller/adapter, and 305 complete Gate Q3.8 tests with four native skips pass at USD 0. The [protected host-status grounding checkpoint](evidence/gateq38-20260903-l-protected-host-status-grounding-checkpoint.md) loads protected context/key/boot inputs, binds prepared state to the exact provider generation, derives authenticated status from the reopened record, serializes prepare/cleanup, and makes terminal cleanup durable before deletion; 119 focused, 328 complete Gate Q3.8, and 1,877 offline tests pass with 3, 4, and 14 skips respectively at USD 0. The [protected IAP and authenticated status-consumer checkpoint](evidence/gateq38-20260903-n-protected-iap-and-status-consumer-checkpoint.md) adds the distinct exact IAP firewall and generation-bracketed canonical HMAC carrier consumer; its firewall candidate passes 180 focused and 368 complete Gate Q3.8 tests, and its consumer candidate passes 101 focused and 376 complete Gate Q3.8 tests, with four native skips in each complete matrix. No provider request occurred. The [instance-key vault and protected-delivery checkpoint](evidence/gateq38-20260903-o-instance-key-vault-and-protected-delivery-checkpoint.md) adds exact-generation controller key vaulting, crash-safe rotation/revocation/cleanup, atomic authenticated host bundles, fixed IAP SSH stdin delivery, generation-stable delivery reads, and authenticated fresh receipts; 147 controller tests with one native skip, 224 focused delivery tests with three native skips, and 412 complete Gate Q3.8 tests with five native skips pass at USD 0. The [native Linux protected-host probe](evidence/gateq38-20260903-p-native-linux-protected-host-probe.md) executes the exact pushed controller, adapter, transport, privileged runtime, and staging matrix as root with read-only inputs and no network; 415 tests pass and only two Windows-only checks skip. It proves native ownership, private-mode, nonroot-traversal, symlink, atomic-delivery, receipt/replay, terminal-cleanup, and fake-provider generation-bracketing behavior, but no live GCP, IAP, metadata, capacity, or paid route. The checked-in ledger contains no Q3.8 reservation marker, so no paid start is authorized. No complete route or stock-parity/recovery/package outcome has passed. | Run a fresh read-only GCP authentication, protected-bootstrap, exact run-resource, quota, accelerator, capacity, and pricing preflight against the pushed native-probe source. If and only if the exact four-L4 plan has a conservative maximum within the remaining USD 44, commit a source-bound readiness reservation before any create. Then run live start, protected IAP delivery, metadata publication, authenticated generation-stable collection, acceptance, and exact cleanup. After the real 64-block route passes, prove stock parity and selected-worker same-session recovery, packaged cold acquisition/cache reuse, representative RTX 30/40/50 measurements, and publish a new signed catalog sequence only after every acceptance outcome passes. | +| 1 | Integrate the active revival branch and make its CI workflows dispatchable from the repository default branch | PASSED | [PR #8](https://github.com/flujo-app/CommunityAI/pull/8) integrated [commit `22b5598`](https://github.com/flujo-app/CommunityAI/commit/22b559836fa5a4c9b228d87a823d1c99dc3939a9) into `main` after [Check style](https://github.com/flujo-app/CommunityAI/actions/runs/32946456633), [Tests](https://github.com/flujo-app/CommunityAI/actions/runs/32946456596), and [Windows/Linux Production desktop](https://github.com/flujo-app/CommunityAI/actions/runs/32946456600) passed. [PR #22](https://github.com/flujo-app/CommunityAI/pull/22) later integrated the accumulated public-alpha path as merge commit `05fa84d`. Its default-branch [test run 33388263559](https://github.com/flujo-app/CommunityAI/actions/runs/33388263559) exposed one nondeterministic Ubuntu MLA paged-cache equivalence failure after the exact PR head had passed. Source `5d29416` isolates that cache contract from MoE expert routing while retaining separate dense/MoE block coverage; [PR #23 run 33388770828](https://github.com/flujo-app/CommunityAI/actions/runs/33388770828) passes the exact MLA test, 627-test Ubuntu suite, and 64-test Linux public-worker contracts. PR #23 merged as exact commit `c90625c`; its default-branch [Tests](https://github.com/flujo-app/CommunityAI/actions/runs/33389270215), [Check style](https://github.com/flujo-app/CommunityAI/actions/runs/33389270333), and [CodeQL](https://github.com/flujo-app/CommunityAI/actions/runs/33389269851) runs all pass. | Keep the same workflows green on follow-up PRs; they are now dispatchable from the default branch | +| 2 | Make Windows/Linux the strict public-alpha qualification matrix | PASSED | Default dispatch, exact-profile aggregation, fleet readiness, and the recovery controller now require Windows CPU/CUDA plus Linux CPU/CUDA; focused contract tests pass | Provision four distinct labelled runners and retain real exact-profile evidence; macOS remains a separate deferred gate | +| 3 | Prepare bounded provider automation and cost controls | PASSED | [PR #9](https://github.com/flujo-app/CommunityAI/pull/9) integrated [commit `1d4f7d4`](https://github.com/flujo-app/CommunityAI/commit/1d4f7d4453eb688994ce21c08e182c1ad8e63ae7) after [style](https://github.com/flujo-app/CommunityAI/actions/runs/32947541300), [tests](https://github.com/flujo-app/CommunityAI/actions/runs/32947541452), and [Windows/Linux production packaging](https://github.com/flujo-app/CommunityAI/actions/runs/32947541637) passed; the 29-test guard prices the serialized 13.5-hour G2/L4 fleet at USD 69 maximum (14-hour N1/T4 at USD 70), binds immutable OS images and hard deletion deadlines, supports split-region CUDA capacity, and excludes `communityai-bootstrap-1` from exact cleanup; provider automation remains passed and native `gcloud`/`flyctl` authentication was valid for the completed Gate 7 work | No further Gate 3 framework work. Revalidate native provider authentication, quota, and the exact ledger reservation immediately before every paid create | +| 4 | Build immutable Qwen3.5 2B and Gemma 4 E2B qualification images/snapshots | PASSED | [Gate 4 attempt `gate4-20260826-b`](evidence/gate4-20260826-b-qualification-image-build-attempt.json) passed both exact snapshot/in-image checks and published source `7660e33` with SLSA provenance and SPDX SBOM. [Qwen evidence](evidence/gate4-20260826-b-qwen3.5-2b-publication-evidence.json) binds `ghcr.io/flujo-app/communityai-qualification-qwen3.5-2b@sha256:129b96fd848b996a5e3a0c918c39c705d328e6e5010b3222a5c25ea10ab142ed` ([metadata](evidence/gate4-20260826-b-qwen3.5-2b-build-metadata.json)): 6,913,811,781 compressed bytes, 6,913,829,173 uncompressed, 9 GB rootfs. [Gemma evidence](evidence/gate4-20260826-b-gemma-4-e2b-publication-evidence.json) binds `ghcr.io/flujo-app/communityai-qualification-gemma-4-e2b@sha256:5f04eb8e923023ff05f64d13fde5b879e8990725518d4e81210b03b4b6047c6f` ([metadata](evidence/gate4-20260826-b-gemma-4-e2b-build-metadata.json)): 11,011,406,681 compressed bytes, 11,011,424,083 uncompressed, 13 GB rootfs. Both isolated builders and the complete retry network were deleted; the protected bootstrap remains. | Use these immutable digests and evidence-bound rootfs sizes for Gates 5 and 6 | +| V | Pass a visible public vertical slice: app observes a remote worker, `auto` selects a model, and inference succeeds | PASSED | [Run `gatev-20260827-a`](evidence/gate-v-20260827-a-public-vertical-slice.json) executed clean source `8200afc` against the immutable Qwen image and exact manifest on a public Linux G2/L4 worker. The [desktop evidence](evidence/gate-v-20260827-a-desktop-models.png) shows signed-catalog `auto` selection, 24/24 blocks, and one verified peer; the localhost OpenAI-compatible request returned one token through Qwen in 15.231 seconds. The real run exposed four bounded fixes, all focused tests and two independent reviews passed, every exact run resource is absent, global GPU usage returned to zero, and the protected bootstrap remains running. | Proceed to Gate 5 using the exact pushed source, revalidated one-L4 quota, immutable Qwen input, a new conservative reservation, sequential CUDA hosts, and complete cleanup evidence. | +| 5 | Qwen3.5 2B Windows/Linux CPU/CUDA qualification | PASSED | [Qualification and cleanup evidence](evidence/gate5-20260827-qwen3.5-2b-qualification.json) and the [strict aggregate](evidence/gate5-20260827-qwen3.5-2b-matrix.json) bind Windows CPU/CUDA and Linux CPU/CUDA passes to exact source `23a4078e17ed9d5ae6f31e7497bae69b83aecef6`, DRIFT `2.3.0.dev2`, Qwen revision `15852e8c16360a2fea060d615a32b45270f8a8fc`, and manifest `sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33`. Every profile proved exact artifacts, 24/24 manifested stock-token parity, selected-worker interruption, and recovery. All Gate 5 instances, disks, and perimeters are absent; L4 usage is zero; the protected bootstrap remains running. | Proceed to Gate 6 under the owner-reset USD 100 budget epoch. | +| 6 | Gemma 4 E2B Windows/Linux CPU/CUDA qualification | PASSED | [Qualification and cleanup evidence](evidence/gate6-20260827-gemma-4-e2b-qualification.json) and the [strict aggregate](evidence/gate6-20260827-gemma-4-e2b-matrix.json) bind Windows CPU/CUDA and Linux CPU/CUDA passes to exact source `a45025a3262a88df65217b630392488e8548aaaf`, DRIFT `2.3.0.dev2`, Gemma revision `3e22461f65e89153144f8adb70e3b8c2cc9845a7`, and manifest `sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd`. Every profile proved exact artifacts, 35/35 manifested stock-token parity, selected-worker interruption, and recovery. All Gate 6 instances, disks, firewall, NATs, routers, subnets, addresses, and VPC are absent; global GPU and regional L4 usage are zero; the protected bootstrap remains running. | Complete; Gate 7 subsequently passed. Proceed to Gate 9. | +| 7 | Provider-level real separate-machine recovery | PASSED | [Run `gate7-tiny-20260828-j`](evidence/gate7-20260828-tinyllama-recovery.json) ran one bootstrap and four CPU-only TinyLlama workers in Fly `gru` with two replicas per block. SIGKILL of `host-a` during generation caused rerouting to `host-c`, activation replay, same-session completion, and exact stock-token parity in 16.395 seconds. All five resources were destroyed and the token was revoked. The [recovery runbook](RECOVERY_TEST_RUNBOOK.md) records the artifact, control-plane, retry, and cleanup lessons. | Proceed to Gate 9. Repeat recovery only in the later clean-install automatic-placement product flow, not once per model. | +| 8 | Per-model duplicate separate-machine recovery | DEFERRED | Gates 5 and 6 already qualify Qwen and Gemma across the supported platform/device matrix; Gate 7 proves the model-independent provider recovery mechanism. Repeating the same Fly topology for each catalog model would test artifact transport rather than a new release property. | No public-alpha action. Model admission uses manifest/artifact and resource-envelope checks; product-level recovery is covered after automatic placement and catalog publication. | +| 9 | Publish edge resource envelopes for selectable profiles | PASSED | [Run `gate9-20260830-e`](evidence/gate9-20260830-e-edge-resource-envelopes.json) publishes all four privacy-safe acquisition and schema-v3 steady-state records at exact runtime source `ba410f7`. Qwen selected 4,571,197,320 bytes and Gemma 10,278,818,149 bytes from empty caches on both Windows Server 2022 and Ubuntu 24.04; every artifact SHA-256 passed with zero resumptions. Qwen measured load/first-token/decode at 25.896 s/1.785 s/1.392 tok/s on Windows and 17.115 s/2.973 s/0.800 tok/s on Linux, with process-tree RSS peaks of 1,883,205,632 and 2,832,244,736 bytes. Gemma measured 64.286 s/1.386 s/1.949 tok/s on Windows and 38.808 s/2.025 s/0.868 tok/s on Linux, with peaks of 1,728,995,328 and 2,818,523,136 bytes. Every workload generated eight tokens without retaining prompts or outputs; Windows Job Objects and Linux process groups were empty, and route/DHT, accelerator, runtime-close, and provider cleanup all passed. All four temporary instances and auto-delete disks are absent; the protected bootstrap and separately authorized Gate 11 route remain running. The Windows Gemma cache-preserving in-place memory retry created no new resource and did not raise the USD 46 Gate 9 ceiling. | Proceed immediately to Gate 13 clean packaged install/inference on Windows and Linux using these envelopes and the live bounded product route. | +| 10 | Implement automatic contributor model and block placement | PASSED | Signed bootstrap now installs one bounded `auto` worker. The local planner filters exact manifested candidates through owner policy and local resource ceilings, requires fresh authenticated replica coverage, targets the least-covered contiguous range with per-node jitter, reconciles exact-manifest launches through the existing artifact-verifying server and `WorkerSupervisor`, applies residency/cooldown/switch hysteresis, exposes placement reasons, and preserves an explicit operator pause across ineligibility or placement changes. A new or migrated worker must sign an expiring exact-manifest/range intent with fixed numeric resource claims and receive a remote DHT store acknowledgement (`exclude_self=True`) before entering the artifact path; invalid, rejected, or failed publication is fail-closed and cannot advance planner state, while a previously admitted placement is retained. Actual completed local generations feed exact-manifest demand, useful-throughput, and reliability through two bounded five-minute aggregate windows; no prompt, output, token ID, key, request ID, address, path, error, or per-request event is retained. Only a closed window with at least four completed routes may be signed by the separate router identity and published under the manifest-bound `demand-v1` DHT key with a 90-second lifetime and `exclude_self=True`. Consumers verify signature, exact schema/digest, lifetime, revocation, and replay ordering. The threshold-signed catalog may authorize 2–32 sorted RSA observer roots; missing or empty roots disable remote demand. Discovery discards unlisted identities before signature/replay work, excludes local and duplicate roots, isolates malformed records, requires two authorized roots, and medians at most 32 quantized observations. Observer keys are never generated or bundled: only a separately provisioned `route-demand.key` matching a signed root may publish, while ordinary nodes can consume without one. Any hot-edited root-list mismatch disables both publication and consumption until restart. Local utility is capped at 6 points and signed remote utility at 2, keeping the combined hint below the 10-point migration margin and 100-point replica step. Verified announcement and route-demand replay watermarks now survive restarts in one Windows-safe journal per raw manifest digest under the node data directory. Each strict journal is capped at 256 active identity scopes and 256 KiB, retains only public record kind, key ID, ordering tuple, record digest, and the bounded replay deadline, and is fsync-written through atomic replacement; malformed, duplicate, oversized, symlinked, non-regular, or unwritable state fails closed. The retained deadline prevents an older still-live record from returning after a short-lived newer record expires. The replay slice's 99-test focused protocol/discovery/planner/node-configuration matrix and 209-pass, 2-skip catalog/node/API superset pass. The Sybil slice's 122-test focused catalog/bootstrap/config/discovery matrix proves that 30 valid attacker keys plus one authorized root cannot reach threshold, two authorized roots aggregate without attacker weight, one high authorized vote cannot inflate a lower second vote, old catalogs remain signature-verifiable with remote demand disabled, and trust-epoch reload mismatches fail closed. A 190-pass, 1-skip catalog/protocol/planner/discovery/node/API superset also passes. Independent verification passed 146 focused tests and a 255-pass, 2-skip broader node/API superset, plus a native-Windows publication-boundary probe; formatting, import-order, import-smoke, and diff checks pass. The [explicit privacy review](AUTOMATIC_PLACEMENT_PRIVACY_V1.md) inventories collection, retention, public-key linkability, DHT/journal/API/log exposure, secure-deletion limits, and residual governance/host risks. Three executable privacy-contract tests fix the aggregate, intent, demand, replay, forbidden-field, and path-free warning schemas; the focused privacy/protocol/planner/discovery/node matrix passes 108 tests and the broader catalog/node/API matrix passes 258 tests with 2 skips. Independent privacy review passed 108 tests with 1 skip and a 225-pass, 2-skip broader subset; every caught observer-key exception and an unauthorized key produced no path, key ID, or exception detail, while prompt and identity-path schema injections failed closed. The [deterministic convergence and load acceptance](AUTOMATIC_PLACEMENT_ACCEPTANCE_V1.md) closes the remaining software gate: equal snapshots use node-specific 32-point model dispersion and range rendezvous ranks; a fixed 512-node cold cohort selects both models and every range below the 85% concentration boundary; two 4,096-node fresh-arrival cohorts remain below that boundary under maximum priority-aligned or standby demand; maximum demand causes zero incumbent migrations; one-replica loss migrates after residency without early reversal; rolling arrivals keep every model/block populated and repair an abrupt block loss. The alpha fails closed above 32 candidates or 512 blocks, permits one `auto` worker, clamps reconciliation to at least one second, and scans each candidate in one bounded pass. The focused planner/convergence/configuration matrix passes 78 tests and the broader catalog/protocol/discovery/node/API matrix passes 214 with 2 skips. A real Windows DHT round trip exposed and fixed a durable-replay multiprocessing regression: replay guards now omit/recreate their thread lock across serialization and reload persistent state; its 15-test protocol/network matrix passes. Independent verification reproduced the 78-test focus, passed an expanded 235-test matrix with 2 skips and the 15-test real-DHT probe, and exercised adversarial score, timing, 32-by-512 load, 1,000-case range-equivalence, and persistent replay-reload boundaries. This slice used no cloud resources and spent USD 0. | Gates 9–11 are passed. Gates 13–14 must now prove the packaged flow and real hardware ceilings using the published envelopes. | +| 11 | Operate initial public alpha routes | PASSED | [Product-node run `route-20260830-j`](evidence/gate11node-20260830-a-lifecycle.json) installed the generic CommunityAI wheel on a bounded G2/L4 VM, verified the signed catalog, downloaded both exact manifested models directly from Hugging Face into one persistent shared cache, and used the product node's automatic workers to expose complete Qwen 24/24 primary and Gemma 35/35 standby routes. No model-specific image, cache mirror, or operator-transferred model artifact was used. The privacy-safe acceptance passed one-token primary inference, deliberate primary pause, automatic Gemma selection in 58.073 seconds, standby inference, Qwen restoration in 32.042 seconds, and restored inference. Both workers were stable before the drill. After Gate 13 released the L4, the preserved route was restored without changing its model cache or source, its ephemeral endpoint was rebound, both product-node services became active, and a fresh acceptance reproved Qwen 24/24 primary inference, automatic Gemma 35/35 fallback/inference, Qwen restoration, and restored inference. The protected bootstrap remains running. A corrected 4,800-second provider DELETE backstop was set for `2026-08-31T05:28:16.516Z`, earlier than the original deadline. [Post-backstop cleanup evidence](evidence/gate11route-20260830-j-backstop-cleanup.json) and an independent recheck prove the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero remaining route availability, and the protected bootstrap still running. The same-host standby is a bounded alpha fallback, not independent infrastructure redundancy; independent redundancy remains post-alpha. | Gate 11 acceptance evidence remains complete, but no product route is live after the corrected DELETE backstop. [Gate 13 run `gate13-20260831-a`](evidence/gate13-20260831-a-cost-authorization.json) now binds refreshed native authentication, fail-closed preflight, and a fresh USD 52 reservation for the replacement route and packaged clients. | +| 12 | Create, publish, and bundle the minimal signed alpha catalog/bootstrap | PASSED | [Run `gate12-20260829-a`](evidence/gate12-20260829-alpha-catalog-publication.json) published the deterministic [`communityai-public-alpha-v1` bundle](../public-alpha/catalog-v1/bundle.json) from source `26be579`. Its threshold-one Ed25519 root signs sequence 1 with the exact qualified Qwen primary and Gemma standby manifests, one pinned public HTTPS mirror, one public seed, a one-route best-effort policy, and no unprovisioned route-demand roots. The canonical bundle binds five members and retains `complete_release_qualification=false`. All three public objects returned HTTP 200 with exact sizes, and a fresh empty consumer fetched them remotely, verified the signature/digests, and created the two-model `auto` node configuration. The private signing key remained ignored and uncommitted. The focused publication suite passes 32 tests, the catalog/bootstrap/model/desktop superset passes 92, and the run spent USD 0. | Preserve the branch-scoped mirror until a newly signed catalog sequence and packaged bootstrap migrate it. The Gate 11 acceptance and Gate 9 envelopes exist; [Gate 13 run `gate13-20260831-a`](evidence/gate13-20260831-a-cost-authorization.json) now authorizes the bounded replacement route and fresh packaged clients under the new epoch. Independent threshold holders and interchangeable mirror/seed governance are post-alpha. | +| 13 | Pass packaged clean-install inference on Windows and Linux | PASSED | [Manual run `gate13-20260831-i`](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) found and fixed the Windows legacy-MAX_PATH blocker while proving the literal clean-host flow. [Automated paid-cloud run `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) translates that flow into two real-window sessions per platform from exact production packages. Formal attempt 1 passed unattended on Windows/Qwen and Linux/Gemma: package verification, four self-tests, inference, policy save, per-model normalization, literal Start, observation, full process restart, Pause, final paused intent, and Linux restart-resume plus second inference. The route passed exact 24/24 and 35/35 stable fences. All run instances, disks, and firewalls are absent, L4 usage is zero, and the protected bootstrap is running. | Proceed to Gate 14 without replaying Gate 13 discovery. Gate 15 owns reinstall/uninstall/retained-data release engineering. | +| 14 | Pass automatic-contribution and resource-control hardware checks | IN PROGRESS | [PR #11](https://github.com/flujo-app/CommunityAI/pull/11) and [PR #12](https://github.com/flujo-app/CommunityAI/pull/12) implemented authenticated node-authoritative sharing controls. [Automated Gate 13 run `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) proves the real packaged Sharing UI starts, survives restart, resumes on Linux, and pauses on both supported platforms without manual UI recovery. The new strict Gate 14 verifier pins that lifecycle evidence plus the exact Windows/Qwen and Linux/Gemma Gate 9 envelopes, platform/OS pair, production package, L4 device profile, all five configured resource classes, suspension/resume, automatic block placement, recovery, unsupported CPU power telemetry, privacy, and exact GCP cleanup. Its durable controller recomputes the reset current-epoch spend ledger under the USD 100 ceiling, rejects expired/foreign/overlapping resources, serializes fresh Windows then Linux hosts, binds reported evidence bytes before collection, rejects rolled-back completed jobs, hidden active reservations, orphan planned disks before every fresh start, and resources that return during teardown, and requires cleanup to match the exact authorization file, controller source, plan, project, zone, resources, and successful terminal state while excluding the protected bootstrap. Exact source [`c0f2342e15aa7e12ca7c2980deca64d613204143`](https://github.com/flujo-app/CommunityAI/commit/c0f2342e15aa7e12ca7c2980deca64d613204143) passed independent adversarial review, the 49-test focused suite, formatting, import-order, compilation, and diff checks; [CodeQL](https://github.com/flujo-app/CommunityAI/actions/runs/33659218953), [style](https://github.com/flujo-app/CommunityAI/actions/runs/33659223681), and [Linux/Windows tests](https://github.com/flujo-app/CommunityAI/actions/runs/33659223645) are green. Its [production desktop run](https://github.com/flujo-app/CommunityAI/actions/runs/33659223622) passed exact-source build, smoke, independent checksum/provenance verification, packaged-node/native-credential/public-seed exercise on Windows, and all four uploads. The retained artifact digests are Windows install `sha256:44ab9faa5dae4537bae60e005ac790ce2b4f3636a2f5ad5327de713302a37b2c` (2,695,084,318 bytes), Windows audit `sha256:55e924e4a36f6deeb0e37bd64979f20debc36227503616f71c1f40622241ccc7` (468,605 bytes), Linux install `sha256:1494cb0bb4c37de1c8825c20d5400e76b4bbc62eefbd4010a161427a85f164c8` (3,360,751,567 bytes), and Linux audit `sha256:5ac2ce726235ef6c1ae3f1dea6faf89c9e42d61a2d0d7cd0f905ef5fe92deb91` (514,731 bytes), expiring 2026-09-09. The follow-up implementation adds thin source-bound Windows/Linux probes and an exact GCP action executor. Fresh calibrated bandwidth and physical-power samples must bind the configured limit, trigger crossing, worker absence, intent preservation, and below-limit recovery. Every calibration must also bind a controller-issued, one-time 15-minute challenge and measured start/end timestamps inside a maximum 120-second sample window. Controller state persists the issued digest and one-time consumption, while interrupted issuance reattaches only the exact still-valid file; stale, missing, future-dated, or cross-challenge evidence fails closed. Every provider mutation now requires a valid controller-bound action plus a fresh authenticated inventory/reconciliation, exact disks bind the full authorized image-project path, and instances with an attached service account are rejected. The focused 67-test Gate 14 suite, compilation, formatting, import-order, wrapper parsing, and diff checks pass locally. A final independent rerun reproduced all 67 tests plus a 10-case adversarial subset covering exact challenge time bounds, one-time state binding, cross-challenge rejection, full image-project and no-service-account bindings, and fresh controller-authorized inventory before mutation. Exact source [`727690c84e96c39742636cce037e099e838c4956`](https://github.com/flujo-app/CommunityAI/commit/727690c84e96c39742636cce037e099e838c4956) is pushed, and [production desktop run 33672294838](https://github.com/flujo-app/CommunityAI/actions/runs/33672294838) passed build, smoke, independent checksum/provenance verification, the Windows packaged-node/native-credential/public-seed exercise, and all four uploads. Its source-bound archives are Windows `sha256:908fd6b279e8152725e8b481d1cc4b2683058da7654a9f38367abbc65ab7eac6` (2,695,088,230 bytes) and Linux `sha256:4caba1d67f636b13303c9efe5318f8a566b7fe03dfae5a02bb0f3284683b44b0` (3,360,757,103 bytes), retained through 2026-09-09. After native authentication was refreshed, a 2026-09-02 read-only GCP preflight proved exactly one active account, the project active, the protected bootstrap running, global GPU quota 1 with usage 0, ready L4/G2 and Windows/Linux image inputs, and zero Gate 14 instances, disks, firewalls, or running L4s. No reservation or provider resource was created, and the current-epoch USD 44 remainder is intact. The next source-bound blocker is now closed in product code: an automatic worker remains policy-blocked until its signed placement intent is both published and remotely acknowledged, the privileged worker status carries both facts for the Gate 14 host probe, and failed publication cannot admit a previously unacknowledged assignment. The focused placement/discovery/supervisor/control suite passes 141 tests, with formatting, import-order, compilation, and diff checks clean. A separate Gate 14 exact-once native host-job namespace now reuses the proven Gate 13 Scheduled Task/systemd process-safety implementation while validating only strict Gate 14 platform evidence. The shared core is canonical-digest-bound by the hashed wrapper, evaluated directly from the verified in-memory bytes with no pathname reopen, and isolated from the Gate 13 module defaults. The combined Gate 13/Gate 14 host-job plus Gate 14 contract suite passes 106 tests, including changed-core, CRLF, namespace, replay, timeout, output-bound, native-binding, and evidence-binding regressions; formatting, import-order, compilation, and diff checks are clean. Exact source [`5632cc528b7a3e39296fd3fef8a0b3fd6dd620d0`](https://github.com/flujo-app/CommunityAI/commit/5632cc528b7a3e39296fd3fef8a0b3fd6dd620d0) passed both jobs in [production desktop run 33679402658](https://github.com/flujo-app/CommunityAI/actions/runs/33679402658): Windows and Linux completed exact-source build/smoke, independent checksum and provenance verification, and both archive-bound uploads; Windows also passed the packaged-node/native-credential/public-seed exercise. All four artifacts are retained through 2026-09-09. A final no-spend audit found and closed a stale Gate 13 Linux HOME/runtime hardcode in the shared desktop-session launcher; the source-bound wrapper now accepts its configured Gate 14 paths, and the focused suite passes 107 tests. The next no-spend slice adds the shared packaged-lifecycle sequencer: its strict configuration contains no pass, suspension, calibration, or cleanup claims; it binds the exact production archive and exact release metadata; validates all non-calibration observations; publishes a no-clobber challenge-ready checkpoint; and accepts only a controller-issued challenge covering that exact checkpoint digest. Retained checkpoints cannot replay a lifecycle. Controller-owned staging is separate from writable lifecycle outputs: POSIX requires root ownership with no qualification-process write access, while Windows requires a SYSTEM/Administrators owner, a protected non-null DACL, and independent denial of delete, DACL/owner change, generic write, child creation, and child deletion rights on the staging root and its parent. Config, package, metadata, and challenge reads use no-follow opened-handle identity checks and are revalidated around action phases. Every failure invokes cleanup; private facts are removed before a pending document may be published; and pending, persisted, and final evidence are strictly reread and compared. The 17-test lifecycle suite and expanded 125-test Gate 13/Gate 14 contract matrix pass locally. The controller and lifecycle now agree on the exact `gate14-lifecycle.json` basename, and the Windows adapter boundary has a source-digest-bound persistent PowerShell action host. One native host and state nonce survive prepare/calibrate/cleanup; bounded canonical controller-bound frames reject duplicate keys, replayed or out-of-order request IDs, and private material; EOF and failure force product-tree cleanup. Production prepare/calibrate remain deliberately unavailable until controller-owned release audit companions and the Gate 9 warm cache are bound, so this bridge cannot manufacture a pass. The expanded no-spend contract matrix passes 154 tests, including native malformed-frame, replay, ordering, cleanup, source mutation, unavailable-handler, and client/server Boolean, numeric, array-coercion, and changed-controller-binding regressions. The controller-owned lifecycle input now binds the complete Actions audit ZIP by exact artifact name/digest/size and exact extracted `SHA256SUMS`, desktop-metrics, provenance, and release-metadata members, then cross-validates their package, source, platform, archive, checksum, smoke, and incomplete unsigned-alpha claims. A separate warm-cache object binds the historical Gate 9 acquisition and envelope identities plus a fresh direct-upstream, no-mirror, empty-cache materialization record with the exact sorted artifact digests, roles, counts, and bytes for Windows/Qwen and Linux/Gemma. Outer ZIP, extracted-member, and materialization-record drift is rechecked at every lifecycle boundary; the immutable checkpoint binds the release-audit, warm-cache, and fresh-record digests. The lifecycle/controller focus passes 75 tests, including nested-schema, repository, non-finite timing, historical-artifact mutation, and substitution attacks. Exact source [`d4586f02530be7ce052ce9298ae95207e4368f05`](https://github.com/flujo-app/CommunityAI/commit/d4586f02530be7ce052ce9298ae95207e4368f05) then passed both jobs in [production desktop run 33695073372](https://github.com/flujo-app/CommunityAI/actions/runs/33695073372): Windows job `100462099399` and Linux job `100462099551` completed exact-source build/smoke, checksum/provenance verification, and both uploads; Windows also passed the packaged node/native-credential/public-seed exercise. The retained exact-head artifacts are Windows install ID `9871982175`, `sha256:3eea3254309fac149de210fa7c397cc94d0bf3f38c8a302e2f8d4b52671caa4e` (2,695,093,223 bytes); Windows audit ID `9871982782`, `sha256:afb5a6201f0a1d5788c43ead11f1fcb0994f8749d6d64525b8aa5ae4e78a2c4f` (468,612 bytes); Linux install ID `9872047850`, `sha256:ee1a9dccba6dbdb800e9b65cca6f0abbe7a8f38cb8f425802cebdb3cbffff47c` (3,360,731,719 bytes); and Linux audit ID `9872048365`, `sha256:210e65d8b7fbe6517813638eecbcc8eddfe639a48f5180e5e9ef8dac80b658d0` (514,751 bytes), all expiring 2026-09-09. A native cross-platform lifecycle entrypoint now connects the sequencer to the source-bound Windows action host and an equivalent persistent Linux Python host. Both transports carry the complete safe challenge digest/revision/time window, enforce bounded operation-specific install/calibration/cleanup timeouts, preserve one process/state identity across the controller wait, reject malformed/replayed/coerced frames, and clean on EOF or failure; Linux adds process-group kill fallback. Production prepare/calibrate remain deliberately unavailable until concrete packaged-cache/control handlers exist, so this boundary cannot manufacture a pass. The 64-test action/entrypoint focus and expanded Gate 14 contract matrix pass locally. No historical physical cache survives cleanup, so both platform profiles still require fresh direct host materialization from the official source. No Gate 14 hardware or cloud-run pass is claimed by this evidence, and this slice created no reservation or cloud resource (USD 0). A [restart-safe Linux product-action checkpoint](evidence/gate14-20260902-c-linux-product-actions-checkpoint.md) replaces the Linux unavailable-handler placeholder with a source-bound package/cache/control implementation. Its 21-test product/transport focus and the prior 201-test full matrix prove controlled prepare/calibrate/cleanup and failure cleanup before and after credential creation. A [Windows product-action checkpoint](evidence/gate14-20260902-d-windows-product-actions-checkpoint.md) now closes the equivalent local platform half with native Job Object membership cleanup, controller-bound package/audit/cache/source/config inputs, no-follow locked cache identity, exact action-specific persistent paths, physical bandwidth/power/schedule handlers, and phased retryable teardown. The Windows product/transport focus passes 50 tests, its Gate 13 lifecycle regression passes 16, and the complete Gate 14 matrix passes 211 tests under independent adversarial review. A [source-bound cache-materialization handoff checkpoint](evidence/gate14-20260902-e-cache-materialization-handoff-checkpoint.md) now adds a two-phase boundary: the ordinary native process validates a protected canonical plan/template before transfer, materializes only the exact official-source cache below its work root, and publishes a source/plan/record-bound handoff; a privileged promoter revalidates the physical cache and handoff before installing controller-protected lifecycle inputs. Windows promotion installs an explicit protected SYSTEM/Administrators DACL, normal lifecycle execution retains qualification-token write-denial checks, and a committed promotion survives partial handoff cleanup for idempotent retry. The cache/lifecycle focus passes 96 tests and the complete Gate 14 matrix passes 253 tests under final independent adversarial review, with formatting, import-order, compilation, and diff checks clean. The checkpoint downloaded no production model bytes and created no reservation or cloud resource (USD 0). A [promoted-input readability checkpoint](evidence/gate14-20260902-f-promoted-input-readability-checkpoint.md) closes the bootstrap-boundary defect found immediately afterward: final controller-owned inputs are now readable but nonwritable by the ordinary host-job identity, using root-owned POSIX mode 0644 or a protected Windows SYSTEM/Administrators-full-control plus Authenticated-Users-generic-read DACL. The cache/lifecycle focus passes 97 tests and the complete Gate 14 matrix passes 254, with formatting, import-order, compilation, and diff checks clean. It created no reservation, cloud resource, or production model download (USD 0). A [packaged-acquirer identity checkpoint](evidence/gate14-20260903-g-packaged-acquirer-identity-checkpoint.md) closes the next source-binding defect: controller mode now streams the once-read, 65,536-byte-bounded manifest under its exact digest, retains verified executable and manifest handles for the complete child lifetime, executes Linux from the verified `/proc/self/fd` while preserving the onedir-compatible original `argv[0]`, and holds a restrictive Windows share-read-only handle through `CreateProcess`. Native Windows write/delete/replacement and real locked-image launch probes pass; a Linux real-ELF fd-substitution probe is present for Linux CI. The focused cache/acquisition suite passes 64 tests with one Linux-native skip on Windows, the complete Gate 14 matrix passes 264 with the same skip, and the packaged-node dispatch regression passes 4; independent adversarial review found no commit blocker. Real packaged execution, complete onedir sidecar-inventory verification, fresh native materialization of the 4,571,197,320-byte Windows/Qwen and 10,278,818,149-byte Linux/Gemma selections, and the hardware run remain unverified. | Wire protected plan/template creation plus ordinary materialization and privileged promotion into the exact Windows/Linux host bootstrap. Before provisioning, verify and protect the complete extracted PyInstaller onedir runtime against the release-audit `SHA256SUMS`, run the native packaged `edge-acquire --help` no-download preflight on both platforms, and prove cleanup. Then revalidate native authentication, GCP inventory/quota/pricing, and the combined USD 100 ledger, record a bounded reservation, and run sequentially on fresh no-public-IP Windows/Linux L4 hosts with exact cleanup. | +| 15 | Complete minimal alpha release engineering | WAITING | The desktop builder now emits a stable sorted `SHA256SUMS` inventory of exact regular-file bytes and safe relative in-bundle file symlinks, source/build/catalog-bound `provenance.json`, and `release-metadata.json` with explicit unsigned public-alpha, no-publisher-signature, no-authenticated-update, Windows/Linux-only, no-credits, and incomplete-qualification claims. Structural verification binds each safe file symlink to its canonical in-bundle target, digest, and size while rejecting changed, missing, extra, absolute, external, broken, cyclic, directory-linked/junction, special, traversal, or case-colliding payloads plus unsupported or noncanonical metadata. Exact-source builds also reject dirty relevant inputs, and the expected-input fresh-process check rejects rewritten commit/tree, workflow, platform, Python, PyInstaller, or catalog evidence. Production desktop CI is configured to verify and bundle the Gate 12 inputs, bind the exact clean Git commit/tree and workflow, revalidate every expected input separately, and upload all evidence on Windows/Linux. The focused release-input/artifact suite passes 15 tests, including fresh-process CLI, dirty-source, and canonical-rewrite checks, and the broader catalog/bootstrap/model/desktop subset passes 134. Independent verification reproduced all 134, passed 58 desktop unittests with two environment skips, formatting/import-order/YAML/diff checks, an expected Gate 12/workflow fresh-process probe, and real Windows junction rejection; no cloud was used. [The first PR #22 production-desktop run](https://github.com/flujo-app/CommunityAI/actions/runs/33273518744) reached packaging on both hosts and exposed two exact cross-platform defects: PyInstaller's legitimate relative internal Qt file symlink on Ubuntu and CRLF-transformed signed Gate 12 JSON on Windows. The follow-up binds safe internal file symlinks without accepting external or directory links, forces `public-alpha/**` to LF at checkout, and includes `.gitattributes` in the clean-source boundary. [The second run](https://github.com/flujo-app/CommunityAI/actions/runs/33274432423) proved the Ubuntu package and the Windows signed-bundle/provenance path, then exposed a stale desktop contribution-status schema 2 contract when the packaged node emitted schema 3 automatic-placement evidence. Source `fcd1f41` now strictly validates schema 3 placement and rejects stale schema 2 plus missing, extra, secret-bearing, or inconsistent placement data; its 50-test node/client/lifecycle/build focus and all 59 desktop unittests passed with two environment skips. [The final run](https://github.com/flujo-app/CommunityAI/actions/runs/33275216332) bound exact source `fcd1f417d1435557addb2d6cded9dac0827c7d8c` and completed both Windows and Ubuntu package jobs, including bundle build/smoke, independent checksum/provenance verification, the Windows packaged-node/native-credential/public-seed smoke, and artifact uploads; every PR style, test, and package check is green. Source `36d85d2` makes generic release-artifact fixtures select the supported Linux archive explicitly instead of inheriting the CI host platform; the 21-test local artifact suite and [PR #22 test run 33372581439](https://github.com/flujo-app/CommunityAI/actions/runs/33372581439) pass, without expanding the supported platform matrix. Gate 13 manual clean-install evidence now exists; Gate 15 still lacks its upgrade/reinstall/uninstall and retained-data release evidence. | Retain the verified Windows/Linux artifacts as engineering evidence, then test clean install, manual upgrade/reinstall, uninstall, retained-data choice for the persistent verified model cache, and recovery instructions on both platforms against a newly authorized live product-node route and the published Gate 9 envelopes. Do not mark passed from metadata/unit tests alone. Publisher signing and automatic authenticated update/rollback are post-alpha. | +| 16 | Complete the bounded public-alpha safety canary | WAITING | [PR #13](https://github.com/flujo-app/CommunityAI/pull/13) and [PR #14](https://github.com/flujo-app/CommunityAI/pull/14) implemented bounded admission, privacy-safe aggregate health, training-off defaults, rollback procedures, and bounded routine rejection logs; no public canary has run. | After Gates 11–15, run a small monitored canary proving finite admission/timeouts, malformed-peer rejection, health reconstruction, privacy disclosure, route/catalog disable, and clean rollback. Exhaustive hostile-load, Sybil/collusion, partition, and long-soak campaigns are post-alpha. | +| 17 | Publish and observe the public alpha | TODO | Owner has authorized a public inference alpha, but preceding mandatory alpha gates are open. | After Gate V, Gate Q3.8, and Gates 1–16 pass, publish with explicit best-effort availability, unsigned-package, support, and prompt-privacy limitations; preserve the disable path and monitor real route/worker failures. | + +## Deferred work + +| Item | Status | Resume condition | +| --- | --- | --- | +| macOS CPU/MPS and packaged application support | DEFERRED | Real Apple-device hosts and testers are available | +| Credits, receipts, balances, spend authorization, earnings, and payouts | DEFERRED | Public inference alpha is live and its reliability/privacy behavior is understood | +| Compute marketplace and jurisdiction-specific payment onboarding | DEFERRED | Accounting threat model, legal review, and independent audit are complete | +| DeepSeek-V4-Flash and GLM-5.3-Flash ladder rungs | DEFERRED | After the Qwen3.8 release gate passes and its public capacity is stable, implement and qualify DeepSeek-V4-Flash, then GLM-5.3-Flash, without destroying the lower complete route | +| Production-SLO model-route redundancy and largest-worker-loss survival | DEFERRED | The best-effort alpha is live and its real route-loss evidence identifies the required topology | +| Independent multi-provider seeds, catalog mirrors, and outage survival | DEFERRED | The alpha seed/catalog dependency is measured and independent operators are available | +| Independent threshold catalog key holders and compromise/rotation governance | DEFERRED | The pinned single-signer alpha catalog is operating and human key holders accept responsibility | +| Publisher-signed installers plus authenticated automatic update/rollback | DEFERRED | Alpha packaging stabilizes and publisher identities/signing credentials are available | +| Exhaustive malicious-load, Sybil/collusion, partition, herd-switching, and long-soak campaigns | DEFERRED | The bounded alpha canary passes and real public telemetry supplies representative workloads | + +## Cloud authorization and spend ledger + +Authorization applies only to CommunityAI qualification and public-alpha infrastructure. +On 2026-09-01 the owner explicitly reset the cloud accounting epoch after reporting that the +prior real-world cloud charge was approximately USD 10 rather than the conservative reserved +maximums. Historical rows and their then-current `CLEANED-COMMITTED` labels remain unchanged +for auditability but consume USD 0 in the reset epoch. The reset epoch has a USD 100 accounting +ceiling; only the exact USD 56 `gate13-20260901-a` reservation is authorized, leaving USD 44 +unreserved and unauthorized for any other run. USD 56 remains a maximum-lifetime safety bound, +not a bill forecast. The existing GCP bootstrap's ordinary baseline cost is tracked separately; +never delete it as test cleanup. + +Before every paid run, add an entry with a conservative maximum. After cleanup, replace +the estimate with observed cost when available. If provider billing is delayed, retain the +maximum estimate until actual cost is known unless the owner explicitly resets the budget +after complete cleanup. On reset, keep historical rows and continue recording later observed +charges for information. `CLEANED-COMMITTED` means resources were absence-proved while the +conservative maximum still consumed the accounting epoch then in force; a later explicit reset +starts a new epoch without rewriting that historical state. + +| Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State | +| --- | --- | --- | ---: | ---: | --- | --- | +| gate13-20260901-a | GCP | Automated Gate 13 real-window replay, finalized against production packages from `e904d36416a4f186c0bec05ff20210df9ca19848`: one bounded L4 route, then sequential ordinary-user Windows/Qwen and Linux/Gemma clients [original plan `sha256:6687b9ba098b3f6676f48f4bf03ebb92bdc6a1278bf5bc1c227819b3a3e7cbb0`] | USD 56.00 | — | [Passed automation and cleanup](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) records both formal attempt-1 passes, exact evidence digests, all instances/disks/firewalls absent, L4 usage zero, and the protected bootstrap running. Provider billing was not yet available; the owner reports comparable real-world use at approximately USD 10. | CLEANED-COMMITTED | +| gate13-20260831-i | GCP | Final Gate 13 manual clean-host playthrough: Gate 11 route acceptance first, then sequential ordinary-user Windows/Qwen and Linux/Gemma desktop qualification with literal UI controls and post-restart inference [plan `sha256:8525c3099f273c099aba26de57c1f610a0c74cac65ed2640589d51e874bd0c44`] | USD 56.00 | — | [Passed qualification and cleanup](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) proves both exact archives, packaged self-tests, real desktop start/share/restart/pause flows, Qwen and Gemma inference, the Windows long-path product fix, all exact resources absent, L4 usage zero, and the protected bootstrap running. | CLEANED-COMMITTED | +| gate13-20260831-h | GCP | Final corrected Gate 13 route-first lifecycle with both four-file release-audit bundles pinned and staged, the bounded Windows user-runtime environment, exact archive preflight, and sequential ordinary-user Windows/Qwen then Linux/Gemma clients [plan `sha256:f243254cc5fb65f44d0c9e707be36feb3284fd6e15b15620882843798fb456b1`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-h-failed-attempt-and-cleanup.json) records passed route acceptance and exact Windows archive verification, one Windows failure at `signed_bootstrap/product_readiness`, no Linux create, exact instance/disk/firewall absence, L4 usage zero, and protected-bootstrap health. | CLEANED-COMMITTED | +| gate13-20260831-g | GCP | Corrected Gate 13 route-first lifecycle with a bounded standard Windows user-runtime environment, one durable foreground host-adapter execution as each ordinary OS user, exact archive preflight, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-g-failed-attempt-and-cleanup.json) records passed route acceptance, the exact Windows archive, a two-second `package_verification` failure caused by four omitted existing audit inputs, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-f | GCP | Fresh Gate 13 route-first lifecycle using one durable foreground host-adapter execution over IAP SSH as each ordinary OS user, exact archive preflight, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:c9a2aafc84940df901a7db1755af2e684f845b78dcdfac04332cfed36388ba25`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-f-failed-attempt-and-cleanup.json) records passed route acceptance, exact Windows archive and staged-input verification, the same bounded `signed_bootstrap` failure under a direct ordinary-user launch as under S4U, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-e | GCP | Fresh Gate 13 route-first lifecycle with pinned reusable route setup, corrected S4U/SID Windows host job, explicit archive download-and-hash prerequisite, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:9ca0fa516017c4a3709a467752f779bcb3bbc0a7c790f9bc61de56d385804c62`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-e-failed-attempt-and-cleanup.json) records passed route acceptance, the exact Windows archive preflight, ordinary-user SSH repair, a durable S4U/Limited lifecycle failure with opaque phase output, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-d | GCP | Fresh Gate 13 route-first lifecycle using the durable controller and host jobs, one bounded route and sequential clients [plan `sha256:d32050a51b8f696aa224fc7e748c9113e174e3c3069c1f8b2bc769b0c5ecea18`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-d-failed-attempt-and-cleanup.json) records passed route acceptance, the corrected headless S4U supervisor, one consumed Windows attempt that failed because its archive had not been downloaded, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-c | GCP | Gate 13 durable route-first lifecycle with the same bounded 16-hour route and sequential 6-hour clients, new exact resources, and corrected explicit IAP target-tag arguments [plan `sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c`] | USD 56.00 | — | [Terminal-state and cleanup proof](evidence/gate13-20260831-c-terminal-state-and-cleanup.json) records a local terminal absence state without a durable provider execution record, retires the run ID without reset or reuse, proves every exact target absent, global GPU usage zero, and the protected bootstrap running. | CLEANED-COMMITTED | +| gate13-20260831-b | GCP | Gate 13 durable route-first lifecycle: one 16-hour G2/L4 product route, then sequential fresh 6-hour Windows/Qwen and Linux/Gemma CPU clients [plan `sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64`] | USD 56.00 | — | [Failed start and cleanup](evidence/gate13-20260831-b-failed-start-and-cleanup.json) records passed preflight and persisted intent, one transient DHT firewall, IAP-tag argument rejection before VM creation, exact firewall cleanup, all run resources absent, and protected-bootstrap health. | CLEANED-COMMITTED | +| gate13-20260831-a | GCP | Gate 13 replacement product-node route plus fresh CPU Windows/Linux packaged lifecycles at route source `f64a388a47b098ac7f69d2affc59816376b43bb1` and exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7` [plan sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69] | USD 52.00 | — | [Failed attempt and cleanup proof](evidence/gate13-20260831-a-failed-attempt-and-cleanup.json) records verified archive downloads but no completed lifecycle, the non-durable orchestration failure, consumed-client semantics, and exact absence of the route, both clients, all three disks, and both firewalls while the protected bootstrap remains running. The USD 52 maximum remains committed; after the owner raised the epoch ceiling to USD 500, USD 448 remains before a new reservation. | CLEANED-COMMITTED | +| gate13-20260830-c | GCP | Gate 13 sequential clean packaged Qwen Windows and Gemma Linux lifecycles at exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7`, temporarily suspending and later restoring the Gate 11 route while reusing its sole global L4 allocation on uniquely named fresh Windows and Linux clients [plan sha256:427bc1ed8a6645ad0650d91aaba7aa753d398fa84f56d57b50aca04c4e0cc955] | USD 26.00 | — | [Cost authorization](evidence/gate13-20260830-c-cost-authorization.json) binds the passed production archives/audits, pushed download-helper/config identities, exact Actions wrapper/inner archives, exact Qwen/Gemma manifests, no service accounts/scopes, direct model transfer, native credential stores, whole-tree containment, all 16 phases, exact cleanup targets, and zero Fly/image/mirror/credits/macOS work. Revision 13 records the final Windows pre-acquisition failure, pushed correction `4818da3`, complete native cleanup, all four exact client instance/disk absences, and successful Gate 11 route restoration. [Privacy-safe final state](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves the package audit and install boundary, zero model-cache bytes, no retained credential/process/path/endpoint/provider output, protected-bootstrap health, active Qwen/Gemma route services, and fresh primary/fallback/restoration inference. The two required 16-phase lifecycles remain incomplete. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 26 maximum is historical and delayed billing remains informational. This record authorizes no later provisioning. | CLEANED-RELEASED | +| gate9-20260830-e | GCP | Gate 9 concurrent Qwen/Gemma Windows/Linux acquisition records and schema-v3 envelopes at pushed source `ba410f74f1cf625f1e1c34734b53e4514fa7c5ec`, reusing the separately authorized product route and using bounded isolated clients [plan sha256:04ba77ee68f4a895ae080a4ddcbf6805b502da6a95a4146734acbddff92de307] | USD 46.00 | — | [Passed envelopes and cleanup](evidence/gate9-20260830-e-edge-resource-envelopes.json) publish all four exact acquisition/envelope records and prove complete client cleanup; [cost authorization](evidence/gate9-20260830-e-cost-authorization.json) binds the exact wheel or exact-commit source archive, signed catalog/bootstrap, Qwen/Gemma manifests, owner-authorized parallel platform/model execution, 60-minute model windows, 90-minute client deletion backstops, exact cleanup targets, protected resources, and zero Fly/image/mirror operations. Native provider authentication was refreshed before the USD 18 Windows-client expansion and again before the zero-ceiling-increase Gemma memory retry; the exact plan permits one cache-preserving in-place resize to `e2-standard-8`. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 46 maximum is historical and delayed billing remains informational. | CLEANED-RELEASED | +| route-20260830-j | GCP | Gate 11 signed-catalog product node route [workload gcp-product-node-route] [source e1d715fd47c852fa12ca50c76e8f4c6a0831fd78] [final runtime source 4cef141746705c3ee8bc8e017693855e0bc4871e] [plan sha256:1a0927e9d83a9a409ac2ea0232c4fceb14821d3f2c5eb87def88b8e7cdcb07d8] | USD 26.00 | — | [Passed live lifecycle](evidence/gate11node-20260830-a-lifecycle.json): generic runtime, signed catalog, direct Hugging Face artifacts, shared persistent cache, complete primary/standby routes, primary/fallback/restoration inference, stable workers, no model image, and protected-bootstrap health. [Gate 13 restoration evidence](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves the route was restored, both product services became active, fresh Qwen/Gemma primary/fallback/restoration inference passed, and a corrected 4,800-second DELETE backstop ended no later than the original deadline. [Post-backstop cleanup](evidence/gate11route-20260830-j-backstop-cleanup.json) proves the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero GPU use, and the protected bootstrap running; acceptance evidence is preserved but no product route is live. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 26 maximum is historical and delayed billing remains informational. | CLEANED-RELEASED | +| cache-20260830-g | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 62be8f1c999b6ebe0ece2a660a0be4757cc83005] [plan sha256:109d2b6958ac8ced31e7202c8eb230387d29615f964d32d6726564b9366eafd7] | USD 10.00 | — | [Live lifecycle](evidence/cache-20260830-g-lifecycle.json) passed public-package/native/provider preflight, exact private repository, keyless identity, reader binding, and builder creation, then failed closed at `cache_warm` after 572.531 seconds with zero cached manifests. Cleanup passed all six exact deletes and absences, removed the ephemeral identity and repository, retained no key, public access, credential, provider output, path, identifier, or argv, and kept the protected bootstrap running. | CLEANED-RELEASED | +| cache-20260830-f | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source bff0c3203191725928246ad3e13deb01ffbab8de] [plan sha256:735cfd847291229571529c8f640fc76005e340a29680806295dad33a7e1a1fb6] | USD 10.00 | — | [Sanitized post-failure verification](evidence/cache-20260830-f-post-failure-verification.json): the private cache and keyless builder reached concurrent warm, then both exact GHCR pulls reported authentication/daemon failure because both upstream packages were still private. The failed controller was interrupted after the startup script's nonzero exit; all six exact cleanup commands and absence checks, repository deletion/absence, no public access/key/retained credential, and protected-bootstrap health passed. | CLEANED-RELEASED | +| cache-20260830-e | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source c0bd81e4e3ced3cd05a642740e343da41d05aceb] [plan sha256:fc13db74e107795c6d2896e0135c4a669a3fd7618a9ef1c4feab54f2425cf948] | USD 10.00 | — | [Live lifecycle](evidence/cache-20260830-e-lifecycle.json) passed exact private repository, ephemeral identity, reader binding, and builder creation, then failed closed at `cache_warm` after 1,653.422 seconds with no cache success claim. All six builder/perimeter/identity absences, identity removal, exact repository deletion, no public access/key/retained credential, and protected-bootstrap health passed. The [bounded acknowledgement diagnostic](evidence/cache-20260830-e-acknowledgement-diagnostic.json) proves the known exact JSON boundary while retaining no failed remote bytes. | CLEANED-RELEASED | +| cache-20260830-d | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 3ae7a094a1e4ca3865d5b6aa463816eac36318f4] [plan sha256:2387d038386ea64e6301d70133aaee4dceedb2c8279e1a341b744ffb1f9fdbc4] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-d-lifecycle.json) passed exact repository creation/configuration, then stopped before builder creation when domain-restricted sharing rejected the planned temporary `allUsers` reader binding. The [bounded policy diagnostic](evidence/cache-20260830-d-domain-policy-diagnostic.json) proves no public binding applied and exact repository deletion; [sanitized post-failure verification](evidence/cache-20260830-d-post-failure-verification.json) proves the repository and all five builder/perimeter targets absent and the protected bootstrap running. | CLEANED-RELEASED | +| cache-20260830-c | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 42241d6fb951cc6274ba991d5762558d67c376ab] [plan sha256:634c4d9db1474655065b1d4d6c2bb4066aeb6c48afa3e2eda7e85d980282104e] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-c-lifecycle.json) stopped at exact repository verification before public binding or builder creation; the [bounded provider-schema diagnostic](evidence/cache-20260830-c-repository-schema-diagnostic.json) proved GCP returns `remoteRepositoryConfig.commonRepository.uri`, deleted the exact diagnostic repository, and re-proved absence; [sanitized post-failure verification](evidence/cache-20260830-c-post-failure-verification.json) proves the API enabled, repository and all five builder/perimeter targets absent, and protected bootstrap running. | CLEANED-RELEASED | +| cache-20260830-b | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source 448196300660174ae8daf5b70bb55c275dcc981d] [plan sha256:861ebeaa2af38e563bdfb736d955b23ea87bd579188636a5512577ee6b35dd52] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-b-lifecycle.json) stopped at the exact enabled-service query before repository or builder creation; [sanitized post-failure verification](evidence/cache-20260830-b-post-failure-verification.json) proves the API enabled, the exact repository and all five builder/perimeter targets absent, and the protected bootstrap running. | CLEANED-RELEASED | +| cache-20260830-a | GCP | Gate 11 private same-region route image cache [workload gcp-public-route-cache] [source a41d9ed72e333057fc017c769ed65f17c92a46e6] [plan sha256:271778431c7553f93d674dffb5131c60133449478d4103c46f366129d7eae2ab] | USD 10.00 | — | [Failed lifecycle](evidence/cache-20260830-a-lifecycle.json) stopped at API enablement before repository or builder creation; [sanitized post-failure verification](evidence/cache-20260830-a-post-failure-verification.json) proves the API enabled, the exact repository and all five builder/perimeter targets absent, and the protected bootstrap running. | CLEANED-RELEASED | +| route-20260830-i | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source fc4c18b045b9143ba455c38fa890eb112429ad3f] [plan sha256:c17ca0aa19f3eb79f1ae837f240b4972a17c821c5c4b8521582e2d38fbd6b99a] | USD 26.00 | — | [Concurrent-prefetch startup-health timeout and cleanup proof](evidence/gate11route-20260830-i-lifecycle.json): native/provider preflight, exact create, bootstrap, protected registry transport, authenticated concurrent prefetch, and both local digest checks passed; the direct GHCR path still exhausted startup before health, so no inference ran; five exact deletes, all six absence checks, registry removal, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-h | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source c09552e7ea0d3f0905857acb35a94affabccedbb] [plan sha256:97ce29d07b3965f8fad4272c9a7b641347622a5940b628d917f3a54fa5a17234] | USD 26.00 | — | [Startup-health timeout and cleanup proof](evidence/gate11route-20260830-h-lifecycle.json): native/provider preflight, exact create, bootstrap, protected registry transport, authenticated prefetch, and both local digest checks passed; sequential pulls exhausted the shared startup window before health, so no inference ran; five exact deletes, all six absence checks, registry removal, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-g | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 108ddbbd4a7da97a426a799e5ced71df87edad36] [plan sha256:52ff4c997508d406b71e0719e4c956829da4a24275d559428de16069c2b37fac] | USD 26.00 | — | [Registry-transport failure and cleanup proof](evidence/gate11route-20260830-g-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, registry removal, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-f | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 77eaa8ad683477ac07498d4c2420d8a959afc1e7] [plan sha256:49b182a304b1cd4dd527345cd9f64c1ec80a74dfedda740b2a198c81279e6ece] | USD 26.00 | — | [Serialized primary image-pull failure and cleanup proof](evidence/gate11route-20260830-f-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-e | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 22b468ad7901edaf85c0ff1c81594c1e90a102bd] [plan sha256:d80db65e522e6955b8d1df9853e961e0c8f0ed7e687152a26fb9d62f7dc1b016] | USD 26.00 | — | [Repeated primary image-pull failure and cleanup proof](evidence/gate11route-20260830-e-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-d | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source cc2cbb393f19e203a4c7eb5e5abfdfe772dacddc] [plan sha256:47efba5556ab8384b892d4310f3dec8760fe5642f7c20466caea77b858e5c285] | USD 26.00 | — | [Classified primary image-pull failure and cleanup proof](evidence/gate11route-20260830-d-lifecycle.json): native/provider preflight, exact create, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-c | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 47dadde939cc869f4b56ea1713127674350ece10] [plan sha256:7a535abd8b3ad6ab42a94538380897b446a280248678cef5c3cd2273020d7261] | USD 26.00 | — | [Failed start-primary and cleanup proof](evidence/gate11route-20260830-c-lifecycle.json): native/provider preflight, exact create, SSH, and bootstrap passed; no health or inference ran; five exact deletes, all six absence checks, and the protected-bootstrap check passed. | CLEANED-RELEASED | +| route-20260830-b | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 5ef5c5a389ce47080b45bebff66408174a09c4fe] [plan sha256:a87056b4659194824b1a2f0fa40d3834abc7040da78167138df217afd758be12] | USD 26.00 | USD 0 | Independent provider-free verification found a one-second float-rounding timeout overshoot after authorization. No provider call or resource creation occurred; source `47dadde` clamps the bound and uses a new run identity. | CANCELED | +| route-20260830-a | GCP | Gate 11 finite Qwen primary and Gemma standby routes [workload gcp-public-route] [source 0ea140f3fe764a6772a3b4217ead4bcd7e93562f] [plan sha256:dc11838569220a3fd7d7afbd3e8e70f49ac9034994071252b11931dd9ad45947] | USD 26.00 | — | [Detached retry B](evidence/gate11route-20260830-a-detached-retry-b-lifecycle.json) and the concurrent [keyring failure](evidence/gate11route-20260830-a-keyring-failure-cleanup.json) both stopped before inference and proved five exact deletes, all six resource classes absent, and the protected bootstrap running. The immutable source-`0ea140f` reservation is released; the corrected source uses a new run identity. | CLEANED-RELEASED | +| gate11pub-20260829-a | GCP | Gate 11 exact Qwen/Gemma public-route image publication from source `d2ea7dea5f3541b86293279b0a650bb46ab82583`; one `e2-standard-4`, 200 GB balanced auto-delete boot disk, six-hour DELETE deadline, registry egress, and contingency | USD 10.00 | — | [Passed publications](evidence/gate11pub-20260829-a-publication-attempt.json) and the [sanitized cleanup verification](evidence/gate11pub-20260829-a-cleanup-verification-attempt.json) bind the strict [Qwen](evidence/gate11pub-20260829-a-qwen3.5-2b-publication-evidence.json) and [Gemma](evidence/gate11pub-20260829-a-gemma-4-e2b-publication-evidence.json) evidence. Registry credentials are absent; native authentication refreshed, the exact builder and auto-delete boot disk are absent, and the protected bootstrap is running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | +| gate9-20260829-d | GCP | Gate 9 sequential Qwen/Gemma edge envelopes at pushed source `480c1fa`: one G2/L4 route and one native Linux client per model, native Windows client local, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-d-edge-envelope-attempt.json): the native Windows Qwen cold cache made no progress after 22,975,832 bytes and stopped at five minutes; Linux and Gemma did not start; exact instances, disks, firewalls, model service, and benchmark process are absent; global and regional L4 usage are zero; protected bootstrap running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | +| gate9-20260829-c | GCP | Owner-authorized clean Gate 9 retry at pushed source `1e845e6`: sequential Qwen/Gemma routes and Windows/Linux cold clients, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-c-edge-envelope-attempt.json): the single Windows Qwen invocation failed before inference after MSYS converted the bootstrap multiaddr; Linux and Gemma did not start; exact instances, disks, firewalls, model service, and benchmark process are absent; GPU usage is zero; protected bootstrap running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | +| gate9-20260829-b | GCP | Owner-authorized Gate 9 attempt: sequential Qwen/Gemma routes and Windows/Linux cold clients, 60-minute model limits, 90-minute DELETE backstops, disks, egress, and contingency | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-b-edge-envelope-attempt.json): Windows Qwen passed; Linux Qwen inference completed but post-close RSS failed the 16 MiB allowance; Gemma was not started; exact instances, disks, firewalls, and benchmark processes are absent; GPU usage is zero; protected bootstrap running. The historical maximum was released by the explicit cleanup-backed owner reset on 2026-08-30; delayed billing remains informational. | CLEANED-RELEASED | +| gate9-20260829-a | GCP | Stopped Gate 9 Qwen attempt after overlapping orchestration launched two Windows cold-client processes | USD 28.00 | — | [Failed attempt and cleanup proof](evidence/gate9-20260829-a-edge-envelope-attempt.json): both benchmark processes stopped without reports; exact instances, disks, firewalls, and model service absent; GPU usage zero; protected bootstrap running. Historical maximum released by explicit owner direction on 2026-08-29 after cleanup. | CLEANED-RELEASED | +| gatev-20260827-a | GCP | Gate V one-host Linux G2/L4 Qwen public vertical slice, 150 GB balanced disk, six-hour hard deadline, headroom, and contingency | USD 17 | — | [Passed run and cleanup proof](evidence/gate-v-20260827-a-public-vertical-slice.json): instance, disk, firewalls, subnet, network, addresses, routers, and resource policies absent at 2026-08-27T09:28:20Z; GPU usage zero; protected bootstrap running. Historical maximum released by explicit owner reset on 2026-08-27; billing remains informational. | CLEANED-RELEASED | +| gate5-20260827-a | GCP | Gate 5 Qwen3.5 2B Windows/Linux qualification and real-run source fixes | USD 69.00 | — | All four exact profile VMs/disks and both network perimeters are absent; GPU usage is zero and `communityai-bootstrap-1` remains running. Historical maximum released by explicit owner reset on 2026-08-27; billing remains informational. | CLEANED-RELEASED | +| gate5-20260827-b | GCP | Same-source `23a4078` Windows/Linux CPU retries; sequential high-memory hosts, private 150 GB disks, one-hour DELETE deadlines, 25% headroom, and fixed contingency | USD 14.00 | — | [Passed qualification and cleanup proof](evidence/gate5-20260827-qwen3.5-2b-qualification.json): Windows used N1; Linux used a lower-cost E2 fallback after N1 capacity failed in every regional zone. Both hosts/disks and the exact firewall, NAT, router, subnet, address, and network are absent; L4 usage is zero; protected bootstrap running. Historical maximum released by explicit owner reset on 2026-08-27. | CLEANED-RELEASED | +| gate6-20260827-a | GCP | Gate 6 Gemma 4 E2B four-profile qualification; serial 48 GB CUDA recovery after a native Windows failover-load crash | USD 79.00 | — | [Passed qualification and cleanup proof](evidence/gate6-20260827-gemma-4-e2b-qualification.json): all four profile hosts/disks and the exact firewall, NATs, routers, subnets, addresses, and network are absent; global GPU and regional L4 usage are zero; protected bootstrap running. Historical maximum released by explicit owner reset on 2026-08-27; billing remains informational. | CLEANED-RELEASED | +| gate7-20260827-a | FLY | Gate 7 CPU-only provider recovery mechanism | USD 30.00 | — | [Passed TinyLlama recovery and cleanup evidence](evidence/gate7-20260828-tinyllama-recovery.json): one bootstrap and four workers ran, one selected worker was killed, the route recovered with exact parity, all five Machines were destroyed, and the token was revoked. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | +| gate7pub-20260827-a | GCP | Gate 7 exact Qwen CPU image publisher after repeat 3,601.7-second Fly registry disconnects; 80 GB disk, four-hour DELETE deadline, egress, and contingency | USD 10.00 | — | [Attempt and cleanup proof](evidence/gate7-20260827-a-separate-machine-attempt.json): exact builder and boot disk absent at 2026-08-28T01:24:30Z; protected bootstrap running. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup; billing remains informational. | CLEANED-RELEASED | +| gate7pub-20260828-b | GCP | Gate 7 exact CPU-only Qwen image republish from verified source `7570d94`; `e2-standard-4`, 80 GB balanced disk, four-hour DELETE deadline, egress, and contingency | USD 10.00 | — | [Publication and cleanup evidence](evidence/gate7-20260828-b-separate-machine-attempt.json) binds the [immutable image report](evidence/gate7-20260828-b-qwen3.5-2b-publication-evidence.json); builder and disk absent, protected bootstrap running. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | +| g7mirror-20260828-c | GCP | Gate 7 immutable Qwen mirror to the isolated Fly registry; `e2-standard-2`, 30 GB disk, two-hour DELETE deadline, egress, contingency | USD 10.00 | — | [Attempt, repository initialization, credential cleanup, and builder cleanup](evidence/g7mirror-20260828-c-fly-registry-attempt.json): the first copy exposed an uninitialized Fly repository; both registry logins were removed, builder and disk are absent, the protected bootstrap remains running, and supported build-only initialization created no Machine. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | +| g7mirror-20260828-d | GCP | Final Gate 7 immutable Qwen mirror after supported Fly repository initialization; `e2-standard-2`, 30 GB disk, intended two-hour DELETE deadline, egress, contingency | USD 10.00 | — | [Failed attempt and cleanup evidence](evidence/g7mirror-20260828-d-fly-registry-attempt.json): copy did not start because GHCR authentication was rejected; the exact builder and disk are absent, Fly has zero Machines/tokens, and the protected bootstrap is running. Historical maximum released by the explicit owner reset on 2026-08-29 after the later Gate 9A cleanup. | CLEANED-RELEASED | +| g7mirror-20260828-e | GCP | Canceled Qwen mirror retry | USD 10.00 | USD 0 | Owner stopped the GCP-to-Fly mirror loop before provisioning; no instance or disk was created. | CANCELED | + +Owner-set accounting baseline on 2026-08-27: **USD 0 spent before `gatev-20260827-a`**. +The removed USD 99 total was a sum of worst-case reservations, not observed provider spend. +This baseline is an owner authorization decision, not a Cloud Billing reconciliation. +Read-only reconciliation at 2026-08-27T15:42:15Z confirmed billing is enabled but the +project has zero queryable BigQuery export datasets, so no observed-cost figure is available +yet. The owner reset the budget again on 2026-08-27 after Gate 6 cleanup was proved, +so its maximum remains historical evidence but no longer consumes the new epoch. The +`gate7-20260827-a` consumed a conservatively reserved USD 30 maximum and is now +cleaned. The additional +`gate7pub-20260827-a` maximum remains committed at USD 10 after its short-lived +CPU-only GCP builder published the exact image and cleanup was proved; observed billing +is still unavailable. The resulting 9 GB rootfs plan exceeded Fly's current 8 GB hard +limit before any Machine was created. Run `gate7pub-20260828-b` consumed a further +USD 10 maximum for the cleaned short-lived CPU builder that published and verified the +8 GB-compatible replacement image. Fly rejected its private external registry reference +before creating a Machine. Run `g7mirror-20260828-c` consumed USD 10 maximum for a +cleaned mirror builder; the copy exposed that the never-deployed Fly app repository first +required Fly's supported build-only initialization. That zero-byte local initialization +created no Machine. Retry `g7mirror-20260828-d` consumed its committed USD 10 maximum after creating the +bounded builder, then failed before copying because GHCR authentication was rejected. +Its exact GCP builder and disk are now proved absent and the protected bootstrap is +running. Retry `g7mirror-20260828-e` was canceled before provisioning. The four +conservatively committed GCP maxima plus the cleaned Fly reservation left USD 30 before +`gate9-20260829-a`. After that attempt's complete cleanup was proved, the owner explicitly +directed immediate continuation on 2026-08-29, resetting the combined test-budget epoch to +USD 100. The cleaned `gate9-20260829-b`, `gate9-20260829-c`, and `gate9-20260829-d` +maxima and the cleaned `gate11pub-20260829-a` publisher maximum consumed that epoch while +billing remained delayed. Native-auth verification at 2026-08-30T00:36:31Z then proved +the publisher's exact builder and disk absent and the protected bootstrap running. With +every run in that epoch cleanup-proved, the owner explicitly authorized a cleanup-backed +reset on 2026-08-30. Those four historical maxima are now `CLEANED-RELEASED`, delayed +charges remain informational, and the new combined authorization epoch starts at **USD 100**. +On 2026-08-30 the owner also designated execution speed as the operating priority: take the +shortest authorized critical path and begin bounded work as soon as its fail-closed preflight +passes. That priority does not raise the USD 100 ceiling or waive exact cleanup, protected- +resource, credential, privacy, or acceptance requirements. Fly credit is not counted as extra +authorization. + +After the Gate 9 clients, Gate 11 product route, and Gate 13 clients were all cleanup-proved, +the owner explicitly authorized another cleanup-backed reset for the next run on 2026-08-31. +Their USD 98 conservative maxima remain historical evidence but no longer consume the new +epoch; delayed observed charges remain informational. The next run starts with a new combined +authorization of **USD 100**. Later on 2026-08-31, the owner raised that current combined +epoch to **USD 500** without releasing the already committed USD 52 maximum. The dated +[authorization record](evidence/owner-budget-authorization-20260831.json) therefore leaves +USD 448 before a new reservation. The cleaned-committed USD 56 run-B maximum plus the fresh USD 56 run-C reservation now leave +USD 336. Every paid create still requires fresh native authentication, +an exact source-bound cost authorization, a conservative ledger reservation, and the existing +fail-closed preflight and cleanup controls. + +## Evidence update rules + +- Link a passed gate to an immutable report, source commit, manifest digest, and relevant + workflow/provider run. +- Never put credentials, prompts, provider output, private paths, or private endpoints here. +- A deterministic unit/integration test may prove implementation readiness, but it cannot + pass a gate that explicitly requires external hardware, multiple hosts, public workers, + packaging, signing, or real cleanup. +- Once the required runner, adapter, or verifier exists and passes its contract tests, + additional test-harness hardening does not count as critical-path progress unless a + real gate attempt exposed the exact defect being fixed. +- When a gate fails, keep the failure evidence, use `IN PROGRESS`, `WAITING`, or `BLOCKED` + accurately, and record the concrete next action. Never lower or bypass the gate merely + to obtain a pass. + + +## Original model-catalog documentation snapshot + +# Signed model catalog and elastic capacity ladder v1 + +Status: strict schema, independent Ed25519 signing keys, threshold verification, +expiry, persistent rollback protection, local rung selection, bounded HTTPS fetching, +exact manifest installation, first-install node configuration, and desktop-sidecar +consumption are implemented. The model-agnostic qualification runner and an exact +bootstrap evidence pin are also implemented; Qwen3 1.7B passed full-artifact audit, +local Windows CPU parity, and selected-worker recovery. That 2025-generation checkpoint +proves the harness but is not a production-ladder candidate. The production backlog was +refreshed against official publisher releases on 2026-08-23. The dense Qwen3.5 text +adapter now has exact synthetic block, cached-decode, nested-wrapper loading, and real +local Hivemind RPC parity. The exact Qwen3.5 2B and Gemma 4 E2B manifests and the first +threshold-one alpha catalog/bootstrap are published. Trust-root rotation, periodic +catalog refresh, larger-rung migration, edge envelopes, and packaged inference remain +open. Gate 11 route operation passed through the generic product node with direct, +manifest-verified Hugging Face artifact delivery. + +`ModelManifest v1` identifies one exact checkpoint and execution profile. A model +catalog answers a separate question: which immutable manifests does one community +approve, and when is each capacity rung healthy enough to become the default for a +new request? + +The catalog is advisory and forkable. It cannot change a manifest digest, allocate a +user's GPU, move an in-flight request to another model, or prevent an installation +from subscribing to another root or selecting an exact manifest. + +## Elastic ladder + +The small-model rungs exist to bootstrap and test the network. They are not the +product destination. The default `auto` policy should advance toward progressively +larger current-generation models as independently measured network capacity becomes +sufficient. + +For a profile using `bytes_per_parameter` and two complete replicas, the weight-only +approximation is: + +```text +maximum parameters = usable contributed VRAM bytes / (2 * bytes_per_parameter) +``` + +Raw VRAM is not usable VRAM. Promotion also reserves capacity for local embeddings and +heads, KV caches, activations, framework overhead, churn, and graceful migration. MoE +rungs are placed by total stored parameters; active parameters describe token-time +compute and do not reduce the bytes required to keep two complete routes available. + +The current qualification backlog is below. Names link to the exact official repository +that a future manifest must pin. Estimates use total parameters and two unquantized BF16 +replicas; an FP8, INT8, or lower-bit artifact is a separate profile with its own manifest +and qualification evidence. + +| Rung by total parameters | Preferred candidate | Standby candidate | Approx. two-replica BF16 weights | +| --- | --- | --- | ---: | +| Edge, 2-5B | [`Qwen/Qwen3.5-2B`](https://huggingface.co/Qwen/Qwen3.5-2B) | [`google/gemma-4-E2B-it`](https://huggingface.co/google/gemma-4-E2B-it), 5.1B total / 2.3B effective | 9.1-20.5 GB | +| Compact, 4-8B | [`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B) | [`google/gemma-4-E4B-it`](https://huggingface.co/google/gemma-4-E4B-it), 8.0B total / 4.5B effective | 18.6-32.0 GB | +| Standard, 9-12B | [`Qwen/Qwen3.5-9B`](https://huggingface.co/Qwen/Qwen3.5-9B) | [`google/gemma-4-12B-it`](https://huggingface.co/google/gemma-4-12B-it) | 38.6-47.8 GB | +| Collective, 27-31B | [`Qwen/Qwen3.8-27B`](https://huggingface.co/Qwen/Qwen3.8-27B) | [`google/gemma-4-31B-it`](https://huggingface.co/google/gemma-4-31B-it) | 111-125 GB | +| Cluster MoE, 109-125B | [`Qwen/Qwen3.5-122B-A10B`](https://huggingface.co/Qwen/Qwen3.5-122B-A10B), about 125B total / 10B active | [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct), about 109B total / 17B active | 435-500 GB | +| Frontier MoE, 397-402B | [`Qwen/Qwen3.5-397B-A17B`](https://huggingface.co/Qwen/Qwen3.5-397B-A17B), about 403B total / 17B active | [`meta-llama/Llama-4-Maverick-17B-128E-Instruct`](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct), about 402B total / 17B active | about 1.61 TB | + +These are candidates, not published approvals. Each exact revision, tokenizer, +runtime profile, quantization, license, artifact inventory, distributed parity, +failure recovery, and edge envelope must pass qualification before its digest enters +a catalog. The Qwen3.5 through Qwen3.8 releases use `qwen3_5` or `qwen3_5_moe`, not the +implemented `qwen3` architecture. Llama 4 uses `llama4`, not the implemented dense +`llama` adapter. Both families need explicit DRIFT adapters. Gemma 4 and Gemma 4 Unified +have DRIFT adapters and focused stock-parity tests, but still need exact real-checkpoint +qualification. Llama 4 artifacts are manually gated on Hugging Face and require a +distribution and operator-access review before catalog use. + +[`Qwen/Qwen3.8-2.4T-A95B`](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) is the current +top Qwen release, with about 2.45T total and 95B active parameters. Two BF16 replicas +alone require roughly 9.8 TB. It remains a frontier preview rather than an activatable +rung because it has no comparable standby, uses the separate `qwen3.8-max` license, and +needs the `qwen3_5_moe_text` adapter plus qualification. Qwen3.5 0.8B may be used for +adapter bring-up but is not a selectable production rung. + +Each rung contains exactly one primary and at least one standby. The standby is an +approved replacement, not a requirement to keep both choices resident in volunteer +VRAM. Hosting two alternatives with two replicas each would double the capacity +requirement and fragment coverage. + +## Promotion evidence + +The selector uses observations for exact manifest digests. It examines the minimum +coverage across all blocks rather than summing advertised VRAM. A model is eligible +only when it simultaneously meets its signed rung policy: + +- minimum bottleneck replicas across every block; +- minimum independent complete routes; +- minimum surviving coverage after removing the largest peer; +- a continuous stability soak; +- a fresh observation window; +- maximum measured p95 time to first token; and +- minimum measured generation throughput. + +The highest eligible rung wins, with its primary preferred over its standby. If no +model in a higher rung qualifies, selection remains on the highest lower rung with +complete evidence. Missing or stale evidence never promotes a model. + +The selector only answers which exact manifest a new `auto` request should use. The +promotion controller still needs to preannounce demand, download and verify artifacts, +establish two independent routes, soak them, atomically update the default alias, and +retain the previous rung as a fallback. Explicit manifest requests and in-flight +requests remain pinned. + +## Catalog trust versus artifact delivery + +The signed catalog should not become a model package or CDN manifest. It authorizes exact +`ModelManifest` digests, promotion policy, public manifest locations, and discovery inputs. +The manifest separately pins the immutable Hugging Face repository revision, artifact +inventory, byte sizes, and SHA-256 values. Nodes use that inventory to download the smallest +whole-file shard set required for their local client tensors or assigned worker blocks. + +This separation keeps the catalog small, auditable, transport-independent, and suitable for +offline threshold signing. Direct Hugging Face delivery is the alpha default, but a later +mirror or peer source is acceptable when it returns the same verified manifest-declared +bytes. The catalog must never sign expiring download URLs, registry credentials, cache paths, +or a model-specific runtime image. + +Catalog v1 already carries exact manifest identities and manifested weight-byte totals, so +Gate 11 requires no catalog schema change. Cache affinity, selected-shard bytes, and download +amplification are local planning or evidence inputs, not catalog authority. See +[ADR 0003](adr/0003-direct-manifested-artifact-delivery.md). + +## Trust root and signatures + +Catalog keys are not worker identities, API keys, bootstrap identities, or credit +keys. `drift catalog keygen` creates a separate offline Ed25519 key. An installation +trusts a local root containing a catalog identifier, a set of public keys, and the +number of distinct valid signatures required. + +For the private testnet, the root may contain one key with threshold one. A later +public root can contain three independently held keys with threshold two. In plain +language, any two maintainers would then have to approve a catalog update. This is an +administrative safety mechanism and has no effect on inference capacity. + +The root is trusted out of band and is never taken from the catalog it verifies. The +signed envelope covers a strict canonical JSON payload with: + +- `catalog_id`, monotonically increasing `sequence`, issue time, and expiry; +- ordered promotion rungs and their complete safety/SLO policy; and +- exact `sha256:` manifest digests, HTTPS manifest mirrors, rung and primary/standby + role, total and active parameter counts, and manifested weight bytes; and +- an optional sorted, bounded set of RSA route-demand authority root key IDs. + +Unknown fields, duplicate JSON keys, duplicate model digests, duplicate signatures, +untrusted signers, malformed keys, non-canonical base64, invalid signatures, +self-authorized keys, excessive lifetimes, and expired catalogs fail closed. The v1 +maximum catalog lifetime is 180 days. + +A persistent rollback guard stores the highest accepted sequence and its payload +digest for each catalog. It rejects an older sequence and rejects a different payload +signed at an already accepted sequence. The state is updated only after the catalog's +schema, time, trust root, and threshold signatures have passed. + +### Route-demand authority roots + +`route_demand_authority_roots` binds the online route observers to the same offline, +threshold-signed catalog decision as the approved manifests. The optional field is +strictly sorted and duplicate-free. It is either empty, which disables remote demand, +or contains between 2 and 32 canonical `sha256:` fingerprints of RSA public keys. +Omitting it preserves the canonical bytes and safe disabled behavior of earlier signed +catalogs. + +An accepted catalog installer copies the exact list into the node configuration. +Discovery discards every unlisted DHT subkey before signature and replay processing, +then requires two distinct listed roots and uses the conservative lower median. A +single listed observer can suppress its own vote but cannot inflate a lower honest +observation; any number of newly generated keys contributes no vote. The remote +placement influence remains capped below migration and coverage margins. + +Observer private keys are online operational credentials, never catalog signing keys or +release assets. A node may consume trusted observations without possessing one. It +publishes only when `route-demand.key` was separately pre-provisioned and its public +fingerprint is listed; node startup never creates that key. Rotation requires a new +threshold-signed catalog list in this first slice. Only public fingerprints are added, +not operator names, network addresses, prompts, request identifiers, or route contents. +Real-world operator independence, collusion, and catalog-key compromise remain governance +and canary risks rather than properties inferred from distinct keys. + +Trust-root rotation is deliberately not smuggled into catalog v1. A later root-update +format must prove old-to-new authorization, expiry and rollback behavior before the +desktop can rotate roots automatically. + +## CLI workflow + +Create the private testnet signing key and export its public half: + +```text +drift catalog keygen catalog-testnet.pem --public-output catalog-testnet.pub.json +``` + +Create a one-signature trust root: + +```text +drift catalog root \ + --catalog-id communityai-testnet \ + --threshold 1 \ + --key catalog-testnet.pub.json \ + --output catalog-root.json +``` + +Sign a strict payload and verify it while recording rollback state: + +```text +drift catalog sign catalog-payload.json \ + --key catalog-testnet.pem \ + --output catalog.signed.json + +drift catalog verify catalog.signed.json \ + --root catalog-root.json \ + --state catalog-state.json +``` + +For a future threshold greater than one, pass multiple public key files when creating +the root. Each maintainer signs the preceding envelope into a new output file until +the required number of distinct signatures is present. + +## Remaining integration work + +1. Publish the Qwen3.5 2B and Gemma 4 E2B Windows/Linux Gate 9 acquisition records and + steady-state edge envelopes through the direct manifested-artifact path in + [`EDGE_RESOURCE_ENVELOPE_RUNBOOK.md`](EDGE_RESOURCE_ENVELOPE_RUNBOOK.md). +2. Preserve the published threshold-one alpha catalog/bootstrap and migrate its + branch-scoped HTTPS mirror only through a newly signed sequence and packaged bootstrap + before deleting the branch. Multiple interchangeable mirrors and independently + operated seeds are post-alpha hardening. +3. Bundle that bootstrap and pass the clean-install packaged inference gate. The + implemented sidecar consumer fetches manifests, verifies their digest against the + catalog, and registers them without trusting catalog display metadata. +4. Extend placement evidence with selected-shard byte cost and verified cache affinity while + preserving user bandwidth/storage ceilings and the existing anti-herding margins. +5. Reconstruct capacity observations from authenticated DHT records and completed route + probes rather than accepting a central capacity total; then add staged promotion, + fallback/downgrade drills, and deterministic churn simulation. +6. Design and validate signed trust-root rotation before the public root has multiple + independent maintainers. + + +## Superseded roadmap model ladder + +### Elastic model ladder + +Small checkpoints are bootstrap and test rungs, not the distributed network's product +ceiling. Community `auto` selection should move monotonically toward larger qualified +models as measured capacity grows: approximately 1-2B, 3-4B, 8B, 27-32B, 70B, and +400B-plus. Each rung approves exactly one primary and at least one standby so the +catalog can replace a model without requiring both alternatives to fragment live VRAM. + +These are catalog capacity classes, not a mandatory sequential qualification staircase. +The original Petals demonstrations and successful TinyLlama, Qwen, and Gemma bring-up +make larger block-sharded inference plausible enough to test directly. They do **not** +prove that an exact 30B or 70B checkpoint is compatible, fits the intended worker/client +memory envelopes, recovers correctly, or performs well enough to use. + +The first post-alpha scaling experiment should therefore use an exact 27-32B candidate +split across independent workers, with no worker required to hold the full model. If one +complete block fits the target worker envelope and that run passes manifest/artifact +checks, stock parity, two complete routes, selected-worker interruption, client and +worker memory limits, TTFT, and decode throughput, proceed directly to an exact roughly +70B candidate. Test a smaller intermediate rung only when it is a useful product fallback +or helps diagnose a concrete failure; do not spend milestones on 4B -> 8B -> 12B merely +as confidence-building prerequisites. + +At INT8, two complete weight replicas require roughly two bytes of aggregate usable +VRAM per parameter: 10 GB for a 5B model, 60 GB for a 30B model, 140 GB for 70B, and +810 GB for 405B before KV-cache, activation, framework, churn, and migration headroom. +Promotion is never inferred from that aggregate alone. The selector requires minimum +per-block replica coverage, independent complete routes, survival after the largest +peer loss, a stability soak, fresh observations, and measured latency and throughput +limits. Total parameters determine MoE storage; active parameters describe per-token +compute and do not make the other expert weights disappear. + +The first candidate ladder and the implemented signed format are specified in +[`MODEL_CATALOG_V1.md`](MODEL_CATALOG_V1.md). The local selector may resolve an `auto` +request to the highest eligible exact manifest, but it never changes an explicit model +selection or an in-flight request. Catalog fetching, DHT-derived observations, staged +worker migration, fallback, and automatic alias updates remain integration work. diff --git a/docs/REVIVAL.md b/docs/REVIVAL.md index a0ccf347d..30041d343 100644 --- a/docs/REVIVAL.md +++ b/docs/REVIVAL.md @@ -1,5 +1,82 @@ # Petals revival: public inference alpha roadmap +September 9 product correction, release `0.1.0-alpha.20260909.3`: consumers send text +without loading input/output model weights. A complete block grid did not prove +that behavior in the earlier released client. Input/output processing runs on contributing +text peers; the small local Qwen fallback stays when the mesh cannot answer. +See [ADR 0004](adr/0004-text-only-community-consumers.md). Earlier generation +proofs remain valid for their tested tensor-client scope and do not establish +fresh, weight-free consumer readiness. + +Current release status and execution order are maintained in +[RELEASE_READINESS.md](RELEASE_READINESS.md), reviewed 2026-09-09. The Qwen3.8 +64-block route, same-session replacement, reference comparison and Windows +packaged short chat, worker-loss recovery and HTTP-blocked cache restart passed +on assigned cloud routes. Bounded autonomous CPU desktop formation and recovery +also passed on September 7. Gate 14 resource controls now pass the bounded +Windows/Linux frozen-package acceptance. Gate 15 installers and login controls +also pass their bounded Windows/Debian/Ubuntu acceptance. The owner accepts the +existing Qwen conversation proof for alpha; additional measurements and frozen +periodic catalog-update qualification are deferred. The smaller September 8 +installers passed installed native checks and removal. Both online installers +passed complete hosted download, verified installation and removal; all four +download options and release metadata are public and verified. Qualified +candidate links and a draft release can proceed during Gate 16 scope review; +the combined public canary remains unexecuted. The [model ladder](COMMUNITY_AI_MODEL_LADDER.md) supersedes the older size-by-size +candidate lists in historical implementation snapshots below. + +September 7 Gate 14 update: **PASSED for the bounded alpha scope.** The +[final Windows/Linux acceptance](evidence/gate14-20260907-final-resource-acceptance.md) +used complete frozen packages and real Qwen load. Both sliders default to 100%, +sharing remains opt-in, lower limits are enforced, Pause removes worker trees, +and settings persist across restart. Low-VRAM rejection now waits without a +restart loop; signed-manifest migration retains cache/resource preferences. +Windows ran non-elevated; Linux used an ordinary Debian/Xvfb session with CUDA +passthrough. Broader hardware and physical desktop coverage are not implied. + +September 8 release continuation: **Gate 15 PASSED for the bounded alpha scope.** +Windows, Debian and Ubuntu installed lifecycles, both frozen sign-in controls, +and normal Windows/Linux signed-catalog startup migration passed. +Earlier failed attempts are retained separately; local canary checks do +not replace the live public route exercise. See the +[current evidence and remaining work](RELEASE_READINESS.md#next-work-in-useful-product-order). + +September 8 owner scope clarification: additional conversation/performance +qualification is not required before alpha. Frozen periodic catalog activation +and active-answer draining tests follow in beta; the owner expects only one or +two more catalog changes this year. Existing real recovery and safety results +must be credited when deciding whether Gate 16 adds useful new evidence. Its +combined public deployment drill remains unexecuted, not passed. + +September 8–9 distribution refresh: both runtimes identify `84205f93`. The new +Windows setup is 2,462,345,104 bytes and passed installed native CUDA checks and +removal. The Linux package is 2,302,428,788 bytes and passed installation, +installed CPU/CUDA/worker checks and removal on Ubuntu 22.04. All nine CI checks passed at +the import-formatting follow-up `fdd8d0b`. Both offline packages are public on +the owner-authorized R2 origin and passed complete hosted download/hash checks. +The 2,107,751-byte Windows online setup passed actual ordinary-user handoff, +installed CPU diagnostics and removal with exact child exit, temporary cleanup +and persisted baseline verified. Its helper, Inno script and builder match the +subsequent source commit `b6c8aad9`; this is separate from runtime source `84205f93`. +[Windows hosted acceptance](evidence/normalized-online-windows-installer-20260908.md). +Linux's 13,662-byte online installer passed its actual hosted download, +protected-copy/APT installation and removal and is published with hash-verified +metadata. [Linux hosted acceptance](evidence/alpha-online-linux-hosted-20260908.md). +Earlier failed attempts remain recorded. Both online files, all 19 curated +platform records, the combined manifest, checksums and metadata ZIP are now +public; all 24 small object bodies matched their hashes. The +[publication audit](evidence/alpha-cloudflare-publication-20260909.json) records +that result. No broad availability guarantee is implied by these single complete +handoffs. +Exact versions, hashes and availability are in the +[installation guide](ALPHA_INSTALL.md); earlier Gate 14/15 evidence is retained. + +The public signed catalog/bootstrap/manifests passed a bounded metadata check +on September 8. The catalog expires on September 28 at 19:35 UTC and its URLs +depend on preserving `codex/gate-v-auto-selection`. This check establishes no +current public worker capacity; community inference remains best effort. +[Metadata evidence](evidence/alpha-public-metadata-20260908.json). + This repository starts from DRIFT-LLM, the most practical maintained continuation of Petals found during the August 2026 fork audit. It preserves the parts that are most valuable for a revival: transformer-block sharding, Hivemind DHT discovery, @@ -48,22 +125,29 @@ agent: - The first supported desktop and qualification matrix is Windows and Linux. macOS is explicitly deferred and must not be claimed as supported until later tests on real Apple devices pass. -- Qwen3.5 2B is the first-rung primary candidate and Gemma 4 E2B is its standby. +- Qwen3.8-27B FP8 is the first community-model release target, with a qualified + local Qwen3.5 fallback. Qwen3.5 2B and Gemma 4 E2B remain historical qualification + fixtures. DeepSeek-V4-Flash and GLM-5.3-Flash are the later community targets. - GCP and Fly Machines are authorized for bounded qualification and public-alpha infrastructure. GCP/local hosts cover the Windows/Linux CPU/CUDA platform matrix. As of 2026-08-27, Fly is authorized only for the existing **CPU-only** Linux separate-machine recovery adapter; Fly supplies no GPU qualification capacity, and a Fly recovery result must never be presented as CUDA or GPU-performance evidence. -- After the first-rung alpha is stable, do not climb every intermediate model size merely - to prove that block sharding scales. Use the accumulated Petals and - TinyLlama/Qwen/Gemma implementation evidence to attempt a real 27-32B split route - directly, then attempt roughly 70B if that passes. This is permission to test those - sizes, not permission to claim that an exact larger checkpoint works before its own - model-specific evidence passes. -- New temporary GCP and Fly test resources share one combined **USD 100 maximum**. - Track conservative estimates and observed cost in - [`RELEASE_READINESS.md`](RELEASE_READINESS.md). Do not start a run that could exceed - the remaining balance. +- Complete the useful Qwen desktop path before implementing the larger model + adapters. Do not qualify arbitrary intermediate sizes merely to demonstrate + sharding. Each actual ladder entry needs its own correctness, memory, recovery, + packaged-delivery, and performance evidence before activation. +- For the September 8 alpha scope, accept the existing packaged Qwen conversation, + performance observations and recovery proof. Additional representative chat or + hardware measurements and frozen periodic catalog activation/draining are + deferred after alpha (catalog-update qualification to beta). Preserve the + measured limits and do not reintroduce these deferred checks as release gates. +- New temporary GCP and Fly test resources share one live owner-authorized combined + ceiling. The baseline is USD 100; on 2026-08-31 the owner raised the current accounting + epoch to **USD 500 maximum**. The already committed USD 52 maximum remains charged to + that epoch, leaving USD 448 before a new reservation. Track conservative estimates and + observed cost in [`RELEASE_READINESS.md`](RELEASE_READINESS.md). Do not start a run that + could exceed the remaining balance. - Use the existing `gcloud`, `flyctl`, and `gh` logins. Do not require the owner to copy provider tokens into environment variables when native CLI authentication works. - On Windows, every registry token, remote credential, and Linux script must follow the @@ -91,8 +175,9 @@ The public alpha still requires: - the client automatically selects an eligible catalog model, while an opted-in contributor automatically selects a model and block range within the user's VRAM, storage, bandwidth, power, schedule, and model-policy limits; -- Qwen3.5 2B and Gemma 4 E2B pass the declared Windows/Linux CPU/CUDA qualification and - real CPU-only separate-machine recovery gates before they are advertised as qualified; +- Qwen3.8 and the selected local Qwen fallback pass their declared product + qualification before being advertised; preserve the prior Qwen3.5/Gemma results + without treating them as qualification for a different model or package; - an alpha catalog is authenticated by at least one pinned CommunityAI release key, manifests and artifacts are content-verified, peer announcements are authenticated, public requests have finite admission/time limits, and operators can disable a bad @@ -109,7 +194,7 @@ The following are post-alpha hardening, not reasons to delay first public use: seed/mirror operators, and multi-provider outage survival; - independent threshold catalog key holders, key-compromise/rotation drills, and interchangeable-mirror governance beyond the alpha's pinned signed catalog; -- operating-system publisher signing/notarization, an authenticated automatic updater, +- macOS notarization, an authenticated automatic updater, automatic rollback, and polished retained-data migration beyond the alpha's manual path; - exhaustive malicious-load, Sybil/collusion, partition, herd-switching, long-soak, and production-style evidence-retention programs; and @@ -139,20 +224,33 @@ shard granularity limit are recorded in ### Non-negotiable launch sequence -The signed catalog and product-node Gate 11 route have passed. Gate 9 is the immediate -critical path: - -1. split each Windows/Linux edge measurement into resumable direct-Hub acquisition and a - supervised steady-state benchmark from the verified persistent cache; -2. publish the four Qwen/Gemma Windows/Linux client envelopes without building or pulling a - model-specific image; -3. pass clean packaged install and inference against a product-node route, including cache - reuse, restart, manual upgrade/reinstall, uninstall, and retained-data choice; -4. prove automatic contribution and resource controls on real packaged Windows/Linux - hardware; and -5. run the bounded public canary and publish the explicitly best-effort alpha. - -Do not resume post-alpha redundancy, publisher-signing/updater, independent-governance, or +Gate V and Gates 1-13 passed for their recorded scopes. On September 7 the owner +selected **Gate 14 resource sliders → installers/lifecycle (15) → canary (16) → +release (17)** as the next product sequence: + +1. deliver two desktop sliders: VRAM and processing usage, both defaulting to + 100% on a new installation. Sharing stays opt-in. Verify lower limits under + load, persistence, live changes, and complete worker shutdown on Pause. + VRAM controls the contribution allocator budget, preserving local-inference + reservations; processing controls contribution compute duty cycle, with + explicit per-step bursts rather than an instantaneous whole-device guarantee. +2. build the Windows and Linux setup artifacts described below with the proven + formation fixes. Verify ordinary startup, upgrade/reinstall, catalog migration, + uninstall, and retain/delete-cache choices. Upgrades must stop the node and + its complete worker trees before replacing files and preserve settings/cache. +3. retain the passed Qwen/resource-control observations and declare their tested + hardware and conversation limits; the September 8 owner decision defers broader + measurements and frozen periodic catalog-update qualification; and +4. resolve Gate 16's remaining deployment scope using the existing recovery/safety + evidence, then publish the best-effort Qwen alpha. The owner has asked why a + further integrated run is needed; do not repeat proven recovery solely to + complete a gate number. + +Combine overlapping product checks in the same real desktop sessions. The full +Qwen runtime and tested same-session recovery already passed; repeat them only +where new source/profile changes or product integration require verification. + +Do not resume post-alpha redundancy, automatic-updater, independent-governance, or exhaustive hostile-network programs while an earlier alpha outcome is unfinished. Preserve completed foundations for those programs, but do not polish them ahead of the usable path. @@ -166,10 +264,67 @@ temporary host. Native `gcloud`, `flyctl`, and `gh` authentication is currently available; re-check it immediately before use rather than relying on an older evidence note. -The next external deliverable is not another image, mirror, harness, or unit-test expansion. -It is the four real Gate 9 client envelopes using the product artifact path. Supporting code -is justified only when it implements the bounded acquisition record, process-supervised -cleanup, or another concrete gap exposed by that real run. +The next external deliverable is the usable Qwen3.8 desktop/local-fallback path. +Supporting code must address a concrete product or observed-run gap; the existing +one-click runners and evidence machinery are the starting point. + +### Distribution decision, September 7 + +The owner's latest decision on September 7 makes working Windows/Linux installers +the alpha requirement and defers Windows publisher signing until after alpha. +Unsigned setup must be clearly labelled and ship checksums/provenance. Existing +engineering packages still need the remaining product acceptance before release. + +| Channel | Deliverable | +| --- | --- | +| Windows download | Inno Setup, ordinary-user installation and working upgrade/uninstall. Unsigned alpha is authorized; trusted setup/uninstaller signing follows after alpha. [Inno Setup capabilities](https://jrsoftware.org/isinfo.php). | +| Microsoft Store | Submit the same installer through the MSI/EXE route. Provide a standalone offline, silent-capable installer at an immutable versioned HTTPS URL; sign installer and PE payloads with a trusted code-signing identity. This route does not provide Store-managed updates. [Package requirements](https://learn.microsoft.com/en-us/windows/apps/publish/publish-your-app/msi/upload-app-packages), [distribution/signing requirements](https://learn.microsoft.com/en-us/windows/apps/package-and-deploy/choose-distribution-path). | +| Ubuntu/Debian | Build a `.deb` and publish a signed HTTPS APT repository, with its key scoped using `Signed-By`. After repository setup, install with `sudo apt install communityai`. [Debian repository guidance](https://wiki.debian.org/DebianRepository/UseThirdParty). | + +Store submission follows trusted signing and is no longer an alpha blocker. +Direct `.deb` installation can precede the hosted APT channel; APT metadata must +still be signed before that repository is offered. A generated installer script +alone does not satisfy install/upgrade/removal acceptance. Model weights +remain verified on-demand data, separate from the bundled executable runtime. + +The owner authorized the block-health grid, available peer details and the user's +own download progress on September 7. The source implementation and bounded +[HTTP/process/Qt checks](evidence/desktop-health-downloads-20260907.md) are complete; +carry this into final packages. Coverage/replicas, signed reservations, joining and +offline/failure states remain separate. Remote download percentages and unused +capacity are unreported. Local progress separates transferred/cached bytes from +verified artifacts and model loading. This is not an additional release gate. + +The owner explicitly deferred Windows publisher signing until after alpha. +Working unsigned direct-download setup files with checksums/provenance are +accepted. Mario Andreschak is an individual based in Colombia; Azure Artifact +Signing Public Trust does not currently support that individual location. +The authorized SignPath eligibility inquiry was sent September 7, with no +enrollment or approval yet. See [signing status](WINDOWS_SIGNING.md). Store and +hosted signed APT distribution follow after alpha; signing does not block Gate 14. + +The final [Gate 14 package matrix](evidence/gate14-20260907-final-resource-acceptance.md) +passed at `76b6d84` on Windows and `bf67f0d` on Linux, with Linux packaging +fixes and unchanged application/catalog source. Acceptance included real Qwen processing load, +both literal sliders, persistence, repeated Pause/Start, four independent +admission guards, local inference and complete owned-process/native-key cleanup. +The [fully frozen Windows installer lifecycle](evidence/gate15-20260907-frozen-windows-installer.json) +also passed product install/upgrade/removal/reinstall assertions and an independent +cleanup audit; its redundant final test-cleanup error is retained explicitly. +The Debian lifecycle and the [Ubuntu retry](evidence/gate15-20260908-frozen-ubuntu-installer.md) +also passed with the same final `.4` package. The earlier Ubuntu unpack timeout +remains recorded. The [Linux frozen sign-in checkbox](evidence/gate15-20260908-frozen-linux-login.md) +passed enable, restart and disable on a private Xvfb display. The +[Windows frozen checkbox](evidence/gate15-20260908-frozen-windows-login.md) then +passed the same cycle on unswitched private desktops, with exact native Run +registration and independent process/credential/original-state cleanup verified. +The [combined Gate 15 acceptance](evidence/gate15-20260908-final-installer-acceptance.md) +records the passed scope and remaining platform limits. +Manual retained-data choices now have a [runbook](DESKTOP_UNINSTALL.md) and +bounded Windows evidence. +The earlier [installer checkpoint](evidence/desktop-installers-20260907.md) includes +the disposable-key APT acceptance/tamper test. No public installer release, Store +submission or production signed APT repository has been published. ### Execution loop @@ -200,10 +355,34 @@ On every implementation run: ends, the release is complete, or that narrow definition applies to every permitted task on the current critical path. +### Durable paid-run contract + +A multi-hour paid qualification must not depend on an operator terminal, SSH/IAP session, +or untracked repair script remaining alive. Before its first create, it must have one +source-bound, persisted, idempotent controller with `start`, `status`, `collect`, and +`cleanup` operations. Every operation begins by inventorying the exact authorized +instances, disks, firewalls, ownership metadata, and absolute deadlines. Matching resources +are reattached; foreign or ambiguous exact-name resources fail closed; missing resources are +never recreated merely because local state was lost. + +Long-running work runs as one named host-local durable service or task and writes only a +bounded sanitized status plus a digest-bound terminal record. Repeating `start` observes the +existing job; it does not launch a second lifecycle. Once a packaged product lifecycle or a +diagnostic product launch begins, any non-pass consumes that client for acceptance. Removing +its files or credentials does not make it fresh again, and phase-level lifecycle resumption +is prohibited. + +For Gate 13, accept the complete product route before creating a client. Run the higher-risk +Windows/Qwen lifecycle first; collect its canonical 16-phase record and delete that client +before creating Linux/Gemma. This is an operational cost/risk sequence, not a relaxation of +the two-platform acceptance contract. Any route failure, ambiguous host job, expired runway, +or client failure goes directly to exact cleanup. A gate passes only after both complete +fresh-host records and final provider absence proof exist. + ### Cloud safety rules - Before provisioning, record a conservative maximum estimate in the spend ledger and - confirm it fits under the combined USD 100 ceiling. + confirm it fits under the live combined ceiling recorded in the readiness tracker. - An explicit owner budget reset starts a new USD 100 accounting epoch only after every prior run is cleanup-proved. Preserve those historical rows as `CLEANED-RELEASED` rather than pretending their actual cost was zero; their maxima no longer consume the new epoch, @@ -225,7 +404,7 @@ Do not block on these while another roadmap item can proceed. Ask the owner only input is on the critical path: - a provider login expires and native CLI reauthentication is required; -- the next bounded cloud run does not fit under the remaining USD 100 ceiling; +- the next bounded cloud run does not fit under the remaining live owner-authorized ceiling; - platform code-signing/notarization credentials or a publisher identity are required; - production catalog signing needs independent human key holders; - an independent seed or mirror operator must accept operational responsibility; or @@ -570,41 +749,32 @@ and then delegates block placement to the existing algorithm. ### Elastic model ladder -Small checkpoints are bootstrap and test rungs, not the distributed network's product -ceiling. Community `auto` selection should move monotonically toward larger qualified -models as measured capacity grows: approximately 1-2B, 3-4B, 8B, 27-32B, 70B, and -400B-plus. Each rung approves exactly one primary and at least one standby so the -catalog can replace a model without requiring both alternatives to fragment live VRAM. - -These are catalog capacity classes, not a mandatory sequential qualification staircase. -The original Petals demonstrations and successful TinyLlama, Qwen, and Gemma bring-up -make larger block-sharded inference plausible enough to test directly. They do **not** -prove that an exact 30B or 70B checkpoint is compatible, fits the intended worker/client -memory envelopes, recovers correctly, or performs well enough to use. - -The first post-alpha scaling experiment should therefore use an exact 27-32B candidate -split across independent workers, with no worker required to hold the full model. If one -complete block fits the target worker envelope and that run passes manifest/artifact -checks, stock parity, two complete routes, selected-worker interruption, client and -worker memory limits, TTFT, and decode throughput, proceed directly to an exact roughly -70B candidate. Test a smaller intermediate rung only when it is a useful product fallback -or helps diagnose a concrete failure; do not spend milestones on 4B -> 8B -> 12B merely -as confidence-building prerequisites. - -At INT8, two complete weight replicas require roughly two bytes of aggregate usable -VRAM per parameter: 10 GB for a 5B model, 60 GB for a 30B model, 140 GB for 70B, and -810 GB for 405B before KV-cache, activation, framework, churn, and migration headroom. -Promotion is never inferred from that aggregate alone. The selector requires minimum -per-block replica coverage, independent complete routes, survival after the largest -peer loss, a stability soak, fresh observations, and measured latency and throughput -limits. Total parameters determine MoE storage; active parameters describe per-token -compute and do not make the other expert weights disappear. - -The first candidate ladder and the implemented signed format are specified in -[`MODEL_CATALOG_V1.md`](MODEL_CATALOG_V1.md). The local selector may resolve an `auto` -request to the highest eligible exact manifest, but it never changes an explicit model -selection or an in-flight request. Catalog fetching, DHT-derived observations, staged -worker migration, fallback, and automatic alias updates remain integration work. +The product progression is **local Qwen3.5 → community Qwen3.8-27B → +DeepSeek-V4-Flash → GLM-5.3-Flash**, as recorded in +[COMMUNITY_AI_MODEL_LADDER.md](COMMUNITY_AI_MODEL_LADDER.md). The first Qwen3.8 +full-route and tested worker-replacement results passed on 2026-09-05. DeepSeek +and GLM require their own adapters and qualification before activation. + +Growth is measured by complete, reachable, sufficiently stable and useful model +routes, not connected-PC count or summed advertised VRAM. Actual per-block +memory, client tensors, context/cache, and migration headroom must fit the +contributors' budgets. Total MoE weights remain relevant even when only a small +subset of experts computes each token. + +The node must keep local fallback useful while opted-in workers stage missing +community spans. After measured readiness, eligible new `auto` requests may move +up; explicit selections and active generations stay pinned. Existing chat text +must be retokenized/prefilled when a later turn changes model. When capacity +falls, selection may move down. Retain lower-route capacity during larger-rung +migration and prevent repeated switching/download storms. + +The working-tree catalog permits one primary and optional standbys per rung. +Measured catalog eligibility is now wired into the desktop node. Verified local +Qwen3.5-0.8B fallback, local-only preference, cancellation and periodic authenticated +catalog refresh are implemented; offline source and packaged Windows GPU inference +passed. Real automatic formation, promotion/downgrade and shared resource-limit +qualification remain open. See [MODEL_CATALOG_V1.md](MODEL_CATALOG_V1.md) and +[QWEN_DESKTOP_PRODUCT_RESULTS.md](QWEN_DESKTOP_PRODUCT_RESULTS.md) for current evidence. ### Canonical model manifest diff --git a/docs/WINDOWS_SIGNING.md b/docs/WINDOWS_SIGNING.md new file mode 100644 index 000000000..2509c577e --- /dev/null +++ b/docs/WINDOWS_SIGNING.md @@ -0,0 +1,104 @@ +# Windows code signing decision and eligibility inquiry + +Status, September 7, 2026: **no enrollment, trusted certificate or signing +approval exists**. The owner prefers a free solution and would otherwise publish +as Mario Andreschak, an individual based in Colombia. + +Later September 7 decision: publisher signing is **post-alpha**. Working unsigned +Windows setup and Debian installers are acceptable for the alpha, with explicit +unsigned labelling and checksums/provenance. Store submission follows signing. +The owner suggested Azure as a possible paid fallback, but its individual-country +restriction still excludes Colombia. Basic is currently $9.99 per month; deleting +the signing account does not affect certificates already used, although future +builds require signing again. Billing is by full month, not prorated. See +[Microsoft pricing](https://learn.microsoft.com/en-us/azure/artifact-signing/how-to-change-sku) +and [unenrollment/billing](https://learn.microsoft.com/en-us/azure/artifact-signing/faq). +No signing account or paid enrollment has been created. + +With the owner's explicit authorization, an eligibility inquiry was sent from +their Gmail account to SignPath's published contact, `info@signpath.io`, on +September 7 at 12:08 Colombia time. Gmail's Sent folder confirmed the message was +sent; receipt or acceptance by SignPath has not been established. +Subject: **CommunityAI: free OSS signing eligibility for CUDA-enabled Windows +packages**. The inquiry requests the free program only; no paid service, +enrollment or signing terms were accepted. + +## Recommendation + +Apply to [SignPath Foundation](https://signpath.org/) for its free open-source +program. Its certificate identifies **SignPath Foundation** as publisher; it does +not create a certificate in the maintainer's personal name. Acceptance is at the +Foundation's discretion. Azure Artifact Signing Public Trust currently supports +individuals in the US and Canada, so it is not available for this individual +location. See [Microsoft's eligibility documentation](https://learn.microsoft.com/en-us/azure/artifact-signing/quickstart). + +The material eligibility question is the CUDA-enabled PyTorch distribution: +our Windows runtime bundles NVIDIA CUDA, cuBLAS and cuDNN DLLs. SignPath's +[conditions](https://signpath.org/terms.html) exclude proprietary components +except System Libraries; the Foundation must decide whether this package fits. +The patched upstream Hivemind build and PyInstaller executables also need their +signing boundaries agreed. Do not relabel or re-sign third-party binaries as +CommunityAI-authored code to avoid these requirements. + +Self-signing can test the pipeline but does not provide a publicly trusted +publisher for Windows downloads or satisfy the Store's trusted-signing requirement. +The existing catalog key authenticates model catalogs; it is separate from +Authenticode and must not be reused for it. + +## Project information supplied in the inquiry + +Project: **CommunityAI** + +Repository and current project documentation: + + +License: MIT; third-party runtime components retain their respective licenses. + +Maintainer: Mario Andreschak, Colombia. The owner supplied the reply address; +it is omitted here to avoid adding personal contact details to public source. + +Description: CommunityAI is a Windows/Linux desktop application for local AI +inference and opt-in community inference sharing. The desktop starts a separate +local node, exposes a loopback API, downloads model artifacts selected by signed +manifests, verifies their hashes, and displays contribution limits, block health, +peer metadata and acquisition progress. Sharing is off by default. + +Build: public GitHub Actions workflow `.github/workflows/desktop.yaml`; Python +and PyInstaller build separate GUI and node executables. The Windows node uses a +patched Hivemind runtime and CUDA-enabled PyTorch. Inno Setup creates a per-user, +silent-capable installer. Current CI outputs are unsigned engineering artifacts; +the first public installer release and its permanent URL are still pending. + +Eligibility inquiry: Can the Foundation sign our authored executables and Inno +installer/uninstaller while the package includes the NVIDIA runtime DLLs supplied +with PyTorch? Which upstream and PyInstaller artifact restrictions apply to this +build? What prior-release evidence is acceptable when the first public installer +is still pending? We offered the dependency/license inventory, workflow and +proposed signing policy before requesting signing approval. + +Outstanding application inputs: eligibility response, permanent installer download +page, dependency/license inventory, confirmed MFA, named reviewer/approver roles, +and a reviewed privacy/code-signing policy. Do not claim sponsorship or display +the Foundation's attribution as an existing relationship before acceptance. + +## Integration after acceptance + +1. Agree which artifacts the provider can sign and record the approved policy. + Set product/version metadata on the authored Windows executables. +2. Use verified GitHub build provenance, protected release inputs and the + provider's required human signing approval. Signing credentials must not be + available to pull-request builds. +3. Sign the approved payloads before packaging. Use Inno's `SignTool` hook and + `SignedUninstaller=yes`; the checked-in Windows builder accepts a signing + command and rejects an invalid resulting installer signature. The provider + integration is pending; this generic hook alone is not a working SignPath + enrollment or end-to-end signing pipeline. +4. Verify the final installer, uninstaller and required PE payload signatures; + generate final checksums after signing. Run the real installer lifecycle and + follow the Store's payload-signing and offline/silent installation rules. + +The Foundation requires a prior release in the intended format and verifiable +project reputation. Build/test artifacts help prepare an application; they do +not guarantee eligibility. If declined, agree on a paid individual signing +provider available in Colombia or revisit the distribution package with the +owner; no purchase is authorized by this document. diff --git a/docs/adr/0004-text-only-community-consumers.md b/docs/adr/0004-text-only-community-consumers.md new file mode 100644 index 000000000..21b5e741f --- /dev/null +++ b/docs/adr/0004-text-only-community-consumers.md @@ -0,0 +1,68 @@ +# ADR 0004: Community consumers send text; contributors own all model stages + +- Status: Accepted; source implementation and short live requests verified +- Date: 2026-09-09 +- Supersedes: ADR 0001's requirement that the consumer owns tensor routing + +## Product behavior + +Open the app and use the LLM. Sharing hardware is optional. A consumer must not +need a GPU, a tokenizer, input/output model weights, or a model download to use +a complete community model. + +Automatic selection prefers the community model when the mesh can answer a +complete request. If it cannot, the existing small local Qwen fallback remains. +When community capacity returns, subsequent requests use the community again. +Explicit local-only selection remains available. An answer already in progress +is never assembled from different models. + +Sharing budgets are independent of fallback. A 100% GPU-memory setting offers +the full physical GPU memory budget to the network; configuring a local fallback +does not subtract a permanent reserve. Local execution checks actual free memory +when needed and again after downloads. Automatic local device selection uses +CPU/RAM when GPU memory is insufficient. + +## Decision + +Contributors can serve transformer blocks and a text-service role. The text peer +owns tokenization, input embeddings, output projection and the generation loop; +its transformer execution uses the existing distributed block route. Consumers +send chat/completion requests and receive text and usage over the existing +authenticated libp2p transport. No central HTTP inference gateway is introduced. + +Text-service announcements are signed by the transport identity and bind the +exact manifest, execution profile, protocol, context/output limits and expiry. +They expire after 40 seconds and are published only while the provider observes +a complete block route. Discovery reports block coverage and chat availability +separately. A green block grid alone is insufficient to claim a usable model. + +The product node uses the text client for community inference. The existing +tensor client remains an internal building block for text peers and the advanced +`drift api` command. It must not be used for desktop community consumers. + +Requests, frames, answers, context and generation time are bounded. A peer admits +one generation at a time. Cancellation is tied to the requesting transport +identity and keeps the generation slot occupied until the computation stops. +Another text peer may be tried before answer text begins; after that a connection +failure is reported without splicing another answer into the same stream. + +## Deployment and limits + +`drift text-peer MANIFEST --initial_peers ... --identity_path ... --cache_dir ...` +starts a dedicated contributor role. It needs memory and storage for input/output +weights in addition to any blocks it serves. Ordinary consumers need neither. +The CLI currently enables this role explicitly; automatic placement and resource +accounting of text roles in desktop sharing remain follow-up work. + +Block availability, text-service availability and spare generation capacity are +different observations. A signed advertisement cannot guarantee the next request +will succeed. Peer failure can still produce a retryable error during an answer. +Expiry removes a lost text service from subsequent automatic selection. + +The local fallback still downloads and executes its own small model when needed. +This decision removes community weight downloads from consumers; it does not +make the local fallback run remotely or establish a new minimum system requirement. + +[September 9 live evidence](../evidence/text-only-mesh-consumer-20260909.md) +records short completion and chat responses from fresh clients with artifact +downloads forbidden. Packaged delivery remains separate. diff --git a/docs/evidence/alpha-artifact-audit-20260907.json b/docs/evidence/alpha-artifact-audit-20260907.json new file mode 100644 index 000000000..4f2e90b17 --- /dev/null +++ b/docs/evidence/alpha-artifact-audit-20260907.json @@ -0,0 +1,130 @@ +{ + "recorded_at_utc": "2026-09-08T00:55:19.206139+00:00", + "scope": "qualified alpha candidates and independent current PR audit; no publication", + "reviewed_head": "e81103663c0c598cdaf43706726fe2a65b4357b6", + "result": "passed-with-distribution-blocker", + "installers": [ + { + "path": ".gate13-runs/gate14-public-windows-installers/communityai-0.1.0-alpha.20260907.2-windows-setup.exe", + "sha256": "c4e8df599f3a6118eab5718a5ad50655b0e07fd6c270aacf7dbb0b3065c5c399", + "size_bytes": 2519046440, + "platform": "windows", + "version": "0.1.0-alpha.20260907.2", + "github_release_asset_eligible": false, + "publisher_signed": false, + "lifecycle_evidence": { + "path": "docs/evidence/gate15-20260907-frozen-windows-installer.json", + "sha256": "ed422c5a313a3e5330ad236d7e8baed9724ccb552b584cbcd81af24eeb621381", + "size_bytes": 2405 + }, + "runtime_source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "publisher": "Mario Andreschak" + }, + { + "path": ".gate13-runs/gate14-public-linux-installers/communityai_0.1.0~alpha.20260907.4_amd64.deb", + "sha256": "a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517", + "size_bytes": 3781591484, + "platform": "linux", + "version": "0.1.0~alpha.20260907.4", + "github_release_asset_eligible": false, + "publisher_signed": false, + "lifecycle_evidence": { + "path": "docs/evidence/gate15-20260907-frozen-debian-installer.json", + "sha256": "d721d22f9c03fe0f2020061c13fca12defa39ff491094e7df455a8a0a3848784", + "size_bytes": 3640 + }, + "runtime_source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_control_source_commit": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb", + "maintainer": "Mario Andreschak " + } + ], + "github_release_asset_limit_bytes_exclusive": 2147483648, + "github_release_limit_source": "https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases#storage-and-bandwidth-quotas", + "ci": { + "run_id": 34171118914, + "head_sha": "e81103663c0c598cdaf43706726fe2a65b4357b6", + "conclusion": "success", + "artifact_provenance": [ + { + "path": ".gate13-runs/release-audit-20260907/ci-windows/provenance.json", + "sha256": "6666ddaebcd185220a2ebf0b7a892840471a512de59fc8fd6c8e616b870f3b80", + "size_bytes": 1220945, + "platform": "windows", + "source_commit": "b4ed322d86e5019d526ee7f5c6c2ccd87a9a9be2", + "source_tree": "0c4032b09b85c8b71b74447d7362a1e47818fd9a", + "archive": { + "artifact_root": "CommunityAI", + "entry_count": 5851, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "c9e7269616d2db17185722fd5b5e2d89d78ede96bea313e612aee46035397cda", + "size_bytes": 2690877285 + }, + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80" + }, + { + "path": ".gate13-runs/release-audit-20260907/ci-linux/provenance.json", + "sha256": "1e3410ce09d81944f210b5e2d01a86eb2a0b80616f05bb634408909a74c18219", + "size_bytes": 1283127, + "platform": "linux", + "source_commit": "b4ed322d86e5019d526ee7f5c6c2ccd87a9a9be2", + "source_tree": "0c4032b09b85c8b71b74447d7362a1e47818fd9a", + "archive": { + "artifact_root": "CommunityAI", + "entry_count": 6118, + "format": "tar.gz", + "path": "communityai-desktop-linux.tar.gz", + "platform": "Linux", + "preserves_executable_modes": true, + "preserves_internal_file_symlinks": true, + "schema_version": 1, + "sha256": "7991b90984fb4900c482fd6487b3e4e658027e1770171878bc1f2d551fca7238", + "size_bytes": 3109948549 + }, + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80" + } + ], + "merge_commit_delta_from_reviewed_head": [ + "README.md" + ] + }, + "limitations": [ + "Current CI artifacts are separate engineering rebuilds; existing installed/runtime qualification does not transfer by filename.", + "Both qualified installers exceed GitHub Releases single-asset size limit; permanent direct-download HTTPS host is not configured.", + "No package upload, release creation, paid service, or catalog mutation occurred." + ], + "qualified_runtime_application_delta_from_reviewed_head": [], + "source_comparison_paths": [ + "desktop/src", + "src", + "public-alpha", + "manifests" + ], + "cloud_storage_read_only_preflight": { + "project": "community-ai-506321", + "result": "authentication-refresh-failed", + "reason": "Reauthentication required; non-interactive gcloud cannot prompt. No buckets or IAM permissions verified." + }, + "preferred_distribution_candidate": { + "provider": "Cloudflare R2", + "storage_class": "Standard", + "installer_total_bytes": 6300637924, + "included_storage_gb_month": 10, + "included_class_a_requests_per_month": 1000000, + "included_class_b_requests_per_month": 10000000, + "direct_egress_charge": 0, + "single_part_upload_limit_bytes": 5368709120, + "public_origin": "custom domain required for production recommendation; not configured", + "existing_account_or_remaining_allowance_verified": false, + "source_urls": [ + "https://developers.cloudflare.com/r2/pricing/", + "https://developers.cloudflare.com/r2/platform/limits/", + "https://developers.cloudflare.com/r2/buckets/public-buckets/" + ], + "decision": "candidate only; no service enabled, bucket created, domain configured or artifact uploaded" + } +} diff --git a/docs/evidence/alpha-cloudflare-publication-20260909.json b/docs/evidence/alpha-cloudflare-publication-20260909.json new file mode 100644 index 000000000..7dc0a1f69 --- /dev/null +++ b/docs/evidence/alpha-cloudflare-publication-20260909.json @@ -0,0 +1,253 @@ +{ + "schema_version": 1, + "result": "passed", + "verified_at_utc": "2026-09-09T02:02:01.346429+00:00", + "public_base_url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/", + "small_public_objects": [ + { + "key": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "size_bytes": 2107751, + "sha256": "8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/octet-stream" + }, + { + "key": "communityai-0.1.0-alpha.20260908.1-linux-online.py", + "size_bytes": 13662, + "sha256": "59a00906d358c1cac0046e9c323a612e7f6fdb824d21ba562de0bae18ba39b0b", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "text/x-python; charset=utf-8" + }, + { + "key": "release-downloads.json", + "size_bytes": 1087, + "sha256": "3ab88fd1dd7b5ccbb19cb226429c5b3646d241a676280bde55795fdc34c8df6a", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "INSTALLER-SHA256SUMS", + "size_bytes": 477, + "sha256": "ba50a7b4502145303beb69b3f1c4a2b25bd5e297817b33d5c0f6d5428e284274", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "text/plain" + }, + { + "key": "metadata/windows/communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe.json", + "size_bytes": 1433, + "sha256": "c3fece282465df8c74907ff4c7c29fd48405b9dc579315a61d340929afe4584e", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/communityai-0.1.0-alpha.20260908.1-windows-setup.exe.json", + "size_bytes": 300, + "sha256": "a8cbf949c099fc5cb110ffe1768bd2f7fa04bffc814e088249272b745271d98d", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/desktop-metrics.json", + "size_bytes": 3786, + "sha256": "35c608d647c72c0201f4a436018d178fe75d017a6a396565ad430dd6d860d35c", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/normalized-online-windows-installer-20260908.json", + "size_bytes": 11035, + "sha256": "df98bf134ce2b49e733ec30b79bec338041146dd0337a69d7f0dd6874adaeace", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/normalized-windows-installer-20260908.json", + "size_bytes": 8369, + "sha256": "88a23622f18df48655e138c74151b7fe9f8b40e8794b2c7b9cb37b5700056a25", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/online-source-commit.json", + "size_bytes": 312, + "sha256": "dc871c18c66b8ccfcb61304b9632d151cccb29cb623333055bef355e01419f94", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/provenance.json", + "size_bytes": 1239925, + "sha256": "d5210329edf7236aaaadc0102c0f9363d3c0560f14ccb46ece3928f61b0ddaa1", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/release-metadata.json", + "size_bytes": 872, + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/windows/SHA256SUMS", + "size_bytes": 673493, + "sha256": "1afc393a0103a43f256181657a062515bb0f9011fb6dbf9bf0c1e84015a2a44d", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "text/plain" + }, + { + "key": "metadata/windows/windows-release-downloads.json", + "size_bytes": 575, + "sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/alpha-normalized-linux-20260908.json", + "size_bytes": 21883, + "sha256": "aa2320fd6b721d70078ba4ef0eedd371685d0db02a4cf7d794f5b3e165d083b5", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/alpha-online-linux-hosted-20260908.json", + "size_bytes": 11794, + "sha256": "a7e69662a24447ad8ac9a60e24731673387c3fce61892fbda9c5cad2a196973b", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/communityai-0.1.0-alpha.20260908.1-linux-online.py.json", + "size_bytes": 915, + "sha256": "a3b9134f69f232cfed6bc5f5fd12e20a0390ef01c08ff7ca2616af962ee8b9d5", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/communityai_0.1.0~alpha.20260908.1_amd64.deb.json", + "size_bytes": 1255, + "sha256": "091325f0b1dfd8a4879d46d700e7ed20c878f5a10c615ca26e12ce9ecf888bc3", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/desktop-metrics.json", + "size_bytes": 3814, + "sha256": "46e0332d0cd8de383d9bd4f60edc31e801b1508dd297fb7e52d79ae62242fde6", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/provenance.json", + "size_bytes": 1244840, + "sha256": "3f8ac9de9f61f663b1c7573a1200aa4c26e5d0e18aa38d0f004d96e95e341db9", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/release-metadata.json", + "size_bytes": 872, + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/runtime-packaging.json", + "size_bytes": 1573, + "sha256": "e5d71cf23a900193d7ab8005291bd22763edb320c64e3a21327ce906ab9e1ade", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/json" + }, + { + "key": "metadata/linux/SHA256SUMS", + "size_bytes": 675799, + "sha256": "b22fd299a36ef208c886d137fe80e7c89e3441c293bcefe02c70b4b412408a99", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "text/plain; charset=utf-8" + }, + { + "key": "communityai-0.1.0-alpha.20260908.1-release-metadata.zip", + "size_bytes": 1009867, + "sha256": "9e3424eb8587b09b02b464dc2456aecd1150f154c4fe719069e39374129bb60b", + "status": 200, + "complete_anonymous_body_verified": true, + "cache_control": "public,max-age=31536000,immutable", + "content_type": "application/zip" + } + ], + "offline_objects": [ + { + "platform": "linux-amd64", + "filename": "communityai_0.1.0~alpha.20260908.1_amd64.deb", + "size_bytes": 2302428788, + "sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "head_status": 200, + "public_sha256_header": null, + "full_body_verification_scope": "Actual online installation evidence, not this HEAD request" + }, + { + "platform": "windows-x64", + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "size_bytes": 2462345104, + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "head_status": 200, + "public_sha256_header": null, + "full_body_verification_scope": "Actual online installation evidence, not this HEAD request" + } + ], + "actual_installation_evidence": [ + "normalized-online-windows-installer-20260908.json", + "alpha-online-linux-hosted-20260908.json" + ], + "limits": [ + "Cloudflare r2.dev is a rate-limited development endpoint; this is observed delivery, not a load/availability guarantee.", + "Installers are unsigned alpha artifacts.", + "The first final-publication audit stopped after all small objects matched because it incorrectly required a custom SHA metadata header on the public endpoint. The public endpoint does not expose this header; full offline byte identity is established by the actual online acceptance records." + ] +} diff --git a/docs/evidence/alpha-installer-size-audit-20260908.md b/docs/evidence/alpha-installer-size-audit-20260908.md new file mode 100644 index 000000000..35ebd8ed7 --- /dev/null +++ b/docs/evidence/alpha-installer-size-audit-20260908.md @@ -0,0 +1,84 @@ +# Qualified alpha installer size audit — September 8, 2026 + +The qualified installers bundle the Python inference engine, PyTorch 2.6.0+cu124, +NVIDIA runtime libraries and Hivemind. Model weights are downloaded separately. +The GPU engine dominates both payloads. Linux also contains substantial duplicate +regular-file data that is a concrete packaging optimization candidate. + +All GB/MB figures below use decimal units. Component rows are subsets of the +unpacked payload; duplicate bytes overlap those components. + +| Measurement | Windows | Linux | +| --- | ---: | ---: | +| Installer download | 2,519,046,440 B | 3,781,591,484 B | +| Unpacked regular files | 4,487,021,701 B | 8,591,203,661 B | +| PyTorch and NVIDIA/CUDA libraries | 3,804,955,387 B | 7,801,051,434 B | +| bitsandbytes | 248,477,193 B | 250,434,156 B | +| Complete desktop portion outside the node | 122,461,606 B | 200,833,265 B | +| Hivemind directory including p2pd | 43,339,835 B | 10,346,393 B | +| Redundant bytes by identical stored SHA-256 and size | 23,211,174 B | 3,260,409,997 B | + +Windows' CUDA libraries reside inside the torch directory. Linux's total combines +the separately classified NVIDIA and PyTorch files, including their duplicate +copies. Other Python libraries and the node executable account for the remainder. +Some Python code is also frozen into the executables, so directory attribution +is not a full module-by-module executable analysis. + +Linux's 62 duplicate hash groups represent 37.951% of its unpacked regular-file +bytes. For example, `libtorch_cuda.so` is 902,652,937 bytes at both +`node/_internal/libtorch_cuda.so` and `node/_internal/torch/lib/libtorch_cuda.so`. +`libtorch_cpu.so` and large NVIDIA libraries are duplicated similarly. The +inventory records zero symlinks, and its use of `lstat` distinguishes actual file +copies from links. Original build output already reports the same 8.591 GB and +4,930 files immediately after PyInstaller collection. The duplication therefore +predates Debian staging, which preserves links; it is not caused by `.deb` +compression or by a later copy onto Windows. + +Windows has 4,944 regular files and only 0.52% redundant content, mostly shared +Python/OpenSSL support. The Windows GUI is about 122 MB, while the node runtime +is 4.365 GB. The signed bootstrap/catalog is about 17 KB. There is no second +large PyTorch copy in the GUI. + +Both platforms include ten bitsandbytes CUDA variants. On Windows, nine variants +other than the bundled PyTorch cu124 profile occupy 223,167,488 bytes unpacked +and approximately 67.6 MB in the existing ZIP. Compatibility with the supported +loader/profile must be checked before pruning these. ZIP compression attribution +does not establish the saving in the solid Inno installer. + +The packaged files include GPU runtime libraries such as cuBLAS/cuDNN and runtime +compilation libraries, not a complete CUDA development toolkit or an NVIDIA +driver installer. No conventional model-weight files were found. The Debian +archive consists almost entirely of compressed runtime payload: its control +archive is only 2,004 bytes. Current compression settings are Inno +`lzma2/fast` with solid compression and Debian XZ level 1; stronger compression +has not been measured on these candidates. + +Recommended next packaging work is to preserve all required loader paths while +eliminating duplicate Linux library data, validate CUDA-variant pruning, and then +measure the rebuilt installers. Keeping one physical copy of every identical +Linux file would leave 5,330,793,664 bytes of unique content; that is an accounting +bound, not a validated replacement bundle or a predicted compressed download. +Removing libraries merely because a model does not explicitly call them may +break stock PyTorch's loading dependencies. A substantially smaller base installer +could instead make the GPU runtime a separate verified download; GPU users would +still download those libraries when enabling that capability. + +## Evidence and limits + +This is an inspection of the qualified Windows `76b6d84` and Linux `bf67f0d` +payloads recorded in [the artifact inventory](alpha-artifact-audit-20260907.json). +No installer or runtime was modified, repacked, executed or requalified. + +Windows filesystem sizes were compared with every entry in the qualified +provenance and ZIP central directory: no missing, extra or size-mismatched files. +Linux used the exact provenance accompanying the `.4` Debian candidate, with +its digest checked against the installer sidecar. Only 21,636 bytes of Debian +headers, XZ index and small control metadata were read, establishing the +8,595,763,200-byte uncompressed tar size. Large binaries were not rehashed or +decompressed. Duplicate groups use their existing recorded SHA-256 identities. +Docker and WSL remained off; no model or desktop window launched. + +The detailed private reports are retained at +`.gate13-runs/installer-size-audit-20260908/windows.json` and `linux.json`. +The Windows metadata audit helper is retained beside them. Exact new installer +savings remain unmeasured and require packaging and frozen-runtime validation. diff --git a/docs/evidence/alpha-normalized-linux-20260908.json b/docs/evidence/alpha-normalized-linux-20260908.json new file mode 100644 index 000000000..efa3d326d --- /dev/null +++ b/docs/evidence/alpha-normalized-linux-20260908.json @@ -0,0 +1,523 @@ +{ + "acceptance_environment": { + "cpus": 2, + "dependencies": "Official Ubuntu python3 and sudo installed with --no-install-recommends before network disconnect", + "distribution": "Ubuntu22.04", + "gpu_access": "CUDA device0 for tiny native operations only", + "memory_bytes": 3221225472, + "offline_network_interfaces": {}, + "ordinary_uid": 1000, + "source_and_release_mounts_readonly": true, + "sudo_fixture": "NOPASSWD rule for qualifier inside this disposable container only" + }, + "build": { + "build_network": "none", + "bundled_offscreen_contract_checks_passed": true, + "cpus": 2, + "independent_release_verification_passed": true, + "memory_bytes": 6442450944, + "result": { + "exit_code": 0, + "full_runtime_qualification": false, + "output": "/environment/release-linux-v3", + "phase": "complete", + "recorded_at_utc": "2026-09-08T23:48:58.108366+00:00", + "result": "passed", + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e" + } + }, + "build_image_id": "sha256:0042575ad9d57e21900044ed3227ca15bb5017b033d84781d6a57f16e9db51a3", + "cleanup": { + "acceptance_container": "retained idle and disconnected for separate online acceptance", + "app_or_package_processes_remaining": 0, + "build_containers": "retained stopped; new output retained in named volume", + "installed_runtime_removed": true, + "native_check_container_removed": true, + "previous_bundles_or_artifacts_changed": false + }, + "frozen_native_checks": { + "checks": [ + { + "arguments": [ + "--native-self-test" + ], + "contract": { + "application": "CommunityAI-Native-Runtime", + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_required": false, + "cuda_test_performed": false, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "elapsed_seconds": 1.961, + "exit_code": 0, + "name": "frozen-native-cpu", + "passed": true, + "stderr": "", + "stdout": "{\"application\": \"CommunityAI-Native-Runtime\", \"cpu_matmul_passed\": true, \"cuda_build\": \"12.4\", \"cuda_required\": false, \"cuda_test_performed\": false, \"frozen\": true, \"model_loading_performed\": false, \"network_join_performed\": false, \"schema_version\": 1, \"torch\": \"2.6.0+cu124\"}\n" + }, + { + "arguments": [ + "--native-self-test", + "--require-cuda" + ], + "contract": { + "application": "CommunityAI-Native-Runtime", + "bitsandbytes_native_library": "libbitsandbytes_cuda124.so", + "bitsandbytes_nf4_maximum_absolute_error": 0.14501953125, + "bitsandbytes_nf4_roundtrip_passed": true, + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_linalg_passed": true, + "cuda_matmul_passed": true, + "cuda_required": true, + "cuda_test_performed": true, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "elapsed_seconds": 2.275, + "exit_code": 0, + "name": "frozen-native-cuda", + "passed": true, + "stderr": "", + "stdout": "{\"application\": \"CommunityAI-Native-Runtime\", \"bitsandbytes_native_library\": \"libbitsandbytes_cuda124.so\", \"bitsandbytes_nf4_maximum_absolute_error\": 0.14501953125, \"bitsandbytes_nf4_roundtrip_passed\": true, \"cpu_matmul_passed\": true, \"cuda_build\": \"12.4\", \"cuda_linalg_passed\": true, \"cuda_matmul_passed\": true, \"cuda_required\": true, \"cuda_test_performed\": true, \"frozen\": true, \"model_loading_performed\": false, \"network_join_performed\": false, \"schema_version\": 1, \"torch\": \"2.6.0+cu124\"}\n" + }, + { + "arguments": [ + "server", + "--self-test" + ], + "contract": { + "application": "CommunityAI-Worker", + "entrypoint": "server", + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "process_lifetime_guard_armed": true, + "schema_version": 1, + "server_class": "Server", + "throughput_mode": "dry_run", + "training_rpcs_enabled": false + }, + "elapsed_seconds": 6.812, + "exit_code": 0, + "name": "frozen-server-contract", + "passed": true, + "stderr": "Sep 08 23:50:06.759 [INFO] p2pd daemons will now receive SIGKILL when the process that spawned them dies (PR_SET_PDEATHSIG)\n", + "stdout": "{\"application\": \"CommunityAI-Worker\", \"entrypoint\": \"server\", \"frozen\": true, \"model_loading_performed\": false, \"network_join_performed\": false, \"process_lifetime_guard_armed\": true, \"schema_version\": 1, \"server_class\": \"Server\", \"throughput_mode\": \"dry_run\", \"training_rpcs_enabled\": false}\n" + } + ], + "full_runtime_qualification": false, + "gpu_access": "all; required-CUDA check selects device 0", + "models_loaded": false, + "network": "none", + "node_executable_sha256": "1e19876ea4f2d7c85336def4a11350c2d8d3b04689c6fa95657f2ecb77746ab7", + "provenance_sha256": "3f8ac9de9f61f663b1c7573a1200aa4c26e5d0e18aa38d0f004d96e95e341db9", + "result": "passed", + "schema_version": 1, + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e" + }, + "installer": { + "build_seconds": 514.229, + "download_bytes_saved": 1479162696, + "filename": "communityai_0.1.0~alpha.20260908.1_amd64.deb", + "host_copy_verified": true, + "host_copy_verified_at_utc": "2026-09-09T00:03:16.0689473Z", + "installed_size_kib": 5040152, + "previous_installer_bytes": 3781591484, + "sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "size_bytes": 2302428788, + "version": "0.1.0~alpha.20260908.1" + }, + "limitations": [ + "No model load, generation, public peer join, cloud mutation or public canary was performed.", + "Installed application checks exercised frozen native and server self-test modes; this is not a full Gate15 GUI/worker lifecycle replay.", + "External user-state sentinel retention was checked; no native credential store was created or changed.", + "Actual Linux online download/sudo/APT acceptance is separate and remains pending in this record.", + "This replay covers Ubuntu22.04 under the stated WSL2/Docker host; it does not claim a new Debian12 or bare-metal OS replay.", + "Signing remains deferred until after alpha." + ], + "normalization": { + "after": { + "logical_file_bytes": 4960280412, + "regular_file_count": 4616, + "symlink_count": 9, + "unique_file_bytes": 4960280412, + "unique_file_count": 4616 + }, + "before": { + "logical_file_bytes": 5185185572, + "regular_file_count": 4625, + "symlink_count": 9, + "unique_file_bytes": 5185185572, + "unique_file_count": 4625 + }, + "bitsandbytes_cuda_version": "124", + "hardlinked_native_libraries": [], + "platform": "Linux", + "removed_bitsandbytes_variants": [ + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda117.so", + "size_bytes": 20878944 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda118.so", + "size_bytes": 26498656 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda120.so", + "size_bytes": 25785952 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda121.so", + "size_bytes": 25794144 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda122.so", + "size_bytes": 25814856 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda123.so", + "size_bytes": 25855816 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda125.so", + "size_bytes": 25185760 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda126.so", + "size_bytes": 25258352 + }, + { + "path": "_internal/bitsandbytes/libbitsandbytes_cuda128.so", + "size_bytes": 23832680 + } + ], + "schema_version": 1, + "scope": "frozen-node-runtime", + "torch_version": "2.6.0+cu124" + }, + "offline_acceptance": { + "checks": [ + { + "elapsed_seconds": 132.119, + "exit_code": 0, + "name": "install" + }, + { + "contract": { + "application": "CommunityAI-Native-Runtime", + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_required": false, + "cuda_test_performed": false, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "elapsed_seconds": 3.265, + "exit_code": 0, + "name": "installed-native-cpu", + "ordinary_uid": 1000 + }, + { + "contract": { + "application": "CommunityAI-Native-Runtime", + "bitsandbytes_native_library": "libbitsandbytes_cuda124.so", + "bitsandbytes_nf4_maximum_absolute_error": 0.14501953125, + "bitsandbytes_nf4_roundtrip_passed": true, + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_linalg_passed": true, + "cuda_matmul_passed": true, + "cuda_required": true, + "cuda_test_performed": true, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "elapsed_seconds": 3.902, + "exit_code": 0, + "name": "installed-native-cuda", + "ordinary_uid": 1000 + }, + { + "contract": { + "application": "CommunityAI-Worker", + "entrypoint": "server", + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "process_lifetime_guard_armed": true, + "schema_version": 1, + "server_class": "Server", + "throughput_mode": "dry_run", + "training_rpcs_enabled": false + }, + "elapsed_seconds": 7.355, + "exit_code": 0, + "name": "installed-server-contract", + "ordinary_uid": 1000 + }, + { + "elapsed_seconds": 0.767, + "exit_code": 0, + "name": "remove" + } + ], + "credential_store_touched": false, + "external_user_state_sentinel_preserved": true, + "full_gate15_replay": false, + "gui_launch_performed": false, + "installed_inventory": { + "every_file_sha256_and_mode_matched": true, + "regular_files": 4900, + "symlinks": [ + { + "path": "_internal/libQt6Core.so.6", + "target": "PySide6/Qt/lib/libQt6Core.so.6" + }, + { + "path": "_internal/libQt6DBus.so.6", + "target": "PySide6/Qt/lib/libQt6DBus.so.6" + }, + { + "path": "_internal/libQt6EglFSDeviceIntegration.so.6", + "target": "PySide6/Qt/lib/libQt6EglFSDeviceIntegration.so.6" + }, + { + "path": "_internal/libQt6EglFsKmsSupport.so.6", + "target": "PySide6/Qt/lib/libQt6EglFsKmsSupport.so.6" + }, + { + "path": "_internal/libQt6Gui.so.6", + "target": "PySide6/Qt/lib/libQt6Gui.so.6" + }, + { + "path": "_internal/libQt6Network.so.6", + "target": "PySide6/Qt/lib/libQt6Network.so.6" + }, + { + "path": "_internal/libQt6OpenGL.so.6", + "target": "PySide6/Qt/lib/libQt6OpenGL.so.6" + }, + { + "path": "_internal/libQt6Pdf.so.6", + "target": "PySide6/Qt/lib/libQt6Pdf.so.6" + }, + { + "path": "_internal/libQt6Qml.so.6", + "target": "PySide6/Qt/lib/libQt6Qml.so.6" + }, + { + "path": "_internal/libQt6QmlMeta.so.6", + "target": "PySide6/Qt/lib/libQt6QmlMeta.so.6" + }, + { + "path": "_internal/libQt6QmlModels.so.6", + "target": "PySide6/Qt/lib/libQt6QmlModels.so.6" + }, + { + "path": "_internal/libQt6QmlWorkerScript.so.6", + "target": "PySide6/Qt/lib/libQt6QmlWorkerScript.so.6" + }, + { + "path": "_internal/libQt6Quick.so.6", + "target": "PySide6/Qt/lib/libQt6Quick.so.6" + }, + { + "path": "_internal/libQt6Svg.so.6", + "target": "PySide6/Qt/lib/libQt6Svg.so.6" + }, + { + "path": "_internal/libQt6Test.so.6", + "target": "PySide6/Qt/lib/libQt6Test.so.6" + }, + { + "path": "_internal/libQt6VirtualKeyboard.so.6", + "target": "PySide6/Qt/lib/libQt6VirtualKeyboard.so.6" + }, + { + "path": "_internal/libQt6VirtualKeyboardQml.so.6", + "target": "PySide6/Qt/lib/libQt6VirtualKeyboardQml.so.6" + }, + { + "path": "_internal/libQt6WaylandClient.so.6", + "target": "PySide6/Qt/lib/libQt6WaylandClient.so.6" + }, + { + "path": "_internal/libQt6Widgets.so.6", + "target": "PySide6/Qt/lib/libQt6Widgets.so.6" + }, + { + "path": "_internal/libQt6WlShellIntegration.so.6", + "target": "PySide6/Qt/lib/libQt6WlShellIntegration.so.6" + }, + { + "path": "_internal/libQt6XcbQpa.so.6", + "target": "PySide6/Qt/lib/libQt6XcbQpa.so.6" + }, + { + "path": "_internal/libicudata.so.73", + "target": "PySide6/Qt/lib/libicudata.so.73" + }, + { + "path": "_internal/libicui18n.so.73", + "target": "PySide6/Qt/lib/libicui18n.so.73" + }, + { + "path": "_internal/libicuuc.so.73", + "target": "PySide6/Qt/lib/libicuuc.so.73" + }, + { + "path": "_internal/libpyside6.abi3.so.6.11", + "target": "PySide6/libpyside6.abi3.so.6.11" + }, + { + "path": "_internal/libshiboken6.abi3.so.6.11", + "target": "shiboken6/libshiboken6.abi3.so.6.11" + }, + { + "path": "node/_internal/libcusparseLt.so.0", + "target": "cusparselt/lib/libcusparseLt.so.0" + }, + { + "path": "node/_internal/libgfortran-040039e1-0352e75f.so.5.0.0", + "target": "scipy.libs/libgfortran-040039e1-0352e75f.so.5.0.0" + }, + { + "path": "node/_internal/libgfortran-83c28eba-468e71e5.so.5.0.0", + "target": "numpy.libs/libgfortran-83c28eba-468e71e5.so.5.0.0" + }, + { + "path": "node/_internal/libgfortran-83c28eba.so.5.0.0", + "target": "scipy.libs/libgfortran-83c28eba.so.5.0.0" + }, + { + "path": "node/_internal/libquadmath-2284e583-a9307bba.so.0.0.0", + "target": "numpy.libs/libquadmath-2284e583-a9307bba.so.0.0.0" + }, + { + "path": "node/_internal/libquadmath-2284e583.so.0.0.0", + "target": "scipy.libs/libquadmath-2284e583.so.0.0.0" + }, + { + "path": "node/_internal/libquadmath-96973f99-934c22de.so.0.0.0", + "target": "scipy.libs/libquadmath-96973f99-934c22de.so.0.0.0" + }, + { + "path": "node/_internal/libscipy_openblas-5f890258.so", + "target": "scipy.libs/libscipy_openblas-5f890258.so" + }, + { + "path": "node/_internal/libscipy_openblas64_-f48b354e.so", + "target": "numpy.libs/libscipy_openblas64_-f48b354e.so" + } + ], + "unique_regular_bytes": 5161115250 + }, + "installed_runtime_removed": true, + "installer_bytes": 2302428788, + "installer_sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "installer_version": "0.1.0~alpha.20260908.1", + "model_loading_performed": false, + "network": "disconnected before this script", + "online_installer_acceptance": false, + "provenance_sha256": "3f8ac9de9f61f663b1c7573a1200aa4c26e5d0e18aa38d0f004d96e95e341db9", + "result": "passed", + "scope": "Ubuntu 22.04 exact offline installer, installed native checks, removal", + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e" + }, + "payload": { + "artifact_logical_bytes_including_symlink_targets": 5528581899, + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5967, + "format": "tar.gz", + "path": "communityai-desktop-linux.tar.gz", + "platform": "Linux", + "preserves_executable_modes": true, + "preserves_internal_file_symlinks": true, + "schema_version": 1, + "sha256": "28f24f55c2e53c44d9235dbc150223e9bbc7b532945dc72ff28298cd82c55c79", + "size_bytes": 3017723439 + }, + "provenance_sha256": "3f8ac9de9f61f663b1c7573a1200aa4c26e5d0e18aa38d0f004d96e95e341db9", + "regular_files": 4900, + "symlinks": 35, + "unique_regular_file_bytes": 5161115250 + }, + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.35", + "post_removal_audit": { + "dpkg_audit_exit": 0, + "dpkg_audit_stdout": "", + "offline_result_sha256": "accfb1fcb70e0f6afdb7921bfea2498913d3b1643d42a374123f70eb37be176c", + "package_query_exit": 1, + "package_status": "", + "resident_app_or_package_processes": [], + "result": "passed", + "runtime_present": false, + "scope": "post-removal filesystem/package/process observation", + "sentinel_preserved": true + }, + "python": "3.12.14", + "raw_evidence_directory": ".gate13-runs/linux-v3-build-20260908", + "raw_evidence_sha256": { + "build-result.json": "a7481793b371b1a24db1c2a39d693da3ad208f33c19e0f62599e6aab96d6e225", + "build.log": "eff1850ef143f4c1afa1f6d00ebd8704870b3cf28dec07ae8fbc655751d26c46", + "deb-build-container-state.json": "6360ff39876974551fbccdee1d22c1996498248f2718fa37d8ee2824c65b1677", + "deb-build-result.json": "091325f0b1dfd8a4879d46d700e7ed20c878f5a10c615ca26e12ce9ecf888bc3", + "deb-build.log": "f61d223f67296486b37039862dd62e15563adf5d295715507ce6438cf526b20c", + "deb-control.txt": "80ae635207c46a4728b2a9c09e1efa3aa12e65945db64bcc3eae85e4d3f0d336", + "dependency-setup.log": "8c6d8bf3f11c3ab650e85e730cbd5493238a842910ecf88385ec24b423343fbc", + "desktop-metrics.json": "46e0332d0cd8de383d9bd4f60edc31e801b1508dd297fb7e52d79ae62242fde6", + "install.log": "c4540878753958947ba99670c2c67eab6ce2253d7dfb06a4c2f5c50d1a5834ba", + "installed-native-cpu.log": "0d3a5fe8f9716612233f980fb6b31e172e31602bc12b25328d29e332833c72df", + "installed-native-cuda.log": "a758fec37e2bdbe2aa182217986fc65f279ccab2b0765072cf893af0c098ec69", + "installed-server-contract.log": "b5fe0b02287b4bc04bebb8b2789fdeb264a73de01ed60c49c421ea8d1168cab9", + "native-check.json": "68ed740f2b42b37fdfebbf9f381714ab09db670f6e8ff4c4ef79f8d43bae3f0d", + "native-container-state.json": "b8f4e63c3e80471e8f1aea70b574a06d1e5016d53e8ad3edf3f8760aaa966c33", + "offline-network-state-after.json": "e3566b3a06430868d71e9287dfd6c6c520a3da027aabea01951d407ee131dc2f", + "offline-network-state.json": "e3566b3a06430868d71e9287dfd6c6c520a3da027aabea01951d407ee131dc2f", + "offline-post-removal-audit.json": "74e74cbecc75c263369eef79953111622e0302390d1073a13410ce33da64218a", + "offline-result.json": "accfb1fcb70e0f6afdb7921bfea2498913d3b1643d42a374123f70eb37be176c", + "provenance.json": "3f8ac9de9f61f663b1c7573a1200aa4c26e5d0e18aa38d0f004d96e95e341db9", + "readonly-pycompile-preflight.json": "886f9d4a2de1ca39caf91a869c3de1a71c217e6b47babdb19d64b17acf17e934", + "remove.log": "248064b6ca6d6e128ce74b97bc6ad5bcae52e7dc2be7ebd3e59c959f0b9b20c4", + "runtime-packaging.json": "e5d71cf23a900193d7ab8005291bd22763edb320c64e3a21327ce906ab9e1ade", + "source-preflight.json": "86ef7ecda2e898907c46545637dea0c6d8004acbb8ca7c94f0e269b5cf8c55d6", + "verification.log": "ca81dcf0f4d4ad864e56a3e3d4af323088782552ef6a5a5dd328fda4c929f5ee" + }, + "result": "passed", + "retained_preflight_failure": "Explicit py_compile attempted a bytecode write to the read-only source mount; a read-only Python3.10 AST parse then passed without weakening the mount.", + "runtime_versions": { + "PyInstaller": "6.22.2", + "PySide6": "6.11.2", + "bitsandbytes": "0.45.5", + "hivemind": "1.1.12", + "torch": "2.6.0+cu124" + }, + "schema_version": 1, + "scope": "Fresh normalized Ubuntu22 Linux bundle, finite CPU/CUDA native checks, exact offline installer acceptance", + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "source_sha256_at_commit": { + "desktop/build_desktop.py": "2ff8469e637639d9978965a3bcd292c714c0e579d4c52d9331e65a4a179c8afd", + "desktop/installers/build_deb.py": "8b26b3d9780fe22b9158b21944b9503636706d524e2b2cc1b763bc1a9add7b57", + "desktop/installers/installation-marker.txt": "517250bef51c65699c4be85617615debd872c06b116fc04f543644063d0c1874", + "desktop/installers/linux_maintenance.py": "4841b09a2a0c60f9e7def7cff7f081da5b82c139d3dbf49363d6e611e7f65fd1", + "desktop/launch_node.py": "b8a9c7418e9691372dd607da2f9a68b911460b2fb2870fca4df7cfe9f931c85c", + "desktop/runtime_packaging.py": "dba6424a90ffe49c8fea71ca95c66d2c741c68bc01f3b20783192ee63b6d2fdc" + }, + "source_tree": "5bd44e4afb6b6911f5ddd1bb3d492dad438883e8" +} diff --git a/docs/evidence/alpha-normalized-linux-20260908.md b/docs/evidence/alpha-normalized-linux-20260908.md new file mode 100644 index 000000000..d56c769b3 --- /dev/null +++ b/docs/evidence/alpha-normalized-linux-20260908.md @@ -0,0 +1,63 @@ +# Normalized Linux installer acceptance — 2026-09-08 + +**Passed for the scope below.** The fresh Linux release was built from clean commit +`84205f93fc73d3babd39e238944b97fab0d11b3e` (tree +`5bd44e4afb6b6911f5ddd1bb3d492dad438883e8`). The source checkout, output and build +directories were new; the previous qualified artifacts were preserved. The +[machine-readable record](alpha-normalized-linux-20260908.json) binds the source, +raw logs, normalization metrics, frozen checks and installed-file verification. + +| Artifact or measure | Actual result | +| --- | --- | +| Installer | `communityai_0.1.0~alpha.20260908.1_amd64.deb` | +| Installer bytes | 2,302,428,788 | +| Installer SHA-256 | `714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576` | +| Previous qualified `.4` installer | 3,781,591,484 bytes | +| Download reduction | 1,479,162,696 bytes, approximately 39.1% | +| Regular-file payload | 5,161,115,250 bytes across 4,900 files | +| Internal symlinks | 35; all preserved by the actual Debian installation | +| Debian Installed-Size | 5,040,152 KiB, including the installation marker and rounding | +| Runtime archive | 3,017,723,439 bytes; SHA-256 `28f24f55c2e53c44d9235dbc150223e9bbc7b532945dc72ff28298cd82c55c79` | + +The node normalization removed nine unused bitsandbytes CUDA variants, saving +224,905,160 bytes and retaining CUDA 12.4. Its regular payload decreased from +5,185,185,572 to 4,960,280,412 bytes. This fresh freeze already contained internal +library symlinks and required zero additional hardlink replacements. The reported +artifact logical total of 5,528,581,899 bytes includes 367,466,649 bytes counted +again through symlink targets; it is not the unique installed payload size. The +earlier hardlink-savings estimate was an inventory estimate for the older artifact. + +The full build used the audited cached Ubuntu 22.04 image +`sha256:0042575ad9d57e21900044ed3227ca15bb5017b033d84781d6a57f16e9db51a3`, +two CPUs, 6 GiB memory, no network and offscreen Qt. The existing dependency +environment remained read-only. The build's bundled startup/UI contract checks +and a separate release/archive/provenance verification passed. A separate +read-only, network-isolated container with two CPUs and 3 GiB then passed the +frozen CPU matrix check, required-CUDA matrix and linalg checks, actual retained +`libbitsandbytes_cuda124.so` NF4 roundtrip, and server entry-point contract. + +The new installer built in 514.229 seconds. In a fresh Ubuntu 22.04 acceptance +container, official `python3` and `sudo` dependencies were installed without +recommends before disconnecting its network. The exact `.deb` installed in +132.119 seconds. Every regular file's SHA-256 and executable mode matched the +verified bundle, and every symlink resolved to the recorded internal target. +As ordinary UID 1000, the installed CPU, CUDA and server checks passed in 3.265, +3.902 and 7.355 seconds respectively. The CUDA NF4 maximum absolute error was +0.14501953125. No weights were loaded and no network was joined. + +Removal passed in 0.767 seconds. An external user-state sentinel survived. A +separate post-removal observation found no installed runtime, package entry, app +or package-manager process, or `dpkg --audit` problem. The credential store was +untouched. The finite native-check container was removed; the acceptance +container was retained idle and disconnected for a separate online acceptance +run. Its passwordless sudo fixture exists only inside that disposable container. +The single Windows-host installer copy was independently size/hash verified. + +This evidence covers the normalized bundle and bounded offline installer/native +acceptance on Ubuntu 22.04 under the recorded WSL2/Docker host. It does not replace +the earlier full Gate 15 GUI/worker lifecycle evidence, claim a new Debian 12 or +bare-metal replay, or establish model generation, periodic catalog draining, +public canary readiness, or the online downloader's sudo/APT handoff. Signing +remains deferred until after alpha. A read-only-mount bytecode-write preflight +failure is retained in the JSON; the corrected read-only AST parse passed, and +no product build/native/install failure occurred in this replay. diff --git a/docs/evidence/alpha-online-linux-hosted-20260908.json b/docs/evidence/alpha-online-linux-hosted-20260908.json new file mode 100644 index 000000000..df2a31809 --- /dev/null +++ b/docs/evidence/alpha-online-linux-hosted-20260908.json @@ -0,0 +1,218 @@ +{ + "container_observation": { + "gpu_math_performed_in_online_replay": false, + "id": "0d34b0f01d06219896235073174016520855209bdc80f4d37f4fe05aaf814d3e", + "image_id": "sha256:0042575ad9d57e21900044ed3227ca15bb5017b033d84781d6a57f16e9db51a3", + "memory_bytes": 3221225472, + "memory_swap_bytes": 3221225472, + "mounts": [ + { + "destination": "/environment", + "writable": false + }, + { + "destination": "/repo", + "writable": false + } + ], + "nano_cpus": 2000000000, + "network_names": [ + null + ], + "status": "running" + }, + "dependency_route": "APT downloaded 24.0 MB from official Ubuntu repositories for the declared gnome-keyring recommendation and its dependencies.", + "execution": { + "container": "communityai-release-linux-v3-acceptance-20260908", + "cpus": 2, + "driver_sha256": "0d40fe1bb294e5b5fa954e0bc9b8dcae71be3967447c3b9526e8ff5773b8159c", + "gpu_math_replay": false, + "gui_or_model_launch": false, + "last_review_detail": "Native stdout and stderr saved separately to preserve strict JSON parsing without treating harmless stderr as stdout", + "memory_bytes": 3221225472, + "network_names": [ + "bridge" + ], + "production_script_sha256": "59a00906d358c1cac0046e9c323a612e7f6fdb824d21ba562de0bae18ba39b0b", + "root_upload_greenlight_received": true, + "scope": "Actual production Linux online acceptance", + "started_at_utc": "2026-09-09T00:54:27.1537002Z" + }, + "first_readonly_audit_failure": "An initial audit expected the removed version in the final not-installed dpkg event; dpkg correctly records . Its failure is retained; the corrected read-only audit verifies installed/removal versions and final absent state.", + "limitations": [ + "APT input is bound to the protected SHA-verified file by actual APT Get output, not by captured process arguments.", + "The original harness failed after the production installer exited0, so its planned post-online CPU check did not run.", + "CPU/CUDA validation is referenced from the prior exact-package offline acceptance; no native replay is claimed for this online attempt.", + "No GUI, model weights, generation, public peer, native credential store or public canary was exercised.", + "Ubuntu 22.04 / Python 3.10 under the recorded Docker/WSL2 host; no new Debian 12 or bare-metal desktop replay.", + "External user-state sentinel retention was checked; full Gate15 settings/worker lifecycle evidence remains separate.", + "Publisher signing remains deferred until after alpha." + ], + "native_validation_reference": { + "evidence": "alpha-normalized-linux-20260908.json", + "evidence_sha256": "aa2320fd6b721d70078ba4ef0eedd371685d0db02a4cf7d794f5b3e165d083b5", + "installed_checks": [ + { + "contract": { + "application": "CommunityAI-Native-Runtime", + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_required": false, + "cuda_test_performed": false, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "elapsed_seconds": 3.265, + "exit_code": 0, + "name": "installed-native-cpu", + "ordinary_uid": 1000 + }, + { + "contract": { + "application": "CommunityAI-Native-Runtime", + "bitsandbytes_native_library": "libbitsandbytes_cuda124.so", + "bitsandbytes_nf4_maximum_absolute_error": 0.14501953125, + "bitsandbytes_nf4_roundtrip_passed": true, + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_linalg_passed": true, + "cuda_matmul_passed": true, + "cuda_required": true, + "cuda_test_performed": true, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "elapsed_seconds": 3.902, + "exit_code": 0, + "name": "installed-native-cuda", + "ordinary_uid": 1000 + } + ], + "installer_sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "replayed_after_this_online_install": false, + "scope": "Previously completed CPU/CUDA checks of the byte-identical offline package" + }, + "network_after": {}, + "original_harness": { + "error": "AssertionError: ", + "failed_assertion": "No APT process arguments were captured by its /proc/exe-based observer.", + "online_native_check_performed": false, + "process_argv_observations": [], + "raw_result_sha256": "3579477dd77a29443df6d69f1a836017c3a1b74590f9b5b7f88fc3f2c4ac7903", + "result": "failed", + "retained_without_rewriting": true + }, + "production_installer": { + "downloaded_artifact": { + "filename": "communityai_0.1.0~alpha.20260908.1_amd64.deb", + "format": "deb", + "kind": "offline-installer", + "platform": "linux-amd64", + "publisher": "Mario Andreschak", + "sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "size_bytes": 2302428788, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai_0.1.0~alpha.20260908.1_amd64.deb", + "version": "0.1.0~alpha.20260908.1" + }, + "elapsed_seconds": 1297.439, + "exit_code": 0, + "filename": "communityai-0.1.0-alpha.20260908.1-linux-online.py", + "sha256": "59a00906d358c1cac0046e9c323a612e7f6fdb824d21ba562de0bae18ba39b0b", + "size_bytes": 13662 + }, + "raw_evidence_directory": ".gate13-runs/linux-v3-build-20260908", + "raw_evidence_sha256": { + "online-container-inspect.json": "12f279f49223a0af5f203c07db2486c7a91c273f89ab639c35aa0e14ba1d133f", + "online-dpkg-transaction.log": "c405e44176e6237706f3e7ce4d0213b5493f690dd953f65f0834aa654fc651c0", + "online-execution-launch.json": "cee2322061d312ff8e87bebbd3f4b377f0a1234d5c320e4287dc9bbc02e75f75", + "online-first-attempt/executed-helper.py": "0d40fe1bb294e5b5fa954e0bc9b8dcae71be3967447c3b9526e8ff5773b8159c", + "online-first-attempt/online-remove.log": "61c2e23ec6f9a836ab4fea4ad2bce0724b5cbe5dc3fdbb28f59b8320edb05a2e", + "online-first-attempt/online-result.json": "3579477dd77a29443df6d69f1a836017c3a1b74590f9b5b7f88fc3f2c4ac7903", + "online-first-attempt/online-wrapper.log": "8301d2dc11423939b980d2bd440e7f3da111b5b9e0cb9efa90600a397bfcb7af", + "online-help.log": "83d1aa3b146a9bce7c5f8a57be3f87e323bdb998943bbafb6e2a2ae3370131dc", + "online-network-state-after.json": "e3566b3a06430868d71e9287dfd6c6c520a3da027aabea01951d407ee131dc2f", + "online-preflight-reviewed.json": "4de355c3e793862cf375092d7d08e140faa989cbbe54ce79b58772a8d7bb23ef", + "online-preflight.json": "5ecf9022d5023b0c50ed08a10c7199c87743046e2ccafb2d9bcfdfc49054640f", + "online-reviewed-default-preflight.json": "83445efb537246ebb4827eb72a29ba7f12b6e393f8a8eeae0504fd4026c9e21c", + "online-scoped-audit-final.json": "0d784a2f50ada1738d5630f40ea8aac44d11537cfdf0adc3eedce44b4d8760a4", + "online-scoped-audit-first-failed.json": "c09c1639aaf7b1408a2597a908d561baf7fc3ea76dad3fb466e38040baed6255", + "online-scoped-audit-first.py": "6ca67bd9ced3fbf3e408d5ccef6cc1c892afc50219b9834afb47d83915e61ff6", + "production-online-script.py": "59a00906d358c1cac0046e9c323a612e7f6fdb824d21ba562de0bae18ba39b0b", + "production-online-script.py.json": "a3b9134f69f232cfed6bc5f5fd12e20a0390ef01c08ff7ca2616af962ee8b9d5" + }, + "repeat_downloads_or_installs_after_harness_failure": 0, + "result": "passed", + "schema_version": 1, + "scope": "Actual public HTTPS download, protected-copy verification, APT installation and removal, accepted through separate log/state audit", + "scoped_audit": { + "apt_input_binding_lines": [ + { + "line": 160, + "text": "Get:78 /var/tmp/communityai-install-e38nkojy/communityai_0.1.0~alpha.20260908.1_amd64.deb communityai amd64 0.1.0~alpha.20260908.1 [2302 MB]" + } + ], + "apt_input_binding_source": "APT Get output in the retained actual wrapper log", + "cleanup": { + "dpkg_audit_exit": 0, + "dpkg_audit_stdout": "", + "package_query_exit": 1, + "package_status": "", + "remaining_online_staging": [], + "resident_app_or_package_processes_by_comm": [], + "runtime_present": false, + "sentinel_sha256": "fe9285e36690c756cc242bdbf3df684483256987d077c20dbc3dde551d58610b", + "unreadable_process_comm_pids": [] + }, + "download_url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai_0.1.0~alpha.20260908.1_amd64.deb", + "dpkg_online_transaction": [ + "2026-09-09 01:13:41 install communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:13:41 status half-installed communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:15:48 status unpacked communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:02 configure communityai:amd64 0.1.0~alpha.20260908.1 ", + "2026-09-09 01:16:02 status unpacked communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:02 status half-configured communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:02 status installed communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:04 status installed communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:04 remove communityai:amd64 0.1.0~alpha.20260908.1 ", + "2026-09-09 01:16:04 status half-configured communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:05 status half-installed communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:05 status config-files communityai:amd64 0.1.0~alpha.20260908.1", + "2026-09-09 01:16:05 status not-installed communityai:amd64 " + ], + "installed_version_confirmed_by_apt_and_dpkg_logs": true, + "installer_bytes": 2302428788, + "installer_sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "installer_version": "0.1.0~alpha.20260908.1", + "native_check_after_online_install_performed": false, + "original_harness_error": "AssertionError: ", + "original_harness_failure_retained": true, + "original_harness_result": "failed", + "original_harness_result_sha256": "3579477dd77a29443df6d69f1a836017c3a1b74590f9b5b7f88fc3f2c4ac7903", + "parent_opt_removal_warning": "The shared /opt parent remains because it also contains image tooling; /opt/communityai is absent.", + "process_argv_observed": false, + "production_wrapper_elapsed_seconds": 1297.439, + "production_wrapper_exit": 0, + "protected_copy": { + "directory_mode": 493, + "directory_uid": 0, + "file_mode": 420, + "file_uid": 0, + "observation": "Independent hash/ownership check while protected APT input existed", + "package_path": "/var/tmp/communityai-install-e38nkojy/communityai_0.1.0~alpha.20260908.1_amd64.deb", + "sha256": "714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576", + "size_bytes": 2302428788 + }, + "recorded_at_utc": "2026-09-09T01:24:16.688407+00:00", + "result": "passed", + "scope": "Actual HTTPS download, independently verified protected copy, APT installation and removal; native validation referenced separately", + "script_sha256": "59a00906d358c1cac0046e9c323a612e7f6fdb824d21ba562de0bae18ba39b0b" + }, + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "sudo_fixture": "Actual sudo ran from ordinary UID 1000 with a NOPASSWD rule only inside this disposable container." +} diff --git a/docs/evidence/alpha-online-linux-hosted-20260908.md b/docs/evidence/alpha-online-linux-hosted-20260908.md new file mode 100644 index 000000000..3401122c1 --- /dev/null +++ b/docs/evidence/alpha-online-linux-hosted-20260908.md @@ -0,0 +1,56 @@ +# Hosted Linux online installer — scoped acceptance + +**Passed for actual HTTPS download, protected-copy verification, APT installation +and removal.** A separate log/state audit supplies the accepted evidence. The +original diagnostic harness failed after the production installer returned zero +because its `/proc/exe` observer captured no APT arguments. That failure remains +unchanged; its planned post-online CPU check did not run. Native validation is +referenced from the [byte-identical offline package acceptance](alpha-normalized-linux-20260908.md). +The [machine-readable record](alpha-online-linux-hosted-20260908.json) preserves +these distinctions and hashes the raw records. + +| Item | Exact identity or result | +| --- | --- | +| Production script | `communityai-0.1.0-alpha.20260908.1-linux-online.py`, 13,662 bytes | +| Script SHA-256 | `59a00906d358c1cac0046e9c323a612e7f6fdb824d21ba562de0bae18ba39b0b` | +| Downloaded package | `communityai_0.1.0~alpha.20260908.1_amd64.deb`, 2,302,428,788 bytes | +| Package SHA-256 | `714b9a7ac9121f3cf3b85f9677d541b488c2f00020bc6e1ce73ba7081f851576` | +| Production script result | Exit 0 after 1,297.439 seconds | +| Ordinary user | UID 1000, Ubuntu 22.04 system Python 3.10 | +| Container limit | Two CPUs, 3 GiB memory; release/source mounts read-only | + +The unmodified production script fetched the complete package from its embedded +[public Cloudflare R2 URL](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai_0.1.0~alpha.20260908.1_amd64.deb). +The actual output records 100% of the pinned byte count and successful SHA-256 +verification. It then used real `sudo` and the embedded protected-copy helper. +The passwordless sudo rule applied only inside this disposable acceptance container. + +The observer independently hashed the protected package to the same SHA-256 and +recorded root ownership, directory mode `0755`, and file mode `0644`. APT's own +`Get:78` output at wrapper-log line 160 names that exact protected package path +and version. This supplies the input binding that the process observer missed. +APT downloaded 24.0 MB of official Ubuntu packages for the declared +`gnome-keyring` recommendation and its dependencies, then installed CommunityAI. +The protected-helper verification message is buffered in the log; its printed +position is not used as a timing measurement. + +The actual `dpkg.log` transaction records installation beginning at +2026-09-09 01:13:41 UTC, the exact version installed at 01:16:02, removal at +01:16:04, and `not-installed` at 01:16:05. The final read-only audit independently +confirmed no package entry, no `/opt/communityai`, no app or package-manager +process, no unreadable process-name entry, no online/protected staging directory, +and an empty `dpkg --audit`. The external user-state sentinel hash was preserved. +The shared `/opt` parent remained because the image also contains tooling there. +The container network was disconnected after the audit. + +No download or installation was repeated to resolve the observer failure. The +first read-only audit also retained a failed expectation that the final removed +package event would still include a version; `dpkg` correctly records ``. +The corrected audit validates the actual installed/removal version events and +the final absent state. + +The earlier offline evidence already records CPU/CUDA native checks for this +exact package hash. No post-online native, GUI, model, public-peer, credential-store, +or public-canary check is claimed here. This does not replace full Gate 15 worker +and settings lifecycle evidence or establish a new Debian 12 or bare-metal replay. +Publisher signing remains deferred until after alpha. diff --git a/docs/evidence/alpha-online-linux-source-20260908.json b/docs/evidence/alpha-online-linux-source-20260908.json new file mode 100644 index 000000000..9b0234add --- /dev/null +++ b/docs/evidence/alpha-online-linux-source-20260908.json @@ -0,0 +1,98 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-08T23:10:02.208289+00:00", + "scope": "Focused Linux source validation of online downloader and runtime archive normalization", + "result": "passed-focused-linux-source-tests", + "release_ready": false, + "new_offline_runtime_built": false, + "real_online_download_executed": false, + "real_sudo_or_apt_executed": false, + "source_base_commit": "076b4b11c7adbca3f0ba5e28bc5586e36e1d696f", + "source_binding": "Current source file hashes; shared working tree contains the reviewed new implementation. No model or runtime artifact was rebuilt.", + "files_sha256": { + "desktop/installers/build_linux_online.py": "a8efe50840786a56e296df72d05b6b378fad3977f2cd2b85e6a6cf12beace68f", + "desktop/installers/linux_online_template.py": "5b94ff2ad46735964212f453d6c47ab103cedb9ff9c25d09add084cfc149eeaa", + "desktop/installers/linux_online_root.py": "8842faa5e2dcf5da54a1e56e50a2eb6bee97b489026213fe646129227c3e540e", + "desktop/installers/release_downloads.py": "393889433822dea2737bdb8fc1026a088869888a1bdc0e9494970c4073323dce", + "tests/test_linux_online_installer.py": "ac7f4bb0ef5adfea8e3a6e708ede87b1ead43ddba3f5247b7c9033c5fd3c682a", + "desktop/runtime_packaging.py": "dba6424a90ffe49c8fea71ca95c66d2c741c68bc01f3b20783192ee63b6d2fdc", + "desktop/tests/test_runtime_packaging.py": "28271054799fddac59180544ae7659137f2c66e146761093229bd0685bbde21d", + "desktop/build_desktop.py": "8b77625c92090f540708271743635486098d8384d1f927d4dc0588b939882a4d", + "desktop/installers/build_deb.py": "8b26b3d9780fe22b9158b21944b9503636706d524e2b2cc1b763bc1a9add7b57", + "scripts/gate13_linux_packaged_lifecycle.py": "6dc53ebbb4f5344803d8b4eda0a09ced41cb7b8803f67b78ac0bca1559d649d8", + "scripts/gateq38_linux_host_runtime.py": "b9fe4b3f0bbc09038dc3f267c2ec11b8b21336d113fe51c36fb9379050b23eb9", + "tests/test_runtime_archive_hardlinks.py": "0df326ca0a249ac46c75740a369e0bdd028505febe6e8a1fd564ec00f040898e" + }, + "tests": { + "files": [ + "tests/test_linux_online_installer.py", + "tests/test_runtime_archive_hardlinks.py", + "desktop/tests/test_runtime_packaging.py" + ], + "passed": 49, + "subtests_passed": 54, + "failed": 0, + "errors": 0, + "skipped": 0, + "pytest_duration_seconds": 2.29, + "junit_duration_seconds": 2.291, + "junit_reported_tests_including_subtests": 103, + "python": "3.12.14", + "pytest": "9.1.1", + "platform": "Linux x86_64, glibc 2.35, cached Ubuntu 22.04 builder image", + "actual_linux_nofollow_case": "test_protected_helper_rejects_symlink_source_on_linux passed without skip", + "command": "python .gate13-runs/run-online-linux-checks-20260908.py", + "runner_sha256": "825e72978ebe327cd7e1d268c996fa5e9265952ac43b983489b18918b83f31e3" + }, + "runtime_resolution": { + "initial_attempt": "Failed before tests: /environment/venv/bin/python links to /usr/local/bin/python, which is absent in this builder image.", + "inspection": "Read-only metadata inspection found an existing matching image interpreter. No venv or volume files changed.", + "interpreter": "/opt/communityai-python/cpython-3.12.14-linux-x86_64-gnu/bin/python3.12", + "cached_site_packages": "/environment/venv/lib/python3.12/site-packages", + "additional_pytest_fallback_used": false + }, + "container": { + "image": "communityai-gate14-linux-builder:20260907-ubuntu22", + "image_id": "sha256:a8b6f5418183d44d731e01b013146d3cda22ce7d14f2f234c42bcc7a71ff2dc3", + "cpus": 2, + "memory_bytes": 2147483648, + "memory_plus_swap_bytes": 2147483648, + "pids_limit": 128, + "uid": "1000:1000", + "network": "none", + "root_filesystem": "read-only", + "repository_mount": "read-only /work", + "cached_environment_volume": "read-only communityai-gate14-linux-environment-20260907 at /environment", + "tmpfs": "/tmp, 128 MiB, rw,nosuid", + "qt_platform": "offscreen", + "gpu_devices": 0, + "dependency_installs": 0, + "models_loaded": 0, + "gui_launches": 0, + "full_builds": 0, + "all_owned_containers_removed": true, + "docker_service": "Already running under root agent authorization; left running for that agent." + }, + "validated_behavior": [ + "Exact URL, size/hash, truncated/oversized and encoded body rejection with inert network responses.", + "Download cancellation and deadline handler cleanup using bounded test fixtures.", + "Protected copy rejects changed source bytes and remains independent when the original file changes afterward.", + "Actual Linux O_NOFOLLOW rejects a symlink source before any package-manager call.", + "Unconfirmed or interrupted APT spawn/exit retains its input; all package-manager calls are inert fixtures.", + "Actual Linux hardlinks preserve inode identity and library lookup paths through normalization, archive writing and both extraction consumers.", + "Unsafe hardlink targets, chains, metadata changes and cross-inventory links fail.", + "Debian staging preserves inode sharing even through the simulated cross-device copy fallback; Installed-Size counts unique content." + ], + "private_logs_sha256": { + ".gate13-runs/online-linux-focused-20260908.log": "5e6ca54546200ada49f11975d257e751f26257a0cbd7a39087554c123687ba8c", + ".gate13-runs/online-linux-path-inspect-20260908.log": "be1a2a4e0cf00358783f2187911eff87bb9535411e0ecfd69db36b6d47cc1c0d", + ".gate13-runs/online-linux-focused-final-20260908.log": "e7e09506669efc6523f41ade48693f9d70b5b9d54695af882344b88fccf3af9c" + }, + "limitations": [ + "The container runs as UID1000. Fixtures emulate the root identity; actual sudo, OS-enforced root-owned staging and APT were not exercised.", + "No real HTTPS download, public origin, cancellation of a blocked external socket or installer publication was performed.", + "Native libraries and Debian payloads are tiny inert fixtures. No full package size measurement, frozen GPU/model execution or normalized runtime acceptance is claimed.", + "The cached Python3.12.14/pytest9.1.1 environment differs from CI environments; CI results remain separate evidence.", + "Existing Windows-only source validation records are unchanged; this is an additional Linux result." + ] +} diff --git a/docs/evidence/alpha-online-linux-source-20260908.md b/docs/evidence/alpha-online-linux-source-20260908.md new file mode 100644 index 000000000..4155b2b30 --- /dev/null +++ b/docs/evidence/alpha-online-linux-source-20260908.md @@ -0,0 +1,34 @@ +# Focused Linux online-installer and archive validation — 2026-09-08 + +**49 tests and 54 subtests passed in 2.29 seconds**, with no failures or skips, +in the cached Ubuntu 22.04 builder image. The [companion record](alpha-online-linux-source-20260908.json) +binds the source and retained logs. This is source validation; no release runtime +was built or published. + +The run covered `tests/test_linux_online_installer.py`, +`tests/test_runtime_archive_hardlinks.py` and +`desktop/tests/test_runtime_packaging.py`. It exercised the Linux-only +`O_NOFOLLOW` source-symlink rejection, protected-copy byte verification, download +and package-manager failure fixtures, actual inode hardlinks, archive writing +and both extraction consumers. Unsafe link targets, metadata mismatches and +cross-inventory links were rejected. The Debian staging fixture preserved +hardlinks through a simulated cross-device copy fallback. + +The disposable container had two CPUs, 2 GiB memory, no network or GPU devices, +read-only repository/environment mounts and a 128 MiB temporary filesystem. It +ran as UID 1000 with Qt configured offscreen. All owned containers were removed. +There were no APT commands, dependency installations, model loads, GUI launches +or full builds. + +The first attempt stopped before test collection because the cached venv's +Python symlink targeted a binary absent from this image. Read-only inspection +found the image's matching Python 3.12.14; the successful run used it with the +existing cached site-packages and pytest 9.1.1. No environment files changed. +The failed start and inspection logs remain separately hashed. + +The protected-copy tests emulate the root identity and keep APT inert. Actual +sudo, operating-system enforcement of root ownership, a real HTTPS package +download and the complete APT handoff remain acceptance work. Library payloads +are tiny fixtures, so this result establishes neither a new compressed package +size nor frozen GPU/runtime behavior. Previous Windows fixture records remain +unchanged. diff --git a/docs/evidence/alpha-online-windows-https-rejection-20260908.json b/docs/evidence/alpha-online-windows-https-rejection-20260908.json new file mode 100644 index 000000000..20ef5dcdf --- /dev/null +++ b/docs/evidence/alpha-online-windows-https-rejection-20260908.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-08T23:19:25.4315022Z", + "scope": "Windows native online downloader real HTTPS hash rejection", + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "compiler": "Inno Setup 6.7.3", + "fixture_setup_sha256": "df3bf54148a1c5abaa9245d2c8779b24f0d356c4d008731e3b12fcc6ce3dfc0d", + "download_url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/_checks/communityai-0.0.0-online-probe-windows-setup.exe", + "download_bytes": 52, + "download_content": "Harmless text fixture; not an executable", + "expected_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "exit_code": 1, + "hash_mismatch_observed": true, + "child_installation_launched": false, + "silent_error_return_passed": true, + "actual_full_installer_download_or_install_tested": false, + "log": ".gate13-runs/online-live-rejection-20260908/download-rejection.log", + "log_sha256": "4bd092f86f9d5effb4ad0273a7e073e1ff0afa9df321e117076dd1580faa91c3", + "temporary_directory_removed": true +} diff --git a/docs/evidence/alpha-online-windows-progress-fix-https-rejection-20260909.json b/docs/evidence/alpha-online-windows-progress-fix-https-rejection-20260909.json new file mode 100644 index 000000000..ebc33ec24 --- /dev/null +++ b/docs/evidence/alpha-online-windows-progress-fix-https-rejection-20260909.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-09T01:29:24.602410+00:00", + "scope": "Presentation-only progress fix: real HTTPS hash rejection", + "fixture_setup_sha256": "b728b49ee5501d6586bf27a18f60becd0afe7f2e0c35ac706bae5690240f02fa", + "fixture_setup_bytes": 2107665, + "source_hashes": { + "builder_script_sha256": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "installer_script_sha256": "d975b6da3a1067e679d9ba55555fa5b6219ae644fce8050225486909600fbbb1", + "download_helper_sha256": "2c0d73c6bc2626f6b70df585d805f63861ca94036e97db35006c079a7135c73a", + "download_helper_source_sha256": "3bfa36e62f2a160b2e258af6d9d6816e815d8a9c5b918c29dd800eb6e6c20d8a" + }, + "download_url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/_checks/communityai-0.0.0-online-probe-windows-setup.exe", + "download_bytes": 52, + "expected_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "exit_code": 1, + "hash_mismatch_observed": true, + "child_installation_launched": false, + "actual_full_installer_download_or_install_tested": false, + "temporary_directory_removed": true, + "log": ".gate13-runs\\resumable-online-v2-https-rejection-20260909\\rejection.log", + "log_sha256": "06e3aec801f076370434a9aebc9843c0904bad066d5a58c2666d733c9e41bfa8" +} diff --git a/docs/evidence/alpha-online-windows-resumable-https-rejection-20260909.json b/docs/evidence/alpha-online-windows-resumable-https-rejection-20260909.json new file mode 100644 index 000000000..b05a9d2d4 --- /dev/null +++ b/docs/evidence/alpha-online-windows-resumable-https-rejection-20260909.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-09T01:11:09.561318+00:00", + "scope": "Final resumable Windows online source: real HTTPS hash rejection", + "fixture_setup_sha256": "4dbfd9448e8638a7bb90cedfb7aa295f22cd111c975fb17c5674e2ffdfb5f1d6", + "fixture_setup_bytes": 2107381, + "source_hashes": { + "download_helper_sha256": "9a609cff0eff3e9cf6381273fbaf43fcbdb20b5eccaef57061b1490ddc60669a", + "builder_script_sha256": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "download_helper_source_sha256": "3df36f4f1273599a3680d3d4981f150f2eaf289d92926b19dbde6620fdad3094", + "installer_script_sha256": "b6c680afa048ba581af6d01a596fa8f9f62837bb7d571979cb9d4dc9fbbf6633" + }, + "download_url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/_checks/communityai-0.0.0-online-probe-windows-setup.exe", + "download_bytes": 52, + "expected_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "exit_code": 1, + "hash_mismatch_observed": true, + "child_installation_launched": false, + "actual_full_installer_download_or_install_tested": false, + "temporary_directory_removed": true, + "log": ".gate13-runs\\resumable-online-final-https-rejection-20260909\\rejection.log", + "log_sha256": "526befce11a2c458251f24fbf36781e6129abca05f5d13cca1bb7865080170bf" +} diff --git a/docs/evidence/alpha-public-metadata-20260908.json b/docs/evidence/alpha-public-metadata-20260908.json new file mode 100644 index 000000000..443f22541 --- /dev/null +++ b/docs/evidence/alpha-public-metadata-20260908.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "scope": "read-only public release catalog, bootstrap and manifest metadata", + "result": "passed", + "checked_at_utc": "2026-09-08T23:52:49.375098+00:00", + "verifier_source_commit": "fdd8d0b799e42b307450b0f0776a88b4aeffe852", + "verifier_source_files_sha256": { + "src/drift/model_catalog.py": "7ee7c631435dce02851e032d4a32844a26502e1d2333d0989868600950df19d0", + "src/drift/model_manifest.py": "d27b0fe02d8c5a89fba146cf4bf52366d8213c19bae5f72c24ca391172b90ec3" + }, + "verification_method": "Loaded the existing source verifier modules directly, without importing drift package initialization, Torch or Hivemind. Verified the bundled signed catalog first, then used its rollback guard and bundled trust root to verify the public catalog. Parsed each public manifest with the existing ModelManifest verifier and compared its canonical digest with the signed catalog entry.", + "catalog": { + "catalog_id": "communityai-public-alpha-v1", + "sequence": 2, + "digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80", + "matches_bundled_catalog": true, + "signature_algorithm": "ed25519", + "trusted_key_id": "sha256:9505d3ac8ec996d4b794bd43d09dd84447acc5da8a0bb1eac1be4e9c9a34b14f", + "signature_threshold": 1, + "signature_validation": "passed", + "time_validation": "passed", + "rollback_and_equivocation_validation_against_bundled_catalog": "passed", + "issued_at_utc": "2026-09-06T00:33:28.765000+00:00", + "expires_at_utc": "2026-09-28T19:35:19.369000+00:00", + "remaining_days_at_check": 19.821 + }, + "public_bootstrap_matches_bundled_bytes": true, + "bundled_bootstrap_path": "public-alpha/catalog-qwen-v2/catalog-bootstrap.json", + "responses": [ + { + "kind": "signed-catalog", + "url": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/catalog.signed.json", + "http_status": 200, + "bytes": 1792, + "file_sha256": "315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315" + }, + { + "kind": "bootstrap", + "url": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/catalog-bootstrap.json", + "http_status": 200, + "bytes": 702, + "file_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410" + }, + { + "kind": "local-model-manifest", + "url": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json", + "http_status": 200, + "bytes": 1711, + "file_sha256": "4536ac2bada7242b758db443b9ebb813a614dd167ab364643fc55c7eb657bb74", + "verified_manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0" + }, + { + "kind": "distributed-model-manifest", + "url": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json", + "http_status": 200, + "bytes": 10936, + "file_sha256": "a2621aa34aa47f0c9074f5baa0254b17549424f1c85d828ae9b1bf6ad9e76bb3", + "verified_manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + } + ], + "privacy_and_scope": { + "peer_connections": 0, + "model_downloads": 0, + "model_loads": 0, + "gpu_runtime_imports": 0, + "credential_access": false, + "private_signing_key_access": false, + "external_mutations": false, + "http_redirects_followed": false + }, + "operational_dependencies": [ + "Preserve the codex/gate-v-auto-selection branch while installed bootstrap and signed manifest URLs depend on it. Merging or deleting that branch must not remove the published metadata endpoints.", + "Publish a renewed valid signed catalog before 2026-09-28T19:35:19.369Z. Current availability does not establish availability after expiry." + ], + "reporting_notes": [ + "The initial read verified all four public endpoints but console result construction failed because it requested a nonexistent ModelCatalog.digest_id attribute. This was a reporting-only failure.", + "A bounded retry produced the successful observation recorded here. Its console formatter redundantly prefixed the already-prefixed catalog digest. This record removes that redundant prefix using the verifier's existing digest value; no third network check was performed." + ], + "limitations": [ + "This is one point-in-time HTTPS metadata observation, not continuous availability monitoring.", + "No bootstrap peer or public worker was probed. Current complete route coverage, usable inference capacity and a combined Gate 16 canary are not established." + ] +} diff --git a/docs/evidence/catalog-signer-replacement-backups-20260906.json b/docs/evidence/catalog-signer-replacement-backups-20260906.json new file mode 100644 index 000000000..3a2052281 --- /dev/null +++ b/docs/evidence/catalog-signer-replacement-backups-20260906.json @@ -0,0 +1,16 @@ +{ + "key_id": "sha256:9505d3ac8ec996d4b794bd43d09dd84447acc5da8a0bb1eac1be4e9c9a34b14f", + "private_material_printed": false, + "project": "community-ai-506321", + "recovered_in_memory": true, + "result": "passed", + "secret": "communityai-catalog-signer-20260906", + "signature_verified": true, + "version": "1", + "schema_version": 1, + "three_backups_verified": true, + "g_backup_retained": true, + "repository_backup_gitignored": true, + "new_catalog_published": false, + "old_key_recovered": false +} diff --git a/docs/evidence/desktop-health-downloads-20260907.md b/docs/evidence/desktop-health-downloads-20260907.md new file mode 100644 index 000000000..497f98f95 --- /dev/null +++ b/docs/evidence/desktop-health-downloads-20260907.md @@ -0,0 +1,48 @@ +# Desktop block health and local download progress + +Date: 2026-09-07. Source implementation and bounded tests passed. This is not +Gate 14 completion, installer qualification or a real Qwen swarm result. + +## Delivered + +- Model cards show numbered, keyboard-accessible block cells. Colors distinguish + online replicas, joining announcements, unexpired signed reservations, offline + announcements/local process failures, missing coverage and unknown/stale views. + Clicking a cell exposes its counts and observed owners. Serving replicas retain + precedence over failed local workers because another peer can cover the block. +- Peer tables show public name or abbreviated PeerID, observed/announced block + ranges, reserved block counts, runtime version/dtype/quantization and relay use + where reported. They also include this computer's configured workers. Remote + download percentages and unused capacity are not inferred. +- Discovery verifies existing intent signatures, exact manifest binding, signer + identity, revocation, range and expiration before displaying reservations. They + never count as serving coverage. Expired reservations disappear at read time. +- Client and contribution-worker acquisition report current file size/progress, + five-second transfer speed, cached/received and verified totals, retries and + resumed bytes. Hash verification and atomic promotion remain authoritative. + Totals cover selected files encountered so far; the UI does not invent a fixed + whole-model total before shard selection. Download/loading/ready remain distinct. +- Supervised worker reporting uses a fresh directory per launch, bounded reads, + allowlisted public fields, atomic throttled writes and shutdown cleanup. Windows + venv launcher and child PIDs are handled by per-launch directory ownership. + Missing display storage does not prevent a worker starting. +- Display refresh preserves selected block and expanded peer tables. The actual + Qt window was rendered and inspected with explicitly synthetic preview data. + +## Validation + +- 265 related tests passed, two platform-specific skips: node, model manager, + verifier/ranged downloader, discovery, worker supervision and all desktop tests. +- Live loopback HTTP tests start from a retained partial, force HTTP 429 retry, + observe progress while transfer is blocked, then verify correct completion or + corrupt-data rejection. Reloading the good cache makes no new HTTP requests. +- A real supervised Python process reports download progress, becomes paused on + Pause, and its temporary progress directory is removed on shutdown. +- Signed-intent tests reject wrong-key/malformed records and remove expired + reservations; Qt tests distinguish serving/joining/reserved/offline/unknown, + retain expansion, render names as text, and keep downloaded bytes unverified. + +The earlier source `5d9eec9` passed both Windows/Linux production engineering +package builds and all CI checks. These display changes need fresh package builds +and real packaged Qwen/resource/lifecycle observations. No cloud qualification, +code-signing enrollment, Store submission or public release occurred here. diff --git a/docs/evidence/desktop-installers-20260907.md b/docs/evidence/desktop-installers-20260907.md new file mode 100644 index 000000000..8db340b93 --- /dev/null +++ b/docs/evidence/desktop-installers-20260907.md @@ -0,0 +1,131 @@ +# Engineering installer checkpoint + +Date: September 7, 2026. Parent source `b09aa2d67b2789f9fd52c91abdea2b9340bb2fe4` +passed Tests, Style and Production desktop, including both Windows/Linux builds +with the new block grid and download reporting. The changes described below are +the installer slice following that parent. Both complete production installer +builds passed at `0b875c19efd952342671371ddacebc09ca3a774c` in +[CI run 34144852840](https://github.com/flujo-app/CommunityAI/actions/runs/34144852840). + +## Implemented + +- Inno Setup per-user install, stable application identity, Start menu entries, + silent install/removal, optional setup/uninstaller signing hook and output hash. + Unsigned engineering builds must be selected explicitly. Unmarked existing + applications are refused; obsolete runtime directories are removed only from + the marked installation. +- Same-user `--prepare-update` IPC requests desktop exit, waits for owned-node + cleanup, and acknowledges only before releasing the instance endpoint/lock. + A failed stop retains the process reference and returns failure. A broken Qt + import returns an error code rather than a windowed traceback dialog. +- Debian builder installs the application, command wrapper and menu entry. + `preinst`/`prerm` select processes from the marked installation and their + observed descendants, signal them using PID handles/start-time identities, + and refuse replacement if any remain. No home-directory state is removed. +- CI builds installers only after the existing release archive/provenance and + runtime checks. Linux build baseline becomes Ubuntu 22.04 (glibc 2.35); + a complete fresh build must confirm the target distribution compatibility. + +## Observed probes + +Windows 10, Python 3.12.9, PySide6, PyInstaller 6.22.2, Inno Setup 6.7.3: + +- Native Qt shutdown test used separate application/helper processes and a + cleanup marker; acknowledgement arrived only after the cleanup callback. +- A real Inno setup installed into an isolated task directory with a distinct + AppId. Re-running it while a source Qt window served the fake-node fixture + requested shutdown, observed normal exit, removed an obsolete application + DLL fixture and preserved external settings/cache sentinels. Silent uninstall + removed the application and retained those sentinels. Probe registration was + removed; no production settings or credentials were provisioned or changed. +- The packaged probe contains the actual frozen GUI and a copied GUI executable + standing in for the node filename. It does **not** run the model/node runtime. + The first locally built probe failed Qt loading because direct PyInstaller + invocation picked up a foreign ICU DLL. Rebuilding through the production + builder's existing restricted-PATH helper resolved it; the upgrade then passed. + That failure motivated explicit non-dialog error handling for maintenance. + +Docker, Python 3.12 on Debian bookworm: + +- Copied an executable into a marked temporary installation, started it and an + unrelated process, stopped the former and preserved the latter. +- A separate process-tree test used a copied shell executable plus its sleep + child; both disappeared from the executable snapshot while unrelated sleep and + external cache bytes remained. An unmarked directory was refused. +- Built and inspected an actual `.deb` from executable fixtures. Root-owned + directories are 0755, launchers/maintenance scripts executable, dependencies and + control records readable. Cross-filesystem copy fallback was exercised before + staging was moved beside the source bundle. Containers were removed after use. + +The final Windows desktop suite passed 101 tests with two Linux-only skips; +the 12 maintenance/lifecycle tests and both Linux ownership tests also passed +in their respective environments. Root Black/isort and diff checks passed. + +## APT signing follow-up + +The repository builder generates a new immutable snapshot directory, validates +CommunityAI/amd64 package identity, signs release metadata with a supplied full +GPG fingerprint, exports only the public key and verifies both signatures using +`gpgv`. A Debian container generated a disposable signing key, accepted the +result through scoped `Signed-By`, selected the fixture package, downloaded it +with matching SHA-256, then rejected tampered signed metadata with `BADSIG`. +The container/private key were removed. No production key or repository was +created. GPG repository signing requires no paid certificate. + +CI at `0b875c1` passed style and Linux tests; its macOS standalone image verifier +hit the pre-existing ten-second subprocess timeout. That test imports the model +runtime from a fresh Python process. The timeout is raised to 30 seconds while +retaining the exact verification assertions. Tests and Style then passed at +`40de496cafd2bca9725c1a9a8a04e60dc6fcf199`, including the macOS test. +Both full Windows/Linux installers also passed at that revision in +[CI run 34145484606](https://github.com/flujo-app/CommunityAI/actions/runs/34145484606). + +## Full Windows runtime follow-up + +A clean detached checkout at `0b875c1` produced and verified the complete GUI/node +bundle: 4,486,446,226 bytes in 4,942 files. The archive is 2,693,786,190 bytes, +SHA-256 `c5d59f4ae8c057315cb50fb0dadc211e36ec3ea58e2952c4493f9e39c4ce29fb`. +The explicitly unsigned Inno setup is 2,518,829,949 bytes, SHA-256 +`95b2d70382ed91b61079581e3fed0c3b12364ebe70576a25eee3231460f8e4d8`. +[Sanitized evidence](qwen-windows-installers-20260907.json) binds the source, +packages, runtime, probe scripts and observed outcomes. + +On Windows 10 with an RTX 2070 SUPER, the source desktop controller drove the +frozen node and a real automatically assigned Qwen3.8 contribution block. Changes +to 20% VRAM/50% processing and 25%/100% stopped the old trees, persisted both +settings and restarted ready workers with the expected allocator ceiling and +processing argument. All 384,054,157 selected artifact bytes were hash-verified; +local Qwen inference continued. Pause removed the worker tree in 0.125 seconds; +restart and the final Pause/cleanup passed. + +The first attempt failed a probe assertion that incorrectly multiplied the +already percentage-limited `vram_pool_bytes` a second time. Correcting the probe +to compare against physical GPU capacity resolved that assertion; no product +enforcement change was needed. This preserves the failed attempt as a harness +error, rather than counting it as a passing product run. + +The complete setup then installed into an isolated directory. Source Qt using +the production `NodeLifecycleSupervisor` started the installed frozen node and +a ready Qwen3.8 worker for block `3:4`. Re-running setup requested shutdown and +removed all seven recorded node/worker/transport processes before replacement. +The settings hash and external cache sentinel were unchanged. Silent uninstall +removed the application and preserved external state. Owned processes, test +native credential, installed executable and installer registration were separately +verified absent afterward. No production credential was changed. + +The health/download Qt view was also rendered against the live packaged node and +inspected. These checks use source Qt/controller integration with frozen runtime; +they do not claim literal frozen-GUI slider interactions, remote Qwen processing +duty-cycle measurement under load, Linux Qwen lifecycle or a broader GPU profile. + +## Remaining release outcomes + +These are bounded engineering results, including the real Windows runtime case +above. Complete Windows/Linux Qwen resource controls under load, +fully frozen desktop lifecycle, Linux real-worker upgrades/removal, login-entry cleanup, +explicit cache deletion policy and supported-distribution measurements remain. +SignPath's CUDA/upstream eligibility, trusted signing, Store account/certification, +release maintainer address and signed HTTPS APT distribution remain unresolved. +An explicitly authorized free-program eligibility inquiry was sent to SignPath +and confirmed in Gmail Sent on September 7; no enrollment, signing approval, +Store submission, public installer release or cloud host was created. diff --git a/docs/evidence/desktop-repair-source-20260909.md b/docs/evidence/desktop-repair-source-20260909.md new file mode 100644 index 000000000..791090f86 --- /dev/null +++ b/docs/evidence/desktop-repair-source-20260909.md @@ -0,0 +1,46 @@ +# Desktop repair source checks — September 9, 2026 + +The installed alpha exposed two functional bugs and a presentation failure. +An existing profile without a contribution policy could make Start Sharing +silently do nothing. Catalogue refresh appended withdrawn managed models, and +an already-installed catalogue skipped the repair. The Home page also exposed +operational details without explaining the selected model or the user's hardware. + +This repair reduces Home to the selected model and its reason, exact processor +and graphics card, configured sharing memory in GB, computing percentage and a +Start/Pause control. Models have collapsed details. Sharing has two sliders, +visible action feedback and optional extra settings. Background status requests +cannot overwrite a later action or disable controls on every poll. + +The node now supplies hardware identity and the actual sharing memory ceiling. +Legacy profiles receive missing sharing defaults while remaining opted out. +Starting persists the user's choice and clears an earlier automatic-worker pause +even while placement is pending; existing resource and placement checks still +apply. Catalogue repair removes only proven withdrawn managed entries and +preserves explicitly pinned and user-added models, settings and cached files. + +## Source verification + +- Full desktop suite: 147 passed, four platform-specific skips. +- Focused backend/hardware/resource suite: 146 passed, two platform-specific skips. +- Catalogue refresh/migration and first-run bootstrap: 50 passed. +- Latest Home interaction checks: five passed, including stale successful and + failed status requests, pending action feedback and disconnected controls. +- Login and Gate 13 UI replay open the collapsed settings through the real + control and verify visible feedback, persisted Start/Pause and restart behavior. + +Counts overlap; they are not separate end-to-end installation runs. Qt checks +ran offscreen against the current source imports. The legacy Start integration +uses the actual authenticated HTTP API and a harmless child process. + +The visual review used a private copy of the installed user's configuration, +catalogue and manifests with the new source node's hardware and model status. +The repaired catalogue contains Qwen3.5 0.8B locally and Qwen3.8 27B for community +execution. Hardware reports an Intel Core i7-9700K and NVIDIA RTX 2070 SUPER, +with 4.5 GB available for sharing from its 8.0 GB physical memory. The remaining +memory is reserved for local inference and its overhead. Computing is 100%. + +This inspection made no inference requests and started no sharing, discovery or +probe services. The user's original profile remained unchanged. These are source +and visual checks; installer qualification and public artifact hashes must be +recorded separately after rebuilding both executables. diff --git a/docs/evidence/desktop-updater-release-20260909.json b/docs/evidence/desktop-updater-release-20260909.json new file mode 100644 index 000000000..0b613fe4b --- /dev/null +++ b/docs/evidence/desktop-updater-release-20260909.json @@ -0,0 +1,407 @@ +{ + "version": "0.1.0-alpha.20260909.2", + "source_commit": "15e1757ade8881c231db0703a4c1ef01b0dbdd74", + "release_url": "https://github.com/flujo-app/CommunityAI/releases/tag/v0.1.0-alpha.20260909.2", + "installers": { + "linux-amd64": { + "filename": "communityai_0.1.0~alpha.20260909.2_amd64.deb", + "format": "deb", + "kind": "offline-installer", + "platform": "linux-amd64", + "publisher": "Mario Andreschak", + "sha256": "a03835d1525126c167b44eec25db609285e5e5197883873b9cc21c6e9ae9d39a", + "size_bytes": 2302548516, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai_0.1.0~alpha.20260909.2_amd64.deb", + "version": "0.1.0~alpha.20260909.2" + }, + "windows-x64": { + "filename": "communityai-0.1.0-alpha.20260909.2-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "0a89756ab829b77019612d4a3e6b0c2bcbc85e8feba98716884ecfe3601f5fa7", + "size_bytes": 2465315581, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-windows-setup.exe", + "version": "0.1.0-alpha.20260909.2" + } + }, + "hosted_objects": [ + { + "anonymous_complete_body_verified": false, + "anonymous_range_samples_verified": true, + "key": "alpha/20260909.2/communityai_0.1.0~alpha.20260909.2_amd64.deb", + "samples": [ + { + "sha256": "dbc73a95ab21d226a1aa77ed483ff4a118c199a84230864b46efedd589a1fbf4", + "size_bytes": 65536, + "start": 0 + }, + { + "sha256": "68337910e49e1200e635f513fe99c9d2d70c2d14e56d8ea50991daf376ada491", + "size_bytes": 65536, + "start": 2302482980 + } + ], + "scope": "Uploaded metadata and first/last 64 KiB match. No complete public re-download; installers/updater verify complete downloads on the client.", + "sha256": "a03835d1525126c167b44eec25db609285e5e5197883873b9cc21c6e9ae9d39a", + "size_bytes": 2302548516, + "uploaded_size_and_checksum_metadata_verified": true, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai_0.1.0~alpha.20260909.2_amd64.deb", + "verified_at_utc": "2026-09-09T05:33:17.151984+00:00" + }, + { + "anonymous_complete_body_verified": false, + "anonymous_range_samples_verified": true, + "key": "alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-windows-setup.exe", + "samples": [ + { + "sha256": "3bebe436fae4d118223044d658128291f2137c5fd7e504b372558504725ecb53", + "size_bytes": 65536, + "start": 0 + }, + { + "sha256": "22078a64b08939dbcae76a5ce62356aa04b32d92f36c1aca66147d26a0f633e0", + "size_bytes": 65536, + "start": 2465250045 + } + ], + "scope": "Uploaded metadata and first/last 64 KiB match. No complete public re-download; installers/updater verify complete downloads on the client.", + "sha256": "0a89756ab829b77019612d4a3e6b0c2bcbc85e8feba98716884ecfe3601f5fa7", + "size_bytes": 2465315581, + "uploaded_size_and_checksum_metadata_verified": true, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-windows-setup.exe", + "verified_at_utc": "2026-09-09T05:33:19.793673+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-windows-online-setup.exe", + "sha256": "b46d9447a9600e07e3be8b3c8fcc258ac83cb1a24369e156e7422e81b1bf07f0", + "size_bytes": 2107749, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-windows-online-setup.exe", + "verified_at_utc": "2026-09-09T05:34:54.817370+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-linux-online.py", + "sha256": "5db5dee70d6e4abddfdc5d4bc02bd8395e40617c504eaabaa84fb550f247bbbf", + "size_bytes": 13900, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-linux-online.py", + "verified_at_utc": "2026-09-09T05:34:55.386137+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/release-downloads.json", + "sha256": "352c859424eb4150e4536926da294450449aa7459c70957c1d178cb02b3c1f1f", + "size_bytes": 1087, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/release-downloads.json", + "verified_at_utc": "2026-09-09T05:34:55.935547+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/source-checksums.json", + "sha256": "c3be263e69f0fb30d6eab9e10dc7c0359c2c610e63cf104522eadf4a1688c844", + "size_bytes": 8351, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/source-checksums.json", + "verified_at_utc": "2026-09-09T05:34:56.358692+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/SOURCE-SHA256SUMS", + "sha256": "b334cd6ad628b2a74044ed4f41705a6e95eeefec9dcdf7be8c82820fba4241aa", + "size_bytes": 2473, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/SOURCE-SHA256SUMS", + "verified_at_utc": "2026-09-09T05:34:56.793973+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/windows/provenance.json", + "sha256": "a8bf7914b9d8b3e956fc4db6a2675ba1d63ce30a5eb0039c05652bd552bfa1b6", + "size_bytes": 1242641, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/windows/provenance.json", + "verified_at_utc": "2026-09-09T05:34:57.842479+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/windows/desktop-metrics.json", + "sha256": "429bdb440584095e901464c69c7747e50f78e5363ca0bee0f3a3d8b69448911f", + "size_bytes": 3786, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/windows/desktop-metrics.json", + "verified_at_utc": "2026-09-09T05:34:58.336517+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/windows/release-metadata.json", + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "size_bytes": 872, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/windows/release-metadata.json", + "verified_at_utc": "2026-09-09T05:34:58.743936+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/windows/SHA256SUMS", + "sha256": "7d6e40575ef3c63756f4e4d90e9bd1d2c9ffefde385870487324f8feaea5000a", + "size_bytes": 674959, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/windows/SHA256SUMS", + "verified_at_utc": "2026-09-09T05:35:00.029347+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/windows/communityai-0.1.0-alpha.20260909.2-windows-setup.exe.json", + "sha256": "6576e4e209ba5eda655ea7d7bbf91375234da3fcedf56e1d6a21cef8f9ec523c", + "size_bytes": 300, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/windows/communityai-0.1.0-alpha.20260909.2-windows-setup.exe.json", + "verified_at_utc": "2026-09-09T05:35:00.537109+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/windows/communityai-0.1.0-alpha.20260909.2-windows-online-setup.exe.json", + "sha256": "fea6249d96169664bafd05dd4ca762507ca7ae585642998428b21c66daf24f5d", + "size_bytes": 1433, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/windows/communityai-0.1.0-alpha.20260909.2-windows-online-setup.exe.json", + "verified_at_utc": "2026-09-09T05:35:00.991568+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/linux/provenance.json", + "sha256": "a10e0391c20436026bf33fb6b62d1d0d75f447212108e6567956b14ecc4b38e0", + "size_bytes": 1244845, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/linux/provenance.json", + "verified_at_utc": "2026-09-09T05:35:02.649518+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/linux/desktop-metrics.json", + "sha256": "385c4ec31a7a7f4d51dd498a6bf43398e4c977637e19cad713c47bd564efbdae", + "size_bytes": 3814, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/linux/desktop-metrics.json", + "verified_at_utc": "2026-09-09T05:35:03.090824+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/linux/release-metadata.json", + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "size_bytes": 872, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/linux/release-metadata.json", + "verified_at_utc": "2026-09-09T05:35:03.521247+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/linux/SHA256SUMS", + "sha256": "e941c35b235fe61f3152c78724f662f85b783a0c87a849a93ee38163716b37dc", + "size_bytes": 675799, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/linux/SHA256SUMS", + "verified_at_utc": "2026-09-09T05:35:04.957086+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/linux/communityai_0.1.0~alpha.20260909.2_amd64.deb.json", + "sha256": "6a8312add7b0b6390787255cc7e2480638127b2e10979f4775b5ad66ab6374dc", + "size_bytes": 1166, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/linux/communityai_0.1.0~alpha.20260909.2_amd64.deb.json", + "verified_at_utc": "2026-09-09T05:35:05.769382+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/metadata/linux/communityai-0.1.0-alpha.20260909.2-linux-online.py.json", + "sha256": "f0d80466749af876c422d9f048eb7e7d2ba2f071b5a38c8b20f7e7883e40878e", + "size_bytes": 915, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/metadata/linux/communityai-0.1.0-alpha.20260909.2-linux-online.py.json", + "verified_at_utc": "2026-09-09T05:35:06.196991+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/INSTALLER-SHA256SUMS", + "sha256": "d26ba5007b305f6ec57c5f04ad20b2dc94886b6b0f4a6077c078ca7e4ad7e93b", + "size_bytes": 473, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/INSTALLER-SHA256SUMS", + "verified_at_utc": "2026-09-09T05:35:06.628201+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-release-metadata.zip", + "sha256": "b03e3695d480d4e42d29e9ce690999b742aa418b56dd7710aed451e44226fda6", + "size_bytes": 996572, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.2/communityai-0.1.0-alpha.20260909.2-release-metadata.zip", + "verified_at_utc": "2026-09-09T05:35:08.035104+00:00" + } + ], + "feed_sha256": "3a7e83f0b9d0d1f154bc08a0c20b883ef1debc220b9a0eea5b7200da29f3adf7", + "updater_source_checks": "197 passed, 5 skipped in the desktop/Linux installer source suites; 12 updater checks also passed under Linux", + "windows_update_handoff": "Passed using a tiny Inno fixture: replace, preserve settings, reopen", + "new_full_desktop_gpu_cloud_tests": false, + "installed_linux_updater_handoff_tested": false, + "portable_bundle_metadata": "The portable bundle's automatic_updates=false describes the uninstalled bundle. Installer-managed applications enable the updater.", + "publication_scope": "Fresh Windows and Linux builds; built-in packaging checks, uploaded size/checksum metadata and public start/end samples for large installers; complete hashes for small public files. Earlier installer/native-runtime evidence remains separate." +} diff --git a/docs/evidence/gate13-20260831-a-cost-authorization.json b/docs/evidence/gate13-20260831-a-cost-authorization.json new file mode 100644 index 000000000..242cf5a40 --- /dev/null +++ b/docs/evidence/gate13-20260831-a-cost-authorization.json @@ -0,0 +1,186 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-a", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "4818da304f4eeafc81978873bcdfa8a41f6cad36", + "linux_lifecycle_helper_commit": "c3dc9234af7980bcaffd481c6f8e4e974ed117d4" + }, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "0.00", + "maximum_estimate_usd": "52.00", + "route_maximum_estimate_usd": "26.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "48.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day previously accepted 14-hour G2/L4 route ceiling plus the same client ceiling; both new clients are CPU-only e2-standard-8 rather than the prior L4 client class" + }, + "immutable_inputs": { + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "sha256": "2272b3bd9a59ccbe66963558f2cee66b41f68403f611f8eff92362eebf8e22b8", + "bytes": 131385 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "sha256": "f50497fa06f465e8dba8b146e963faa11d102bc7c7fc8e5073e6c0b9e1beaf94", + "bytes": 113260 + } + }, + "provider_plan_digest": "sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-a-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 50400, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-a-dht", + "route-20260831-a-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-a-node", + "gate13-20260831-a-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-a-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image_family": "windows-2025", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-windows-qwen-e", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-a-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-a-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image_family": "ubuntu-2404-lts-amd64", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-linux-gemma-e", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-a-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "focused_software_tests_required_before_create": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-a-node", + "route-20260831-a-node boot disk", + "route-20260831-a-dht", + "route-20260831-a-iap", + "gate13-20260831-a-win", + "gate13-20260831-a-win boot disk", + "gate13-20260831-a-linux", + "gate13-20260831-a-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 2, + "previous_provider_plan_digest": "sha256:d3854d047beba36d1415382b296f90f2aaf0fc135d45c7613852c69b85ccf5a1", + "reason": "target both fresh client hosts through the same exact run-scoped IAP-only firewall without reusing the route DHT tag or creating another resource", + "resource_set_changed": false, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-a-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-a-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..17d86b07f --- /dev/null +++ b/docs/evidence/gate13-20260831-a-failed-attempt-and-cleanup.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-attempt", + "run_id": "gate13-20260831-a", + "gate": 13, + "result": "failed", + "recorded_at": "2026-08-31T18:03:21Z", + "authorization": { + "provider_plan_digest": "sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69", + "maximum_estimate_usd": "52.00", + "route_source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_source_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_helper_commit": "4818da304f4eeafc81978873bcdfa8a41f6cad36", + "linux_helper_commit": "c3dc9234af7980bcaffd481c6f8e4e974ed117d4" + }, + "attempt": { + "route_created": true, + "windows_client_created": true, + "linux_client_created": true, + "durable_run_controller_present": false, + "state_aware_reattachment_available": false, + "windows": { + "package_download_verified": true, + "package_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "package_bytes": 2695065068, + "clean_preflight_passed": true, + "complete_lifecycle_record_present": false + }, + "linux": { + "package_download_verified": true, + "package_sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "package_bytes": 3360717934, + "initial_archive_preflight_passed": false, + "retry_exit_code": 2, + "bounded_diagnostic_failed": true, + "complete_lifecycle_record_present": false + }, + "windows_16_phase_lifecycle_complete": false, + "linux_16_phase_lifecycle_complete": false, + "acceptance_passed": false + }, + "failure": { + "class": "non_durable_orchestration", + "summary": "The paid qualification depended on transient operator sessions and one-off repair scripts, had no authoritative persisted state for crash reattachment, and advanced both clients before a single bootstrap and route path had passed.", + "fresh_host_evidence_reusable": false, + "gate_status": "IN PROGRESS", + "later_gates_unblocked": false + }, + "required_correction": { + "single_idempotent_controller_actions": [ + "reconcile", + "start", + "status", + "collect", + "cleanup" + ], + "reconcile_before_mutation": true, + "host_jobs_survive_operator_disconnect": true, + "route_and_transport_probe_before_full_clients": true, + "windows_lifecycle_collected_and_client_deleted_before_linux_create": true, + "failed_or_diagnostically_modified_client_reused_for_acceptance": false, + "all_16_phases_still_required": true, + "controller_state_contract_commit": "ddfb7c617b428a97b33d2b28e42f4fb75f3509ce", + "controller_contract_tests_passed": 15, + "gate13_regression_tests_passed": 187 + }, + "cleanup": { + "performed_at": "2026-08-31T18:03:21Z", + "route_instance_absent": true, + "route_boot_disk_absent": true, + "route_dht_firewall_absent": true, + "route_iap_firewall_absent": true, + "windows_instance_absent": true, + "windows_boot_disk_absent": true, + "linux_instance_absent": true, + "linux_boot_disk_absent": true, + "all_exact_run_resources_absent": true, + "protected_bootstrap_running": true + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-b-cost-authorization.json b/docs/evidence/gate13-20260831-b-cost-authorization.json new file mode 100644 index 000000000..73b6f64ce --- /dev/null +++ b/docs/evidence/gate13-20260831-b-cost-authorization.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-b", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "52.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "392.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day accepted Gate 13 ceilings scaled conservatively by hard duration: the 16-hour G2/L4 route is rounded up to USD 30 from the prior 14-hour USD 26 ceiling, and the two sequential 6-hour e2-standard-8 CPU clients retain the prior USD 26 ceiling" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + } + }, + "provider_plan_digest": "sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-b-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-b-dht", + "route-20260831-b-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-b-node", + "gate13-20260831-b-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-b-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-b-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-b-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-b-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-b-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-b-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T20:15:48Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 219, + "independent_review_matrix_before_final_reservation_guard": 218, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-b-node", + "route-20260831-b-node boot disk", + "route-20260831-b-dht", + "route-20260831-b-iap", + "gate13-20260831-b-win", + "gate13-20260831-b-win boot disk", + "gate13-20260831-b-linux", + "gate13-20260831-b-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "previous_provider_plan_digest": null, + "reason": "fresh durable route-first, sequential-client run under the owner-raised USD 500 epoch ceiling; no failed-run resource or authorization is reused", + "resource_set_changed": true, + "cost_ceiling_changed": true + } +} diff --git a/docs/evidence/gate13-20260831-b-durable-host-job-prerequisite.json b/docs/evidence/gate13-20260831-b-durable-host-job-prerequisite.json new file mode 100644 index 000000000..9b8ec3d44 --- /dev/null +++ b/docs/evidence/gate13-20260831-b-durable-host-job-prerequisite.json @@ -0,0 +1,122 @@ +{ + "schema_version": 1, + "scope": "gate13-durable-native-host-job-prerequisite", + "recorded_at": "2026-08-31T20:27:03Z", + "result": "passed_software_prerequisite", + "gate": 13, + "gate_status": "in_progress", + "source": { + "commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "base_commit": "2fe0de9e1591e36be918fcbac82ecf72c96f8959", + "branch": "codex/gate13-20260831-a", + "pushed": true + }, + "artifacts": { + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "run_controller": { + "path": "scripts/gate13_run_controller.py", + "sha256": "32b31a3380f4a0e295c0185b9d6c2078b072b601d48a846acc119fbc555f2f8d", + "bytes": 33795 + }, + "linux_lifecycle_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_lifecycle_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + } + }, + "contract": { + "action_intent_persisted_before_mutation": true, + "route_acceptance_intent_never_rearmed": true, + "windows_start_intent_never_rearmed": true, + "linux_start_intent_never_rearmed": true, + "client_attempt_ordinal_maximum": 1, + "clients_sequential": true, + "windows_supervisor": "Scheduled Task", + "windows_principal": "exact current ordinary qualification user", + "windows_logon_type": "Interactive", + "windows_run_level": "Limited", + "linux_supervisor": "transient systemd service", + "linux_principal": "gate13", + "linux_no_new_privileges": false, + "linux_sudo_required_for_inner_owned_cgroups": true, + "native_command_and_safety_settings_exactly_inventoried": true, + "lifecycle_config_path_and_digest_bound": true, + "windows_lifecycle_config_exactly_beside_entrypoint": true, + "linux_exec_start_structure_exactly_bound": true, + "stdout_maximum_bytes": 1048576, + "stderr_maximum_bytes": 262144, + "timeout_and_overflow_tree_shutdown": true, + "linux_termination_enters_finally_cleanup": true, + "terminal_record_contains_evidence_digest_only": true, + "successful_collection_revalidates_canonical_evidence": true + }, + "verification": { + "broad_gate13_matrix": { + "passed": 217, + "failed": 0, + "command_scope": [ + "tests/test_gate13_host_job.py", + "tests/test_gate13_run_controller.py", + "tests/test_gate13_packaged_lifecycle.py", + "tests/test_gate13_windows_packaged_lifecycle.py", + "tests/test_gate13_linux_packaged_lifecycle.py", + "tests/test_gate13_linux_localhost_inference.py", + "tests/test_gate13_download_artifact.py", + "desktop/tests/test_build_desktop.py", + "desktop/tests/test_credentials.py" + ] + }, + "focused_host_job_matrix": { + "passed": 23, + "failed": 0 + }, + "independent_broad_gate13_matrix": { + "passed": 217, + "failed": 0, + "warnings": 29 + }, + "windows_powershell_native_parser": "passed", + "python_compile": "passed", + "black_check": "passed", + "isort_check": "passed", + "git_diff_check": "passed" + }, + "provider_preflight": { + "mutation_performed": false, + "native_authentication": "passed", + "protected_bootstrap_running": true, + "next_run_namespace": "gate13-20260831-b", + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "g2_standard_8_available": true, + "e2_standard_8_available": true, + "windows_2025_image_resolved": true, + "ubuntu_2404_image_resolved": true, + "l4_quota_limit": 1, + "l4_quota_usage": 0 + }, + "cost": { + "cloud_resources_created": 0, + "cloud_resources_changed": 0, + "cloud_spend_usd": "0.00", + "current_epoch_committed_before_next_run_usd": "52.00", + "current_epoch_remaining_usd": "48.00" + }, + "claims": { + "paid_run_authorized_by_this_record": false, + "clean_host_lifecycle_completed": false, + "gate_13_passed": false, + "credits_work": false, + "macos_work": false + } +} diff --git a/docs/evidence/gate13-20260831-b-failed-start-and-cleanup.json b/docs/evidence/gate13-20260831-b-failed-start-and-cleanup.json new file mode 100644 index 000000000..5cea9d854 --- /dev/null +++ b/docs/evidence/gate13-20260831-b-failed-start-and-cleanup.json @@ -0,0 +1,62 @@ +{ + "schema_version": 1, + "scope": "gate13-route-start-failure-and-cleanup", + "run_id": "gate13-20260831-b", + "gate": 13, + "result": "failed_cleaned", + "recorded_at": "2026-08-31T21:04:21Z", + "source": { + "controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "authorization_commit": "c1e1f86", + "provider_plan_digest": "sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64" + }, + "stages": { + "native_auth_and_provider_preflight": "passed", + "intent_persisted_before_mutation": true, + "dht_firewall_create": "passed", + "iap_firewall_create": "failed", + "route_instance_create": "not_attempted", + "client_create": "not_attempted", + "cleanup": "passed" + }, + "failure": { + "code": "iap_target_tags_collapsed_by_operator_shell", + "classification": "operator_command_boundary", + "provider_or_environment_failure": false, + "detail": "The two exact IAP target tags reached gcloud as one space-joined value. The command failed before route instance creation." + }, + "cleanup": { + "route_instance_absent": true, + "route_disk_absent": true, + "windows_instance_and_disk_absent": true, + "linux_instance_and_disk_absent": true, + "dht_firewall_absent": true, + "iap_firewall_absent": true, + "protected_bootstrap_running": true + }, + "controller_terminal": { + "phase": "CLEANED_FAILURE", + "failure_code": "resources_disappeared_before_completion", + "cleanup_verified": true, + "revision": 2 + }, + "cost": { + "maximum_estimate_usd": "56.00", + "billable_instance_created": false, + "observed_cost_usd": null, + "maximum_remains_committed": true + }, + "privacy": { + "credentials_retained": false, + "provider_output_retained": false, + "private_paths_retained": false, + "endpoints_retained": false + }, + "claims": { + "route_accepted": false, + "lifecycle_started": false, + "gate_13_passed": false, + "credits_work": false, + "macos_work": false + } +} diff --git a/docs/evidence/gate13-20260831-c-cost-authorization.json b/docs/evidence/gate13-20260831-c-cost-authorization.json new file mode 100644 index 000000000..accbb13ec --- /dev/null +++ b/docs/evidence/gate13-20260831-c-cost-authorization.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-c", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "108.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "336.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day accepted Gate 13 ceilings scaled conservatively by hard duration: the 16-hour G2/L4 route is rounded up to USD 30 from the prior 14-hour USD 26 ceiling, and the two sequential 6-hour e2-standard-8 CPU clients retain the prior USD 26 ceiling" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + } + }, + "provider_plan_digest": "sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-c-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-c-dht", + "route-20260831-c-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-c-node", + "gate13-20260831-c-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-c-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-c-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-c-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-c-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-c-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-c-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T21:06:16Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 219, + "independent_review_matrix_before_final_reservation_guard": 218, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-c-node", + "route-20260831-c-node boot disk", + "route-20260831-c-dht", + "route-20260831-c-iap", + "gate13-20260831-c-win", + "gate13-20260831-c-win boot disk", + "gate13-20260831-c-linux", + "gate13-20260831-c-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "previous_provider_plan_digest": null, + "reason": "fresh replacement after the prior identity failed before VM creation; exact resource names change and the corrected IAP target tags remain two explicit values", + "resource_set_changed": true, + "cost_ceiling_changed": true + } +} diff --git a/docs/evidence/gate13-20260831-c-terminal-state-and-cleanup.json b/docs/evidence/gate13-20260831-c-terminal-state-and-cleanup.json new file mode 100644 index 000000000..12162d0c7 --- /dev/null +++ b/docs/evidence/gate13-20260831-c-terminal-state-and-cleanup.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "scope": "gate13-run-controller-terminal-and-cleanup", + "run_id": "gate13-20260831-c", + "gate": 13, + "result": "failed_cleaned", + "recorded_at": "2026-08-31T21:38:14Z", + "source": { + "authorization_commit": "0dc2345f7c0880bbb63b1d8951187e4e4b744a84", + "authorization_sha256": "sha256:3bb9f79d91d9ef4df2e0082f0358f5edfef199e80f15359560013a10a8b758bd", + "provider_plan_digest": "sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c" + }, + "controller_terminal": { + "phase": "CLEANED_FAILURE", + "failure_code": "resources_disappeared_before_completion", + "cleanup_verified": true, + "revision": 2, + "next_action": "none", + "windows_consumed": false, + "linux_consumed": false + }, + "classification": { + "category": "local_orchestration_state", + "durable_provider_run_record_present": false, + "product_failure_evidenced": false, + "reusable_run_id": false, + "detail": "The reserved run was armed locally and reached a terminal absence state without a durable provider execution record. It is retired rather than reset or reused." + }, + "provider_reconciliation": { + "checked_at": "2026-08-31T21:38:14Z", + "route_instance_absent": true, + "route_disk_absent": true, + "windows_instance_and_disk_absent": true, + "linux_instance_and_disk_absent": true, + "dht_firewall_absent": true, + "iap_firewall_absent": true, + "global_gpu_limit": 1, + "global_gpu_usage": 0, + "protected_bootstrap_running": true + }, + "cost": { + "maximum_estimate_usd": "56.00", + "observed_cost_usd": null, + "maximum_remains_committed": true + }, + "privacy": { + "credentials_retained": false, + "provider_output_retained": false, + "private_paths_retained": false, + "endpoints_retained": false + }, + "claims": { + "route_accepted": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_passed": false, + "gate_13_passed": false + } +} diff --git a/docs/evidence/gate13-20260831-d-cost-authorization.json b/docs/evidence/gate13-20260831-d-cost-authorization.json new file mode 100644 index 000000000..5915c2f36 --- /dev/null +++ b/docs/evidence/gate13-20260831-d-cost-authorization.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-d", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "0dc2345f7c0880bbb63b1d8951187e4e4b744a84", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "164.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "280.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day accepted Gate 13 ceilings scaled conservatively by hard duration: the 16-hour G2/L4 route is rounded up to USD 30 from the prior 14-hour USD 26 ceiling, and the two sequential 6-hour e2-standard-8 CPU clients retain the prior USD 26 ceiling" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + } + }, + "provider_plan_digest": "sha256:d32050a51b8f696aa224fc7e748c9113e174e3c3069c1f8b2bc769b0c5ecea18", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-d-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-d-dht", + "route-20260831-d-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-d-node", + "gate13-20260831-d-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-d-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-d-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-d-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-d-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-d-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-d-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T21:38:14Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 219, + "independent_review_matrix_before_final_reservation_guard": 218, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-d-node", + "route-20260831-d-node boot disk", + "route-20260831-d-dht", + "route-20260831-d-iap", + "gate13-20260831-d-win", + "gate13-20260831-d-win boot disk", + "gate13-20260831-d-linux", + "gate13-20260831-d-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "previous_provider_plan_digest": null, + "reason": "fresh replacement after the prior identity failed before VM creation; exact resource names change and the corrected IAP target tags remain two explicit values", + "resource_set_changed": true, + "cost_ceiling_changed": true + } +} diff --git a/docs/evidence/gate13-20260831-d-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-d-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..118cca646 --- /dev/null +++ b/docs/evidence/gate13-20260831-d-failed-attempt-and-cleanup.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-d", + "recorded_at": "2026-08-31T22:29:33Z", + "result": "failed_cleanup_verified", + "gate": 13, + "route_acceptance": { + "completed_before_client_create": true, + "result": "passed", + "evidence_digest": "sha256:1a8f1adc3bdf876466e4c4a6f66e35a1650286476cc0fd499092bd393f333624", + "primary_model": "Qwen3.5 2B", + "standby_model": "Gemma 4 E2B IT", + "fallback_and_restoration_passed": true, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-d-windows", + "attempt_ordinal": 1, + "result": "failed", + "failure_code": "windows_packaged_lifecycle_failed", + "terminal_failure_code": "lifecycle_failed", + "terminal_exit_code": 2, + "canonical_lifecycle_evidence_present": false, + "failure_record_sha256": "sha256:8d44d0f9529cf358e8941d1114265d654f537ab1cdc75a6944abe6a3b7f9b53e", + "linux_client_created": false + }, + "findings": [ + { + "class": "host_supervisor", + "cause": "The Interactive scheduled task never ran from a headless SSH session, and principal observation compared a scheduler-normalized leaf name to a qualified identity.", + "proof": "The old task remained Ready with Task Scheduler result 0x41303 and no status, terminal, evidence, or stderr record.", + "correction_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "correction": "Use S4U with Limited run level and compare resolved principal SID to the current identity SID.", + "repaired_before_lifecycle_attempt": true + }, + { + "class": "client_prerequisite", + "cause": "The exact package downloader and config were staged, but the 2,695,065,068-byte archive itself was not downloaded before the one-attempt job was armed.", + "proof": "The repaired S4U supervisor ran, wrote its status and terminal records, and the lifecycle failed immediately with its bounded failure record; no canonical phase record was produced.", + "correction": "Download and verify the exact archive on the fresh client before host-job start; verify its fixed size and SHA-256 before consuming the attempt.", + "gcp_related": false + } + ], + "gate11_comparison": { + "repeated_gate11_gcp_issue": false, + "gate11_quota_fix_reused": true, + "route_product_acceptance_passed": true, + "gcp_route_or_quota_blocker": false, + "delay_class": "gate13_operator_and_supervisor_orchestration" + }, + "cleanup": { + "native_windows_task_absent": true, + "exact_instances_absent": true, + "exact_disks_absent": true, + "exact_firewalls_absent": true, + "regional_l4_limit": 1, + "regional_l4_usage": 0, + "verified_at": "2026-08-31T22:29:33Z", + "protected_bootstrap_deleted": false + }, + "cost": { + "maximum_committed_usd": "56.00", + "observed_cost_usd": null, + "additional_supervisor_repair_cost_usd": "0.00" + }, + "claims": { + "gate_13_passed": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_started": false, + "run_id_reusable": false, + "resources_reusable": false + } +} diff --git a/docs/evidence/gate13-20260831-d-windows-supervisor-repair-authorization.json b/docs/evidence/gate13-20260831-d-windows-supervisor-repair-authorization.json new file mode 100644 index 000000000..008ba5454 --- /dev/null +++ b/docs/evidence/gate13-20260831-d-windows-supervisor-repair-authorization.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "scope": "gate13-windows-supervisor-repair-authorization", + "run_id": "gate13-20260831-d", + "recorded_at": "2026-08-31T22:20:51Z", + "objective": "Launch the already-authorized Windows packaged lifecycle exactly once under a headless ordinary-user supervisor.", + "finding": { + "task_name": "communityai-gate13-gate13-20260831-d-windows", + "registered_principal": "M", + "registered_logon_type": "Interactive", + "registered_run_level": "Limited", + "scheduler_state": "Ready", + "scheduler_last_result_decimal": 267011, + "scheduler_last_result_hex": "0x41303", + "scheduler_interpretation": "task_has_not_yet_run", + "status_record_present": false, + "terminal_record_present": false, + "evidence_record_present": false, + "lifecycle_attempt_consumed": false, + "additional_cloud_resources_required": false + }, + "root_cause": [ + "Interactive scheduled tasks do not start from this headless SSH operator session.", + "Task Scheduler canonicalized the registered principal to a leaf account name, while the observer compared it to the qualified current identity string." + ], + "repair_binding": { + "implementation_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "host_adapter_path": "scripts/gate13_host_job.py", + "host_adapter_sha256": "sha256:922a9365269a2c1f0aee09473b15fdcc9b9a08522cfd5e9934145b4966b0ad31", + "host_adapter_bytes": 41592, + "replacement_logon_type": "S4U", + "principal_match": "resolved_principal_sid_equals_current_identity_sid", + "unchanged_entrypoint_sha256": "sha256:9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "unchanged_lifecycle_config_sha256": "sha256:a926514b35b10baab742ff270db606fef2f7ddf58317e1023f12b1cd66affbd0" + }, + "authorized_actions": [ + "Verify the exact old task name, action, ordinary-user principal, Interactive logon type, Limited run level, Ready state, 0x41303 last result, and absence of status, terminal, and evidence records.", + "Unregister only that exact never-run task.", + "Replace only the host adapter and its digest binding in host-job.json.", + "Register and start the same exact task once with S4U, Limited run level, and SID-based principal verification.", + "Fail closed if any lifecycle output appeared before repair or if the old task binding differs." + ], + "invariants": [ + "The paid Windows VM is not recreated.", + "The lifecycle entrypoint, package, model, route acceptance, evidence contract, and one-attempt ceiling are unchanged.", + "No second lifecycle attempt is authorized.", + "Windows evidence must still be collected and the Windows VM deleted before Linux is created." + ], + "added_cost_usd": "0.00", + "tests": { + "command": ".\\.venv-cuda\\Scripts\\python.exe -m pytest -q tests/test_gate13_host_job.py", + "result": "23 passed" + } +} diff --git a/docs/evidence/gate13-20260831-e-cost-authorization.json b/docs/evidence/gate13-20260831-e-cost-authorization.json new file mode 100644 index 000000000..99820a7ee --- /dev/null +++ b/docs/evidence/gate13-20260831-e-cost-authorization.json @@ -0,0 +1,230 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-e", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "220.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "224.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; each exact package archive must be downloaded and hash-verified before its one-attempt host job is armed" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "sha256": "922a9365269a2c1f0aee09473b15fdcc9b9a08522cfd5e9934145b4966b0ad31", + "bytes": 41592 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + } + }, + "provider_plan_digest": "sha256:9ca0fa516017c4a3709a467752f779bcb3bbc0a7c790f9bc61de56d385804c62", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-e-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-e-dht", + "route-20260831-e-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-e-node", + "gate13-20260831-e-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-e-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-e-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-e-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-e-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-e-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-e-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T22:40:11Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 23, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-e-node", + "route-20260831-e-node boot disk", + "route-20260831-e-dht", + "route-20260831-e-iap", + "gate13-20260831-e-win", + "gate13-20260831-e-win boot disk", + "gate13-20260831-e-linux", + "gate13-20260831-e-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 4, + "previous_provider_plan_digest": "sha256:0b8c423090e27ec792c55287676d753db4042a5817d25d550d2ef5f2e4b4119b", + "reason": "the controller requires route and sequential clients in one exact zone; after the us-central1-a L4 stockout created no VM or disk, the complete run moves to provider-recommended us-central1-b at unchanged cost", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-e-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-e-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..0092ed872 --- /dev/null +++ b/docs/evidence/gate13-20260831-e-failed-attempt-and-cleanup.json @@ -0,0 +1,95 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-e", + "recorded_at": "2026-08-31T23:45:44Z", + "result": "failed_cleanup_verified", + "gate": 13, + "route_acceptance": { + "completed_before_client_create": true, + "result": "passed", + "evidence_digest": "sha256:064e1699bff2d0998c37ec6a3be5d37f8b6400a12de9627ea13fddb302508e55", + "primary_model": "Qwen3.5 2B", + "standby_model": "Gemma 4 E2B IT", + "fallback_and_restoration_passed": true, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-e-windows", + "attempt_ordinal": 1, + "result": "failed", + "failure_code": "windows_packaged_lifecycle_failed", + "terminal_failure_code": "lifecycle_failed", + "terminal_exit_code": 2, + "started_at_unix": 1788219070, + "finished_at_unix": 1788219581, + "elapsed_seconds": 511, + "canonical_lifecycle_evidence_present": false, + "failure_record_sha256": "sha256:8d44d0f9529cf358e8941d1114265d654f537ab1cdc75a6944abe6a3b7f9b53e", + "failure_stderr_bytes": 0, + "linux_client_created": false + }, + "pre_start_proofs": { + "windows_archive_bytes": 2695065068, + "windows_archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "archive_verified_before_attempt": true, + "ordinary_ssh_identity_verified": true, + "ordinary_ssh_identity_admin": false, + "scheduler_task_s4u": true, + "scheduler_task_run_level": "Limited", + "scheduler_running_observed_before_operator_demoted_account": true + }, + "findings": [ + { + "class": "windows_ssh_authorization", + "cause": "The Windows OpenSSH service allowed only Administrators and OpenSSH Users; removing M from Administrators without adding it to OpenSSH Users made the valid public key appear rejected.", + "proof": "The OpenSSH operational log reported that M was not allowed because none of the user's groups were listed in AllowGroups. Adding M to OpenSSH Users restored SSH while an independent token check reported is-admin=false.", + "correction": "Provision M in the existing OpenSSH Users group before removing it from Administrators.", + "gcp_related": false + }, + { + "class": "windows_supervisor_boundary", + "cause": "Windows Server 2025 denies an ordinary account access to the ScheduledTasks CIM provider and denies task registration in the root folder, even when the task principal is the same ordinary user.", + "proof": "Get-ScheduledTask failed with CIM access denied and a direct Task Scheduler COM registration probe failed with E_ACCESSDENIED. A privileged bootstrap could register the exact S4U/Limited task, which then ran durably.", + "correction": "Do not ask the ordinary account to provision its own native supervisor. For the next run, keep the bounded host adapter in the ordinary foreground and keep the IAP SSH transport durable at the operator boundary.", + "gcp_related": false + }, + { + "class": "lifecycle_diagnosability", + "cause": "The Windows lifecycle catch path emitted one generic failure record and failure cleanup removed the phase workspace, so the completed attempt could not identify the failed acceptance phase.", + "proof": "After 511 seconds the only lifecycle output was the 91-byte generic failure record and stderr was empty.", + "correction": "Emit only the hard-coded current phase name with the generic failure code; never emit exception text, paths, endpoints, prompts, outputs, or credentials.", + "gcp_related": false + } + ], + "gate11_comparison": { + "repeated_gate11_gcp_issue": false, + "gate11_quota_fix_reused": true, + "route_product_acceptance_passed": true, + "gcp_route_or_quota_blocker": false, + "delay_class": "gate13_windows_operator_boundary_and_packaged_lifecycle" + }, + "cleanup": { + "native_windows_task_absent_with_instance": true, + "exact_instances_absent": true, + "exact_disks_absent": true, + "exact_firewalls_absent": true, + "regional_l4_limit": 1, + "regional_l4_usage": 0, + "verified_at": "2026-08-31T23:45:44Z", + "protected_bootstrap_deleted": false + }, + "cost": { + "maximum_committed_usd": "56.00", + "observed_cost_usd": null + }, + "claims": { + "gate_13_passed": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_started": false, + "run_id_reusable": false, + "resources_reusable": false + } +} diff --git a/docs/evidence/gate13-20260831-f-cost-authorization.json b/docs/evidence/gate13-20260831-f-cost-authorization.json new file mode 100644 index 000000000..d55afdf6b --- /dev/null +++ b/docs/evidence/gate13-20260831-f-cost-authorization.json @@ -0,0 +1,232 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-f", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "2b895f59cabb8b294c8afad09dbcacfb51a0db6b", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "2b895f59cabb8b294c8afad09dbcacfb51a0db6b", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "276.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "168.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; each exact package archive is downloaded and hash-verified before one durable foreground host-adapter execution over IAP SSH as the ordinary OS user" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "sha256": "922a9365269a2c1f0aee09473b15fdcc9b9a08522cfd5e9934145b4966b0ad31", + "bytes": 41592 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "2b895f59cabb8b294c8afad09dbcacfb51a0db6b", + "sha256": "d6363fe00867f2b6ffccc855197ee078d808fa7890ec0234e73050db9d5aa6e1", + "bytes": 132564 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + } + }, + "provider_plan_digest": "sha256:c9a2aafc84940df901a7db1755af2e684f845b78dcdfac04332cfed36388ba25", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-f-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-f-dht", + "route-20260831-f-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-f-node", + "gate13-20260831-f-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-f-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-f-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-f-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-f-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-f-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-f-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "foreground_execute_over_durable_iap_ssh" + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T23:49:33Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 39, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-e-node", + "route-20260831-e-node boot disk", + "route-20260831-e-dht", + "route-20260831-e-iap", + "gate13-20260831-e-win", + "gate13-20260831-e-win boot disk", + "gate13-20260831-e-linux", + "gate13-20260831-e-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "replace the Windows Scheduled Tasks provisioning boundary with one durable foreground host-adapter execution over IAP SSH as the ordinary user; retain exact archives, route-first ordering, all sixteen phases, costs, privacy, and cleanup", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-f-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-f-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..bde995797 --- /dev/null +++ b/docs/evidence/gate13-20260831-f-failed-attempt-and-cleanup.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-f", + "recorded_at": "2026-09-01T00:46:17Z", + "result": "failed_cleanup_verified", + "gate": 13, + "route_acceptance": { + "completed_before_client_create": true, + "result": "passed", + "evidence_digest": "sha256:eb614c1b281a3661560536825edde1ad0e960c0c564e5aabd4900dd41bb2dea4", + "primary_model": "Qwen3.5 2B", + "standby_model": "Gemma 4 E2B IT", + "fallback_and_restoration_passed": true, + "total_duration_ms": 295357, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-f-windows", + "attempt_ordinal": 1, + "execution_transport": "durable_iap_ssh_foreground_as_ordinary_user", + "ordinary_user_admin": false, + "result": "failed", + "failure_code": "windows_packaged_lifecycle_failed", + "failure_phase": "signed_bootstrap", + "terminal_failure_code": "lifecycle_failed", + "terminal_exit_code": 2, + "started_at_unix": 1788222739, + "finished_at_unix": 1788223261, + "elapsed_seconds": 522, + "failure_record_bytes": 126, + "failure_stderr_bytes": 0, + "canonical_lifecycle_evidence_present": false, + "linux_client_created": false + }, + "pre_start_proofs": { + "windows_archive_bytes": 2695065068, + "windows_archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "archive_verified_before_attempt": true, + "staged_file_hashes_verified": 12, + "host_config_validated": true, + "status_absent_before_start": true, + "terminal_absent_before_start": true, + "evidence_absent_before_start": true + }, + "findings": [ + { + "class": "supervisor_hypothesis_eliminated", + "finding": "The direct ordinary-user foreground execution failed at the same elapsed time and same signed_bootstrap phase as the S4U/Limited execution.", + "conclusion": "Scheduled Tasks, S4U, SSH account authorization, and the privileged provisioning boundary are not the packaged lifecycle failure.", + "gcp_related": false + }, + { + "class": "windows_runtime_environment", + "finding": "The host adapter passed only SYSTEMROOT, WINDIR, TEMP, TMP, and USERPROFILE into the Windows lifecycle. It removed standard non-secret user runtime variables including APPDATA, LOCALAPPDATA, PROGRAMDATA, COMSPEC, PATH, and PATHEXT before the packaged desktop was started.", + "timing_inference": "The roughly 300-second difference between preliminary package work and terminal failure is consistent with Wait-Gate13ProductStatus exhausting its 300-second readiness bound in signed_bootstrap.", + "correction": "Keep a fixed allowlist of standard non-secret Windows runtime variables while continuing to exclude arbitrary variables and credential/token names.", + "gcp_related": false + }, + { + "class": "bounded_failure_localization", + "finding": "The phase-bounded failure record localized the deterministic defect to signed_bootstrap without retaining exception text, paths, endpoints, prompts, outputs, or credentials.", + "correction": "Add a hard-coded operation name inside signed_bootstrap for the next run while preserving the same privacy boundary.", + "gcp_related": false + } + ], + "gate11_comparison": { + "repeated_gate11_gcp_issue": false, + "gate11_quota_fix_reused": true, + "route_product_acceptance_passed": true, + "gcp_route_or_quota_blocker": false, + "delay_class": "gate13_windows_host_environment_contract" + }, + "cleanup": { + "exact_instances_absent": true, + "exact_disks_absent": true, + "exact_firewalls_absent": true, + "regional_l4_limit": 1, + "regional_l4_usage": 0, + "verified_at": "2026-09-01T00:46:17Z", + "protected_bootstrap_deleted": false + }, + "cost": { + "maximum_committed_usd": "56.00", + "observed_cost_usd": null + }, + "claims": { + "gate_13_passed": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_started": false, + "run_id_reusable": false, + "resources_reusable": false + } +} diff --git a/docs/evidence/gate13-20260831-g-cost-authorization.json b/docs/evidence/gate13-20260831-g-cost-authorization.json new file mode 100644 index 000000000..dec7ad6b3 --- /dev/null +++ b/docs/evidence/gate13-20260831-g-cost-authorization.json @@ -0,0 +1,232 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-g", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "332.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "112.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; each exact package archive is downloaded and hash-verified before one durable foreground host-adapter execution over IAP SSH as the ordinary OS user" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "bytes": 42051 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "bytes": 133351 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + } + }, + "provider_plan_digest": "sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-g-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-g-dht", + "route-20260831-g-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-g-node", + "gate13-20260831-g-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-g-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-g-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-g-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-g-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-g-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-g-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "foreground_execute_over_durable_iap_ssh" + } + }, + "preflight": { + "read_only_checked_at": "2026-09-01T00:51:42.840Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 40, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-g-node", + "route-20260831-g-node boot disk", + "route-20260831-g-dht", + "route-20260831-g-iap", + "gate13-20260831-g-win", + "gate13-20260831-g-win boot disk", + "gate13-20260831-g-linux", + "gate13-20260831-g-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "preserve a bounded allowlist of standard per-user Windows runtime variables required by the installed desktop package; retain direct ordinary-user execution, exact archives, route-first ordering, all sixteen phases, costs, privacy, and cleanup", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-g-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-g-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..f3592182b --- /dev/null +++ b/docs/evidence/gate13-20260831-g-failed-attempt-and-cleanup.json @@ -0,0 +1,90 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-g", + "recorded_at": "2026-09-01T01:24:46.392Z", + "result": "failed_cleanup_verified", + "provider_plan_digest": "sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032", + "route_acceptance": { + "result": "passed", + "evidence_sha256": "04271d6ac93410974d26818f3c3f930f0031877ae6c6d11441c7e75569453633", + "total_duration_ms": 298408, + "qwen": { + "covered_blocks": 24, + "total_blocks": 24, + "peer_count": 1 + }, + "gemma": { + "covered_blocks": 35, + "total_blocks": 35, + "peer_count": 1 + }, + "primary_inference": true, + "fallback_inference": true, + "restoration_inference": true, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained_in_evidence": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-g-windows", + "attempt_ordinal": 1, + "execution_transport": "durable_iap_ssh_foreground_as_ordinary_user", + "ordinary_user": "M", + "ordinary_user_admin": false, + "package_preflight": { + "archive_bytes": 2695065068, + "archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "download_result": "passed", + "url_retained": false + }, + "committed_adapter_sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "committed_entrypoint_sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "started_at_unix": 1788225618, + "finished_at_unix": 1788225620, + "exit_code": 2, + "failure_code": "windows_packaged_lifecycle_failed", + "failure_phase": "package_verification", + "failure_operation": "package_verification", + "bounded_failure_record_bytes": 173, + "stderr_bytes": 0 + }, + "root_cause": { + "class": "operator_staging_omission", + "finding": "the fresh staging upload contained the eight executable and configuration files but omitted the four already-pinned release audit inputs required by Test-Gate13PackageAudit", + "missing_inputs": [ + "audit/desktop-metrics.json", + "audit/provenance.json", + "audit/release-metadata.json", + "audit/SHA256SUMS" + ], + "product_or_route_regression": false, + "next_attempt_change": "stage and hash-verify the four existing audit inputs before archive download and execution; no lifecycle or route redesign" + }, + "linux_attempt": { + "instance_created": false, + "attempt_ordinal": 0 + }, + "cleanup": { + "route_instance_absent": true, + "route_disk_absent": true, + "windows_instance_absent": true, + "windows_disk_absent": true, + "linux_instance_absent": true, + "linux_disk_absent": true, + "route_firewalls_absent": true, + "gpus_all_regions": { + "limit": 1, + "usage": 0 + }, + "protected_bootstrap_running": true + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-h-cost-authorization.json b/docs/evidence/gate13-20260831-h-cost-authorization.json new file mode 100644 index 000000000..ee9fa7d66 --- /dev/null +++ b/docs/evidence/gate13-20260831-h-cost-authorization.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-h", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "0c67f0b8912ce8323cae2008642783b4aa23f436", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "388.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "56.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; both pinned four-file release-audit bundles and each exact package archive are staged and hash-verified before one durable foreground host-adapter execution as the ordinary OS user" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "bytes": 42051 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "bytes": 133351 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + }, + "windows_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "953fc814d3d7d6787cbe7ecc25e8ab9f60c68515b94c80575207e15e78d69549", + "bytes": 3795 + }, + "audit/provenance.json": { + "sha256": "ac04b71d35493ba4967628af1ac05ca290b1af09aab4e8955ac09031c87ce7f8", + "bytes": 1241883 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "a458760c1e4636956c9fba5a7da93ed869544f4817d5ab883b5d8b34cc9ad964", + "bytes": 674380 + } + }, + "linux_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "5d2b261505e949a15c332c6e5bb817611e340bf897f9b1952ebde8e573e07bd2", + "bytes": 3798 + }, + "audit/provenance.json": { + "sha256": "c9b5e47017b003f6b2d81c9ab8273fcbf3c72f7743f1ebbf99383ae7cd5accda", + "bytes": 1357051 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "4766c587a1d8e430f892128b8667c04ae7e83bde869a3098b24012c4f99cb74d", + "bytes": 737970 + } + } + }, + "provider_plan_digest": "sha256:f243254cc5fb65f44d0c9e707be36feb3284fd6e15b15620882843798fb456b1", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-h-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-h-dht", + "route-20260831-h-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-h-node", + "gate13-20260831-h-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-h-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-h-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-h-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-h-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-h-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-h-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "foreground_execute_over_durable_iap_ssh" + } + }, + "preflight": { + "read_only_checked_at": "2026-09-01T01:25:51.454Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 40, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-h-node", + "route-20260831-h-node boot disk", + "route-20260831-h-dht", + "route-20260831-h-iap", + "gate13-20260831-h-win", + "gate13-20260831-h-win boot disk", + "gate13-20260831-h-linux", + "gate13-20260831-h-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "stage and hash-verify the four already-pinned release audit inputs for each platform before archive download and ordinary-user execution; retain the corrected Windows runtime allowlist, exact archives, route-first ordering, all sixteen phases, costs, privacy, and cleanup", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-h-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-h-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..2b5f5e1ae --- /dev/null +++ b/docs/evidence/gate13-20260831-h-failed-attempt-and-cleanup.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-h", + "gate": 13, + "result": "failed-cleaned", + "recorded_at": "2026-09-01T02:28:35.232Z", + "source": { + "authorization_sha256": "sha256:6714a7a33b671c3fc177e182c354b80e3fb54c6657d5008eff74080c088be523", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "host_job_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20" + }, + "acceptance": { + "route_acceptance_passed": true, + "windows_archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "windows_archive_bytes": 2695065068, + "windows_attempts": 1, + "windows_result": "failed", + "failure_phase": "signed_bootstrap", + "failure_operation": "product_readiness", + "linux_created": false, + "linux_attempts": 0, + "gate_passed": false + }, + "conclusion": "The exact route and package prerequisites passed. The opaque non-interactive Windows launch timed out waiting for product readiness and retained no useful product error. This run does not show a Gate 11 or GCP route failure.", + "cleanup": { + "verified_at": "2026-09-01T02:28:35.232Z", + "instances_absent": [ + "route-20260831-h-node", + "gate13-20260831-h-win", + "gate13-20260831-h-linux" + ], + "disks_absent": [ + "route-20260831-h-node", + "gate13-20260831-h-win", + "gate13-20260831-h-linux" + ], + "firewalls_absent": [ + "route-20260831-h-dht", + "route-20260831-h-iap" + ], + "global_l4_quota_limit": 1, + "global_l4_quota_usage": 0, + "protected_bootstrap_status": "RUNNING" + }, + "next_action": "Run the exact Windows package manually in a real interactive console session with visible node and desktop diagnostics; do not invoke the lifecycle wrapper until the manual flow passes.", + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-i-cost-authorization.json b/docs/evidence/gate13-20260831-i-cost-authorization.json new file mode 100644 index 000000000..8a66f5118 --- /dev/null +++ b/docs/evidence/gate13-20260831-i-cost-authorization.json @@ -0,0 +1,291 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-i", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "9ad67da6728430b965add981d088821d4d600027", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "444.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "0.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceiling: one bounded 16-hour G2/L4 route and two sequential bounded 6-hour CPU clients; Windows is exercised first in a real interactive console session with virtual display and visible product diagnostics, one step at a time, before any lifecycle automation is permitted" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "bytes": 42051 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "bytes": 133351 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + }, + "windows_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "953fc814d3d7d6787cbe7ecc25e8ab9f60c68515b94c80575207e15e78d69549", + "bytes": 3795 + }, + "audit/provenance.json": { + "sha256": "ac04b71d35493ba4967628af1ac05ca290b1af09aab4e8955ac09031c87ce7f8", + "bytes": 1241883 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "a458760c1e4636956c9fba5a7da93ed869544f4817d5ab883b5d8b34cc9ad964", + "bytes": 674380 + } + }, + "linux_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "5d2b261505e949a15c332c6e5bb817611e340bf897f9b1952ebde8e573e07bd2", + "bytes": 3798 + }, + "audit/provenance.json": { + "sha256": "c9b5e47017b003f6b2d81c9ab8273fcbf3c72f7743f1ebbf99383ae7cd5accda", + "bytes": 1357051 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "4766c587a1d8e430f892128b8667c04ae7e83bde869a3098b24012c4f99cb74d", + "bytes": 737970 + } + } + }, + "provider_plan_digest": "sha256:8525c3099f273c099aba26de57c1f610a0c74cac65ed2640589d51e874bd0c44", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-i-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-i-dht", + "route-20260831-i-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-i-node", + "gate13-20260831-i-client" + ], + "enable_virtual_display": true + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-i-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-i-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-i-client", + "host_execution": "manual_interactive_console_as_ordinary_user", + "enable_virtual_display": true + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-i-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-i-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-i-client", + "host_execution": "manual_foreground_shell_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "manual_phase_by_phase_before_adapter", + "manual_windows_desktop_required": true, + "automation_prohibited_until_manual_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-09-01T02:28:35.230Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 22, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-i-node", + "route-20260831-i-node boot disk", + "route-20260831-i-dht", + "route-20260831-i-iap", + "gate13-20260831-i-win", + "gate13-20260831-i-win boot disk", + "gate13-20260831-i-linux", + "gate13-20260831-i-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "replace the opaque host lifecycle launch with a literal clean-host playthrough: verify and extract the exact package, run bootstrap and node visibly, launch the Windows desktop in an interactive session, exercise the required controls and inference, restart and exercise the second control, then repeat on Linux; translate only proven commands back into adapters", + "resource_set_changed": true, + "cost_ceiling_changed": false + }, + "manual_execution": { + "windows": [ + "verify exact archive and four audit records", + "extract into an empty per-user install root", + "run packaged self-tests", + "run signed bootstrap with visible output", + "start packaged node directly and inspect visible readiness", + "launch CommunityAI.exe in a real interactive ordinary-user console", + "exercise sharing control and public inference", + "restart the desktop", + "exercise the second control and re-run inference", + "uninstall/reinstall/cache and cleanup phases" + ], + "linux": [ + "repeat the proven phase sequence in one ordinary-user foreground session" + ], + "wrapper_use_before_manual_pass": false, + "retain_private_prompts_or_outputs": false + } +} diff --git a/docs/evidence/gate13-20260831-i-linux-paused.png b/docs/evidence/gate13-20260831-i-linux-paused.png new file mode 100644 index 000000000..19e2ab7b6 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-linux-paused.png differ diff --git a/docs/evidence/gate13-20260831-i-linux-ready.png b/docs/evidence/gate13-20260831-i-linux-ready.png new file mode 100644 index 000000000..627e036a4 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-linux-ready.png differ diff --git a/docs/evidence/gate13-20260831-i-linux-sharing.png b/docs/evidence/gate13-20260831-i-linux-sharing.png new file mode 100644 index 000000000..93cbe6267 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-linux-sharing.png differ diff --git a/docs/evidence/gate13-20260831-i-manual-qualification-and-cleanup.json b/docs/evidence/gate13-20260831-i-manual-qualification-and-cleanup.json new file mode 100644 index 000000000..83c8a4907 --- /dev/null +++ b/docs/evidence/gate13-20260831-i-manual-qualification-and-cleanup.json @@ -0,0 +1,203 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-clean-install-manual-qualification", + "gate": 13, + "run_id": "gate13-20260831-i", + "result": "passed", + "recorded_at": "2026-09-01T05:25:34.3103224Z", + "goal": "Prove that a normal user can install, open, use, restart, and control the packaged CommunityAI desktop on clean Windows and Linux hosts against the public route.", + "decision": { + "manual_playthrough_is_acceptance_source": true, + "opaque_lifecycle_wrapper_required_for_gate_decision": false, + "reason": "The manual playthrough exercised the actual desktop and exposed the real product defect hidden by the wrappers. Reinstall, uninstall, retained-data choice, and publisher release work remain Gate 15." + }, + "source": { + "published_package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_path_fix_commit": "f1dc3a0e38b0b2ee12150fe403fd1de435c49f71", + "windows_path_fix": "Use extended-length Windows paths for long manifest artifact partial, final, and lock paths without changing their on-disk layout." + }, + "route": { + "accepted_before_client_creation": true, + "qwen": { + "model": "Qwen3.5 2B", + "manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "blocks": 24, + "peer_count": 1, + "primary_inference_passed": true, + "restored_inference_passed": true + }, + "gemma": { + "model": "Gemma 4 E2B IT", + "manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "blocks": 35, + "peer_count": 1, + "fallback_inference_passed": true + }, + "primary_fallback_restoration_total_duration_ms": 337038, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows": { + "result": "passed", + "image": "windows-server-2025-dc-v20260814", + "machine_type": "e2-standard-8", + "ordinary_user": true, + "is_admin_during_product_run": false, + "interactive_console_session": true, + "published_archive": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068, + "download_verified_before_install": true + }, + "packaged_self_tests": { + "runtime": "passed", + "application": "passed", + "ui": "passed", + "onboarding_ui": "passed" + }, + "manual_flow": [ + "installed the verified archive into an empty ordinary-user install root", + "opened CommunityAI.exe in the real interactive console", + "observed complete Qwen and Gemma routes", + "ran one-token Qwen inference", + "closed the desktop and verified the desktop and node stopped", + "restarted the desktop in the same ordinary-user console", + "edited sharing limits through the UI", + "selected Qwen and clicked Start sharing", + "clicked Pause sharing after restart" + ], + "initial_defect": { + "classification": "product", + "not_gcp": true, + "not_ssh_or_scheduler": true, + "error": "Windows legacy MAX_PATH rejected a manifest artifact lock path under the default per-user data directory.", + "reproduced_with_published_package": true, + "short_data_root_control_inference": { + "passed": true, + "model": "Qwen3.5 2B", + "completion_tokens": 1, + "duration_ms": 239683 + } + }, + "fixed_default_path_inference": { + "passed": true, + "model": "Qwen3.5 2B", + "completion_tokens": 1, + "duration_ms": 204348, + "data_root": "default per-user data root", + "fixed_node_source": "f1dc3a0e38b0b2ee12150fe403fd1de435c49f71", + "prompt_retained": false, + "output_retained": false, + "credential_retained": false + }, + "screenshots": { + "ready": { + "path": "gate13-20260831-i-windows-ready.png", + "sha256": "19926d36d8d9fa1c2a8b36449e891e31dd7dfe244ec85ad8921e8793ea9e8d24" + }, + "paused_after_restart": { + "path": "gate13-20260831-i-windows-paused.png", + "sha256": "f9a1da9a94a6d9d158f0479b8e3b1647b39a7a8aa19f08328c7f533f820463b8" + } + } + }, + "linux": { + "result": "passed", + "image": "ubuntu-2404-noble-amd64-v20260826", + "machine_type": "e2-standard-8", + "ordinary_user": true, + "sudo_available_during_product_run": false, + "display": "Xvfb interactive X11 display", + "native_credential_store": "GNOME Secret Service", + "published_archive": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934, + "download_verified_before_install": true + }, + "packaged_self_tests": { + "runtime": "passed", + "application": "passed", + "ui": "passed", + "onboarding_ui": "passed" + }, + "manual_flow": [ + "installed the verified archive into an empty ordinary-user install root", + "opened the real Linux desktop on an X11 display", + "observed complete Qwen and Gemma routes", + "ran one-token Gemma inference", + "edited sharing limits through the UI", + "selected Gemma and clicked Start sharing", + "stopped the complete app and node process tree", + "restarted the desktop with the native credential store", + "observed Gemma sharing resume", + "clicked Pause sharing", + "ran one-token Gemma inference again after restart and pause" + ], + "initial_inference": { + "passed": true, + "model": "Gemma 4 E2B IT", + "completion_tokens": 1, + "duration_ms": 197652, + "prompt_retained": false, + "output_retained": false, + "credential_retained": false + }, + "post_restart_inference": { + "passed": true, + "model": "Gemma 4 E2B IT", + "completion_tokens": 1, + "duration_ms": 54352, + "prompt_retained": false, + "output_retained": false, + "credential_retained": false + }, + "screenshots": { + "ready": { + "path": "gate13-20260831-i-linux-ready.png", + "sha256": "1c51f20269a032de813a309e8230abae0d87759e30288d2347c8de46f48d2e6a" + }, + "sharing": { + "path": "gate13-20260831-i-linux-sharing.png", + "sha256": "a0ef7ec97bc186d3a3ecffcb2baf9241074cf382ff2bb441bcf6853da86857fc" + }, + "paused_after_restart": { + "path": "gate13-20260831-i-linux-paused.png", + "sha256": "343abc26d180578ba97723e2f31e1717dbed2eb247b81ae23aaab69f53207a0b" + } + } + }, + "cleanup": { + "exact_instances_absent": [ + "route-20260831-i-node", + "gate13-20260831-i-win", + "gate13-20260831-i-linux" + ], + "exact_disks_absent": [ + "route-20260831-i-node", + "gate13-20260831-i-win", + "gate13-20260831-i-linux" + ], + "exact_firewalls_absent": [ + "route-20260831-i-dht", + "route-20260831-i-iap" + ], + "global_l4_quota": { + "metric": "GPUS_ALL_REGIONS", + "limit": 1, + "usage": 0 + }, + "protected_bootstrap": { + "name": "communityai-bootstrap-1", + "status": "RUNNING" + }, + "passed": true + }, + "privacy": { + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false, + "signed_urls_retained": false, + "provider_endpoints_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-i-windows-paused.png b/docs/evidence/gate13-20260831-i-windows-paused.png new file mode 100644 index 000000000..3e9846db3 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-windows-paused.png differ diff --git a/docs/evidence/gate13-20260831-i-windows-ready.png b/docs/evidence/gate13-20260831-i-windows-ready.png new file mode 100644 index 000000000..3c14a3002 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-windows-ready.png differ diff --git a/docs/evidence/gate13-20260901-a-automated-qualification-and-cleanup.json b/docs/evidence/gate13-20260901-a-automated-qualification-and-cleanup.json new file mode 100644 index 000000000..841d622fd --- /dev/null +++ b/docs/evidence/gate13-20260901-a-automated-qualification-and-cleanup.json @@ -0,0 +1,238 @@ +{ + "schema_version": 1, + "scope": "gate13-automated-paid-cloud-qualification-and-cleanup", + "gate": 13, + "run_id": "gate13-20260901-a", + "result": "passed", + "recorded_at": "2026-09-02T08:54:19Z", + "goal": "Replay the successful manual Gate 13 clean-host desktop procedure automatically on Windows and Linux against the paid public route, without manual UI recovery.", + "authorization": { + "evidence": "gate13-20260901-a-cost-authorization.json", + "owner_reset_recorded": true, + "maximum_lifetime_reservation_usd": "56.00", + "maximum_is_not_a_bill_forecast": true, + "cost_note": "The owner reports the comparable real-world replay cost is approximately USD 10. Final provider billing for this run was not available at cleanup time." + }, + "production_packages": { + "requested_source_commit": "e904d36416a4f186c0bec05ff20210df9ca19848", + "workflow_merge_commit": "f83c19997d6180c784e2a85f8d5d68c4361ad99e", + "workflow_merge_parents": [ + "f64a388a47b098ac7f69d2affc59816376b43bb1", + "e904d36416a4f186c0bec05ff20210df9ca19848" + ], + "workflow_run": 33600715239, + "workflow_result": "success", + "style_run": 33600715224, + "style_result": "success", + "test_run": 33600715198, + "test_result": "success", + "windows": { + "workflow_artifact": "communityai-desktop-install-windows", + "artifact_id": 9835635064, + "wrapper_sha256": "sha256:d2e6a90b881838b6f738924c3dc222cb7056bf121dc9ed91c56ea055760aa329", + "wrapper_bytes": 2695087981, + "archive_sha256": "sha256:965c24c3235dd5e4621961376e0d563bb50e81ed214297e2826c0a2454accfe5", + "archive_bytes": 2695087805, + "audit_artifact_id": 9835635695, + "audit_artifact_sha256": "sha256:2007bca3fd77d7debacf2fec7f2975b6ee3e3a7d02f29e8baa7014915d57b407" + }, + "linux": { + "workflow_artifact": "communityai-desktop-install-linux", + "artifact_id": 9835679452, + "wrapper_sha256": "sha256:2712b6adc33f9b932b359afe97764a4361dc6f286342c52b46d09aa536a7389d", + "wrapper_bytes": 3360754507, + "archive_sha256": "sha256:9f7c8629f3f91f1a1b291e2f3f7e1019d1497440ed399b08c837c887f2b0107a", + "archive_bytes": 3360754329, + "audit_artifact_id": 9835680252, + "audit_artifact_sha256": "sha256:dc47d2406b84fba40811b9d2bec44e2e99c18992e049179712ea6bb76485d543" + } + }, + "manual_findings_translated": [ + { + "source": "FLUJO conversation 264a0383-8cbf-4e0a-9073-1ae6072d19fe narration and tool calls", + "finding": "The first inference can return Model unavailable while the exact selected model is still becoming complete.", + "automation": "Poll /v1/models every five seconds for up to 90 seconds, then replay the exact one-token model:auto request once." + }, + { + "source": "FLUJO manual desktop sequence", + "finding": "Sharing controls are page-scoped and the manual run navigated to Sharing before editing policy or clicking master controls.", + "automation": "Open the Sharing page before every sharing action and retain the accessible-name, toggle, selection, legacy, and focus/Enter control fallbacks." + }, + { + "source": "FLUJO sequence 17517-17532", + "finding": "The manual run saved the policy, toggled Share compute with the selected model, clicked Start sharing, observed it, then clicked Pause sharing.", + "automation": "If policy reconciliation already enabled sharing, click the exact checked per-model control to restore the paused baseline before exercising literal Start and Pause." + } + ], + "automation_source": { + "inference_recovery_commit": "05fbe4fd40e1daa9f33fabfe6ca5fedc9a6798d6", + "sharing_page_commit": "984aef348c2b09e1a8383bb74873525d62065db8", + "automatic_sharing_normalization_commit": "e904d36416a4f186c0bec05ff20210df9ca19848", + "route_service_timeout_commit": "b093b850625235b9fa6a10605d15005511ee15f6", + "route_stale_advertisement_commit": "66f440bc02d7920b0a697b4095c243c0ff17ae78", + "fresh_linux_supervisor_commit": "4c6eaca8b0de8d20885c932e5bdcd1f50fc67947", + "automated_playthrough": { + "path": "scripts/gate13_automated_playthrough.py", + "sha256": "sha256:9ffda923a37ef64631898ad82139457a329393f15b78afd45709611f7f4a087f", + "bytes": 20545 + }, + "lifecycle_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "sha256": "sha256:c899ffb162aef49e7dc54c1e62a86505713652e341bd32f77f123e76bde8d1d4", + "bytes": 33278 + }, + "final_host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "sha256": "sha256:79a3220ff06faa1c359feb771903d3d842c004b122244cfef161945403fe91d9", + "bytes": 45781 + }, + "final_route_fence": { + "path": "scripts/gate13_route_fence.py", + "sha256": "sha256:e90f48bbf2e582ef7bae47e71c8e635e69f94dc86c8a7b5197a0d079af1bd5a5", + "bytes": 9907 + } + }, + "prequalification_diagnostics": { + "manual_ui_assistance_accepted_as_qualification": false, + "client_attempts_started_before_final_windows_run": 0, + "client_attempts_started_before_final_linux_run": 0, + "findings": [ + "A Windows diagnostic run proved that clicking master controls while the Sharing page was hidden could wait indefinitely; navigation was added and tested.", + "A later Windows diagnostic showed policy reconciliation can auto-start the selected worker before the automation reaches Start; the exact FLUJO per-model toggle sequence was restored.", + "The first Linux route fence failed closed after an API-side bootstrap startup failure. A later live retry exposed and fixed the bounded service-action and stale-DHT-advertisement cases.", + "The Linux host adapter rejected the fresh host before creating a unit because Ubuntu omits ExecStart for LoadState=not-found; the exact fresh inventory is now accepted while extra fields remain rejected." + ] + }, + "route": { + "instance": "route-20260901-a-node", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "runtime_source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "windows_fence": { + "result": "passed", + "target": "windows", + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "covered_blocks": 24, + "total_blocks": 24, + "peer_count_minimum": 1, + "stable_rechecks": 2, + "standby_service_stopped": true + }, + "linux_fence": { + "result": "passed", + "target": "linux", + "model_id": "Gemma 4 E2B IT", + "manifest_digest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "covered_blocks": 35, + "total_blocks": 35, + "peer_count_minimum": 1, + "stable_rechecks": 2, + "standby_service_stopped": true + } + }, + "windows": { + "result": "passed", + "attempt_ordinal": 1, + "instance": "gate13-20260901-a-win", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "ordinary_user": "M", + "interactive_console_session": true, + "manual_ui_actions_after_launch": 0, + "evidence_digest": "sha256:764c63b24b0339ded38862d26787b236699ab7c2fc4c15ddaf2717218c4e5adb", + "real_window_sessions": 2, + "localhost_inference_count": 1, + "policy_dialog_saved": true, + "start_clicked": true, + "start_observation_seconds": 25.0, + "pause_control_observed": true, + "pause_clicked": true, + "sharing_intent_paused": true, + "session_duration_seconds": { + "initial": 260.828, + "restart": 66.328 + }, + "qualification_temporaries_removed": true + }, + "linux": { + "result": "passed", + "attempt_ordinal": 1, + "instance": "gate13-20260901-a-linux", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "ordinary_user": "gate13", + "display": "TCP-disabled Xvfb X11 with private D-Bus and native Secret Service", + "manual_ui_actions_after_launch": 0, + "evidence_digest": "sha256:624bcfd01763ac81b1ebfac50aee196988bbbdd30f94610997464faa6dfe637c", + "real_window_sessions": 2, + "localhost_inference_count": 2, + "policy_dialog_saved": true, + "start_clicked": true, + "start_observation_seconds": 20.0, + "restart_resume_observed": true, + "pause_control_observed": true, + "pause_clicked": true, + "sharing_intent_paused": true, + "session_duration_seconds": { + "initial": 229.269578, + "restart": 44.956091 + }, + "qualification_temporaries_removed": true + }, + "validation": { + "desktop_gate13_tests": { + "passed": 215, + "failed": 0 + }, + "route_fence_tests": { + "passed": 5, + "failed": 0 + }, + "host_job_tests": { + "passed": 28, + "failed": 0 + }, + "latest_source_style_run": 33610030286, + "latest_source_style_result": "success", + "latest_source_test_run": 33610030231, + "latest_source_test_result": "success", + "windows_evidence_validator_result": "passed", + "linux_evidence_validator_result": "passed" + }, + "cleanup": { + "exact_instances_absent": [ + "route-20260901-a-node", + "gate13-20260901-a-win", + "gate13-20260901-a-linux" + ], + "exact_disks_absent": [ + "route-20260901-a-node", + "gate13-20260901-a-win", + "gate13-20260901-a-linux" + ], + "exact_firewalls_absent": [ + "route-20260901-a-dht", + "route-20260901-a-iap", + "route-20260901-a-relay" + ], + "regional_l4_quota": { + "metric": "NVIDIA_L4_GPUS", + "limit": 1, + "usage": 0 + }, + "protected_bootstrap": { + "name": "communityai-bootstrap-1", + "status": "RUNNING" + }, + "passed": true + }, + "privacy": { + "prompts_retained": false, + "outputs_retained": false, + "token_identifiers_retained": false, + "credentials_retained": false, + "signed_urls_retained": false, + "provider_endpoints_retained": false + } +} diff --git a/docs/evidence/gate13-20260901-a-cost-authorization.json b/docs/evidence/gate13-20260901-a-cost-authorization.json new file mode 100644 index 000000000..90318ac33 --- /dev/null +++ b/docs/evidence/gate13-20260901-a-cost-authorization.json @@ -0,0 +1,220 @@ +{ + "schema_version": 1, + "scope": "gate13-automated-paid-cloud-authorization", + "gate": 13, + "run_id": "gate13-20260901-a", + "result": "authorized", + "recorded_at": "2026-09-01T20:58:50Z", + "source": { + "durable_controller_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "host_job_adapter_commit": "d5dc3537eb4a1e405ed9a6bfde0236bda0a58d7f", + "client_session_bootstrap_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "route_setup_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "package_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "production_workflow_run": 33582031380 + }, + "authorization": { + "owner_reset_recorded": true, + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "0.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "44.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-09-01", + "pricing_basis": "The owner reports the prior real-world replay cost was approximately USD 10. USD 56 is retained only as a fail-safe maximum-lifetime reservation, not as a bill forecast." + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "sha256": "5ab9bf2c49267e3b2b7cefcb2d2c9b1eb331a652970e3345ab443f09db4cf8aa", + "bytes": 34281 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "d5dc3537eb4a1e405ed9a6bfde0236bda0a58d7f", + "sha256": "3fd232fa291c849fb45b0432b556bf01a6cb22650d7911389eb533e2f95ce3ff", + "bytes": 45573 + }, + "windows_client_startup": { + "path": "scripts/gate13_windows_client_startup.ps1", + "source_commit": "e60c3577c7205ff434cad6e9396f89555626aceb", + "sha256": "3f8600c42a3c0765e100963c2e28cdef7c6b248992924ff3406941aefce7cf47", + "bytes": 8779 + }, + "linux_client_startup": { + "path": "scripts/gate13_linux_client_startup.sh", + "source_commit": "d5dc3537eb4a1e405ed9a6bfde0236bda0a58d7f", + "sha256": "72ac32fb78946ac09b60bbef571a944a018d790871fafbb818ec7006bee292c6", + "bytes": 2138 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "sha256": "c899ffb162aef49e7dc54c1e62a86505713652e341bd32f77f123e76bde8d1d4", + "bytes": 33278 + }, + "automated_playthrough": { + "path": "scripts/gate13_automated_playthrough.py", + "source_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "sha256": "9ffda923a37ef64631898ad82139457a329393f15b78afd45709611f7f4a087f", + "bytes": 20545 + }, + "route_client_fence": { + "path": "scripts/gate13_route_fence.py", + "source_commit": "238122692655c083d534f2a8359635f6588931e7", + "sha256": "6d42e80a30aaacd3f7b80c89be15af435ab6427b1c62a338cadf11ac32237772", + "bytes": 9367 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "sha256": "045372ea0be9c4a8f31756a502b2a9ec799087eeaac294ebad2b34eccfe0affc", + "bytes": 3371 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "7a42803811289e14f69835331e0fbab69dd353c70c835131c10bdfa96ca5f111", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "127ea96d5eafa908aa6221e11e86af1c05e4183e5cef05696a0a58ea381ebbc0", + "bytes": 2695083895, + "workflow_artifact": "communityai-desktop-install-windows" + }, + "linux_package": { + "sha256": "9791ffa6d3cfa86ef8aabdec518918cef65a961eb69439880306880b741cfe20", + "bytes": 3360741913, + "workflow_artifact": "communityai-desktop-install-linux" + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd" + }, + "provider_plan_digest": "sha256:3fd1e3907d431222a876fa192e8dbc08611dfcfad46cd5c865110eccf01fe547", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260901-a-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [31337, 31338], + "firewalls": ["route-20260901-a-dht", "route-20260901-a-iap"], + "service_account": false, + "scopes": [] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260901-a-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260901-a-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260901-a-client", + "host_execution": "automated_real_window_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260901-a-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260901-a-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260901-a-client", + "host_execution": "automated_real_window_dbus_session_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "route_fenced_and_revalidated_for_each_client": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": false, + "automated_gate13_replay_required": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "production_packages_passed": true + }, + "prohibited": { + "fly_resources": 0, + "macos_hosts": 0, + "service_accounts": 0, + "client_gpu_instances": 0, + "concurrent_clients": 0, + "credit_operations": 0 + }, + "privacy": { + "retain_prompts": false, + "retain_outputs": false, + "retain_credentials": false, + "retain_signed_urls": false, + "retain_provider_endpoints": false + }, + "completion": { + "result": "passed", + "recorded_at": "2026-09-02T08:54:19Z", + "qualification_evidence": "gate13-20260901-a-automated-qualification-and-cleanup.json", + "package_source_commit": "e904d36416a4f186c0bec05ff20210df9ca19848", + "final_route_fence_commit": "66f440bc02d7920b0a697b4095c243c0ff17ae78", + "final_host_job_adapter_commit": "4c6eaca8b0de8d20885c932e5bdcd1f50fc67947", + "windows_attempt_ordinal": 1, + "linux_attempt_ordinal": 1, + "manual_ui_recovery_accepted": false, + "reservation_state": "cleaned-committed-pending-provider-billing", + "provider_billing_available": false, + "cost_note": "The USD 56 value was a maximum-lifetime fail-safe reservation, not an estimate of the provider bill. The owner reports comparable real-world use at approximately USD 10.", + "exact_run_instances_absent": true, + "exact_run_disks_absent": true, + "exact_run_firewalls_absent": true, + "regional_l4_usage": 0, + "protected_bootstrap_running": true + } +} diff --git a/docs/evidence/gate13-20260905-linux-route-worker-lifecycle.md b/docs/evidence/gate13-20260905-linux-route-worker-lifecycle.md new file mode 100644 index 000000000..325c187f3 --- /dev/null +++ b/docs/evidence/gate13-20260905-linux-route-worker-lifecycle.md @@ -0,0 +1,60 @@ +# Gate 13 Linux route-worker failure, 2026-09-05 + +Run: `g13-20260905-152936-2b9a`. Inspection used read-only SSH, +journal/process listings and authenticated localhost GETs. No VM process, +service or metadata was changed; no cloud run or desktop build was dispatched. + +## Observed + +- Windows qualification passed in 375.797 seconds. +- Linux startup and its initial UI session passed. Initial inference returned + one token with all 35 Gemma blocks available (181.338214 seconds). +- The restarted Linux UI waited in `wait_ready`. Its node reported fresh + discovery observations with 0/35 Gemma blocks and no complete route. +- The route's Gemma systemd service stayed active with `NRestarts=0`, but its + internally supervised worker repeatedly shut down and restarted. At 16:41:14 + UTC it announced its blocks offline and reported a shutdown signal. +- Subsequent workers reported that the configured worker identity was already + taken. A surviving DHT process (PID 21636, PPID 1) still owned p2pd PID 21652 + using that same identity and public port 31338. The client also had multiple + orphaned worker descendants. +- The restarted UI eventually progressed but failed inference with HTTP 500 + after 1277.018252 seconds. The launcher collected this in + `linux-host-job-failure-output.json`, then deleted its run-scoped resources. + Final cleanup passed; the protected bootstrap remained running. + +## Fix + +`WorkerLaunch` equality previously included `placement_reason`. After the +planner's 15-minute residency, a fresh observation changes that explanation +(for example, minimum replicas 0 to 1) without changing the model or range. +The reconciler therefore stopped and replaced an unchanged assignment. The +regression test reproduces this with the real planner at timestamps 0 and 901. +Explanatory text is now excluded from launch equality; commands, resource +limits, admission and exact artifact bindings remain compared. + +On Linux, workers now own separate process sessions. Both graceful/forced +stops and observed worker exits kill any remaining members of that worker's +process group before allowing replacement, releasing stale DHT identities and +ports. Windows process creation and termination retain their previous path. + +The source fix was also backported onto the existing pinned route source: +`d2c93af74311bddbee516b5f35e65789449b8b07`. The replacement local wheel is +389449 bytes, SHA-256 +`edfd4598c293719d4d7701c9613b64f47f9fd20c3a2dc2e4c0fcacacad3c493a`. +Comparison with the previous wheel finds only `worker_supervisor.py` and its +wheel RECORD changed after normalizing line endings. The old wheel is retained. +The one-click config and pinned setup helper both select the replacement. + +## Validation and limits + +- Worker tests: 31 passed on Windows, including the planner reproduction and + mocked Linux/Windows graceful-stop, forced-stop and crash cleanup paths. +- Two real Linux descendant-port-release tests are present but skipped on this + Windows host; they were not represented as Linux execution. +- Backported route runtime's existing worker suite: 22 passed. Its added + lifecycle paths also passed the six platform/exit-mode regression cases. +- Gate 13 runner/provider/startup/fence/lifecycle tests: 238 passed. +- No new end-to-end GCP result is claimed. The next normal one-click run needs + a newly built desktop package to include the client-runtime source fix; + the replacement route wheel has already been built locally. diff --git a/docs/evidence/gate14-20260902-c-linux-product-actions-checkpoint.md b/docs/evidence/gate14-20260902-c-linux-product-actions-checkpoint.md new file mode 100644 index 000000000..480d84ebb --- /dev/null +++ b/docs/evidence/gate14-20260902-c-linux-product-actions-checkpoint.md @@ -0,0 +1,106 @@ +# Gate 14 Linux product-action restart checkpoint + +Recorded: 2026-09-02 (America/Bogota) + +Status: implementation checkpoint only. Gate 14 remains `IN PROGRESS`. This +document is not hardware acceptance evidence and does not authorize a paid run. + +## Why this checkpoint exists + +The operator requested a harness rebuild/restart while the concrete Gate 14 +platform handlers were being implemented. This checkpoint makes the partial +Linux slice reproducible and leaves the Windows half and all physical +qualification work explicitly open. + +GitHub Issues are disabled for `flujo-app/CommunityAI`, so [draft PR #26](https://github.com/flujo-app/CommunityAI/pull/26) +is the restart ticket. It carries the remaining implementation, verification, +staging, budget-revalidation, hardware-run, and cleanup checklist. + +## Implemented Linux slice + +- `gate14_linux_action_transport.py` source-binds and passes the concrete + product-action helper to the persistent host. +- `gate14_linux_lifecycle_actions.py` executes the exact verified helper bytes, + lazily creates one product-action session for production `prepare`, carries + it across the controller challenge into `calibrate`, and delegates exact + cleanup on success, error, malformed input, or EOF. +- `gate14_linux_product_actions.py` implements package/audit verification, + fresh warm-cache adoption and digest verification, systemd-owned packaged + startup, native credential lifecycle, exact policy and automatic-placement + observations, low-VRAM and unsupported-CPU-power rejection, worker crash + recovery, operator pause, packaged restart/cache reuse, challenge-bound + bandwidth/power/schedule suspension calibration, and exact cleanup. +- The transport contract now distinguishes a present handler with missing + physical inputs (`product-prepare-failed`) from the removed + `action-handler-unavailable` placeholder. + +Normalized source SHA-256 values: + +- product actions: + `7a904b1c4653eb2a392bb64b1f97404beaee3fff9a2102f3c9c7949f0a2aa973` +- persistent action host: + `495028ae72a6a1c37a8c718356ed7e7bbae437156b0cc8b7328ddb4fce8e1a36` +- action transport: + `e7247c57dd01498dfbdfe695079e595e5755ab99c4f9dd01c17ad200cd3c8730` +- transport test: + `1e1f50ffc9f7fe017fbb6d5662bc4f499ab649a93afcaac27ecf64486fb2f612` +- isolated product-action test after restart: + `93f16dee2d7652bc74dd5c8f80daa1c63c4197d7c40f7b187b44dd00dde4ebcb` + +## Verification completed before restart + +Using the repository's existing environments: + +```text +.venv-cuda/Scripts/python.exe -m py_compile \ + scripts/gate14_linux_product_actions.py \ + scripts/gate14_linux_lifecycle_actions.py \ + scripts/gate14_linux_action_transport.py + +.venv-cuda/Scripts/python.exe -m pytest \ + tests/test_gate14_linux_action_transport.py -q +``` + +Result: `18 passed`. An expanded transport/sequencer/entrypoint run covering +Linux, Windows, the shared lifecycle, and native entrypoint then passed `118` +tests. Black 22.3.0 reports the three implementation files and the focused test +unchanged. An independent helper also AST-parsed the partial Linux implementation +and identified successful product-action coverage as the main missing local proof. + +After restart, `tests/test_gate14_linux_product_actions.py` added controlled +Gate 13 and control-API boundaries for the complete prepare/calibrate/cleanup +success path plus failure cleanup both before and after credential creation. The +product-action and transport focus passes `21` tests, the full +`tests/test_gate14_*.py` matrix passes `201`, Black/isort checks pass in the +repository formatter environment, and Python +compilation passes. This remains no-spend software evidence, not a physical pass. + +## Deliberately incomplete + +- The Linux handler has not run against a real packaged archive, fresh Gate 9 + warm cache, systemd desktop session, L4 device, or physical resource crossing. +- The equivalent concrete Windows `prepare`/`calibrate` handler is not + implemented. +- Controller-side fresh direct-upstream cache materialization into the + `gate14-warm-cache` convention still needs to be connected and tested. +- No no-public-IP IAP staging, protected ACL installation, native remote job, + challenge checkpoint, hardware acceptance, or cleanup run was attempted. +- No readiness gate was marked passed and Gate 15 remains waiting. +- No cloud reservation or resource was created. Spend for this checkpoint is + USD 0. + +## Restart order + +1. Rebuild/restart the harness, then re-check the branch, pinned hashes, and + unrelated dirty worktree entries before editing. +2. Implement the equivalent source-bound Windows product actions and matching + contract tests. +3. Connect fresh official-source warm-cache materialization and protected + staging for both platforms. +4. Only after both platform halves are locally runnable, revalidate + authentication, inventory, quota, current pricing, and the combined USD 100 + ledger. Record a new bounded reservation before any create. +5. Run Windows then Linux sequentially on fresh no-public-IP L4 hosts, verify + exact cleanup, and only then evaluate Gate 14 acceptance. + +Do not work on credits or macOS. diff --git a/docs/evidence/gate14-20260902-d-windows-product-actions-checkpoint.md b/docs/evidence/gate14-20260902-d-windows-product-actions-checkpoint.md new file mode 100644 index 000000000..693abb3f0 --- /dev/null +++ b/docs/evidence/gate14-20260902-d-windows-product-actions-checkpoint.md @@ -0,0 +1,109 @@ +# Gate 14 Windows product-action checkpoint + +Recorded: 2026-09-02 (America/Bogota) + +Status: implementation checkpoint only. Gate 14 remains `IN PROGRESS`. This +document is not hardware acceptance evidence and does not authorize a paid run. + +## Result + +The Windows lifecycle bridge now has a concrete, persistent packaged-product +`prepare`/`calibrate`/`cleanup` implementation equivalent to the verified +Linux half. The implementation is fail-closed and was developed on top of source +`30891396916e61071492cfa056c15d1f4d94e547`. + +No GCP, Fly.io, or GitHub provider resource was created or changed while +implementing or verifying this checkpoint. No reservation was recorded and +spend was USD 0 under the combined USD 100 ceiling. + +## Implemented Windows slice + +- The action transport opens and retains Windows read handles that deny source + write/delete, verifies normalized source digests, and carries the verified + lifecycle configuration handle across the complete persistent session. +- The product action audits and installs the exact production package, runs the + packaged self-tests, adopts only the exact controller-bound warm-cache + inventory, starts the real desktop/node path, and creates exactly one native + control credential only after startup is ready. +- Initial and restart launches use the action-specific persistent node + configuration, node-data directory, and packaged bootstrap configuration. + No implicit user-profile node path may satisfy the contract. +- The handler verifies the exact sharing policy, remotely acknowledged automatic + placement, low-VRAM rejection, unsupported CPU-power behavior, crash recovery, + operator pause, packaged restart, and cache reuse. +- Bandwidth, physical-power, and schedule calibrations remain bound to the + controller challenge and verify limit crossing, worker absence, preserved + owner intent, and below-limit recovery. +- Cache inventory and deletion use native no-follow handles for the root and + every descendant. File identity, exact path/count/bytes/digest, and reparse + rejection are proved while locks remain held through drills. +- Native Job Object helpers prove membership and terminate only exact members. + Failed Job or power-burn cleanup retains its owner handle for retry. +- Cleanup is phased in process/burn, credential, cache-lock, and root order. + A failed phase preserves the package deletion tool and roots; one-shot process, + burn, and credential failures are each proved to succeed on a second cleanup. + +## Source bindings + +Normalized source SHA-256 values used by the persistent transport: + +- Gate 13 Windows packaged lifecycle: + `aa549335b63f43ef2e68f40881635ab077e916878bc472b8674424aa087a6dda` +- Gate 13 Windows packaged inference: + `2d53424c886ff4a70367a3a0844e33a234bc6c290828a21b70a134b5bf115611` +- Windows product actions: + `3a29f13ecd855fbdb21d42b21ffd3e793e8a3c1086f816a28d20f9e8cfbb2e23` +- Persistent Windows action host: + `4ebf68d5fbeb3afad9cd52a7e062162de61da4f6ecee0cd113585a20c84fdab5` + +The final Windows action transport content SHA-256 is +`ebe455967082c2ee8f93b499d9b73aff92854bd0a625a85527c038328c24abfc`. +The isolated product-action test content SHA-256 is +`7598606087777f78f1bf6799e39a85f1e0dc503efc0ba2b5b1484524983681f0`. + +## Verification + +Using the repository's existing environments: + +```text +.venv-cuda/Scripts/python.exe -m pytest \ + tests/test_gate14_windows_product_actions.py \ + tests/test_gate14_windows_action_transport.py -q +# 50 passed + +.venv-cuda/Scripts/python.exe -m pytest \ + tests/test_gate13_windows_packaged_lifecycle.py -q +# 16 passed + +.venv-cuda/Scripts/python.exe -m pytest tests/test_gate14_*.py -q +# PowerShell-expanded file list: 211 passed +``` + +PowerShell parsing passed for the Gate 13 lifecycle, Gate 14 product actions, +and persistent action host. Black, isort, Python compilation, and +`git diff --check` passed. An independent adversarial review reran the +50-test Windows focus and 16-test Gate 13 regression and returned PASS after +confirming the retryable power-burn cleanup invariant. + +## Deliberately incomplete + +- Neither platform handler has executed this checkpoint against the retained + production archives on fresh clean hosts. +- The historical Gate 9 physical caches were cleaned. Fresh direct + official-source cache materialization, its exact artifact record, and + controller-side staging are still required for both platform profiles. +- No no-public-IP Windows or Linux L4 host was created, no hardware calibration + was attempted, and no Gate 14 acceptance pass is claimed. +- Native authentication, inventory, quota, pricing, and the current-epoch + ledger must be revalidated immediately before any new bounded reservation. +- Gate 14 remains `IN PROGRESS`; Gate 15 remains waiting. + +## Next unblocked gate + +Connect and test fresh direct official-source cache materialization for the +exact Windows/Qwen and Linux/Gemma profiles. Do not create paid hosts until +that source-bound input is runnable. Then revalidate the provider and USD 100 +budget boundaries, reserve a bounded amount, run Windows then Linux +sequentially, and prove exact cleanup. + +Do not work on credits or macOS. diff --git a/docs/evidence/gate14-20260902-e-cache-materialization-handoff-checkpoint.md b/docs/evidence/gate14-20260902-e-cache-materialization-handoff-checkpoint.md new file mode 100644 index 000000000..0082bf7c5 --- /dev/null +++ b/docs/evidence/gate14-20260902-e-cache-materialization-handoff-checkpoint.md @@ -0,0 +1,130 @@ +# Gate 14 cache-materialization handoff checkpoint + +Recorded: 2026-09-02 (America/Bogota) + +Status: implementation checkpoint only. Gate 14 remains `IN PROGRESS`. This +document is not fresh-cache, packaged-execution, or hardware acceptance evidence +and does not authorize a paid run. + +## Result + +Gate 14 now has a fail-closed, two-phase official-source cache boundary for the +exact Windows/Qwen and Linux/Gemma profiles. The implementation was developed on +top of source `f686016b8c41d7756682deabee5967d661be5c73`. + +The ordinary materialization phase can read a controller-owned canonical plan +and lifecycle template, download only the exact manifest-selected files from +the official source, verify the exact cache tree, and leave a canonical +record/binding/handoff below its writable work root. It cannot write the +controller staging root. A separate privileged promotion phase revalidates the +plan, source files, template, record, binding, handoff, and physical cache +before creating the protected lifecycle record and configuration. + +No GCP, Fly.io, GitHub provider, reservation, or cloud resource was created or +changed. No production model artifact was downloaded. This checkpoint spent +USD 0 under the combined USD 100 ceiling. + +## Implemented boundary + +- The protected plan binds the platform, exact manifest filename, source commit, + disjoint absolute work/staging roots, lifecycle-template digest, and normalized + SHA-256 of the materializer, lifecycle, acquisition, and manifest sources. +- Template canonical form, digest, platform/model/manifest/source identity, and + exact roots are verified before any transport check, cache creation, or + acquisition call. A predictably unpromotable input therefore cannot trigger + the multi-gigabyte transfer. +- Official transport rejects Hugging Face endpoint overrides, HTTP/HTTPS/all + proxies from both environment and system proxy discovery, Requests/cURL CA + overrides, and `SSL_CERT_FILE`/`SSL_CERT_DIR`. Acquisition uses no token, + requires direct upstream, and permits at most three resumptions. +- The cache verifier accepts only the exact manifest-artifacts tree. It rejects + symlinks/reparse points, special or extra entries, case collisions, changed + directory inventories, wrong sizes/digests, and incomplete selections. + Windows uses no-follow share-read-only native handles; POSIX uses + `O_NOFOLLOW` handles. Open-handle and path identities are rechecked before + release. +- The materialization record must prove an empty cold cache, complete + direct-upstream/no-mirror transfer, exact repository/revision/dtype, + privacy-safe output, and the target runtime platform. Its lifecycle binding + now carries the source commit, plan digest, and complete materializer-source + digest. +- Promotion structurally validates root/SYSTEM-or-Administrators ownership and + protected modes/DACLs without imposing the qualification token's write-denial + rule on the controller itself. Normal lifecycle loading and every action + boundary retain the stricter proof that the qualification process cannot + mutate staging. +- Newly promoted Windows files receive an explicit protected DACL owned by + Administrators and granting full access only to SYSTEM and Administrators + before structural validation. POSIX promoted files must be root-owned and + mode `0600`. +- Verified staged record/config creation is an explicit commit point. + Interrupted source-handoff deletion preserves those protected outputs; a + retry revalidates the staged files and exact cache, then idempotently removes + whatever handoff members remain. Protection or pre-commit write failure + removes staged partials while leaving the complete work handoff retryable. +- Cleanup and no-clobber writes use bounded retries and return only generic + errors/results without private paths, URLs, credentials, or response bodies. + +## Source and test identities + +Final working-tree SHA-256 values: + +- Cache materializer: + `e6066464fc7aecc78a8df18fef8ce5919b82fe19e6922737e2b29b824f60172e` +- Packaged lifecycle: + `a62469ea1ca1a764606add76b4e4913284d260014d6273cc9650ee722f0719bb` +- Cache-materializer tests: + `e4cc177c456cb1f234fb9c81e1a9584cb5fade0cbdedf36b8b0e8f562e1c1fa4` +- Packaged-lifecycle tests: + `9084f7f94d0f2336ab9e6b1e2812423072c71ea4efaac4b33c3d17e269a6beb2` + +The controller plan records fresh normalized source digests at creation time and +both phases recompute them before accepting or promoting a handoff. Those plan +values, not this narrative, are the runtime trust input. + +## Verification + +Using the repository's existing environments: + +```text +.venv-cuda/Scripts/python.exe -m pytest \ + tests/test_gate14_cache_materializer.py \ + tests/test_gate14_packaged_lifecycle.py -q +# 96 passed + +.venv-cuda/Scripts/python.exe -m pytest tests/test_gate14_*.py -q +# PowerShell-expanded file list: 253 passed +``` + +Black, isort, Python compilation, and scoped `git diff --check` passed. An +independent adversarial review reproduced the 96-test focus and 253-test full +matrix. It returned PASS after verifying privileged default promotion, explicit +Windows output protection, rollback before the commit point, retry after partial +handoff cleanup, and template rejection before acquisition. + +## Deliberately incomplete + +- Neither exact production profile has been freshly materialized from the + official source on its native clean host. The Windows/Qwen and Linux/Gemma + selected caches total 14,850,015,469 bytes across the two sequential runs. +- The GCP host bootstrap/controller does not yet create the protected plan and + lifecycle template or invoke the ordinary materializer and privileged + promoter in the required identities. That integration must be source-bound + and tested before provisioning. +- No retained production archive has passed this new boundary on a fresh host, + no hardware calibration was attempted, and no Gate 14 acceptance pass is + claimed. +- Native authentication, inventory, quota, pricing, and the current-epoch + ledger must be revalidated immediately before any future bounded reservation. +- Gate 14 remains `IN PROGRESS`; Gate 15 remains waiting. + +## Next unblocked gate + +Wire controller-owned plan/template creation and the two materialization phases +into the exact Windows/Linux host bootstrap, then exercise the no-download +preflight and native cleanup paths. Do not create paid hosts until that +source-bound integration is runnable. Afterward, revalidate provider and USD +100 boundaries, record a bounded reservation, run Windows then Linux +sequentially, and prove exact cleanup. + +Do not work on credits or macOS. diff --git a/docs/evidence/gate14-20260902-f-promoted-input-readability-checkpoint.md b/docs/evidence/gate14-20260902-f-promoted-input-readability-checkpoint.md new file mode 100644 index 000000000..37312e910 --- /dev/null +++ b/docs/evidence/gate14-20260902-f-promoted-input-readability-checkpoint.md @@ -0,0 +1,34 @@ +# Gate 14 promoted-input readability checkpoint + +- Date: 2026-09-02 (America/Bogota) +- Gate: 14, automatic contribution and resource-control hardware checks +- Result: software checkpoint passed; Gate 14 remains in progress +- Cloud spend: USD 0 +- Provider resources created: none +- Production model bytes downloaded: none + +## Blocker closed + +Independent bootstrap-boundary review found that promotion made the final lifecycle configuration and cache-materialization record unreadable by the ordinary identity that must execute the durable host job. POSIX promotion used root-owned mode `0600`; the Windows protected DACL granted access only to SYSTEM and Administrators. Both forms preserved controller ownership but prevented `gate14` on Linux or the limited Windows desktop user from loading the promoted inputs. + +Promotion now preserves the privilege split while allowing the required read path: + +- POSIX files are root-owned mode `0644`: all identities may read, but only the controller owner may modify them. +- Windows files retain a protected DACL with SYSTEM and Administrators full control and Authenticated Users generic read: the ordinary job identity can read, but receives no write, delete, owner-change, or DACL-change grant. +- The existing lifecycle verifier still checks structural controller ownership, and the ordinary lifecycle path still performs the independent write-denial probes. + +## Verification + +- Focused cache/lifecycle suite: `97 passed` +- Complete Gate 14 suite: `254 passed` +- Black, isort, py_compile, and diff checks: passed +- Independent adversarial review: passed; it confirmed no ACL or deletion blocker and reproduced 97 focused tests plus the targeted protection subset + +Verified working-tree SHA-256 values: + +- `scripts/gate14_cache_materializer.py`: `0e79c4acf8c3cb39af4d7e5787636187e733c8ffd2cee1cdf985b7965077a0c7` +- `tests/test_gate14_cache_materializer.py`: `3314437c0b0fe7062cf84454501b7db90b75c2bccd8e802ab75e42f13eac249b` + +## Remaining Gate 14 work + +Build and verify the source-bound Windows/Linux host bootstrap around the corrected readable/nonwritable staging contract. The bootstrap must prepare exact package/audit/source inputs, run ordinary materialization then privileged promotion, start the ordinary durable host job, transport the checkpoint/challenge/evidence without broadening write access, and prove exact cleanup before any paid host is created. Matching production desktop artifacts for the eventual integration source are also still required. diff --git a/docs/evidence/gate14-20260903-g-packaged-acquirer-identity-checkpoint.md b/docs/evidence/gate14-20260903-g-packaged-acquirer-identity-checkpoint.md new file mode 100644 index 000000000..7ba73634d --- /dev/null +++ b/docs/evidence/gate14-20260903-g-packaged-acquirer-identity-checkpoint.md @@ -0,0 +1,39 @@ +# Gate 14 packaged-acquirer identity checkpoint + +- Date: 2026-09-03 (America/Bogota) +- Gate: 14, automatic contribution and resource-control hardware checks +- Result: software checkpoint passed; Gate 14 remains in progress +- Cloud spend: USD 0 +- Provider resources created: none +- Production model bytes downloaded: none + +## Blocker closed + +Independent review found that the first clean-host acquisition bridge hashed the packaged node and manifest through temporary handles, closed those handles, and then reopened both inputs by pathname during launch. A substituted executable could therefore run before the post-execution digest check detected the change. + +The controller-only acquisition path now binds verified bytes to child execution: + +- The materializer reads the bounded 65,536-byte manifest once through a no-follow locked handle, retains that handle for the child lifetime, and sends those exact bytes through stdin. +- `edge-acquire --manifest_stdin_sha256` requires the exact raw-byte digest, rejects path-plus-stdin, empty, oversized, changed, malformed UTF-8/JSON, and duplicate-key inputs, and parses the captured bytes without reopening a pathname. The ordinary `edge-acquire ` interface remains compatible. +- Windows retains a `CreateFileW` handle opened for read with share-read only while `CreateProcess` launches the original path. Native probes prove write, delete, and replacement fail while the handle is held, a copied executable still launches, and mutation succeeds only after release. +- Linux launches with the original onedir-compatible `argv[0]` but overrides the executed image with `/proc/self/fd/`, validates the procfs descriptor identity, and passes only that descriptor to the child. +- Both handles are revalidated after child completion and released in unconditional cleanup. Child input, output, timeout, shell, environment, working directory, and anonymous-Hub behavior remain bounded. + +## Verification + +- Focused cache-materializer/acquisition suite: `64 passed, 1 skipped`; the skip is the Linux-native fd execution probe on this Windows host. +- Complete Gate 14 suite: `264 passed, 1 skipped`. +- Packaged-node dispatch regression: `4 passed`. +- Black, isort, py_compile, and diff checks: passed. +- Independent adversarial review: passed with no commit blocker. + +Verified staged-blob SHA-256 values: + +- `scripts/gate14_cache_materializer.py`: `7f7a27f8f07a48afbe4d27a376512e89c141df405880fbb5c0d4d26a409cfb56` +- `src/drift/cli/run_edge_acquisition.py`: `255989467ad245ca24d2f2f0f360b3faa0e9ac5c93a4646bbf61a6698732e942` +- `tests/test_gate14_cache_materializer.py`: `d67984e7805054f3604d32af903fcd51b9a460feb9eab772039d58e75a355a0d` +- `tests/test_edge_acquisition.py`: `82826dbb9c2729228300616f9baf572f6f6c7cb02e29167c0f7e9ea96be3a7df` + +## Remaining Gate 14 work + +This checkpoint does not prove a real packaged-node acquisition or a fresh model download. The next no-spend bootstrap slice must verify the complete extracted PyInstaller onedir runtime against the existing release-audit `SHA256SUMS` inventory, protect every executable and sidecar from the ordinary qualification identity, stage the exact plan/template/manifest, run the packaged `edge-acquire --help` preflight on native Windows and Linux, and prove cleanup. Only after that bridge passes may the controller revalidate authentication, inventory, quota, pricing, and the combined USD 100 ceiling before creating sequential fresh L4 hosts. diff --git a/docs/evidence/gate14-20260907-codeql-triage.md b/docs/evidence/gate14-20260907-codeql-triage.md new file mode 100644 index 000000000..669c351be --- /dev/null +++ b/docs/evidence/gate14-20260907-codeql-triage.md @@ -0,0 +1,43 @@ +# September 7 CodeQL key-handling review + +Source: `76b6d84fc52342af4fd2315926b187aaa36b1378`. +Python analysis: `1737656484`. The CodeQL summary initially failed with two +high-severity findings, despite passing functional, style and package jobs. +The exact SARIF source-to-sink paths were inspected; both are false positives +for the implemented credential types. The scan and its rules remain enabled. + +## Alert 11: secret-resource name classified as private material + +The logging rule starts at the literal assigned to `SECRET` in +`scripts/catalog_key_backup.py:13`, carries that value through the report's +`secret` field and reaches the final JSON print. That literal identifies a +Secret Manager resource; it is not its secret payload. The private PEM bytes +returned by the CLI are captured, parsed in memory and omitted from the report. +Subprocess output is also excluded from error messages. The printed signing +identity is the SHA-256 of the public key's DER encoding. + +This review did not retrieve the online private key or execute the recovery +drill. It inspected source and the scanner's own data-flow record. + +## Alert 6: bearer tokens classified as passwords + +The hash store in `src/drift/node/keys.py` holds opaque API bearer tokens. +Created client keys, bootstrap keys and the Fly probe use +`secrets.token_urlsafe(32)`, with 256 random bits. Native credential storage's +`get_password` API returns the generated control token; its API name does not +make the stored value a human password. The control-token class is validated. +Domain-separated SHA-256 and constant-time comparison are appropriate for those +high-entropy token values; a slow password KDF is not needed for generated tokens. + +The advanced headless `--api-key` import also accepts caller-provided bearer +tokens. Their entropy remains the operator's responsibility. Use a generated +token with at least 32 random bytes; do not substitute a human password or a +short memorable string. No claim is made that SHA-256 strengthens weak tokens +or that the importer can prove a caller's randomness. Ordinary desktop users +receive generated keys through the API-key creation flow. + +All ten API-key regression tests passed, including key creation, persistence, +revocation and rotation behavior. No hashing/storage implementation, test, +security rule or branch protection was changed for this review. The two +specific alerts were dismissed with these explanations. GitHub's API confirmed +`state: dismissed` and `dismissed_reason: false positive` for alerts 11 and 6. diff --git a/docs/evidence/gate14-20260907-final-resource-acceptance.json b/docs/evidence/gate14-20260907-final-resource-acceptance.json new file mode 100644 index 000000000..97f212a16 --- /dev/null +++ b/docs/evidence/gate14-20260907-final-resource-acceptance.json @@ -0,0 +1,1028 @@ +{ + "date": "2026-09-07", + "feature_source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "scope": "Gate 14 bounded Windows/Linux resource acceptance", + "result": "passed", + "platforms": { + "windows": { + "scope": "native-packaged-Qwen-resource-controls", + "result": "passed", + "platform": "nt", + "device": "cuda:0", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "observations": [ + { + "stage": "initial", + "step": 0, + "action": "observe", + "vram_percent": 100, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 1, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 2, + "action": "start", + "worker_state": "running", + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.062000000005355105, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 46.329787234042556, + "baseline_sample_count": 15, + "baseline_mean_percent": 9, + "workload_sample_count": 94 + }, + "request_count": 325, + "mean_seconds": 0.0588553846157335, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 3, + "action": "limits", + "vram_percent": 25, + "processing_percent": 50, + "old_worker_tree_gone": true, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.09400000001187436, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 33.361702127659576, + "baseline_sample_count": 15, + "baseline_mean_percent": 8.733333333333333, + "workload_sample_count": 94 + }, + "request_count": 173, + "mean_seconds": 0.1095606936422793, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 4, + "action": "limits", + "vram_percent": 20, + "processing_percent": 25, + "old_worker_tree_gone": true, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 1717921382, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.21899999998277053, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 23.412371134020617, + "baseline_sample_count": 15, + "baseline_mean_percent": 11.533333333333333, + "workload_sample_count": 97 + }, + "request_count": 77, + "mean_seconds": 0.2582857142861739, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 5, + "action": "limits", + "vram_percent": 1, + "processing_percent": 25, + "old_worker_tree_gone": true, + "worker_state": "paused", + "policy_persisted": true, + "low_memory_blocked_without_worker": true, + "resource_reason": "selected blocks exceed the VRAM budget; increase VRAM or contribute fewer blocks", + "no_restart_loop": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 6, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 7, + "action": "pause", + "old_worker_tree_gone": true, + "worker_state": "paused", + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 8, + "action": "start", + "worker_state": "running", + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 9, + "action": "pause", + "old_worker_tree_gone": true, + "worker_state": "paused", + "local_inference_tokens": 3 + }, + { + "stage": "restart", + "step": 0, + "action": "observe", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + } + ], + "initial_owned_tree_gone": true, + "restart_owned_tree_gone": true, + "same_Qwen_outputs_at_all_processing_limits": true, + "admission_guards": { + "result": "passed", + "packaged": true, + "scope": "resource-admission-guards", + "checks": { + "schedule": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": false, + "schedule_reason": "outside the configured contribution schedule", + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + }, + "power": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "power usage 63.95 W exceeds the 1.00 W contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": 63.955, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + }, + "bandwidth": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "bandwidth usage 19.31 Mbps exceeds the 0.00 Mbps contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": 19.306816, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + }, + "storage": { + "state": "paused", + "pid": null, + "policy_admitted": false, + "policy_reason": "Qwen3.8 27B FP8 Dequant: exact block artifact planning failed: ManifestError", + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + } + }, + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "complete_gate14": false, + "limitations": [ + "Power and bandwidth are sampled host telemetry pause guards, not OS hard caps or traffic shapers.", + "Tiny thresholds prove blocked admission; sustained load, overshoot and automatic resumption are separate checks.", + "Storage checks declared manifested artifact admission, not total disk-cache quota.", + "Control API admission test; literal desktop slider acceptance is recorded separately." + ], + "platform": "nt", + "node_stopped": true + }, + "test_credential_removed": true, + "gui_stopped": true, + "ui": { + "initial": { + "scope": "packaged-resource-controls", + "frozen": true, + "result": "passed", + "steps": [ + { + "index": 0, + "action": "observe", + "vram_percent": 100, + "processing_percent": 100, + "seconds": 21.031, + "action_seconds": 0.203, + "saved_vram": "100%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "100%", + "processing_display": "100%", + "message": "Sharing stays off until you start it. Use Pause sharing to stop completely." + }, + { + "index": 1, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "seconds": 34.031, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Sharing remains paused." + }, + { + "index": 2, + "action": "start", + "seconds": 40.734, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Sharing remains paused." + }, + { + "index": 3, + "action": "limits", + "vram_percent": 25, + "processing_percent": 50, + "seconds": 83.187, + "action_seconds": 0.406, + "saved_vram": "25%", + "saved_processing_percent": 50.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "50%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 4, + "action": "limits", + "vram_percent": 20, + "processing_percent": 25, + "seconds": 130.515, + "action_seconds": 0.406, + "saved_vram": "20%", + "saved_processing_percent": 25.0, + "sharing_intent": true, + "vram_display": "20%", + "processing_display": "25%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 5, + "action": "limits", + "vram_percent": 1, + "processing_percent": 25, + "seconds": 182.109, + "action_seconds": 1.016, + "saved_vram": "1%", + "saved_processing_percent": 25.0, + "sharing_intent": true, + "vram_display": "1%", + "processing_display": "25%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 6, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "seconds": 208.718, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 7, + "action": "pause", + "seconds": 228.218, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 8, + "action": "start", + "seconds": 229.64, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 9, + "action": "pause", + "seconds": 249.14, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + } + ], + "opt_in_policy_dialog": true + }, + "restart": { + "scope": "packaged-resource-controls", + "frozen": true, + "result": "passed", + "steps": [ + { + "index": 0, + "action": "observe", + "vram_percent": 25, + "processing_percent": 100, + "seconds": 17.297, + "action_seconds": 0.203, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Sharing stays off until you start it. Use Pause sharing to stop completely." + } + ] + } + }, + "package": { + "source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "source_tree": "474edae23a04c7be8d55241cab7fdbe8255b215b", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5950, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "6a56cbc578f08e6fc56643767240c25a2bbf6ad8c16a79b575a970746c6a562e", + "size_bytes": 2694038865 + } + }, + "environment": "Windows 10 Pro 10.0.19045; non-elevated user; RTX 2070 SUPER 8 GiB" + }, + "linux": { + "scope": "native-packaged-Qwen-resource-controls", + "result": "passed", + "platform": "posix", + "device": "cuda:0", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "desktop_sha256": "41383b5e995a2f200f7c96b0fdb8764b7d9edae89ba8efbba0a93d80d883f0aa", + "node_sha256": "313f8cbe8b1329b0025e98e64cd12eb2abbadb5e0083f5de5360d997b23d4aa0", + "observations": [ + { + "stage": "initial", + "step": 0, + "action": "observe", + "vram_percent": 100, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 1, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 2, + "action": "start", + "worker_state": "running", + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.04737583298992831, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 56.45263157894737, + "baseline_sample_count": 15, + "baseline_mean_percent": 13.866666666666667, + "workload_sample_count": 95 + }, + "request_count": 384, + "mean_seconds": 0.0486751173381208, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 3, + "action": "limits", + "vram_percent": 25, + "processing_percent": 50, + "old_worker_tree_gone": true, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.07881049650313798, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 36.126315789473686, + "baseline_sample_count": 15, + "baseline_mean_percent": 10.733333333333333, + "workload_sample_count": 95 + }, + "request_count": 238, + "mean_seconds": 0.08113217690312441, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 4, + "action": "limits", + "vram_percent": 20, + "processing_percent": 25, + "old_worker_tree_gone": true, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 1717921382, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.2036138964976999, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 25.63157894736842, + "baseline_sample_count": 15, + "baseline_mean_percent": 12, + "workload_sample_count": 95 + }, + "request_count": 86, + "mean_seconds": 0.23134880359311652, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 5, + "action": "limits", + "vram_percent": 1, + "processing_percent": 25, + "old_worker_tree_gone": true, + "worker_state": "paused", + "policy_persisted": true, + "low_memory_blocked_without_worker": true, + "resource_reason": "selected blocks exceed the VRAM budget; increase VRAM or contribute fewer blocks", + "no_restart_loop": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 6, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 7, + "action": "pause", + "old_worker_tree_gone": true, + "worker_state": "paused", + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 8, + "action": "start", + "worker_state": "running", + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 9, + "action": "pause", + "old_worker_tree_gone": true, + "worker_state": "paused", + "local_inference_tokens": 3 + }, + { + "stage": "restart", + "step": 0, + "action": "observe", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + } + ], + "initial_owned_tree_gone": true, + "restart_owned_tree_gone": true, + "same_Qwen_outputs_at_all_processing_limits": true, + "admission_guards": { + "result": "passed", + "packaged": true, + "scope": "resource-admission-guards", + "checks": { + "schedule": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": false, + "schedule_reason": "outside the configured contribution schedule", + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + }, + "power": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "power usage 53.81 W exceeds the 1.00 W contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": 53.808, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + }, + "bandwidth": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "bandwidth usage 0.05 Mbps exceeds the 0.00 Mbps contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": 0.04778019502166443, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + }, + "storage": { + "state": "paused", + "pid": null, + "policy_admitted": false, + "policy_reason": "Qwen3.8 27B FP8 Dequant: exact block artifact planning failed: ManifestError", + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147401728, + "start_rejected": true, + "local_tokens": 3 + } + }, + "node_sha256": "313f8cbe8b1329b0025e98e64cd12eb2abbadb5e0083f5de5360d997b23d4aa0", + "complete_gate14": false, + "limitations": [ + "Power and bandwidth are sampled host telemetry pause guards, not OS hard caps or traffic shapers.", + "Tiny thresholds prove blocked admission; sustained load, overshoot and automatic resumption are separate checks.", + "Storage checks declared manifested artifact admission, not total disk-cache quota.", + "Control API admission test; literal desktop slider acceptance is recorded separately." + ], + "platform": "posix", + "node_stopped": true + }, + "test_credential_removed": true, + "gui_stopped": true, + "ui": { + "initial": { + "scope": "packaged-resource-controls", + "frozen": true, + "result": "passed", + "steps": [ + { + "index": 0, + "action": "observe", + "vram_percent": 100, + "processing_percent": 100, + "seconds": 11.906, + "action_seconds": 0.2, + "saved_vram": "100%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "100%", + "processing_display": "100%", + "message": "Sharing stays off until you start it. Use Pause sharing to stop completely." + }, + { + "index": 1, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "seconds": 59.505, + "action_seconds": 0.199, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Sharing remains paused." + }, + { + "index": 2, + "action": "start", + "seconds": 64.521, + "action_seconds": 0.215, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Sharing remains paused." + }, + { + "index": 3, + "action": "limits", + "vram_percent": 25, + "processing_percent": 50, + "seconds": 109.506, + "action_seconds": 1.6, + "saved_vram": "25%", + "saved_processing_percent": 50.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "50%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 4, + "action": "limits", + "vram_percent": 20, + "processing_percent": 25, + "seconds": 149.105, + "action_seconds": 1.199, + "saved_vram": "20%", + "saved_processing_percent": 25.0, + "sharing_intent": true, + "vram_display": "20%", + "processing_display": "25%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 5, + "action": "limits", + "vram_percent": 1, + "processing_percent": 25, + "seconds": 188.706, + "action_seconds": 1.2, + "saved_vram": "1%", + "saved_processing_percent": 25.0, + "sharing_intent": true, + "vram_display": "1%", + "processing_display": "25%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 6, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "seconds": 201.712, + "action_seconds": 0.207, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 7, + "action": "pause", + "seconds": 215.905, + "action_seconds": 1.399, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 8, + "action": "start", + "seconds": 216.905, + "action_seconds": 0.2, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": true, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + }, + { + "index": 9, + "action": "pause", + "seconds": 230.905, + "action_seconds": 1.399, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Limits saved. Previously selected sharing resumed." + } + ], + "opt_in_policy_dialog": true + }, + "restart": { + "scope": "packaged-resource-controls", + "frozen": true, + "result": "passed", + "steps": [ + { + "index": 0, + "action": "observe", + "vram_percent": 25, + "processing_percent": 100, + "seconds": 11.256, + "action_seconds": 0.2, + "saved_vram": "25%", + "saved_processing_percent": 100.0, + "sharing_intent": false, + "vram_display": "25%", + "processing_display": "100%", + "message": "Sharing stays off until you start it. Use Pause sharing to stop completely." + } + ] + } + }, + "package": { + "source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "source_tree": "436c44cc1aa0ae4f4e05fdefb7bf17caa39d9a5f", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5957, + "format": "tar.gz", + "path": "communityai-desktop-linux.tar.gz", + "platform": "Linux", + "preserves_executable_modes": true, + "preserves_internal_file_symlinks": true, + "schema_version": 1, + "sha256": "e554d38e53fa75361b91c60da949f92d6f04c9067d493899e2c77846a9201bbc", + "size_bytes": 5007861125 + } + }, + "environment": "Debian 12 container on WSL2; ordinary UID 1000; X11/Xvfb; native Secret Service; RTX 2070 SUPER GPU passthrough. Archive built locally on Ubuntu 22.04 with Python 3.12.14; libxcb-shape0 installed and optional Triton JIT excluded from the frozen runtime.", + "supplemental_runtime_probes": { + "cpu": { + "scope": "synthetic-production-runtime-processing-budget", + "device": "cpu", + "complete_gate14": false, + "runs": [ + { + "requested_percent": 100, + "measured_compute_duty_percent": 95.08319450571467, + "wall_seconds": 4.016823772995849, + "compute_seconds": 3.8193243610294303, + "steps": 137 + }, + { + "requested_percent": 50, + "measured_compute_duty_percent": 48.2654796181736, + "wall_seconds": 4.003402884001844, + "compute_seconds": 1.932261603011284, + "steps": 69 + }, + { + "requested_percent": 25, + "measured_compute_duty_percent": 24.566197332666366, + "wall_seconds": 4.0159086149942596, + "compute_seconds": 0.9865560350590385, + "steps": 36 + } + ], + "result": "passed", + "limitations": [ + "Compute duty cycle, not an instantaneous whole-device utilization guarantee", + "No Qwen model, download, cold startup or final-package lifecycle exercised", + "Local inference, downloads and other applications are outside the sharing compute budget" + ] + }, + "gpu": { + "scope": "synthetic-production-runtime-processing-budget", + "device": "cuda:0", + "complete_gate14": false, + "runs": [ + { + "requested_percent": 100, + "measured_compute_duty_percent": 96.55541056562619, + "wall_seconds": 4.020132609002758, + "compute_seconds": 3.8816555459052324, + "steps": 153 + }, + { + "requested_percent": 50, + "measured_compute_duty_percent": 48.83378123868669, + "wall_seconds": 4.0422293479932705, + "compute_seconds": 1.973973436965025, + "steps": 78 + }, + { + "requested_percent": 25, + "measured_compute_duty_percent": 24.487676398175363, + "wall_seconds": 4.0381050940050045, + "compute_seconds": 0.9888381080381805, + "steps": 39 + } + ], + "hardware": "NVIDIA GeForce RTX 2070 SUPER", + "memory_allocator": { + "cap_bytes": 268435456, + "under_limit_succeeded": true, + "over_limit_rejected": true, + "device_total_bytes": 8589606912 + }, + "result": "passed", + "limitations": [ + "Compute duty cycle, not an instantaneous whole-device utilization guarantee", + "No Qwen model, download, cold startup or final-package lifecycle exercised", + "Local inference, downloads and other applications are outside the sharing compute budget" + ] + } + } + } + }, + "source_comparison": { + "windows_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "linux_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "linux_parents": [ + "d2608aed6e89b82c34c6f769341559103c517d19" + ], + "changed_paths": [ + ".github/workflows/desktop.yaml", + "desktop/build_desktop.py", + "desktop/installers/README.md", + "desktop/installers/build_deb.py", + "docs/NODE_CONFIG_V1.md", + "docs/evidence/gate14-20260907-codeql-triage.md", + "docs/evidence/gate15-20260907-frozen-windows-installer.json", + "scripts/qualify_qwen_resource_desktop.py" + ], + "application_and_catalog_inputs_identical": true, + "linux_packaging_changes": "Declare libxcb-shape0 and exclude optional Triton JIT; add X11 CI smoke and bounded qualification cleanup." + }, + "limitations": [ + "Real Qwen load uses one assigned block, repeated 128-token prefill in one admitted session; full-route/conversation evidence belongs to Q3.8.", + "Processing limits pace sharing compute, with brief bursts; whole-GPU samples include other applications.", + "VRAM bounds cover the contribution allocator; driver/context overhead and local inference are separate.", + "Power and bandwidth use sampled host pause guards; these are not OS traffic shapers or hard power caps.", + "Linux graphical acceptance used Xvfb in a Debian/WSL2 container, not a physical Ubuntu desktop or Wayland session.", + "Installer lifecycle, broader hardware coverage and the release canary have separate acceptance boundaries." + ] +} diff --git a/docs/evidence/gate14-20260907-final-resource-acceptance.md b/docs/evidence/gate14-20260907-final-resource-acceptance.md new file mode 100644 index 000000000..92225227f --- /dev/null +++ b/docs/evidence/gate14-20260907-final-resource-acceptance.md @@ -0,0 +1,107 @@ +# Gate 14: final Windows/Linux resource acceptance + +Date: 2026-09-07. **PASSED for the bounded alpha resource-control scope.** +The Windows complete frozen GUI/node package came from `76b6d84fc52342af4fd2315926b187aaa36b1378`, +source tree `474edae23a04c7be8d55241cab7fdbe8255b215b`. Linux was rebuilt on Ubuntu +22.04 from `bf67f0d68df067f43797b4cd47a98f1fea49b2bc`, tree `436c44cc1aa0ae4f4e05fdefb7bf17caa39d9a5f`. +Application and catalog source files are unchanged between these commits; +Linux packaging fixes declare an X11 dependency and exclude optional Triton JIT. +The earlier CI artifact used a README-only PR merge commit and then failed native +X11/GPU acceptance; it is not substituted for this corrected package. Independent release +verification checked the archive and complete file inventory, modes, source +identity and signed catalog inputs. The [portable record](gate14-20260907-final-resource-acceptance.json) +contains package hashes, every checkpoint and cleanup evidence. + +## Acceptance + +Both packages passed all 11 actual Qt-control checkpoints: fresh 100%/100% +defaults with sharing off, explicit sharing opt-in, live slider changes, repeated +Pause/Start, retained settings and paused intent after full desktop restart. +The host driver waited for real node/worker observations before acknowledging +each UI step. All selected worker artifact bytes were verified before load. +Local Qwen3.5-0.8B generated three real tokens at every checkpoint. + +VRAM limits bounded the worker allocator at 25% and 20%. At 1%, an assignment +that could not fit waited with a clear VRAM reason, no worker PID and no restart +loop, while preserving the user's sharing selection. Raising the budget to 25% +restored sharing. Pause removed the complete observed worker tree. Final GUI/node +trees and the test native credentials were removed on both platforms. + +The same packages separately rejected Start under schedule, power, bandwidth and +storage limits, with the other guards open and local inference still usable. +Existing [Windows power recovery evidence](qwen-power-recovery-20260906.json) +also covers automatic resumption after a measured load subsided. + +## Real processing load + +Each setting ran at least 20 seconds of repeated 128-token prefill through the +same assigned Qwen3.8 block, in one admitted RPC session. Requests rewound the +session before repeating the same input. All outputs were finite and bit-identical +within each platform across settings. Warmup requests were excluded. + +| Platform | Processing | Requests | Median request | Mean whole-GPU activity | +| --- | ---: | ---: | ---: | ---: | +| Windows | 100% | 325 | 62.0 ms | 46.3% | +| Windows | 50% | 173 | 94.0 ms | 33.4% | +| Windows | 25% | 77 | 219.0 ms | 23.4% | +| Linux | 100% | 384 | 47.4 ms | 56.5% | +| Linux | 50% | 238 | 78.8 ms | 36.1% | +| Linux | 25% | 86 | 203.6 ms | 25.6% | + +Supplementary production-runtime tensor probes measured Linux CPU duty of +95.1/48.3/24.6% and CUDA duty of 96.6/48.8/24.5% at requested 100/50/25%. +The CUDA allocator accepted 32 MiB below a 256 MiB ceiling and rejected 257 MiB. +Earlier [Windows tensor probes](gate14-20260907-resource-sliders.md) independently +exercised the same limiter and allocator. + +## Fixes found by acceptance + +- An insufficient VRAM budget previously caused repeated worker restarts. A + distinct memory-budget exit signal now leaves that launch configuration waiting + until the budget or assignment changes; ordinary crash recovery remains intact. +- Catalog migration previously lost cache/resource preferences when an identical + manifest moved into the managed directory. Refresh now matches verified manifest + identity, retains those preferences and adopts the managed path. A different + manifest digest cannot inherit preferences through a reused model name. +- Minimal Linux installations lacked `libxcb-shape0`, preventing Qt's X11 plugin + loading. The Debian package now declares it, and CI exercises the frozen UI and + onboarding through X11/Xvfb in addition to the existing offscreen checks. +- Optional PEFT/bitsandbytes imports initialized Triton's compiler on GPU hosts, + which failed inside the frozen runtime. The Linux package now excludes this + optional JIT and retains the approved eager/native kernels; contributors do not + need a compiler or Python development headers for these profiles. + +The [earlier Linux failures and cleanup](gate14-20260907-linux-failed-attempts.json) +remain recorded alongside their retained private logs. The staged Windows +[checkpoint](gate14-20260907-real-qwen-windows.json) is separate from this final +catalog-bearing package acceptance. Catalog migration tests passed on Windows and +Linux (36 each); Linux resource regressions passed (68), and the desktop suite +passed (101, with two Linux-specific skips on Windows). Test/style jobs and both +production package/installer jobs passed for this source. Two CodeQL findings were +reviewed against their complete data-flow paths and dismissed as false positives +with [specific rationale](gate14-20260907-codeql-triage.md); scanning remains enabled. + +## Environment and limits + +Windows ran as a non-elevated user on Windows 10 Pro 10.0.19045. Linux ran as +ordinary UID 1000 on Debian 12 in a WSL2 Docker container, with X11/Xvfb and a real +Secret Service credential store; its archive was built on Ubuntu 22.04. Both used +the host RTX 2070 SUPER with 8 GiB VRAM. Linux used actual CUDA passthrough and +the frozen desktop/node executables. This does not claim a physical Ubuntu or +Wayland desktop playthrough, another GPU generation or broad hardware support. + +Processing is paced sharing compute time. Brief bursts, loading, downloads, local +inference and other applications are outside an instantaneous whole-device cap; +whole-GPU samples include other applications. VRAM bounds cover the contribution +allocator, with driver/context overhead and local inference separate. Power and +bandwidth are sampled pause guards rather than hard power caps or traffic shapers. +This is one-block resource acceptance, not a full-route conversation benchmark. + +The block-health grid, reported peer metadata and local client/worker download +progress are included in these packages; their detailed +[display/integrity tests](desktop-health-downloads-20260907.md) remain the evidence +for all display states. Remote download percentages and unreported spare capacity +are not inferred. Installer lifecycle is Gate 15; Windows's fully frozen +[installation result](gate15-20260907-frozen-windows-installer.json) and the +[corrected Debian lifecycle](gate15-20260907-frozen-debian-installer.json) are separate. +The canary and public release remain later gates. Signing is owner-deferred. diff --git a/docs/evidence/gate14-20260907-linux-failed-attempts.json b/docs/evidence/gate14-20260907-linux-failed-attempts.json new file mode 100644 index 000000000..836aff58e --- /dev/null +++ b/docs/evidence/gate14-20260907-linux-failed-attempts.json @@ -0,0 +1,38 @@ +{ + "date": "2026-09-07", + "result": "failed attempts retained; superseded by corrected package acceptance", + "failed_package": { + "source_commit": "55301c12edb3e70460249a00652d75c7d005aed2", + "source_tree": "c3c2de877218f63c309df25878911c928439f2d7", + "install_archive_sha256": "9d2d2e757281d234bd466461bb31654676bc08c4c733546a8f15afa8be2bf806", + "install_archive_bytes": 3275696149, + "provenance_verification": "passed after identifying GitHub's README-only PR merge commit" + }, + "attempts": [ + { + "phase": "first frozen X11 launch", + "result": "failed before checkpoint 0", + "cause": "The Qt X11 plugin required libxcb-shape.so.0, absent on the minimal Debian host and omitted from the installer dependency list.", + "correction": "Declare libxcb-shape0 in Debian metadata and build prerequisites; add native X11 UI/onboarding smoke checks.", + "gui_stopped": true, + "native_test_credential_removed": true + }, + { + "phase": "first frozen GPU contribution launch after X11 correction", + "result": "failed after defaults and local inference passed", + "cause": "PEFT imported bitsandbytes, which initialized optional Triton GPU JIT and failed in its compiler subprocess inside the frozen runtime. The contribution worker repeatedly restarted before readiness.", + "correction": "Exclude optional Triton from the frozen Linux runtime used by approved eager/native profiles; preserve native PyTorch/bitsandbytes kernels.", + "intervention": "Stopped the owned desktop. The qualification driver's fallback cleanup then encountered its own unreaped GUI PID; the driver now reaps its child, records cleanup failures and fails promptly on repeated startup crashes.", + "original_driver_exit": 1, + "independent_cleanup_audit": { + "result": "passed", + "native_credential_already_absent": true, + "frozen_runtime_processes_absent": true, + "dht_daemons_absent": true + } + } + ], + "passing_replacement_source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "passing_evidence": "gate14-20260907-final-resource-acceptance.json", + "private_raw_logs_retained": true +} diff --git a/docs/evidence/gate14-20260907-real-qwen-windows.json b/docs/evidence/gate14-20260907-real-qwen-windows.json new file mode 100644 index 000000000..e524b545c --- /dev/null +++ b/docs/evidence/gate14-20260907-real-qwen-windows.json @@ -0,0 +1,206 @@ +{ + "scope": "native-packaged-Qwen-resource-controls", + "result": "passed", + "platform": "nt", + "device": "cuda:0", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "desktop_sha256": "85d6a3d693cc5cf4467e20bd8573c5866a46dd18f5f6878bff6d0fafd6dc5bdb", + "node_sha256": "7f1aef2f8f3aa43ae577ee520840b750f6949ffbc861fc2c849a4ff1beaa6ab6", + "observations": [ + { + "stage": "initial", + "step": 0, + "action": "observe", + "vram_percent": 100, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 1, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 2, + "action": "start", + "worker_state": "running", + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.062000000005355105, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 43.59139784946237, + "baseline_samples": 15, + "baseline_mean_percent": 3.8, + "workload_samples": 93 + }, + "requests": 300, + "mean_seconds": 0.06327666666698253, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 3, + "action": "limits", + "vram_percent": 25, + "processing_percent": 50, + "old_worker_tree_gone": true, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.15599999998812564, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 30.72340425531915, + "baseline_samples": 15, + "baseline_mean_percent": 9.933333333333334, + "workload_samples": 94 + }, + "requests": 131, + "mean_seconds": 0.15040458015273753, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 4, + "action": "limits", + "vram_percent": 20, + "processing_percent": 25, + "old_worker_tree_gone": true, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 1717921382, + "verified_download_bytes": 384054157, + "rpc_load": { + "block_indices": "60:61", + "tokens_per_request": 128, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "median_seconds": 0.25, + "finite_outputs": true, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "workload_mean_percent": 23.91578947368421, + "baseline_samples": 15, + "baseline_mean_percent": 10.533333333333333, + "workload_samples": 95 + }, + "requests": 67, + "mean_seconds": 0.29897014925355414, + "unique_output_hashes": [ + "b28f7f6a078101062dfac452f23f36d5414efb8cdd2b9e3330df0b01557d7ae0" + ] + }, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 5, + "action": "limits", + "vram_percent": 1, + "processing_percent": 25, + "old_worker_tree_gone": true, + "worker_state": "paused", + "policy_persisted": true, + "low_memory_blocked_without_worker": true, + "resource_reason": "selected blocks exceed the VRAM budget; increase VRAM or contribute fewer blocks", + "no_restart_loop": true, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 6, + "action": "limits", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "running", + "policy_persisted": true, + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 7, + "action": "pause", + "old_worker_tree_gone": true, + "worker_state": "paused", + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 8, + "action": "start", + "worker_state": "running", + "block_indices": "60:61", + "max_vram_bytes": 2147401728, + "verified_download_bytes": 384054157, + "local_inference_tokens": 3 + }, + { + "stage": "initial", + "step": 9, + "action": "pause", + "old_worker_tree_gone": true, + "worker_state": "paused", + "local_inference_tokens": 3 + }, + { + "stage": "restart", + "step": 0, + "action": "observe", + "vram_percent": 25, + "processing_percent": 100, + "worker_state": "paused", + "policy_persisted": true, + "local_inference_tokens": 3 + } + ], + "initial_owned_tree_gone": true, + "restart_owned_tree_gone": true, + "test_credential_removed": true, + "gui_stopped": true, + "date": "2026-09-07", + "complete_gate14": false, + "same_Qwen_outputs_at_all_processing_limits": true, + "hardware": "Windows 10 Pro 10.0.19045, RTX 2070 SUPER 8 GiB", + "packaging": "New frozen node and frozen GUI staged together for the bounded check; final clean release archive qualification follows.", + "limitations": [ + "One real Qwen block (60:61), repeated 128-token prefill in one admitted session; not a full 64-block conversation.", + "Whole-GPU activity includes unrelated applications. Processing limits pace contribution compute, not instantaneous whole-device utilization.", + "Windows-only checkpoint; Linux and final source-bound archive checks remain." + ] +} diff --git a/docs/evidence/gate14-20260907-resource-sliders.json b/docs/evidence/gate14-20260907-resource-sliders.json new file mode 100644 index 000000000..b932ef9e3 --- /dev/null +++ b/docs/evidence/gate14-20260907-resource-sliders.json @@ -0,0 +1,105 @@ +{ + "date": "2026-09-07", + "scope": "source-slider-controls-and-native-windows-synthetic-runtime-probes", + "complete_gate14": false, + "base_commit": "16043d8a69edc1dede7b5b080098e104cebd968e", + "source_sha256": { + "desktop/src/communityai_desktop/client.py": "294acbad74eedafa02910f4120ac670a8cf2ad21e57f0d47ee350daa611fc541", + "desktop/src/communityai_desktop/controller.py": "72400b074609ddc3028879412e0cebee534627c8f6dc381cf58b2b350ef7ea1a", + "desktop/src/communityai_desktop/pyside_shell.py": "c5095911841216d8faa512d8b187a9eb7869477349c0d031d4df6c86b4736078", + "desktop/src/communityai_desktop/resource_controls.py": "0ea3ecaeeb64de3ccfa7b9aa902cc68d2f378530d6fcd267f57237def0143b0e", + "desktop/src/communityai_desktop/gate13_playthrough.py": "13b1c00c822ab89ee715e87510572b832703d6e830380d475f067b6c4aba29d0", + "src/drift/node/config.py": "4da68e1c78c8a4109edaa034bebffe5b97bf309adf2ce1ce0fce513c18b6133a", + "src/drift/node/catalog_bootstrap.py": "f9983e74b4f14ee78250eca3276c47534ae93f1a35eed6cf5c1d956af7c63677", + "src/drift/cli/run_node.py": "fbff54f30906e4481db578cc826bc1e236c9fa2f52438c1c01bd8ed4759c2f7c", + "src/drift/cli/run_server.py": "246131a5517ee9a7e8005d9de44be850cb9d12814034f65b49c52da2aeffe79f", + "src/drift/server/server.py": "ef3a2730a88683c7bde54e81e15cf082322eef90ca2eb898871698707585f2f4", + "src/drift/server/processing_budget.py": "5fda1f6131ab0c48410a9ec750c2da77a5e81e39b07f521c65f3dea1178f5426", + "scripts/qualify_resource_processing.py": "4b115e5118f99836ff53f18f27f5dde9fb0d541a9234cfa75565cb44c34626d6" + }, + "cpu": { + "scope": "synthetic-production-runtime-processing-budget", + "device": "cpu", + "complete_gate14": false, + "runs": [ + { + "requested_percent": 100, + "measured_compute_duty_percent": 100.0, + "wall_seconds": 4.0, + "compute_seconds": 4.0, + "steps": 128 + }, + { + "requested_percent": 50, + "measured_compute_duty_percent": 49.59999998100102, + "wall_seconds": 4.0, + "compute_seconds": 1.9839999992400408, + "steps": 64 + }, + { + "requested_percent": 25, + "measured_compute_duty_percent": 24.79999999050051, + "wall_seconds": 4.0, + "compute_seconds": 0.9919999996200204, + "steps": 32 + } + ], + "result": "passed", + "limitations": [ + "Compute duty cycle, not an instantaneous whole-device utilization guarantee", + "No Qwen model, download, cold startup or final-package lifecycle exercised", + "Local inference, downloads and other applications are outside the sharing compute budget" + ] + }, + "cuda": { + "scope": "synthetic-production-runtime-processing-budget", + "device": "cuda:0", + "complete_gate14": false, + "runs": [ + { + "requested_percent": 100, + "measured_compute_duty_percent": 98.42500000013388, + "wall_seconds": 4.0, + "compute_seconds": 3.937000000005355, + "steps": 126 + }, + { + "requested_percent": 50, + "measured_compute_duty_percent": 49.59999998100102, + "wall_seconds": 4.0, + "compute_seconds": 1.9839999992400408, + "steps": 64 + }, + { + "requested_percent": 25, + "measured_compute_duty_percent": 24.79999999050051, + "wall_seconds": 4.0, + "compute_seconds": 0.9919999996200204, + "steps": 32 + } + ], + "hardware": "NVIDIA GeForce RTX 2070 SUPER", + "memory_allocator": { + "cap_bytes": 268435456, + "under_limit_succeeded": true, + "over_limit_rejected": true, + "device_total_bytes": 8589606912 + }, + "result": "passed", + "limitations": [ + "Compute duty cycle, not an instantaneous whole-device utilization guarantee", + "No Qwen model, download, cold startup or final-package lifecycle exercised", + "Local inference, downloads and other applications are outside the sharing compute budget" + ] + }, + "validation": { + "related_tests_passed": 278, + "platform_skips": 2, + "black": "passed", + "isort": "passed", + "ui_preview": "inspected using synthetic acceptance server" + }, + "cloud_resources_created": false, + "downloads_started": false, + "result": "checkpoint-passed" +} diff --git a/docs/evidence/gate14-20260907-resource-sliders.md b/docs/evidence/gate14-20260907-resource-sliders.md new file mode 100644 index 000000000..8c4f6361c --- /dev/null +++ b/docs/evidence/gate14-20260907-resource-sliders.md @@ -0,0 +1,73 @@ +# Gate 14 resource-slider implementation and native Windows probes + +Date: 2026-09-07. Status: source implementation and bounded Windows probes passed. +Gate 14 remains in progress until current Windows/Linux packages pass real sharing +acceptance. No installer, signing, Store submission or cloud run occurred here. + +## Delivered + +- Two real Qt sliders, VRAM and processing usage, covering 1–100%. Fresh catalog + installs default both to 100% and keep contribution opt-in. Existing explicit + limits, including absolute VRAM budgets, remain intact until changed. +- Apply stops all configured workers before the existing atomic, revision-bound + policy save. It resumes only workers selected before the operation. Stale edits + are rejected before stopping anything; failed saves leave sharing paused. +- Processing percentage is persisted and passed through the ordinary worker CLI + into the production runtime. Every inference/forward/backward batch goes through + synchronized compute and proportional cooldown when capped. Workers share a + native OS lock across compute and cooldown; Pause interrupts limiter waits. +- The advanced policy editor preserves the processing setting. The legacy Gate 13 + UI replay accepts the additive default-100 field without weakening its existing + policy comparison. Older nodes remain viewable but cannot silently accept the + new controls. + +## Measured evidence + +The reusable `scripts/qualify_resource_processing.py` executed real tensor work +through `RuntimeWithDeduplicatedPools.process_batch` on this Windows host. Each +setting ran for four seconds, after warmup, with finite-output checks. + +| Requested compute percentage | CPU measured duty | RTX 2070 SUPER measured duty | +| --- | ---: | ---: | +| 100% | 100.0% | 98.4% | +| 50% | 49.6% | 49.6% | +| 25% | 24.8% | 24.8% | + +The actual CUDA allocator also accepted a 32 MiB allocation with a 256 MiB ceiling +and rejected a 257 MiB allocation. The ceiling was restored within that isolated +probe process before processing tests. This was an 8 GiB RTX 2070 SUPER; no other +GPU profile or OS execution is implied. + +Related tests exercise configuration/CLI binding, fresh-install defaults, catalog +preservation, policy persistence, real process stop/replacement, Pause intent, +stale revisions, failed saves, native shared-lock contention, interruptible waits, +and the two Qt controls. The actual application window was rendered and inspected +using the existing synthetic acceptance server; its screenshot is a UI preview, +not live swarm evidence. + +Raw retained results: + +The [portable evidence](gate14-20260907-resource-sliders.json) contains both probe +results and the source-file SHA-256 inventory. Related validation passed 278 tests +with two platform-specific skips; Black, isort and diff checks passed. A later +desktop shutdown check exposed a pending Qt status callback after closure; the +window now stops refreshes on quit and discards late callback delivery. + +- `.gate13-runs/gate14-sliders-20260907-cpu/result.json` +- `.gate13-runs/gate14-sliders-20260907-gpu/result.json` +- `.gate13-runs/gate14-resource-sliders-preview.png` + +## Limits and next acceptance + +Compute duty is synchronized compute time divided by elapsed time. Individual +steps can burst above the selected percentage. This is not an instantaneous +whole-device utilization cap, and does not throttle local inference, downloads, +model loading or unrelated applications. Tiny budgets can increase request latency +or leave insufficient VRAM for even one block. Existing finite request deadlines +and allocator rejection still apply. + +The probes use synthetic tensor workloads, not Qwen inference or final frozen +packages. Next, run the literal sliders with actual Qwen sharing on both fresh +Windows/Linux packages; measure loading and request behavior, low-memory failure, +whole worker-tree cleanup, repeated Pause/resume and retained settings/cache. +Do not mark Gate 14 complete from this checkpoint. diff --git a/docs/evidence/gate15-20260907-debian-late-helper-failure.json b/docs/evidence/gate15-20260907-debian-late-helper-failure.json new file mode 100644 index 000000000..2aa6c337b --- /dev/null +++ b/docs/evidence/gate15-20260907-debian-late-helper-failure.json @@ -0,0 +1,18 @@ +{ + "date": "2026-09-07", + "result": "failed independent cleanup audit after passing product-phase assertions", + "runtime_source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_source_commit": "03169d5bc548f7f5dd0e2e7999499774a2d8fb7a", + "installer_version": "0.1.0~alpha.20260907.3", + "installer_sha256": "27408b573e0c17d4c218f72becaa2f0869a581ebfc367bfb8ede27c6c18fc29a", + "installer_bytes": 3781591288, + "environment": "Debian 12 Docker with SYS_PTRACE for root package maintenance; ordinary-user X11 frozen GUI and native Secret Service", + "product_phase_assertions": "Install, active replacement, removal, reinstall and final removal passed. Each launch generated three local Qwen tokens and verified one sharing block; all 11 initially recorded processes stopped, with settings/cache/credential retained.", + "independent_failure": "A newly created installed node helper and its p2pd child survived final removal. The fixed initial process snapshot did not discover them during shutdown.", + "remaining_installed_processes": 2, + "correction": "Continuously discover installed executables and descendants during shutdown, signal newly observed identities, and require repeated quiet observations before replacement/removal can proceed.", + "regression": "A real shell fixture forks an installed helper on SIGTERM and exits. The old implementation fails the no-helper-left assertion; the corrected implementation passes all four installer tests.", + "operator_cleanup": "The two exact process identities/executable paths were verified and stopped using kernel PID handles. The subsequent independent audit passed for native credentials, installed runtime/DHT processes, package removal and retained cache sentinel.", + "private_raw_logs_retained": true, + "gate14_resource_acceptance_unchanged": true +} diff --git a/docs/evidence/gate15-20260907-debian-replacement-failure.json b/docs/evidence/gate15-20260907-debian-replacement-failure.json new file mode 100644 index 000000000..5b1f6715e --- /dev/null +++ b/docs/evidence/gate15-20260907-debian-replacement-failure.json @@ -0,0 +1,32 @@ +{ + "date": "2026-09-07", + "result": "failed", + "source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_version": "0.1.0~alpha.20260907.2", + "installer_sha256": "5a7ab8aad0b60121118ea4bac128910280a6fea8977db5708879ae4090895dde", + "installer_bytes": 3781591204, + "environment": "Debian 12 Docker, root dpkg and ordinary UID 1000 frozen GUI, default container capabilities without SYS_PTRACE", + "passed_before_failure": { + "initial_install": true, + "frozen_gui_and_node": true, + "local_qwen_tokens": 3, + "verified_worker_bytes": 384054157, + "worker_blocks": "60:61" + }, + "failure": "dpkg replacement returned success but the installed GUI remained alive. The lifecycle driver's 60-second exit assertion failed.", + "confirmed_cause": "Container root lacked permission to read the ordinary user's /proc/PID/exe. A separate disposable cross-user process probe reproduced PermissionError. The installer snapshot silently skipped that error and selected no application processes.", + "correction": "Refuse package maintenance on process-inspection permission errors. The target-desktop-equivalent container rerun grants SYS_PTRACE to root; the GUI remains an ordinary user.", + "regression": "Three Linux installer tests passed, including refusal before signaling when executable ownership is unreadable and preservation of unrelated processes/data.", + "post_run_cleanup_audit": { + "result": "passed", + "package_not_installed": true, + "installed_executable_absent": true, + "native_credential_absent": true, + "frozen_runtime_processes_absent": true, + "dht_processes_absent": true, + "external_cache_sentinel_preserved": true + }, + "failed_container_stopped": true, + "private_raw_logs_retained": true, + "gate14_resource_acceptance_unchanged": true +} diff --git a/docs/evidence/gate15-20260907-frozen-debian-installer.json b/docs/evidence/gate15-20260907-frozen-debian-installer.json new file mode 100644 index 000000000..f55e50617 --- /dev/null +++ b/docs/evidence/gate15-20260907-frozen-debian-installer.json @@ -0,0 +1,73 @@ +{ + "result": "passed", + "frozen_gui_and_node": true, + "ordinary_uid": 1000, + "scope": "Debian 12 same-version package replacement, removal and reinstall", + "full_gate15": false, + "phases": [ + { + "phase": "initial-installation", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 11, + "local_tokens": 3 + }, + { + "phase": "replaced-installation", + "verified_download_bytes": 384054157, + "block_indices": "36:37", + "owned_process_count": 11, + "local_tokens": 3 + }, + { + "phase": "reinstalled", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 11, + "local_tokens": 3 + } + ], + "replacement_stopped_owned_tree_and_retained_state": true, + "removal_stopped_owned_tree_and_retained_state": true, + "reinstall_with_retained_settings_cache_and_credentials": true, + "native_qualification_credential_removed": true, + "installer_sha256": "a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517", + "installer_version": "0.1.0~alpha.20260907.4", + "source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_builder_source_commit": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb", + "installed_executable_removed": true, + "post_run_cleanup_audit": { + "result": "passed", + "lifecycle_result": "passed", + "package_not_installed": true, + "installed_executable_absent": true, + "native_credential_absent": true, + "frozen_runtime_processes_absent": true, + "dht_processes_absent": true, + "external_cache_sentinel_preserved": true + }, + "date": "2026-09-07", + "qualification_root_capability": "SYS_PTRACE; permits cross-user executable inspection", + "previous_failure_records": [ + "gate15-20260907-debian-replacement-failure.json", + "gate15-20260907-debian-late-helper-failure.json" + ], + "installer_packaging": { + "method": "Replace only Debian control archive; compressed runtime payload copied unchanged", + "source_installer_sha256": "27408b573e0c17d4c218f72becaa2f0869a581ebfc367bfb8ede27c6c18fc29a", + "installer_source_commit": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb", + "maintenance_sha256": "4841b09a2a0c60f9e7def7cff7f081da5b82c139d3dbf49363d6e611e7f65fd1", + "compressed_payload_sha256": "ee091347be36af6fb723d364e6f073f43cd8b7d696deb4299d00bb087e956e7f", + "installer_sha256": "a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517", + "installer_bytes": 3781591484, + "version": "0.1.0~alpha.20260907.4" + }, + "environment": "Debian 12 in WSL2 Docker; ordinary UID 1000; X11/Xvfb and native Secret Service; RTX 2070 SUPER 8 GiB CUDA passthrough", + "build_environment": "Ubuntu 22.04, Python 3.12.14; complete independently verified frozen bundle", + "limitations": [ + "Package maintenance used root dpkg; the installed GUI and node ran as an ordinary user.", + "Replacement used the same version and exact package, exercising active-process shutdown and file replacement; this is not a different-version upgrade claim.", + "The test began with existing model caches and a private loopback DHT. It generated three local Qwen tokens and verified all artifacts for one real Qwen3.8 sharing block after install, replacement and reinstall. Automatic placement can select a different block; each observed range is recorded.", + "Ubuntu installation lifecycle, physical desktop coverage, explicit cache deletion/login-entry cleanup, canary and publication remain separate. Signing is owner-deferred." + ] +} diff --git a/docs/evidence/gate15-20260907-frozen-windows-installer.json b/docs/evidence/gate15-20260907-frozen-windows-installer.json new file mode 100644 index 000000000..4d265a067 --- /dev/null +++ b/docs/evidence/gate15-20260907-frozen-windows-installer.json @@ -0,0 +1,54 @@ +{ + "result": "passed", + "frozen_gui_and_node": true, + "full_gate15": false, + "old_installer_sha256": "95b2d70382ed91b61079581e3fed0c3b12364ebe70576a25eee3231460f8e4d8", + "installer_sha256": "c4e8df599f3a6118eab5718a5ad50655b0e07fd6c270aacf7dbb0b3065c5c399", + "phases": [ + { + "phase": "old-installation", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 9, + "local_tokens": 3 + }, + { + "phase": "upgraded-installation", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 9, + "local_tokens": 3 + }, + { + "phase": "reinstalled", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 9, + "local_tokens": 3 + } + ], + "upgrade_stopped_owned_tree_and_retained_state": true, + "uninstall_stopped_owned_tree_and_retained_state": true, + "reinstall_with_retained_settings_cache_and_credentials": true, + "native_qualification_credential_removed": true, + "installer_registration_removed": true, + "date": "2026-09-07", + "source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "environment": "Windows 10 Pro 10.0.19045; non-elevated ordinary user; RTX 2070 SUPER 8 GiB", + "installer_version": "0.1.0-alpha.20260907.2", + "installer_app_identifier": "CommunityAI.Desktop", + "authenticode_status": "NotSigned; owner-authorized for alpha", + "post_run_cleanup_audit": { + "result": "passed", + "installation_directory_absent": true, + "registration_absent": true, + "native_credential_absent": true, + "installation_executables_absent": true, + "external_cache_sentinel_preserved": true + }, + "harness_limitations": [ + "All product-phase assertions passed. The original driver exited 1 after it attempted a redundant uninstall while Inno was deleting its own executable. The final-uninstall log independently reports success and all removed; the subsequent native credential/registry/process/filesystem audit passed.", + "The driver now checks remaining installer registration before cleanup; no installer product change was needed. Raw failed-driver output is retained.", + "This is Windows installation lifecycle evidence; Linux, explicit cache deletion/login-entry cleanup, canary and publication remain separate." + ] +} diff --git a/docs/evidence/gate15-20260907-ubuntu-unpack-diagnosis.md b/docs/evidence/gate15-20260907-ubuntu-unpack-diagnosis.md new file mode 100644 index 000000000..0c8a80ce3 --- /dev/null +++ b/docs/evidence/gate15-20260907-ubuntu-unpack-diagnosis.md @@ -0,0 +1,85 @@ +# Ubuntu package-unpack timeout diagnosis + +The first Ubuntu 22.04 lifecycle attempt exceeded its **540-second package +installation deadline**, before launching the installed GUI. This is a failed +qualification attempt, not evidence that installation completed. The same `.4` +package has separate passing Debian 12 lifecycle evidence. This note investigates +the difference without running another installation or changing package bytes. + +**September 8 follow-up:** the unchanged `.4` package subsequently passed the +complete Ubuntu lifecycle with a two-core/6 GiB resource cap and a longer finite +operation deadline. Initial installation took 329.971 seconds. The earlier +failure remains recorded; its cause is unconfirmed. The diagnostic proposals +below are retained as history. [Passing retry](gate15-20260908-frozen-ubuntu-installer.md). + +## Verified package facts + +`communityai_0.1.0~alpha.20260907.4_amd64.deb` is 3,781,591,484 bytes, SHA-256 +`a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517`. +The builder invokes `dpkg-deb --root-owner-group -Zxz -z1 --build`. Its staging +hardlinks avoid a build-time copy; they do not make installation free of file +extraction. The `.4` control-only repack preserved the compressed runtime payload +exactly. [Debian package provenance](gate15-20260907-frozen-debian-installer.json). + +A bounded read of the host archive's headers and XZ index found: + +| Observation | Value | +| --- | ---: | +| `data.tar.xz` compressed bytes | 3,781,589,288 | +| Uncompressed tar bytes | 8,595,763,200 | +| XZ blocks | 2,733 | +| XZ index bytes | 19,432 | +| Control members | `.`, `./control`, `./preinst`, `./prerm` | + +There is no packaged `md5sums` control file. This matches the builder source and +its retained `dpkg-deb --info` output. Debian documents that dpkg generates this +information during unpacking when the package does not supply it. MD5 here is +package-integrity metadata, not the release's security boundary; the existing +SHA-256 provenance verification remains necessary. +[Debian `deb-md5sums(5)`](https://manpages.debian.org/bookworm/dpkg-dev/deb-md5sums.5.en.html). + +## Relevant version differences, not a proven root cause + +Upstream dpkg **1.21.13** added multithreaded XZ decompression, requiring liblzma +5.4.0 or newer, under Debian issue 956452. Ubuntu Jammy's 1.21.1 line predates +that change; Debian 12's 1.21.22 follows it. This archive has thousands of XZ +blocks, so it contains work that a parallel decoder can distribute. Dpkg +1.21.10 also switched its MD5 implementation fully to libmd. The changelog does +not establish an MD5 speed regression or fix for this particular package. +[Upstream dpkg changelog](https://launchpad.net/debian/+source/dpkg/+changelog), +[Ubuntu Jammy package version](https://manpages.ubuntu.com/manpages/jammy/man1/dpkg-deb.1.html). + +The reported CPU activity and continuing partial extraction are consistent with +a slow unpack stage. They do not distinguish decoder work, digest calculation, +filesystem synchronization, or host/container overhead. No comparative CPU +profile or controlled timing was captured by this audit. Do not label the +timeout a confirmed dpkg MD5 defect. + +## Next bounded qualification + +1. Keep the failed attempt and its cleanup record. Begin the next attempt with + adequate host space, a fresh owned Ubuntu environment and no concurrent large + hashing/build work. Record exact dpkg/liblzma versions, filesystem/mount, + cgroup CPU limits and the verified installer SHA-256. +2. Time installation separately from the GUI lifecycle. Capture per-process CPU + and read/write counters for dpkg and its decoder children, plus extracted + byte progress and a bounded syscall summary if needed. Give the diagnostic + install an explicit longer deadline; 540 seconds has already proved + insufficient here. Retain the same full package/provenance verification. +3. If XZ decoding dominates, prepare a separately versioned **zstd-compressed** + package from the same verified runtime. Jammy supports zstd packages and + Debian 12 does too. Verify the complete decoded file inventory, permissions, + symlinks and new package hash, then rerun both installed lifecycles. This + changes packaging and requires new evidence; it does not require weakening + any integrity check. [Jammy compression support](https://manpages.ubuntu.com/manpages/jammy/man1/dpkg-deb.1.html), + [Debian 12 compression support](https://manpages.debian.org/bookworm/dpkg/dpkg-deb.1.en.html). +4. Add a complete deterministic `DEBIAN/md5sums` inventory during packaging as a + separate candidate improvement, then measure its effect. Do not assume that + it alone resolves the timeout. Keep the strong SHA-256 release inventory and + post-install checks. + +Do not substitute raw tar extraction for `dpkg -i`, skip verification, disable +normal filesystem safety, or upgrade Ubuntu's package manager solely to obtain a +passing acceptance result. Those would change the tested product or baseline. +No installer, runtime, operating-system package or cloud resource was changed +by this diagnosis. diff --git a/docs/evidence/gate15-20260908-final-installer-acceptance.json b/docs/evidence/gate15-20260908-final-installer-acceptance.json new file mode 100644 index 000000000..1d3094143 --- /dev/null +++ b/docs/evidence/gate15-20260908-final-installer-acceptance.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "date": "2026-09-08", + "gate": 15, + "result": "passed", + "scope": "bounded-Windows-10-Debian-12-Ubuntu-22.04-alpha-installers-and-login-controls", + "windows_installer": { + "file": "communityai-0.1.0-alpha.20260907.2-windows-setup.exe", + "bytes": 2519046440, + "sha256": "c4e8df599f3a6118eab5718a5ad50655b0e07fd6c270aacf7dbb0b3065c5c399", + "runtime_source": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "upgrade_scope": "active different-version upgrade from earlier full installer", + "authenticode": "unsigned; owner-deferred after alpha" + }, + "linux_installer": { + "file": "communityai_0.1.0~alpha.20260907.4_amd64.deb", + "bytes": 3781591484, + "sha256": "a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517", + "runtime_source": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_control_source": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb", + "upgrade_scope": "active same-version replacement on Debian12 and Ubuntu22.04 Xvfb/container sessions" + }, + "component_evidence": { + "windows_installer": { + "path": "docs/evidence/gate15-20260907-frozen-windows-installer.json", + "sha256": "ed422c5a313a3e5330ad236d7e8baed9724ccb552b584cbcd81af24eeb621381" + }, + "debian_installer": { + "path": "docs/evidence/gate15-20260907-frozen-debian-installer.json", + "sha256": "d721d22f9c03fe0f2020061c13fca12defa39ff491094e7df455a8a0a3848784" + }, + "ubuntu_installer": { + "path": "docs/evidence/gate15-20260908-frozen-ubuntu-installer.json", + "sha256": "e4074cbbcbc61418cc4656cdcc1147d74c288912cdfba98ccaebabbaa10083e6" + }, + "windows_frozen_login": { + "path": "docs/evidence/gate15-20260908-frozen-windows-login.json", + "sha256": "1bdfe4cdf70310a6c00e83960823b2fbd68e17e46d8d3d917f662a6b60fbb553" + }, + "linux_frozen_login": { + "path": "docs/evidence/gate15-20260908-frozen-linux-login.json", + "sha256": "fde0fbd597e7b58faaee9d8f3d7110885db9f2dceef4dfdcc66a84209e44097d" + }, + "manual_data_choices": { + "path": "docs/evidence/gate15-20260908-windows-data-login-choices.json", + "sha256": "60c1b100e0c6980631dfcc7111d8e2ccd2785b93091bba6ec506190bc9eba6bc" + }, + "windows_catalog_startup": { + "path": "docs/evidence/qwen-catalog-desktop-20260907.json", + "sha256": "99e41a1993cd5cf08c47ed7f3f60a57af11708aa5dba724e4aa230c8d0f6296e" + }, + "linux_catalog_startup": { + "path": "docs/evidence/qwen-catalog-linux-startup-20260908.json", + "sha256": "5b7d5a5e0a42f68c87b41552079b0235bdf8564858391318614ba51293fbf2e6" + }, + "windows_login_independent_cleanup": { + "path": "docs/evidence/gate15-windows-cycle-independent-audit-20260908.json", + "sha256": "dac54963061652544b302d2ee56d58caa1e23668ee527d87b3989e2a544767fd" + } + }, + "executed_windows_cycle_helpers_match_workspace": true, + "manual_disable_login_before_uninstall_required": true, + "automatic_uninstaller_login_or_cache_deletion": false, + "actual_os_logout_login_qualified": false, + "remaining_separate_work": [ + "Q3.8 representative conversation/client measurements and frozen periodic catalog activation/draining", + "Gate16 monitored public canary", + "Gate17 public artifact hosting, publication and observation" + ], + "retained_failures": "Individual component records preserve failed attempts and later independent cleanup; no failure is replaced." +} diff --git a/docs/evidence/gate15-20260908-final-installer-acceptance.md b/docs/evidence/gate15-20260908-final-installer-acceptance.md new file mode 100644 index 000000000..4af186978 --- /dev/null +++ b/docs/evidence/gate15-20260908-final-installer-acceptance.md @@ -0,0 +1,51 @@ +# Gate 15: installer and login-startup acceptance + +**PASSED for the bounded Windows/Debian/Ubuntu alpha scope, September 8, 2026.** +The final Windows frozen sign-in cycle closes the remaining installer gate. +This acceptance combines the following real runs; their original failures and +scope limits remain in the individual records. + +| Outcome | Tested result | +| --- | --- | +| Windows setup lifecycle | Non-elevated Windows 10 Pro 19045: installation, upgrade from an earlier full setup while sharing was active, removal, reinstall and final removal. Three installed GUI/node launches generated local Qwen tokens and verified sharing artifacts. Settings/cache/credential retention and independent cleanup passed. [Evidence](gate15-20260907-frozen-windows-installer.json). | +| Debian setup lifecycle | Debian 12, ordinary-user Xvfb/native Secret Service and CUDA passthrough: install, active same-version replacement, removal, reinstall and final removal. All three installed launches generated local tokens and verified a sharing block; complete process shutdown, retention and independent cleanup passed. [Evidence](gate15-20260907-frozen-debian-installer.json). | +| Ubuntu setup lifecycle | Ubuntu 22.04 in a disposable local WSL2/Docker container, two CPU cores and 6 GiB RAM: the same unchanged `.deb` passed the full lifecycle and all three real inference/artifact checks. Independent cleanup passed and the container was removed. Initial installation took 329.971 seconds. [Evidence](gate15-20260908-frozen-ubuntu-installer.md). | +| Frozen Windows sign-in control | The literal checkbox changed Off→On; a new GUI retained On and changed it to Off. The exact frozen executable's `REG_SZ` startup command was verified. Both normal shutdowns preserved configuration/native credential, loaded no models and left empty jobs. Independent audit confirmed all 18 identities gone, credential removed and original Run state restored. [Evidence](gate15-20260908-frozen-windows-login.md). | +| Frozen Linux sign-in control | Literal checkbox enable, new GUI reading enabled state, disable, and an explicit login-flag launch passed on private Xvfb/AT-SPI. The exact XDG entry, credential continuity and all three normal shutdowns passed. Independent cleanup passed. [Evidence](gate15-20260908-frozen-linux-login.md). | +| Manual retained-data choices | Retain, cache-only deletion and full reset passed on disposable Windows state. The actual frozen credential-deletion command passed; unrelated state was preserved. [Evidence](gate15-20260908-windows-data-login-choices.json). | +| Signed catalog startup migration | Normal frozen Windows and Linux startup independently migrated signed sequence 1 to exact sequence 2, preserving preferences/cache/credential and rejecting the old trust root. [Windows](qwen-catalog-desktop-20260907.md), [Linux](qwen-catalog-linux-startup-20260908.md). | + +The [machine-readable record](gate15-20260908-final-installer-acceptance.json) +binds the component evidence and qualified artifact identities. The candidates are: + +| Installer | SHA-256 | +| --- | --- | +| `communityai-0.1.0-alpha.20260907.2-windows-setup.exe` (2,519,046,440 bytes) | `c4e8df599f3a6118eab5718a5ad50655b0e07fd6c270aacf7dbb0b3065c5c399` | +| `communityai_0.1.0~alpha.20260907.4_amd64.deb` (3,781,591,484 bytes) | `a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517` | + +Windows runtime source is `76b6d84`; Linux runtime source is `bf67f0d`. +Linux maintenance controls come from `61ab7b1`, with the compressed runtime +payload unchanged. Later CI builds remain distinct artifacts. + +The Windows lifecycle's redundant final driver-cleanup error, two earlier Debian +shutdown failures, Ubuntu's first unpack timeout, the Linux accessibility actor +and display-wrapper errors, and the first Linux catalog-startup failure remain +linked in their records. The Windows reader's four harness failures and passing +read-only prerequisite are also [preserved separately](gate15-windows-private-read-20260908.md). +No failed attempt is reclassified as a pass. + +Users must **disable sign-in startup before uninstalling**, and perform native +credential reset while the executable remains installed. The alpha uses explicit +manual cache/state deletion; uninstall retains those files and does not remove +the per-user login entry automatically. Follow the [removal runbook](../DESKTOP_UNINSTALL.md). + +Linux acceptance covers the tested container/Xvfb/NVIDIA profiles and same-version +replacement; Windows additionally covers a different-version upgrade. This does +not qualify physical Wayland sessions, other GPUs, Ubuntu 24.04, macOS, actual +OS logout/login, or unobserved tray/minimized presentation. Frozen periodic catalog +activation during generation and broader Qwen measurements remain separate. + +Unsigned setup is owner-authorized for alpha. Publisher signing, Store, hosted +signed APT and automatic software updates follow after alpha. Gate 16's live +canary and Gate 17 publication/observation remain open; these installers are +qualified candidates, not a published public service. diff --git a/docs/evidence/gate15-20260908-frozen-linux-login.json b/docs/evidence/gate15-20260908-frozen-linux-login.json new file mode 100644 index 000000000..3a896a9bf --- /dev/null +++ b/docs/evidence/gate15-20260908-frozen-linux-login.json @@ -0,0 +1,110 @@ +{ + "result": "passed", + "scope": "unmodified-frozen-Linux-Qt-sign-in-checkbox-via-AT-SPI", + "desktop_sha256": "41383b5e995a2f200f7c96b0fdb8764b7d9edae89ba8efbba0a93d80d883f0aa", + "node_sha256": "313f8cbe8b1329b0025e98e64cd12eb2abbadb5e0083f5de5360d997b23d4aa0", + "helper_sha256": "c2f359145f43d5e0cdfaa7c76746b78a3e356d4d6b20368d0f7a81dc1f23a18a", + "ordinary_user": true, + "isolated_xvfb_verified": true, + "mock_telemetry": false, + "phases": [ + { + "phase": "enable", + "authenticated_node_ready": true, + "sharing_paused": true, + "resident_models": 0, + "model_cache_file_count": 0, + "model_cache_bytes": 0, + "checked_before": false, + "checked_after": true, + "action": "Toggle", + "exact_frozen_autostart_file": true + }, + { + "phase": "restart-and-disable", + "authenticated_node_ready": true, + "sharing_paused": true, + "resident_models": 0, + "model_cache_file_count": 0, + "model_cache_bytes": 0, + "checked_before": true, + "checked_after": false, + "action": "Toggle", + "autostart_file_removed": true, + "native_credential_preserved": true + }, + { + "phase": "started-at-login", + "authenticated_node_ready": true, + "sharing_paused": true, + "resident_models": 0, + "model_cache_file_count": 0, + "model_cache_bytes": 0, + "owned_application_exposed": true, + "started_at_login_argument": true + } + ], + "shutdowns": [ + { + "normal_shutdown": true, + "forced_cleanup": false, + "gui_returncode": 0, + "owned_identities_stopped": true + }, + { + "normal_shutdown": true, + "forced_cleanup": false, + "gui_returncode": 0, + "owned_identities_stopped": true + }, + { + "normal_shutdown": true, + "forced_cleanup": false, + "gui_returncode": 0, + "owned_identities_stopped": true + } + ], + "final_model_cache": { + "model_cache_file_count": 0, + "model_cache_bytes": 0 + }, + "recorded_runtime_identities_stopped": true, + "recorded_runtime_identity_count": 14, + "private_autostart_entry_absent": true, + "native_qualification_credential_absent": true, + "limitations": [ + "Linux Xvfb/AT-SPI, not Windows frozen-checkbox acceptance or physical desktop sign-in.", + "Minimized/tray presentation without a window manager is not qualified; executed launch modes are in phases.", + "No model acquisition, inference, or sharing work was requested." + ], + "schema_version": 1, + "date_utc": "2026-09-08", + "environment": "Ubuntu 22.04 container, ordinary UID 1000, isolated Xvfb and D-Bus/Secret Service session; 2 CPUs and 6 GiB container memory limit", + "runtime_source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "full_gate15": false, + "full_windows_frozen_login_acceptance": false, + "python_driver_exit_code": 0, + "outer_xvfb_wrapper_exit_code": 1, + "independent_process_audit": { + "result": "passed", + "private_session_processes_absent": true, + "autostart_entry_absent": true, + "model_cache_files": 0, + "model_cache_bytes": 0, + "user_scope": 1000 + }, + "independent_fresh_secret_service_audit": { + "result": "passed", + "native_qualification_credential_absent_in_fresh_secret_service_session": true + }, + "harness_limitations": [ + "The actual Python product driver and all three normal product shutdowns passed. Auxiliary session cleanup stopped its Xvfb server before xvfb-run completed, so the outer wrapper returned 1. The independent subsequent audit found no task-session processes or autostart entry; a new Secret Service session independently confirmed the native credential absent.", + "This uses the verified unpacked frozen Linux bundle, not the installed /opt path; package maintenance is covered by separate installer lifecycle records.", + "The earlier actor rejected a valid Qt control exposing both Toggle and Press actions; that failed attempt is retained separately. The corrected actor explicitly selects Press for navigation and Toggle for the checkbox." + ], + "deterministic_tests": { + "file": "tests/test_login_startup_linux.py", + "passed": 12, + "duration_seconds": 0.17 + } +} diff --git a/docs/evidence/gate15-20260908-frozen-linux-login.md b/docs/evidence/gate15-20260908-frozen-linux-login.md new file mode 100644 index 000000000..e372710d5 --- /dev/null +++ b/docs/evidence/gate15-20260908-frozen-linux-login.md @@ -0,0 +1,53 @@ +# Frozen Linux sign-in checkbox acceptance + +**Passed for the tested Linux scope.** The unmodified frozen Qt application +enabled, persisted and disabled sign-in startup through its AT-SPI accessibility +interface. [Machine-readable evidence](gate15-20260908-frozen-linux-login.json). + +The test used the verified Linux bundle from `bf67f0d` as an ordinary Ubuntu +22.04 container user, on a private Xvfb display and D-Bus/Secret Service session. +Private XDG directories kept the test registration separate from other desktop +state. There was no host Windows window or Computer Use interaction. + +The actual application bootstrapped the signed Qwen catalog and connected to its +actual authenticated node. Sharing was paused and local-only mode was selected. +Each observed phase reported zero resident models and zero cache files/bytes; +the final cache check was also empty. No fixture telemetry was supplied. + +The accessibility actor selected the exact owned GUI process, pressed its Sharing +navigation control, and invoked the reported Toggle action of **Start CommunityAI +when I sign in**. It checked both the checkbox state and the exact autostart-file +contents naming the qualified frozen executable with `--started-at-login`. +A new GUI process read the saved enabled state; toggling it again removed the +file. A third real launch with `--started-at-login` reached authenticated node +readiness and exposed the owned accessibility application. Minimized/tray +appearance and actual OS sign-out/sign-in were not qualified without a window +manager. + +All three product shutdowns completed normally with exit code 0. Fourteen recorded +GUI/node identities stopped, and the private credential and autostart file were +removed. An independent process audit passed; a fresh Secret Service session +independently confirmed that the test credential was absent. + +Two harness limitations remain explicit. The +[first actor attempt](gate15-20260908-linux-login-actor-failure.json) refused a +valid control exposing both Press and Toggle. The corrected actor selects Press +for navigation and Toggle for the checkbox, with a regression test. After the +passing replay, auxiliary cleanup stopped Xvfb before its wrapper finished, so +the outer wrapper returned 1 even though the product driver returned 0. The +independent cleanup audits passed afterward. Neither issue is reported as a +product failure or silently discarded. + +The maintained helper is `scripts/qualify_login_startup_linux.py`; its 12 tests +cover exact PID selection, unique name/role lookup, bounded malformed trees, +private-display validation and explicit Qt action selection. Run it only inside +an owned Xvfb session with `QT_QPA_PLATFORM=xcb`, a private `XAUTHORITY`, +`XDG_CONFIG_HOME`, `XDG_DATA_HOME`, `XDG_RUNTIME_DIR`, unlocked Secret Service, +and `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1`. System `python3-pyatspi` supplies the +accessibility actor; the desktop Python environment supplies the host's client +and credential libraries. The helper refuses root and unbound/inherited displays, +limits acceptance to 540 seconds plus finite cleanup, and treats forced runtime +cleanup as a failed acceptance. + +The package's installation/removal lifecycle remains separate. This Linux result +also does not close the [frozen Windows checkbox gap](gate15-20260908-source-login-checkbox.md). diff --git a/docs/evidence/gate15-20260908-frozen-ubuntu-installer.json b/docs/evidence/gate15-20260908-frozen-ubuntu-installer.json new file mode 100644 index 000000000..e12749bfa --- /dev/null +++ b/docs/evidence/gate15-20260908-frozen-ubuntu-installer.json @@ -0,0 +1,149 @@ +{ + "result": "passed", + "frozen_gui_and_node": true, + "ordinary_uid": 1000, + "scope": "Ubuntu 22.04 same-version package replacement, removal and reinstall", + "full_gate15": false, + "phases": [ + { + "phase": "initial-installation", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 11, + "local_tokens": 3 + }, + { + "phase": "replaced-installation", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 11, + "local_tokens": 3 + }, + { + "phase": "reinstalled", + "verified_download_bytes": 384054157, + "block_indices": "60:61", + "owned_process_count": 11, + "local_tokens": 3 + } + ], + "replacement_stopped_owned_tree_and_retained_state": true, + "removal_stopped_owned_tree_and_retained_state": true, + "reinstall_with_retained_settings_cache_and_credentials": true, + "native_qualification_credential_removed": true, + "installer_sha256": "a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517", + "installer_version": "0.1.0~alpha.20260907.4", + "source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_builder_source_commit": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb", + "installed_executable_removed": true, + "package_operation_timings": [ + { + "action": "install", + "seconds": 329.971 + }, + { + "action": "replace", + "seconds": 390.383 + }, + { + "action": "remove", + "seconds": 31.038 + }, + { + "action": "reinstall", + "seconds": 349.084 + }, + { + "action": "final-remove", + "seconds": 30.988 + } + ], + "date": "2026-09-08", + "environment": "Ubuntu 22.04 in WSL2 Docker; ordinary UID 1000; X11/Xvfb and native Secret Service; RTX 2070 SUPER 8 GiB CUDA passthrough", + "qualification_container": "communityai-gate15-ubuntu-retry-20260908", + "qualification_container_id": "0b24686ae5dd6c38384c978e33e80bf32f39c771ecda1c9a937d485346f1e1ab", + "qualification_container_removed": true, + "container_absence_independently_verified": true, + "qualification_root_capability": "SYS_PTRACE permits cross-user executable inspection", + "package_maintenance_interpreter": "Ubuntu system Python 3.10.12", + "resource_bounds": { + "cpus": 2, + "memory_bytes": 6442450944, + "swap_bytes": 0, + "pids": 512, + "package_operation_seconds": 1800, + "container_backstop_seconds": 14400 + }, + "package_manager": { + "dpkg": "Debian 'dpkg' package management program version 1.21.1 (amd64).", + "liblzma": "5.2.5-2ubuntu1.1" + }, + "post_run_cleanup_audit": { + "result": "passed", + "lifecycle_result": "passed", + "package_not_installed": true, + "installed_executable_absent": true, + "native_credential_absent": true, + "frozen_runtime_processes_absent": true, + "dht_processes_absent": true, + "external_cache_sentinel_preserved": true + }, + "manual_login_entry_choice": { + "result": "passed", + "ordinary_uid": 1000, + "scope": "Installed executable registration via production startup functions and real XDG files; no simulated login or frozen UI click claim", + "installed_command_registered": true, + "registration_file_mode": "0o600", + "manual_disable_removed_entry": true, + "unrelated_settings_preserved": true, + "source_commit": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb" + }, + "previous_failed_attempt": "gate15-20260908-ubuntu-install-timeout.json", + "harness_sha256": { + "run-ubuntu-retry.py": "9fc300853c8bafa93a750630275f6fd23fbc61dcd959a7a2dfebba70a385a5e4", + "qualify-ubuntu-retry.py": "bdcd08bc1dcd9eb0e9ab6f2282c16ccaf884ece349fe35242a74e78cab39004e", + "gate15-ubuntu-retry-session.sh": "a59ed2da168a822a6e107ceaf018e9f6c28b644240cf06822f3a9e389f93c203", + "audit-ubuntu-retry.py": "2cf4bb2b6e1cc1811dcde8f265489a924fabfa4a980eff7a1c4602d42f1e9cd4", + "qualify-ubuntu-retry-login.py": "e0a77f935ef0e9e3e07e7610cad3a9664b5d1a435eb24bb4a1985468bd0c0c67" + }, + "raw_evidence_sha256": { + "0-done.json": "113fa6a0d266498cc39a6c1615cd598137534c665d8970a9f41d08d397b370f7", + "0-request.json": "547dc9b63adef26218a022f6b1c2d064eb3a18a4e72814ff77128d0e564307cb", + "1-done.json": "113fa6a0d266498cc39a6c1615cd598137534c665d8970a9f41d08d397b370f7", + "1-request.json": "b48e7a6e05e6cdf2f6fd19f65dfbdc5a928953b541a7e62417194840a4442a36", + "2-done.json": "113fa6a0d266498cc39a6c1615cd598137534c665d8970a9f41d08d397b370f7", + "2-request.json": "bdb636b7cd572d7788f6f602c1224d8d11610c8b66c4c2203bb7b71dd56e0c49", + "3-done.json": "113fa6a0d266498cc39a6c1615cd598137534c665d8970a9f41d08d397b370f7", + "3-request.json": "5e7f921f3a5fe2223185dece7bf1be7624caad6f663730eb09746e6e87cdeced", + "4-done.json": "113fa6a0d266498cc39a6c1615cd598137534c665d8970a9f41d08d397b370f7", + "4-request.json": "9b94a59de206ae406250893a5eec46363f036b770f75103281ca4a083560870c", + "environment.json": "a5ca68c45f200133900ed94f2e94987db238fe60fee5ff6d1fdf34df7e61f5cb", + "final-remove-dpkg.log": "750e04462f154683fb67a6712a74c3f0f60a03a070134b3ff270b54840eccec7", + "final-remove-samples.jsonl": "196b8223bbb2ec6d86a24767e28d78117b5849bf0794a819b65e5307dfc23e64", + "initial-installation-gui.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "install-dpkg.log": "f903dbc18d2601619c508a36beb31d05319554df78d3d8faf37a00afee004a5b", + "install-samples.jsonl": "da508a8e4bf81b33c2d5970a933c6a6c0362fb0052edfb454064d655898053bc", + "login-choice.json": "1525c0a5180e2498144080c6b17812ff2e6b85c437927f76bd85b94e48aa8008", + "operation-timings.json": "95c50fc1785e672f3d5682bdd5812443b5d73055fc523d571fdd9366e954dac3", + "ordinary-user.log": "a7cd671d3799213071409cde5e9ea88d1af56873cc620a928fa84148477085b4", + "post-run-audit.json": "a0080b48def918e276c9ffc4acad6b0461a234bedffdb3bd46fe759953351b56", + "reinstall-dpkg.log": "f903dbc18d2601619c508a36beb31d05319554df78d3d8faf37a00afee004a5b", + "reinstall-samples.jsonl": "b0b2151b1791eed52941405c7e8256b674c357eadac33f9e52cb807845532d48", + "reinstalled-gui.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "remove-dpkg.log": "750e04462f154683fb67a6712a74c3f0f60a03a070134b3ff270b54840eccec7", + "remove-samples.jsonl": "1a4bb38ffcc48078aa73a495c1887d56bf4a2de417adfc1ecea446c84d18841d", + "replace-dpkg.log": "8504044c27cbd0175686801ca68131f874badc917cc132414a2e6ff11335fb54", + "replace-samples.jsonl": "8756771ae4f2582e57d85ab65418b6ce3e8586d48cc512bbc5ebee1b8e65859c", + "replaced-installation-gui.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "result.json": "029b320d03220197b94826a66cb1e5190a1edadba36331f085f8dfe16d99d2dd" + }, + "limitations": [ + "Ubuntu 22.04/Xvfb container with CUDA passthrough, not a physical Wayland desktop, Ubuntu 24.04 or broader GPU qualification.", + "Root dpkg installed/replaced/removed the package; the actual installed frozen GUI/node and inference ran as ordinary UID 1000.", + "Active replacement used the same version and exact package; different-version Windows upgrade has separate evidence.", + "Existing verified model caches and a private loopback DHT were used; this was not cold model acquisition or a complete community route.", + "Login registration here used production source functions and real XDG files; frozen checkbox acceptance is a separate record.", + "The prior 540-second initial-unpack timeout was not reproduced; host/resource differences prevent a proven causal attribution.", + "The installer retains per-user cache/settings/credentials; deletion is the explicit manual alpha workflow in DESKTOP_UNINSTALL.md." + ] +} diff --git a/docs/evidence/gate15-20260908-frozen-ubuntu-installer.md b/docs/evidence/gate15-20260908-frozen-ubuntu-installer.md new file mode 100644 index 000000000..68c65ff39 --- /dev/null +++ b/docs/evidence/gate15-20260908-frozen-ubuntu-installer.md @@ -0,0 +1,53 @@ +# Ubuntu 22.04 frozen installer lifecycle + +**Passed**, using the unchanged qualified +`communityai_0.1.0~alpha.20260907.4_amd64.deb`, SHA-256 +`a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517`. +The [machine-readable evidence](gate15-20260908-frozen-ubuntu-installer.json) +binds the runtime, installer source, harness and raw observations. + +The disposable Ubuntu 22.04 environment ran locally through WSL2/Docker with +two CPU cores, a 6 GiB memory ceiling, no container swap, CUDA passthrough, +ordinary UID 1000 and a private Xvfb/native-keyring session. Package maintenance +ran as root with `SYS_PTRACE` for cross-user process inspection. Each package +operation had an 1,800-second deadline; the container had a four-hour backstop. +Older duplicate Windows build directories were moved to a separate local drive +before installation, preserving their archives and the qualified candidates. + +| Package operation | Result | Seconds | +| --- | --- | ---: | +| Initial installation | Passed | 329.971 | +| Replacement while the installed app and sharing worker were active | Passed | 390.383 | +| Removal while active | Passed | 31.038 | +| Reinstallation with retained state | Passed | 349.084 | +| Final removal while active | Passed | 30.988 | + +Each of the three actual installed GUI/node launches generated three local +Qwen3.5-0.8B tokens and verified 384,054,157 bytes for the Qwen3.8 sharing block +`60:61`. Each owned runtime tree had 11 observed processes. Replacement and +removal stopped those trees and preserved config bytes, the native credential +and a retained-data sentinel. The real cached model files stayed outside the +installation directory. This used existing caches and a private loopback DHT. + +An additional ordinary-user check created and removed the actual XDG login +entry using production startup functions and the installed executable command. +The [frozen Linux checkbox replay](gate15-20260908-frozen-linux-login.md) separately +exercises the literal Qt control through accessibility. + +After the lifecycle, a fresh independent audit found no installed executable, +installed package, frozen runtime, DHT process or test credential. The retained +sentinel survived. The exact disposable container was then stopped and removed; +absence was independently checked. The qualification volume, model caches and +raw evidence remain available. + +The earlier [540-second unpack timeout](gate15-20260908-ubuntu-install-timeout.json) +remains a separate failed attempt. The retry completed initial installation in +approximately five and a half minutes with the same package bytes. Resource +and host-load differences prevent assigning a proven cause to the earlier +timeout; no package-manager or compression change was needed for this pass. + +This qualifies Ubuntu 22.04/Xvfb with the tested NVIDIA device and same-version +replacement. Physical Wayland desktops, Ubuntu 24.04, broader hardware and a +different-version Linux upgrade retain their separate scope. The Windows +different-version upgrade has its own passing evidence. Signing remains +owner-deferred after alpha. diff --git a/docs/evidence/gate15-20260908-frozen-windows-login.json b/docs/evidence/gate15-20260908-frozen-windows-login.json new file mode 100644 index 000000000..b879a3006 --- /dev/null +++ b/docs/evidence/gate15-20260908-frozen-windows-login.json @@ -0,0 +1,249 @@ +{ + "result": "passed", + "scope": "frozen-Windows-private-desktop-login-enable-restart-disable", + "explicit_mutation_opt_in": true, + "run_entry_original_present": false, + "expected_run_type": "REG_SZ", + "replay_sha256": "9130c36aef4765a2e812262f04b075bf5b32d0f879ee2908a8a96ef0ba3697ea", + "run_state_guard_sha256": "dd6db75b16ee2c4cf36148de8e6212e1dcbf25085aa906b4c3ed93abf7a55f7c", + "phases": [ + { + "action": "enable", + "result": "passed", + "owned_processes_stopped": true, + "scope": "frozen-Windows-private-desktop-login-checkbox-enable", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "9d841ecaa30b1eeeafbf7164852df79fe43a2035992a31d3f04b12e14fb8031c", + "uia_source_sha256": "7287870248706850bc31ae9f21cbedcd4fb510356acca7cf8049d28a6893f49a", + "non_elevated": true, + "run_entry_original_present": false, + "registry_mutation": true, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "private_desktop_name": "CommunityAIReadOnly-e32d91cbb52f4bc9a17bbea824ec9f42", + "input_desktop_before": "Default", + "last_authenticated_status": { + "status": "running", + "resident_models": 0, + "worker_states": [ + "paused" + ] + }, + "seconds_to_uia_and_authenticated_status": 17.359, + "uia": { + "result": "passed", + "checkbox_name": "Start CommunityAI when I sign in", + "control_type": "ControlType.CheckBox", + "initial_state": "Off", + "state": "On", + "enabled": "True", + "registry_mutation": "True", + "login_action": "enable", + "enumeration_retries": "2", + "last_enumeration_error": "0", + "actor_private_desktop_verified": "True", + "navigation_control_type": "ControlType.CheckBox", + "navigation_supported_patterns": "InvokePatternIdentifiers.Pattern,ValuePatternIdentifiers.Pattern,TogglePatternIdentifiers.Pattern", + "navigation_action": "InvokePattern.Invoke" + }, + "resident_models": 0, + "private_model_weight_files": 0, + "worker_states": [ + "paused" + ], + "catalog_sequence": 2, + "private_desktop_helper": { + "result": "passed", + "error": "", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "job_assigned_before_resume": "True", + "actor_job_assigned_before_resume": "True", + "job_active_processes_at_close": "0", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "gui_launch": { + "pid": 34320, + "creation_filetime": 134333344965453848 + }, + "uia_actor_launch": { + "pid": 71432, + "creation_filetime": 134333344965494145 + }, + "expected_run_state_verified": true, + "owned_process_identities": [ + [ + 10708, + 1788860897.9718237 + ], + [ + 14956, + 1788860897.9516923 + ], + [ + 34320, + 1788860896.545385 + ], + [ + 34356, + 1788860914.023944 + ], + [ + 51432, + 1788860906.0571911 + ], + [ + 66904, + 1788860906.0750787 + ], + [ + 68160, + 1788860913.477782 + ], + [ + 71432, + 1788860896.5494144 + ], + [ + 72056, + 1788860896.511415 + ] + ], + "native_credential_unchanged_after_shutdown": true, + "private_config_unchanged_after_shutdown": true + }, + { + "action": "disable", + "result": "passed", + "owned_processes_stopped": true, + "scope": "frozen-Windows-private-desktop-login-checkbox-disable", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "9d841ecaa30b1eeeafbf7164852df79fe43a2035992a31d3f04b12e14fb8031c", + "uia_source_sha256": "7287870248706850bc31ae9f21cbedcd4fb510356acca7cf8049d28a6893f49a", + "non_elevated": true, + "run_entry_original_present": true, + "registry_mutation": true, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "private_desktop_name": "CommunityAIReadOnly-e055acf1997044258d32a53ad7b26db4", + "input_desktop_before": "Default", + "last_authenticated_status": { + "status": "running", + "resident_models": 0, + "worker_states": [ + "paused" + ] + }, + "seconds_to_uia_and_authenticated_status": 17.375, + "uia": { + "result": "passed", + "checkbox_name": "Start CommunityAI when I sign in", + "control_type": "ControlType.CheckBox", + "initial_state": "On", + "state": "Off", + "enabled": "True", + "registry_mutation": "True", + "login_action": "disable", + "enumeration_retries": "2", + "last_enumeration_error": "0", + "actor_private_desktop_verified": "True", + "navigation_control_type": "ControlType.CheckBox", + "navigation_supported_patterns": "InvokePatternIdentifiers.Pattern,ValuePatternIdentifiers.Pattern,TogglePatternIdentifiers.Pattern", + "navigation_action": "InvokePattern.Invoke" + }, + "resident_models": 0, + "private_model_weight_files": 0, + "worker_states": [ + "paused" + ], + "catalog_sequence": 2, + "private_desktop_helper": { + "result": "passed", + "error": "", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "job_assigned_before_resume": "True", + "actor_job_assigned_before_resume": "True", + "job_active_processes_at_close": "0", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "gui_launch": { + "pid": 57756, + "creation_filetime": 134333345152925017 + }, + "uia_actor_launch": { + "pid": 26644, + "creation_filetime": 134333345152967062 + }, + "expected_run_state_verified": true, + "owned_process_identities": [ + [ + 16912, + 1788860916.683412 + ], + [ + 26644, + 1788860915.2967062 + ], + [ + 30896, + 1788860916.7000582 + ], + [ + 41068, + 1788860932.7433004 + ], + [ + 57756, + 1788860915.2925017 + ], + [ + 61852, + 1788860932.2786412 + ], + [ + 66484, + 1788860915.2590582 + ], + [ + 68904, + 1788860924.491645 + ], + [ + 69700, + 1788860924.489276 + ] + ], + "native_credential_unchanged_after_shutdown": true, + "private_config_unchanged_after_shutdown": true + } + ], + "actual_os_sign_in_exercised": false, + "restart_preserved_enabled_checkbox": true, + "private_credential_continuity": true, + "owned_processes_stopped": true, + "original_run_state_restored": true, + "restoration_required_registry_write": false, + "private_credential_removed": true, + "source_result_sha256": "0602540f4a5250768df29ebe9760767c674fb6539d357f8edc30d0a05cd66888", + "expected_startup_argv_redacted": [ + "", + "--started-at-login" + ], + "public_redactions": [ + "Unique private native credential service/account identifiers", + "Absolute local qualified executable path" + ] +} diff --git a/docs/evidence/gate15-20260908-frozen-windows-login.md b/docs/evidence/gate15-20260908-frozen-windows-login.md new file mode 100644 index 000000000..f460bab97 --- /dev/null +++ b/docs/evidence/gate15-20260908-frozen-windows-login.md @@ -0,0 +1,63 @@ +# Frozen Windows sign-in checkbox cycle — 2026-09-08 + +**Passed:** the unmodified qualified Windows desktop enabled login startup, +preserved it across a normal shutdown and fresh GUI launch, then disabled it. +The [sanitized acceptance record](gate15-20260908-frozen-windows-login.json) contains +the exact desktop/node/bootstrap and qualification source hashes, both phase +results, owned PID/creation identities, and cleanup proof. +Its source-result SHA-256 identifies the retained local raw record. The public +copy omits only the unique credential service/account identifiers and absolute +local executable path; it retains the verified startup argument and registry type. + +| Phase | Real Qt checkbox | Native registry verification | Readiness | +| --- | --- | --- | --- | +| Enable | Off → On | `HKCU\Software\Microsoft\Windows\CurrentVersion\Run\CommunityAI` exactly matches the qualified frozen executable plus `--started-at-login`, type `REG_SZ` | 17.359 seconds | +| Fresh launch, then disable | Initially On → Off | The CommunityAI Run value is absent | 17.375 seconds | + +Both launches used the same fresh signed sequence-2 state and the same unique +Windows native credential. The credential and private node configuration remained +unchanged through each normal shutdown. Each authenticated normal managed node +reported running, zero resident models, local-only inference and paused +contribution; no model weight files were downloaded. + +Each GUI and its UI Automation actor ran on a new unswitched private Windows +desktop. The actor found exactly one checkbox with the expected accessible name +and role. Before each `TogglePattern.Toggle`, Python verified the exact expected +Run state through a ready/go handshake. The user's input desktop remained +`Default`; no input injection or desktop switching was used. The qualified app +and its UI were not patched. + +Normal `--prepare-update` shutdown stopped each owned tree. Each kill-on-close job +had **zero active processes before closing**, and its desktop/job handles closed. +All 18 recorded process identities were gone before final credential removal. The +original Run state was absence and was restored by the product's disable action; +the restoration guard verified it without an additional registry write. The +temporary native credential was then deleted and independently read as absent. +The [separate root-agent audit](gate15-windows-cycle-independent-audit-20260908.json) +confirmed all 18 exact process identities were gone, both jobs were empty before +closing, the credential was absent, the original Run absence was restored, and +the executable hashes matched the qualified Windows artifacts. + +The default replay remains read-only. The cycle requires explicit opt-in: + +```powershell +$env:PYTHONPATH = Join-Path (Get-Location) 'desktop/src' +$env:OMP_NUM_THREADS = '1' +$env:MKL_NUM_THREADS = '1' +.gate13-runs/qwen-product-venv/Scripts/python.exe scripts/qualify_login_startup_windows.py ` + --desktop .gate13-runs/gate14-release-windows-v2-output/CommunityAI/CommunityAI.exe ` + --output .gate13-runs/gate15-windows-login-cycle-NEW-RUN ` + --node-url http://127.0.0.1:18118 --exercise-login-startup +``` + +Use a new output directory and an unused localhost port with all other +CommunityAI desktops stopped. The original Run value/type is saved privately for +recovery. Restoration refuses an observed unrelated registry change; missing job +cleanup proof retains the private credential and fails acceptance. The 46 +focused ownership, restoration and cycle-finalizer tests passed before this run; +the pure safety tests also passed on Linux. + +This verifies the shipped checkbox and native registration across an application +restart. It does not perform a Windows logout/login or claim that an actual OS +sign-in session was exercised. The earlier read-only diagnostic attempts remain +available in the [read-only evidence](gate15-windows-private-read-20260908.md). diff --git a/docs/evidence/gate15-20260908-linux-login-actor-failure.json b/docs/evidence/gate15-20260908-linux-login-actor-failure.json new file mode 100644 index 000000000..a76286634 --- /dev/null +++ b/docs/evidence/gate15-20260908-linux-login-actor-failure.json @@ -0,0 +1,40 @@ +{ + "date_utc": "2026-09-08", + "result": "failed-qualification-actor; cleanup-passed", + "scope": "first frozen Linux sign-in checkbox attempt", + "desktop_sha256": "41383b5e995a2f200f7c96b0fdb8764b7d9edae89ba8efbba0a93d80d883f0aa", + "node_sha256": "313f8cbe8b1329b0025e98e64cd12eb2abbadb5e0083f5de5360d997b23d4aa0", + "helper_sha256": "726cd08ab7219cd4cc2abeb7e1962bd54fa6d58266a9e22ac8598a102101a43c", + "phases": [ + { + "phase": "enable", + "authenticated_node_ready": true, + "sharing_paused": true + } + ], + "failure": "Sharing navigation exposed Toggle, Press and SetFocus; the first actor required exactly one of several permitted action names and refused before any checkbox interaction.", + "checkbox_action_completed": false, + "started_at_login_launch_reached": false, + "resident_models_observed": false, + "independent_model_cache_files": 0, + "independent_model_cache_bytes": 0, + "shutdowns": [ + { + "normal_shutdown": true, + "forced_cleanup": false, + "gui_returncode": 0, + "owned_identities_stopped": true + } + ], + "native_qualification_credential_absent": true, + "private_autostart_entry_absent": true, + "independent_cleanup_audit": { + "result": "passed", + "private_session_processes_absent": true, + "autostart_entry_absent": true, + "model_cache_files": 0, + "model_cache_bytes": 0, + "user_scope": 1000 + }, + "reporting_correction": "The original generic limitations string mentioned the login argument even though that phase was not reached. Phase records establish that only initial startup completed; no login-argument pass is claimed." +} diff --git a/docs/evidence/gate15-20260908-source-login-checkbox.json b/docs/evidence/gate15-20260908-source-login-checkbox.json new file mode 100644 index 000000000..2862f7738 --- /dev/null +++ b/docs/evidence/gate15-20260908-source-login-checkbox.json @@ -0,0 +1,42 @@ +{ + "date_utc": "2026-09-08", + "result": "passed-source-only", + "frozen_gate_passed": false, + "environment": "Windows 10 Pro 19045; non-elevated ordinary user; Qt offscreen", + "source_head": "23a1f99e81df5f98ee3260b7df2529332547ccca", + "startup_and_shell_source_unchanged_from": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "helper_sha256": "597231eca8f97dfecc883471679d699de6829f53613d17080fb9322e5c9ced0d", + "observations": { + "literal_source_qt_checkbox_enabled_native_isolated_registration": true, + "recreated_window_read_saved_enabled_state": true, + "literal_source_qt_checkbox_disabled_registration": true, + "ui_detail_changed_off_enabled_off": true, + "qualification_registry_key_removed": true, + "real_communityai_login_entry_unchanged": true, + "visible_windows": false, + "native_credentials_created": false, + "nodes_workers_inference_or_fixture_telemetry_started": false + }, + "regression_tests": { + "command": "python -m unittest discover -s desktop/tests -p test_login_startup_ui.py -v", + "passed": 3, + "duration_seconds": 1.018, + "checks": [ + "Literal checkbox state and label persist across source window recreation.", + "A failed registration write reverts the checkbox and reports the failure.", + "An unreadable registration disables the checkbox." + ] + }, + "frozen_read_only_inspection": { + "executable_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "method": "Read PyInstaller embedded PYZ code objects without executing or modifying them.", + "embedded_resource_actions": ["observe", "limits", "start", "pause"], + "login_toggle_action_available": false, + "frozen_checkbox_exercised": false + }, + "scope_limits": [ + "Native writes were redirected to a fresh non-autostart HKCU qualification key. The user's actual Run entry was never changed.", + "This is source Qt wiring and native registration evidence, not frozen Windows checkbox or OS sign-in acceptance.", + "The existing frozen CLI/resource hooks cannot target the sign-in checkbox. No frozen code injection, product modification, rebuild or Computer Use was attempted." + ] +} diff --git a/docs/evidence/gate15-20260908-source-login-checkbox.md b/docs/evidence/gate15-20260908-source-login-checkbox.md new file mode 100644 index 000000000..adb436af2 --- /dev/null +++ b/docs/evidence/gate15-20260908-source-login-checkbox.md @@ -0,0 +1,35 @@ +# Offscreen sign-in checkbox regression + +**Source regression passed.** Frozen Windows acceptance was still open when this +record was made; the later [unmodified frozen cycle](gate15-20260908-frozen-windows-login.md) +passed separately. The +[evidence](gate15-20260908-source-login-checkbox.json) preserves that distinction. + +The actual source Qt checkbox enabled a registration in a newly created native +Windows qualification key. A recreated window read it as enabled, and a second +literal checkbox action disabled it. Labels followed Off → Enabled for this +user → Off. The test key was removed, while the user's real CommunityAI login +entry stayed unchanged. The key was outside Windows' autostart location, so it +could not launch anything at sign-in. No credential, node, inference, fixture +telemetry or visible window was created. + +Three offscreen desktop tests also cover saved-state reflection, failed-write +reversion and disabled controls when registration cannot be read. They run with +the normal desktop unittest suite. + +The helper read the final Windows executable's embedded PyInstaller bytecode +without running or changing it. Its resource qualification hook accepts only +`observe`, `limits`, `start` and `pause`; its CLI has no sign-in-toggle test action. +Source testing cannot close that frozen boundary. The earlier interrupted +[Windows UI attempt](gate15-20260908-windows-data-login-choices.json) is retained. + +Repeat the bounded source check with the desktop development environment: + +```powershell +python scripts/qualify_login_startup_source.py ` + --desktop /CommunityAI.exe ` + --output +``` + +The executable is used only for read-only hook inspection. Qt is forced +offscreen, and the helper refuses an already initialized native Qt platform. diff --git a/docs/evidence/gate15-20260908-source-login-portability.json b/docs/evidence/gate15-20260908-source-login-portability.json new file mode 100644 index 000000000..297b4dda1 --- /dev/null +++ b/docs/evidence/gate15-20260908-source-login-portability.json @@ -0,0 +1,85 @@ +{ + "result": "passed", + "scope": "offscreen-source-Qt-login-checkbox-Linux-portability-and-suite-ordering", + "date_utc": "2026-09-08", + "source_base_commit": "64e4cf070dd8e5d19b9591e910e654094fa0f60b", + "source_helper_sha256_before": "597231eca8f97dfecc883471679d699de6829f53613d17080fb9322e5c9ced0d", + "source_helper_sha256_after": "1681ada82e0948e3731b99290f2cc0ea17183c7adc612f622ce20162ed4a5079", + "source_ui_tests_sha256_after": "8dfc671d73b343130325c3866665333fc59172d6123c52c2f8c82402aa17f298", + "ci_failure": { + "run_id": 34212907804, + "job_id": 102017840345, + "tests_run": 108, + "failures": 2, + "skipped": 2, + "failed_test_names": [ + "test_failed_native_write_reverts_checkbox_and_reports_failure", + "test_literal_checkbox_reflects_saved_state_after_window_recreation" + ] + }, + "linux_baseline_reproduction": { + "tests_run": 3, + "failures": 2, + "seconds": 1.091, + "checkbox_visible": false, + "widget_rect_xywh": [0, 0, 640, 480], + "style_click_rect_xywh": [0, 228, 271, 24], + "default_widget_center_inside_click_rect": false + }, + "fix": { + "sharing_page_shown_offscreen": true, + "checkbox_scrolled_into_view": true, + "literal_QTest_mouse_event_retained": true, + "click_position": "QStyle.SE_CheckBoxIndicator center, validated against widget and style click region", + "visibility_assertions_added": true, + "watchdog_and_exercise_timers_owned_stopped_and_disconnected": true, + "static_product_auto_close_timer_requested": false + }, + "intermediate_full_suite": { + "tests_run": 105, + "failures": 1, + "errors": 3, + "skipped": 2, + "issues": [ + "Two source checkbox callbacks were interrupted by earlier sessions' uncancelled static auto-close timers.", + "The isolated test wrapper omitted the repository root from Python imports.", + "The isolated nonroot wrapper needed an in-process Git safe.directory setting for the read-only repository mount." + ], + "all_issues_resolved_before_final_run": true + }, + "final_windows_focused": { + "tests_run": 3, + "failures": 0, + "errors": 0, + "seconds": 0.865, + "exit_code": 0 + }, + "final_linux_full_desktop_suite": { + "tests_run": 108, + "failures": 0, + "errors": 0, + "skipped": 2, + "skip_reason": "Existing dpkg process-maintenance tests require root; the qualification runs as UID 1000.", + "source_checkbox_tests_passed_in_full_suite_order": true, + "seconds": 27.099, + "exit_code": 0, + "private_log_sha256": "8e6f4e24d0b4fc637490ab3ba496c49aec252c0540b886bbcc6dbc30309b4daf" + }, + "linux_isolation": { + "network": "none", + "cpu_limit": 2, + "memory_limit_bytes": 2147483648, + "uid": 1000, + "repository_and_dependency_mounts_read_only": true, + "home_and_XDG_paths": "private container tmpfs", + "Qt_platform": "offscreen", + "temporary_containers_removed": true + }, + "black_check": "passed", + "isort_check": "passed", + "git_diff_check": "passed", + "product_source_changed": false, + "real_native_login_entry_or_credential_mutated": false, + "historical_windows_source_evidence_preserved": true, + "frozen_acceptance_replayed": false +} diff --git a/docs/evidence/gate15-20260908-source-login-portability.md b/docs/evidence/gate15-20260908-source-login-portability.md new file mode 100644 index 000000000..fda6bf4f2 --- /dev/null +++ b/docs/evidence/gate15-20260908-source-login-portability.md @@ -0,0 +1,34 @@ +# Source checkbox portability follow-up + +**Passed:** the corrected source helper passes all three checkbox tests on +Windows and the complete Linux desktop suite: 108 tests in 27.099 seconds, with +the two existing root-only installer tests skipped. The +[structured evidence](gate15-20260908-source-login-portability.json) preserves +the failed baseline and final checks. + +The production packaging job at commit `64e4cf0` exposed two Linux failures. +Reproduction showed that the helper left the checkbox on the hidden Sharing +page, with an unlaid-out 640 × 480 widget. Its default center was outside the +Linux style's 271-pixel clickable region. The mouse event therefore triggered +neither a state change nor the expected failed-write warning. + +The helper now shows Sharing in its offscreen window, scrolls the checkbox into +view and sends a literal `QTest.mouseClick` to the style-defined indicator +center. It checks visibility and that the point lies inside both the widget and +the clickable region. The tests retain their saved-state, write-history and +warning assertions. + +A full-suite rerun also exposed earlier sessions' uncancelled static auto-close +timers interrupting subsequent event loops. The source helper now owns its +watchdog and exercise timers, stopping and disconnecting them on every return. +The final full suite passed after correcting the isolated runner's import path +and Git safe-directory configuration. Black, isort and whitespace checks passed. + +Linux runs used disposable, network-disabled containers limited to two CPUs and +2 GiB, with read-only repository/dependency mounts and private temporary home +and XDG directories. Product source, real login entries and native credentials +were unchanged. The earlier +[Windows source evidence](gate15-20260908-source-login-checkbox.json), including +its original helper hash, remains intact. The separately recorded +[frozen Windows acceptance](gate15-20260908-frozen-windows-login.md) remains the +packaged checkbox result. diff --git a/docs/evidence/gate15-20260908-ubuntu-install-timeout.json b/docs/evidence/gate15-20260908-ubuntu-install-timeout.json new file mode 100644 index 000000000..a835ee346 --- /dev/null +++ b/docs/evidence/gate15-20260908-ubuntu-install-timeout.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "date_utc": "2026-09-08", + "result": "failed", + "full_gate15": false, + "phase": "initial Ubuntu package unpacking", + "installer_version": "0.1.0~alpha.20260907.4", + "installer_sha256": "a2cc0548cd51f98ed7a9c208be18b53a701a9317cbc63293d4bf7d1e14151517", + "runtime_source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "installer_source_commit": "61ab7b1df61c82c5a306cefaf4e14b56356f5feb", + "environment": "Ubuntu 22.04 in WSL2 Docker, amd64; system dpkg 1.21.1; root package operation and planned ordinary UID 1000 GUI; SYS_PTRACE and CUDA passthrough", + "command": [ + "dpkg", + "-i", + "communityai_0.1.0~alpha.20260907.4_amd64.deb" + ], + "harness_timeout_seconds": 540, + "observed_partial_unpacked_size_approx_gib": 2.6, + "cause": "Unpacking exceeded the fixed harness deadline; underlying performance cause has not been established.", + "claims": { + "installer_success": false, + "frozen_gui_or_node_launched": false, + "inference_exercised": false, + "replacement_or_reinstall_exercised": false + }, + "cleanup": { + "result": "passed", + "lifecycle_result": "failed", + "native_credential_absent": true, + "runtime_dht_and_xvfb_absent": true, + "no_installed_gui_was_launched": true, + "package_state_before_container_removal": "install reinstreq half-installed", + "partial_package_requires_owned_container_removal": true, + "external_cache_sentinel_preserved": true, + "exact_owned_container_stopped_and_removed": true, + "partial_installation_removed_with_container_layer": true, + "qualification_volume_and_model_caches_preserved": true, + "paid_cloud_resources_created": false + }, + "raw_evidence_sha256": { + "result.json": "0e5583136eaadc714ef3c6a82595a59bff71ee9a5bdc7f655c40a24e37fcf4c4", + "install-dpkg.log": "b31b3be5729728729c3e879e0094119e5d7d44d7c492e821e901a0e7b43104c7", + "ordinary-user.log": "6912f6479b94023a0a57b126347c0a3d502ab5f8800368d93ba4ce0f85508849", + "timeout-cleanup-audit.json": "f4cafe801ff99c519f556928fc80428708b99060c62e5ce66b50ec7901b4e929" + }, + "root_log_sha256": "94ee852ef1f15587c0a47b290fd6237cea6dbc5cc03653955049d9b71e90025c", + "next_step": "Diagnose Ubuntu unpacking speed and use a finite measured installation deadline; rerun the complete ordinary-user installed lifecycle. Do not infer Ubuntu acceptance from the passing Debian run." +} diff --git a/docs/evidence/gate15-20260908-windows-data-login-choices.json b/docs/evidence/gate15-20260908-windows-data-login-choices.json new file mode 100644 index 000000000..ba76aa1c0 --- /dev/null +++ b/docs/evidence/gate15-20260908-windows-data-login-choices.json @@ -0,0 +1,70 @@ +{ + "schema_version": 1, + "date_utc": "2026-09-08", + "result": "manual_data_choices_passed; frozen_login_ui_acceptance_interrupted", + "full_gate15": false, + "environment": "Windows 10 Pro 19045, non-elevated ordinary user", + "runtime_source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "frozen_gui_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "related_installer_sha256": "c4e8df599f3a6118eab5718a5ad50655b0e07fd6c270aacf7dbb0b3065c5c399", + "runbook": "docs/DESKTOP_UNINSTALL.md", + "manual_data_choices": { + "result": "passed", + "disposable_state_only": true, + "retain_choice_preserves_config_cache_native_credential": true, + "cache_only_choice_preserves_settings_credential_and_other_paths": true, + "frozen_delete_control_key_removed_native_credential": true, + "frozen_delete_control_key_exit_code": 0, + "full_reset_removes_only_selected_node_state": true, + "unrelated_drift_and_custom_cache_preserved": true, + "independent_native_credential_absence": true, + "scope": "A newly created, task-owned home/node/cache layout and separate native credential. Filesystem fixtures stand in for model bytes. The actual qualified frozen executable performed credential deletion. No real user cache or credential was deleted. No download, inference or reinstall is claimed by this probe." + }, + "startup_contract_checks": { + "result": "passed", + "command": "python -m unittest desktop.tests.test_startup.LoginStartupTests -v", + "tests": 13, + "duration_seconds": 5.644, + "scope": "Existing source contract tests cover exact frozen/source command construction, Windows registry roundtrip and errors, Linux XDG autostart files, minimized login launch, and single-instance ownership/activation. These are not an actual Windows sign-out/sign-in test." + }, + "frozen_startup_ui_attempt": { + "result": "interrupted; no pass claimed", + "actual_frozen_window_opened": true, + "fixture_api_explicitly_selected": true, + "node_management_disabled_for_fixture": true, + "fixture_data_was_not_live_community_telemetry": true, + "fixture_source": "desktop/src/communityai_desktop/acceptance.py", + "startup_toggle_actions_completed": 0, + "login_registration_changed": false, + "observations": [ + "The explicitly configured localhost fixture displayed its demo Llama/Qwen/Mistral models and peer counts. The normal production entrypoint was not exercised by this fixture.", + "Computer Use could read the accessibility tree, but its click failed because coordinate input geometry was unavailable. Screenshot capture then failed with E_NOINTERFACE.", + "Computer Use reported the user's physical Escape interruption. All further Computer Use stopped. No startup toggle acceptance was claimed.", + "The abort cleanup's prepare-update command returned exit 2. The remaining exact owned GUI was independently identified by executable, parent and process creation identity and terminated." + ], + "cleanup_audit": { + "result": "passed", + "owned_gui_helper_host_processes_absent": true, + "native_test_credential_absent": true, + "login_registration_absent": true, + "unrelated_processes_targeted": false + } + }, + "alpha_policy": { + "installer_preserves_settings_cache_and_credentials": true, + "data_deletion_is_explicit_manual_choice": true, + "disable_sign_in_toggle_before_uninstall": true, + "automatic_uninstaller_cache_or_login_entry_removal": false, + "custom_cache_paths_require_individual_review": true + }, + "remaining": [ + "Actual frozen Windows sign-in toggle enable/restart/disable acceptance remains open; the interrupted fixture run does not establish it.", + "OS sign-out/sign-in and other hardware/desktop profiles are outside this record.", + "Ubuntu installation and per-user cleanup observations are recorded separately." + ], + "privacy": { + "user_data_deleted": false, + "credentials_or_private_endpoints_retained": false, + "prompt_or_response_content_retained": false + } +} diff --git a/docs/evidence/gate15-windows-cycle-independent-audit-20260908.json b/docs/evidence/gate15-windows-cycle-independent-audit-20260908.json new file mode 100644 index 000000000..e1833360e --- /dev/null +++ b/docs/evidence/gate15-windows-cycle-independent-audit-20260908.json @@ -0,0 +1,16 @@ +{ + "date_utc": "2026-09-08T09:49:38.392544+00:00", + "scope": "independent-Windows-frozen-login-cycle-cleanup-audit", + "source_result_sha256": "0602540f4a5250768df29ebe9760767c674fb6539d357f8edc30d0a05cd66888", + "audit_helper_sha256": "7c80eb061e5640ef2ad428dddfdb3dd26c81760246b811a7d8c064f142c6c621", + "recorded_identity_count": 18, + "recorded_identities_stopped": true, + "native_credential_absent": true, + "original_run_state_restored": true, + "both_phase_jobs_empty_before_close": true, + "both_phase_jobs_and_desktops_closed": true, + "qualified_artifact_identities_match": true, + "separate_gui_launches": true, + "no_model_loads_or_model_weight_files": true, + "result": "passed" +} diff --git a/docs/evidence/gate15-windows-private-enumeration-20260908.txt b/docs/evidence/gate15-windows-private-enumeration-20260908.txt new file mode 100644 index 000000000..e27e11932 --- /dev/null +++ b/docs/evidence/gate15-windows-private-enumeration-20260908.txt @@ -0,0 +1,16 @@ +empty_binding_warmup=False,127,0,0 +empty_0=False,0,0,0 +empty_1=False,0,0,0 +empty_2=False,0,0,0 +populated_binding_warmup=True,0,13,13 +populated_0=True,0,13,13 +populated_1=True,0,13,13 +populated_2=True,0,13,13 +result=passed +error= +input_before=Default +input_after=Default +owned_child_stopped=True +desktop_closed=True +communityai_launched=false +registry_mutation=false diff --git a/docs/evidence/gate15-windows-private-read-20260908.json b/docs/evidence/gate15-windows-private-read-20260908.json new file mode 100644 index 000000000..518d1159d --- /dev/null +++ b/docs/evidence/gate15-windows-private-read-20260908.json @@ -0,0 +1,107 @@ +{ + "result": "passed", + "scope": "frozen-Windows-private-desktop-read-only-login-checkbox", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "0897dd7a943431a3c262b0c925986ab942527c36b149e9d1c4c89e7f7a93bb63", + "uia_source_sha256": "1128e841b39d66b15c889c962d4a537207bf0f25395e31d8aa77428d2b01d04a", + "non_elevated": true, + "run_entry_original_present": false, + "registry_mutation": false, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "private_desktop_name": "CommunityAIReadOnly-1d92b2903451485dbf65e46a0a013a95", + "input_desktop_before": "Default", + "last_authenticated_status": { + "status": "running", + "resident_models": 0, + "worker_states": [ + "paused" + ] + }, + "seconds_to_uia_and_authenticated_status": 16.765, + "uia": { + "result": "passed", + "checkbox_name": "Start CommunityAI when I sign in", + "control_type": "ControlType.CheckBox", + "state": "Off", + "enabled": "True", + "registry_mutation": "false", + "enumeration_retries": "2", + "last_enumeration_error": "0", + "actor_private_desktop_verified": "True", + "navigation_control_type": "ControlType.CheckBox", + "navigation_supported_patterns": "InvokePatternIdentifiers.Pattern,ValuePatternIdentifiers.Pattern,TogglePatternIdentifiers.Pattern", + "navigation_action": "InvokePattern.Invoke" + }, + "resident_models": 0, + "private_model_weight_files": 0, + "worker_states": [ + "paused" + ], + "catalog_sequence": 2, + "private_desktop_helper": { + "result": "passed", + "error": "", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "job_assigned_before_resume": "True", + "actor_job_assigned_before_resume": "True", + "job_active_processes_at_close": "0", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "owned_processes_stopped": true, + "gui_launch": { + "pid": 49056, + "creation_filetime": 134333332766714881 + }, + "uia_actor_launch": { + "pid": 47620, + "creation_filetime": 134333332766751187 + }, + "run_entry_unchanged": true, + "private_credential_removed": true, + "owned_process_identities": [ + [ + 17384, + 1788859685.7475908 + ], + [ + 25572, + 1788859693.5638793 + ], + [ + 33256, + 1788859676.6390364 + ], + [ + 40536, + 1788859685.7448056 + ], + [ + 47620, + 1788859676.6751187 + ], + [ + 49056, + 1788859676.671488 + ], + [ + 64380, + 1788859678.0901115 + ], + [ + 66168, + 1788859678.0877764 + ], + [ + 71432, + 1788859693.1988335 + ] + ] +} diff --git a/docs/evidence/gate15-windows-private-read-20260908.md b/docs/evidence/gate15-windows-private-read-20260908.md new file mode 100644 index 000000000..d111b702d --- /dev/null +++ b/docs/evidence/gate15-windows-private-read-20260908.md @@ -0,0 +1,45 @@ +# Frozen Windows checkbox read on a private desktop — 2026-09-08 + +The unmodified qualified Windows desktop exposed **Start CommunityAI when I sign +in** as an enabled UI Automation checkbox, initially **Off**. Its normal managed +node authenticated successfully in 16.765 seconds with zero resident models, +paused contribution, and signed catalog sequence 2. See the +[passing record](gate15-windows-private-read-20260908.json) for exact executable, +bootstrap and replay hashes. + +The GUI and a windowless MTA UI Automation actor ran on the same new private +Windows desktop. Both were created suspended and assigned to a kill-on-close job +before resuming. The helper never requested desktop-switching rights or injected +input. The user's input desktop remained `Default`. The normal desktop maintenance +command stopped the managed tree; the job contained zero active processes before +closure, both desktop/job handles closed, and the unique native credential was +removed. The real Run value was absent before and after this read-only test. A +[separate root-agent audit](gate15-windows-read-independent-audit-20260908.json) +confirmed all five attempts' recorded identities were gone and their unique +credentials absent. + +Four earlier failed harness attempts remain recorded: + +1. [Initial enumeration failure](gate15-windows-private-read-failed-20260908.json). +2. [Thread-only desktop attachment attempt](gate15-windows-private-read-second-failed-20260908.json). +3. [Same-desktop actor, first native binding error](gate15-windows-private-read-third-failed-20260908.json). +4. [Successful enumeration, incorrect Sharing role locator](gate15-windows-private-read-fourth-failed-20260908.json). + +The native-boundary diagnostic separately reproduced .NET Framework's first +callback binding reporting error 127 on an empty private desktop. Reusing a +callback and warming that binding before measured calls produced false/error 0 +for an empty desktop and successful enumeration of a controlled private window. +Its [result](gate15-windows-private-enumeration-20260908.txt) used no CommunityAI +process or registry mutation. Subsequent measured errors remain fatal except +bounded empty-desktop retries. + +Qt exposes the checkable Sharing navigation button as `ControlType.CheckBox`. +The corrected locator matches its exact name within the exact owned window and +uses the supported `InvokePattern.Invoke`. This exposed the actual sign-in +checkbox. These failures diagnose the qualification harness; they do not establish +a product checkbox failure. No sign-in checkbox mutation occurred in these five +attempts, so this record alone does not establish enable/restart/disable behavior. + +The approach follows Microsoft's guidance on a +[windowless MTA UIA client](https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-threading) +and [process/thread desktop association](https://learn.microsoft.com/en-us/windows/win32/winstation/thread-connection-to-a-desktop). diff --git a/docs/evidence/gate15-windows-private-read-failed-20260908.json b/docs/evidence/gate15-windows-private-read-failed-20260908.json new file mode 100644 index 000000000..685ad9480 --- /dev/null +++ b/docs/evidence/gate15-windows-private-read-failed-20260908.json @@ -0,0 +1,39 @@ +{ + "result": "failed", + "scope": "frozen-Windows-private-desktop-read-only-login-checkbox", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "3625e82647189800d40207acfac81dc5c4e6ea85d261821cfe9f9ab219ff1efa", + "uia_source_sha256": "aaefd2fa5a20f831136175325cc036e2a2563714c2c12ec620e2ef491b3b9be4", + "non_elevated": true, + "run_entry_original_present": false, + "registry_mutation": false, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "error_type": "RuntimeError", + "cleanup_error_type": "AssertionError", + "owned_processes_stopped": true, + "private_desktop_helper": { + "result": "failed", + "error": "Exception:desktop_enumeration", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "checkbox_state": "", + "job_assigned_before_resume": "True", + "job_active_processes_at_close": "1", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "run_entry_unchanged": true, + "private_credential_removed": true, + "owned_process_identities": [ + [ + 29552, + 1788857813.5293157 + ] + ] +} diff --git a/docs/evidence/gate15-windows-private-read-fourth-failed-20260908.json b/docs/evidence/gate15-windows-private-read-fourth-failed-20260908.json new file mode 100644 index 000000000..54862b6dc --- /dev/null +++ b/docs/evidence/gate15-windows-private-read-fourth-failed-20260908.json @@ -0,0 +1,124 @@ +{ + "result": "failed", + "scope": "frozen-Windows-private-desktop-read-only-login-checkbox", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "0897dd7a943431a3c262b0c925986ab942527c36b149e9d1c4c89e7f7a93bb63", + "uia_source_sha256": "aaaef238deccd25fbbd034393af8c4b622faf8c9e37ba0e5b7637217da22bf9a", + "non_elevated": true, + "run_entry_original_present": false, + "registry_mutation": false, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "private_desktop_name": "CommunityAIReadOnly-84b7cdf08f6a4af092957430db217f50", + "input_desktop_before": "Default", + "last_authenticated_status": { + "status": "running", + "resident_models": 0, + "worker_states": [ + "paused" + ] + }, + "error_type": "TimeoutError", + "cleanup_error_type": "AssertionError", + "owned_processes_stopped": true, + "private_desktop_helper": { + "result": "failed", + "error": "Exception:actor_read_failed", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "job_assigned_before_resume": "True", + "actor_job_assigned_before_resume": "True", + "job_active_processes_at_close": "0", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "gui_launch": { + "pid": 67392, + "creation_filetime": 134333330402244298 + }, + "uia_actor_launch": { + "pid": 34808, + "creation_filetime": 134333330402280058 + }, + "run_entry_unchanged": true, + "private_credential_removed": true, + "owned_process_identities": [ + [ + 14596, + 1788859440.1919928 + ], + [ + 18784, + 1788859508.8244715 + ], + [ + 24164, + 1788859514.6049757 + ], + [ + 26916, + 1788859441.7350175 + ], + [ + 34808, + 1788859440.228006 + ], + [ + 37712, + 1788859502.9342039 + ], + [ + 50008, + 1788859449.6423848 + ], + [ + 50024, + 1788859481.2508116 + ], + [ + 58480, + 1788859457.2976267 + ], + [ + 59444, + 1788859492.788256 + ], + [ + 60216, + 1788859475.154192 + ], + [ + 62400, + 1788859487.0274587 + ], + [ + 64416, + 1788859449.6217396 + ], + [ + 65004, + 1788859441.75912 + ], + [ + 66964, + 1788859469.319122 + ], + [ + 67392, + 1788859440.2244298 + ], + [ + 67952, + 1788859515.72166 + ], + [ + 71984, + 1788859463.421945 + ] + ] +} diff --git a/docs/evidence/gate15-windows-private-read-second-failed-20260908.json b/docs/evidence/gate15-windows-private-read-second-failed-20260908.json new file mode 100644 index 000000000..302944c68 --- /dev/null +++ b/docs/evidence/gate15-windows-private-read-second-failed-20260908.json @@ -0,0 +1,105 @@ +{ + "result": "failed", + "scope": "frozen-Windows-private-desktop-read-only-login-checkbox", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "3625e82647189800d40207acfac81dc5c4e6ea85d261821cfe9f9ab219ff1efa", + "uia_source_sha256": "4c263219299f6ac6523ef04ca70d5382b5ecaf7bd869ae42ce3fa1678561db58", + "non_elevated": true, + "run_entry_original_present": false, + "registry_mutation": false, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "private_desktop_name": "CommunityAIReadOnly-499f9598228047739adf0eafddc474ef", + "input_desktop_before": "Default", + "error_type": "TimeoutError", + "cleanup_error_type": "AssertionError", + "owned_processes_stopped": true, + "private_desktop_helper": { + "result": "failed", + "error": "ElementNotAvailableException:Das Zielelement entspricht einer Benutzeroberfl\u00c3\u00a4che, die nicht mehr verf\u00c3\u00bcgbar ist (z. B., weil das \u00c3\u00bcbergeordnete Fenster geschlossen wurde).", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "checkbox_state": "", + "job_assigned_before_resume": "True", + "job_active_processes_at_close": "0", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "run_entry_unchanged": true, + "private_credential_removed": true, + "owned_process_identities": [ + [ + 23780, + 1788858024.280406 + ], + [ + 30012, + 1788858018.4529374 + ], + [ + 31332, + 1788858041.1007996 + ], + [ + 35312, + 1788857989.1070948 + ], + [ + 35648, + 1788858000.8867 + ], + [ + 42760, + 1788857965.8523092 + ], + [ + 45244, + 1788857967.345239 + ], + [ + 50988, + 1788857965.8843665 + ], + [ + 54820, + 1788857982.9737396 + ], + [ + 58824, + 1788858012.5469985 + ], + [ + 59608, + 1788858036.046422 + ], + [ + 59720, + 1788857967.3484116 + ], + [ + 65028, + 1788858030.2094517 + ], + [ + 65600, + 1788857975.481908 + ], + [ + 70516, + 1788857975.4994905 + ], + [ + 70948, + 1788857994.9033594 + ], + [ + 71756, + 1788858006.7285523 + ] + ] +} diff --git a/docs/evidence/gate15-windows-private-read-third-failed-20260908.json b/docs/evidence/gate15-windows-private-read-third-failed-20260908.json new file mode 100644 index 000000000..0e3ac000b --- /dev/null +++ b/docs/evidence/gate15-windows-private-read-third-failed-20260908.json @@ -0,0 +1,47 @@ +{ + "result": "failed", + "scope": "frozen-Windows-private-desktop-read-only-login-checkbox", + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_sha256": "0897dd7a943431a3c262b0c925986ab942527c36b149e9d1c4c89e7f7a93bb63", + "uia_source_sha256": "0cc3dd8ab9237834373aa6bf425be85003c227df521ae2dcf96cc57234fb0e45", + "non_elevated": true, + "run_entry_original_present": false, + "registry_mutation": false, + "visible_input_desktop_acceptance": false, + "gui_probe_deadline_seconds": 120, + "helper_containment_required": true, + "error_type": "RuntimeError", + "cleanup_error_type": "AssertionError", + "owned_processes_stopped": true, + "private_desktop_helper": { + "result": "failed", + "error": "Exception:actor_read_failed", + "input_before": "Default", + "input_after": "Default", + "input_desktop_unchanged": "True", + "desktop_handle_closed": "True", + "job_assigned_before_resume": "True", + "actor_job_assigned_before_resume": "True", + "job_active_processes_at_close": "1", + "job_handle_closed": "True", + "job_cleanup_verified": "True" + }, + "gui_launch": { + "pid": 4440, + "creation_filetime": 134333322452412300 + }, + "uia_actor_launch": { + "pid": 60008, + "creation_filetime": 134333322452450044 + }, + "run_entry_unchanged": true, + "private_credential_removed": true, + "owned_process_identities": [ + [ + 71764, + 1788858645.2075198 + ] + ] +} diff --git a/docs/evidence/gate15-windows-read-independent-audit-20260908.json b/docs/evidence/gate15-windows-read-independent-audit-20260908.json new file mode 100644 index 000000000..fccf860d4 --- /dev/null +++ b/docs/evidence/gate15-windows-read-independent-audit-20260908.json @@ -0,0 +1,43 @@ +{ + "date_utc": "2026-09-08T09:36:33.542982+00:00", + "scope": "independent-read-only-Windows-probe-cleanup-audit", + "real_run_entry_absent": true, + "runs": [ + { + "run": "gate15-windows-private-read-20260908", + "recorded_identity_count": 1, + "recorded_identities_stopped": true, + "native_credential_absent": true, + "saved_job_cleanup_verified": true + }, + { + "run": "gate15-windows-private-read-20260908-actor", + "recorded_identity_count": 1, + "recorded_identities_stopped": true, + "native_credential_absent": true, + "saved_job_cleanup_verified": true + }, + { + "run": "gate15-windows-private-read-20260908-boundary", + "recorded_identity_count": 18, + "recorded_identities_stopped": true, + "native_credential_absent": true, + "saved_job_cleanup_verified": true + }, + { + "run": "gate15-windows-private-read-20260908-locator", + "recorded_identity_count": 9, + "recorded_identities_stopped": true, + "native_credential_absent": true, + "saved_job_cleanup_verified": true + }, + { + "run": "gate15-windows-private-read-20260908-retry", + "recorded_identity_count": 17, + "recorded_identities_stopped": true, + "native_credential_absent": true, + "saved_job_cleanup_verified": true + } + ], + "result": "passed" +} diff --git a/docs/evidence/gate16-20260907-local-preflight.json b/docs/evidence/gate16-20260907-local-preflight.json new file mode 100644 index 000000000..a72547f47 --- /dev/null +++ b/docs/evidence/gate16-20260907-local-preflight.json @@ -0,0 +1,232 @@ +{ + "schema_version": 1, + "result": "local-prerequisites-passed", + "complete_gate16": false, + "date_local": "2026-09-07", + "source_commit": "e81103663c0c598cdaf43706726fe2a65b4357b6", + "source_note": "Source suite uses working src via PYTHONPATH. Relevant file hashes are recorded; the frozen runtime retains its own qualified source identity.", + "source_files": { + "scripts/gate16_local_preflight.py": "d0100314633d41e376bea2fafc385a27292f5a306e219dd1ce793bc2ee44e5d7", + "tests/test_gate16_catalog_drill.py": "cd5b52d78bc0e1fe49f7e8afb9f3dd2a1cdd8b4cef780e1122bef74fccb844e8", + "src/drift/server/admission.py": "228021a291ad46ae24eb2e9ee886c71ce263264baa790ddcd8b481808df7d904", + "src/drift/server/handler.py": "9aa18b62a7aeaeeb0fda87b9b505e1e513de7dc47ad03c4f3b707bb064b1058d", + "src/drift/server/rejection_logging.py": "8b15251165ed65a563cf18e0a807e95d155e9d215fc093c0a282a43768a67f32", + "src/drift/protocol_identity.py": "0f35b4729929250875649a835af6b7fb38b9006f8f6e71e494ccbb906488373d", + "src/drift/node/catalog_bootstrap.py": "8722bc8b8db4766abf9530425b0a4aa5ce0c95100f304bffb6d9d2bdfeac9be3", + "src/drift/node/catalog_refresh.py": "92412ad5915a812b28cff20aabf28e3d3e854cc88dcf7c90f0f93fdc53048bf3", + "src/drift/node/model_selection.py": "5eb6dec25116a675490209e3283b33d3eabf70efb4895ba230ceb844f7352f92", + "src/drift/node/route_health.py": "8c5ab3ed0f4fb58470650b46e0006305b7d388c7a1fe758ba3b3a38094d5d0b4", + "src/drift/node/discovery.py": "f4261f63a0a1ac4058cf898d29b0ab8af2d0baf2c65e6cb537293dad66c2066f", + "src/drift/node/server.py": "c1d4029705a0dae1721b0bdedb5c2332836630c368ecd279840eead0d6487f50", + "src/drift/node/model_manager.py": "4182962cad854eaf934d29f942a8fc8901370295bdcce84949c910860ba05b61", + "src/drift/cli/run_node.py": "fbff54f30906e4481db578cc826bc1e236c9fa2f52438c1c01bd8ed4759c2f7c", + "src/drift/node/keys.py": "3bf8d6209b2e8994f86011d4230f2f88e0f051e3f133ab8790f6f3fd4ed6662d" + }, + "frozen_runtime_source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "frozen_runtime_source_tree": "474edae23a04c7be8d55241cab7fdbe8255b215b", + "platform": "Windows 10 Pro build 19045, ordinary user, headless node", + "source_safety_suite": { + "tests": 130, + "failures": 0, + "errors": 0, + "skipped": 0, + "duration_seconds": 42.708, + "junit_sha256": "8b1e87ac8b5a9e6b1fb34af709e29bdb7ae6ab1bb0ba27c18abcdb0c2bfc25ed", + "modules": { + "tests.test_server_admission": 55, + "tests.test_protocol_identity": 15, + "tests.test_protocol_identity_network": 1, + "tests.test_public_worker_health": 15, + "tests.test_discovery": 14, + "tests.test_route_health": 4, + "tests.test_measured_model_selection": 5, + "tests.test_catalog_refresh": 7, + "tests.test_route_metrics": 11, + "tests.test_automatic_placement_privacy": 3 + } + }, + "signed_catalog_drill": { + "tests": 1, + "failures": 0, + "errors": 0, + "skipped": 0, + "duration_seconds": 18.187, + "junit_sha256": "d5d44eeecd8d80a37dacd30388973ca0645123a75de12a7c7c431a320d7c610e", + "modules": { + "tests.test_gate16_catalog_drill": 1 + } + }, + "frozen_local_api": { + "schema_version": 1, + "result": "passed", + "scope": "frozen-node-local-api-preflight", + "complete_gate16": false, + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "manifest_sha256": "4536ac2bada7242b758db443b9ebb813a614dd167ab364643fc55c7eb657bb74", + "packaged": true, + "checks": { + "control_requires_auth": { + "status_code": 401, + "duration_seconds": 0.0 + }, + "client_cannot_control": { + "status_code": 401, + "duration_seconds": 0.0 + }, + "control_cannot_infer": { + "status_code": 401, + "duration_seconds": 0.0 + }, + "client_requires_auth": { + "status_code": 401, + "duration_seconds": 0.0 + }, + "client_models": { + "status_code": 200, + "duration_seconds": 0.016 + }, + "control_status": { + "status_code": 200, + "duration_seconds": 0.015 + }, + "malformed_json": { + "status_code": 422, + "duration_seconds": 0.0 + }, + "unknown_model_bounded_rejection": { + "status_code": 404, + "duration_seconds": 0.0 + }, + "malformed_chat": { + "status_code": 422, + "duration_seconds": 0.0 + }, + "invalid_inference_mode": { + "status_code": 422, + "duration_seconds": 0.0 + }, + "control_policy_requires_json": { + "status_code": 415, + "duration_seconds": 0.016 + }, + "oversized_control_policy": { + "status_code": 413, + "duration_seconds": 0.0 + }, + "stale_policy_revision": { + "status_code": 412, + "duration_seconds": 0.0 + }, + "mode_auto": { + "status_code": 200, + "duration_seconds": 0.047 + }, + "mode_local_only": { + "status_code": 200, + "duration_seconds": 0.015 + }, + "create_disposable_client_key": { + "status_code": 201, + "duration_seconds": 0.016 + }, + "disposable_key_works": { + "status_code": 200, + "duration_seconds": 0.016 + }, + "revoke_disposable_key": { + "status_code": 200, + "duration_seconds": 0.0 + }, + "revoked_key_rejected": { + "status_code": 401, + "duration_seconds": 0.0 + }, + "original_key_preserved": { + "status_code": 200, + "duration_seconds": 0.015 + }, + "owned_process_images": [ + "CommunityAI-Node.exe", + "conhost.exe" + ], + "no_model_or_worker_loaded": true, + "owned_runtime_process_count": 2 + }, + "limitations": [ + "No public swarm, model load, inference, GPU work, or public mutation was performed.", + "HTTP deadlines bound this probe, not stalled generation or worker RPC execution.", + "Local inference mode persistence is not signed catalog withdrawal or remote route disable.", + "File credentials isolate this headless-node probe; native desktop credential lifecycle is covered separately." + ], + "cleanup": { + "owned_processes_stopped": true, + "credential_files_removed": true + }, + "privacy": { + "secrets_absent_from_log": true, + "synthetic_prompt_absent_from_log": true, + "raw_log_exported": false, + "prompt_or_output_exported": false + }, + "duration_seconds": 13.657, + "independent_cleanup_audit": { + "result": "passed", + "recorded_process_identities": 2, + "live_owned_identities": 0, + "credential_files_absent": true, + "model_cache_absent": true, + "process_identities_exported": false + }, + "result_sha256": "2d7124daab9c0557e7e2d494d50152b4dbeb6f3063e22e16650cb008e4244cbd" + }, + "failed_harness_attempts": [ + { + "result": "failed", + "error_code": "unexpected_child_process", + "result_sha256": "22f633afa55594460c812b5d0a64135a5bd05fca2919fa5711ee9dd23415f948", + "completed_http_checks": 18, + "cleanup": { + "owned_processes_stopped": true, + "credential_files_removed": true + }, + "correction": "The harness incorrectly rejected every child process.", + "product_defect": false + }, + { + "result": "failed", + "error_code": "unexpected_child_image", + "result_sha256": "26892b202cb9dfd6678279a7e68d2f294c3b1b09984237ca6418f2d7db66a9ce", + "completed_http_checks": 18, + "cleanup": { + "owned_processes_stopped": true, + "credential_files_removed": true + }, + "correction": "The harness incorrectly excluded Windows conhost.exe; it is an owned runtime child.", + "product_defect": false + } + ], + "catalog_drill_initial_assertion": { + "result": "failed", + "reason": "The first test expected withdrawn models to be removed from all configuration. Existing documented behavior retains exact manual selectors while removing automatic priority and contribution approval.", + "correction": "The final test checks signed catalog withdrawal, automatic-priority removal, forward restore and preserved local preferences/cache. No product code changed.", + "junit_sha256": "0446b74fcee5ea9bd182010dac4d654e764d8fa625b168e99ad2f55078c08be2", + "product_defect": false + }, + "privacy": { + "credentials_exported": false, + "raw_prompts_or_outputs_exported": false, + "peer_identities_exported": false, + "host_paths_exported": false, + "raw_logs_or_junit_exported": false + }, + "remaining_live_requirements": [ + "Actual installed clients on a monitored, explicitly owned public canary route.", + "Finite active-session admission and idle/stalled-worker timeout release under the effective live policy.", + "Allowlisted live malformed-RPC injector, rejection counters and post-rejection inference recovery.", + "Block-health reconstruction after owned peer removal and return.", + "Privacy disclosure observed in the actual installed UI.", + "Isolated signed canary-channel withdrawal and higher-sequence restore through ordinary packaged refresh.", + "Exact owned route stop, automatic local fallback and explicit-selector unavailability observation.", + "Independent full canary process/resource/native-credential cleanup." + ] +} diff --git a/docs/evidence/gate16-20260908-driver-preparation.json b/docs/evidence/gate16-20260908-driver-preparation.json new file mode 100644 index 000000000..a360bb147 --- /dev/null +++ b/docs/evidence/gate16-20260908-driver-preparation.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-08T08:40:29.662722+00:00", + "scope": "Gate 16 driver preparation and local source validation only", + "result": "passed-local-driver-validation", + "complete_gate16": false, + "live_public_canary_executed": false, + "source_base_commit": "23a1f99e81df5f98ee3260b7df2529332547ccca", + "source_binding": "Exact file hashes below; shared working tree has unrelated changes. No release artifact rebuilt.", + "files_sha256": { + "scripts/gate16_live_rpc.py": "440b8da49169796c4161c61e5f21fc6171b3c0dad891ef4afe14efacd3aeeff1", + "scripts/gate16_catalog_channel.py": "521cd9362c088f4b16e2d1bfea09d09c80c35b25e96e7d17bf662ef729a75e99", + "tests/test_gate16_live_rpc.py": "4947f5df260379b2921057aa4b0f3fd246dea4d54de93fb8165a03a199cc3887", + "tests/test_gate16_catalog_channel.py": "e942292e15c73c5cbefbab2895a7d7ee96ec116f8dc9ae362263a22d638ec613", + "docs/gate16-rpc-policy.example.json": "77b66b5981fdc0bc2dadf3e59d2211a5e436332ceafcabd58f962476c63a1022", + "docs/GATE16_CANARY.md": "13b6f4e90308c8216753db7f058786083201b523516df9fcfef9081b40f951cf" + }, + "validation": { + "command": "python -m pytest tests/test_gate16_live_rpc.py tests/test_gate16_catalog_channel.py -q --disable-warnings", + "python": "3.12.9", + "platform": "win32", + "passed": 24, + "failures": 0, + "errors": 0, + "skipped": 0, + "duration_seconds": 11.003, + "private_junit_sha256": "140c4ac99e053b0f099c0dcda3322fa7d84736cf1cb54032db1271d3f4bb00eb", + "private_log_sha256": "46d7d9bd010619e860239c3a54b4a3d0cf8ff4b25fb5bba39140a1456371433b", + "black_22_3_check": "passed for both scripts and both test files", + "isort_5_10_check": "passed for both scripts and both test files", + "actual_transport": "Real loopback TLS p2pd server/client using the actual TransformerConnectionHandler and RPC stubs, with a non-compute fake backend whose cache allocator raises on use", + "catalog_execution": "Actual catalog crypto/installer against temporary local state; mocked authenticated consumer status and in-memory fetch, no live HTTPS channel", + "regressions": [ + "dry preflight cannot create transport", + "overload cannot pass malformed rejection", + "rate refill before per-peer session-cap test; missing cap fails", + "stale health and wrong manifest fail", + "file-only catalog update cannot pass", + "later node serving stale config revision cannot pass", + "stopping node cannot pass", + "child inspection and HTTP cleanup failures retain failed result JSON" + ], + "cleanup": "Loopback client/server shutdown checked by owned p2pd child return codes; temporary fixture state only; native credential getter mocked in cleanup-fault test", + "earlier_failed_attempts": "Private logs retained. Initial test fixture used an incorrect P2P handler registration signature and too-short local deadlines. Loopback test then exposed Hivemind waiting for the request producer after remote close; driver now waits for independently observed worker lease release before finishing that producer. Final validation is a separate run." + }, + "limits": { + "rpc_operation_seconds": 600, + "rpc_cleanup_seconds": 80, + "rpc_calls": 20, + "rpc_input_bytes": 131072, + "catalog_observer_default_seconds": 420, + "catalog_observer_maximum_seconds": 900, + "catalog_validity_seconds": 7200 + }, + "privacy": { + "model_loads": 0, + "gpu_jobs": 0, + "gui_launches": 0, + "public_route_mutations": 0, + "cloud_resources_provisioned": 0, + "external_catalog_publications": 0, + "raw_logs_exported": false, + "native_credentials_created": 0, + "private_signing_keys_retained": 0 + }, + "remaining_live_prerequisites": [ + "Already-owned formed route with exact worker allowlist and live health access", + "Recorded effective launch settings and operator-verified health-to-peer pairing", + "Existing authorized HTTPS canary channel and publication workflow", + "Qualified installed consumers under explicit owning lifecycle with disposable native credentials" + ], + "remaining_gate16_observations": [ + "Real local/community inference before and after probes", + "Installed-client disclosure and block-health reconstruction", + "Live signed catalog withdrawal and forward restore through ordinary consumer refresh", + "Actual owned-worker route disable and no unintended restart", + "Independent full process/resource/credential cleanup and retained settings/cache proof" + ] +} diff --git a/docs/evidence/gate16-20260908-linux-idle-transport-fix.json b/docs/evidence/gate16-20260908-linux-idle-transport-fix.json new file mode 100644 index 000000000..d77c06386 --- /dev/null +++ b/docs/evidence/gate16-20260908-linux-idle-transport-fix.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-08T10:17:09.836605+00:00", + "scope": "Gate 16 source-driver idle transport compatibility fix; local non-compute validation only", + "result": "passed-focused-local-validation", + "complete_gate16": false, + "release_artifacts_rebuilt": false, + "ci_failure": { + "commit": "64e4cf070dd8e5d19b9591e910e654094fa0f60b", + "run_id": 34212907798, + "job_id": 102017840609, + "platform": "Linux", + "python": "3.12.3", + "pytest": "6.2.5", + "hivemind": "1.1.12", + "case": "test_probe_uses_real_handler_admission_and_rejects_without_cache_allocation[loopback-tls]", + "error_type": "ConnectionResetError", + "failed": 1, + "passed": 80, + "skipped": 4, + "duration_seconds": 13.8, + "diagnosis": "After independent health showed the idle lease released, producer completion caused upstream Hivemind to write END_OF_STREAM to the expired connection. The initial exception arose in await next_reply; reawaiting the same task during cleanup made the traceback point at finally." + }, + "before_fix_reproduction": { + "platform": "Linux", + "failed": 1, + "duration_seconds": 11.19, + "error_type": "ConnectionResetError" + }, + "source_sha256": { + "scripts/gate16_live_rpc.py": "57a2b5cc3afa184963b74a38988f380922b83597eefa012b544c96ceb53957ec", + "tests/test_gate16_live_rpc.py": "527e292730254f9204f5aa1a3c275a25be9ff21275c5bc5d6f983bb28de5e6dc" + }, + "changes": [ + "Require observed idle release no earlier than step_timeout and no later than step_timeout plus five seconds, measured using the client monotonic clock from stream opening.", + "Check the exact accepted and rejected admission deltas before finishing the request producer.", + "Reject a reply task already completed with a reset, cancellation or any result other than normal EOF before producer release.", + "Only accept ConnectionResetError from awaiting the producer-triggered closure after those independent checks; record the fixed transport_closure category.", + "Keep admission and malformed-request reset failures, cache-capacity equality checks, RPC budgets and owned-process cleanup unchanged." + ], + "validation": { + "command": "python -m pytest tests/test_gate16_live_rpc.py -q -p no:cacheprovider --disable-warnings", + "linux": { + "passed": 26, + "failed": 0, + "skipped": 0, + "duration_seconds": 12.65, + "python": "3.12.14", + "pytest": "9.1.1", + "hivemind": "1.1.12", + "transport": "upstream Linux Hivemind with actual loopback TLS" + }, + "windows": { + "passed": 26, + "failed": 0, + "skipped": 0, + "duration_seconds": 9.5, + "python": "3.12.9", + "pytest": "6.2.5", + "hivemind": "1.1.12", + "transport": "locally patched Windows Hivemind with actual loopback TLS" + }, + "first_fix_before_final_negative_regression": { + "linux": { + "passed": 25, + "duration_seconds": 12.45 + }, + "windows": { + "passed": 25, + "duration_seconds": 12.12 + } + }, + "black_22_3_exit": 0, + "isort_5_10_exit": 0, + "git_diff_check": "passed for both source files", + "independent_read_only_review": "Passed after adding the already-completed reset negative regression.", + "new_regressions": [ + "Post-timeout ConnectionResetError accepted only after health/counter proof.", + "Normal EOF remains accepted after the same proof.", + "Early lease release fails for both reset and EOF.", + "Counter contamination fails before producer release.", + "Already-completed reset with otherwise-valid elapsed time and counters fails before producer release.", + "An unrelated RuntimeError after producer release remains a failure.", + "Resets during admission and malformed probes remain failures." + ], + "real_handler_scope": "Actual TransformerConnectionHandler and admission state with a non-compute fake backend whose allocator raises on use; actual loopback TLS server/client in both platforms. New proof-ordering regressions additionally use deterministic health/clock fixtures." + }, + "linux_container_limits": { + "image_id": "sha256:5a8edd63065b163d724195b1afb2554418a4d194b2ea3d9b07c58ceae64896f7", + "cpus": 2, + "memory_mib": 2048, + "memory_plus_swap_mib": 2048, + "pids": 128, + "network": "none; loopback within the disposable container only", + "user": "1000:1000", + "read_only_root": true, + "read_only_repository": true, + "read_only_existing_venv_volume": true, + "tmpfs_mib": 128, + "gpu_devices": 0, + "model_cache_mounts": 0, + "pull_policy": "never", + "all_owned_containers_removed": true + }, + "private_logs_sha256": { + "ci_failure": { + "path": ".gate13-runs/ci-linux-64e4cf0.log", + "sha256": "296a519816485f2c7a8e8220d3a9e52dece96240fb7430d4feb0d3e87ec0e0ec" + }, + "linux_before_fix": { + "path": ".gate13-runs/gate16-linux-idle-before-20260908.log", + "sha256": "bc6be7a7abf5e020b41be462c3179ec5c669a4504e3667b1b582dc0d97d4d56b" + }, + "linux_first_fix": { + "path": ".gate13-runs/gate16-linux-idle-after-20260908.log", + "sha256": "f7e004a6a4dfaace8c7214278b28ac1d6eb6e08fefaa07d547e86307ea591c8e" + }, + "windows_first_fix": { + "path": ".gate13-runs/gate16-windows-idle-fix-20260908.log", + "sha256": "4a664ba95eb053abd0e8adfb23b94cc13f7cc739b2fe985b364b81660f6da439" + }, + "linux_final": { + "path": ".gate13-runs/gate16-linux-idle-final-20260908.log", + "sha256": "8df2cbb325f3811878ff500bfc790445f8abc9bd5e37f6553cda48f498a6310a" + }, + "windows_final": { + "path": ".gate13-runs/gate16-windows-idle-fix-final-20260908.log", + "sha256": "fad9e63ef854b21adb065237d10523a99e0eede0dbe28fa36a3f3b0e8f097c07" + }, + "black": { + "path": ".gate13-runs/gate16-idle-fix-black-20260908.log", + "sha256": "83abf4b47e6b6e216d698b30f1f030922562c86836a54764d86a0e463467807e" + }, + "isort": { + "path": ".gate13-runs/gate16-idle-fix-isort-20260908.log", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "historical_evidence_unchanged": { + "path": "docs/evidence/gate16-20260908-driver-preparation.json", + "sha256": "e9e26234b48c581abd7832527743a0ada1cd8b1d5eddbccefe032349c2553911", + "original_validation": "24 passed on Windows; exact historical source hashes retained there." + }, + "limits": [ + "This is local source-driver validation, not a live public canary or a frozen release qualification.", + "The Linux reproduction uses the existing Python 3.12.14 / pytest 9.1.1 environment, while CI used Python 3.12.3 / pytest 6.2.5. The original Linux transport failure reproduced before the fix. A fresh CI result is separate evidence.", + "The idle timing proof is client-observed; public health is sampled and does not expose a per-session timer or a transport close reason. It is timeout-consistent evidence, not proof of the remote close cause.", + "Health-to-peer pairing remains operator supplied because public health exposes no peer identity.", + "No model loads, GPU work, GUI launches, external network traffic, public route changes, native credential changes or cloud provisioning occurred.", + "The remaining Gate 16 live prerequisites and observations in the canary runbook remain open." + ] +} diff --git a/docs/evidence/gate16-20260908-linux-idle-transport-fix.md b/docs/evidence/gate16-20260908-linux-idle-transport-fix.md new file mode 100644 index 000000000..45f42b5a1 --- /dev/null +++ b/docs/evidence/gate16-20260908-linux-idle-transport-fix.md @@ -0,0 +1,28 @@ +# Gate 16 Linux idle-stream transport follow-up — 2026-09-08 + +The focused RPC driver suite passes on Linux and Windows after a narrow idle-stream closure fix: **26 passed on each platform**. This is local source validation. Gate 16 remains open and no release artifact was rebuilt. Exact source and retained private-log hashes are in the [companion JSON](gate16-20260908-linux-idle-transport-fix.json). + +At commit `64e4cf070dd8e5d19b9591e910e654094fa0f60b`, [Linux CI run 34212907798, job 102017840609](https://github.com/flujo-app/CommunityAI/actions/runs/34212907798/job/102017840609) reported one failure, 80 passes and four skips in the control-contract step. The failed case used an actual loopback TLS connection and the real admission handler. After the worker had released its idle lease, upstream Hivemind raised `ConnectionResetError` while the client completed its request producer and wrote end-of-stream to the expired connection. The same failure reproduced locally before the fix: one failed case in 11.19 seconds. + +The driver now requires the client-observed idle duration to fall between the configured step timeout and that timeout plus five seconds, and verifies the exact accepted/rejected admission-counter changes before completing the producer. A reply already completed with a reset or another unexpected result fails. Only a `ConnectionResetError` raised while awaiting the subsequent producer-triggered closure is accepted, with a fixed `transport_closure` result category. Admission and malformed-request errors retain their existing strict handling; allocator guards, cache-capacity checks and cleanup remain in place. + +| Focused validation | Result | +| --- | --- | +| Linux, upstream Hivemind 1.1.12, Python 3.12.14, pytest 9.1.1 | 26 passed in 12.65 seconds | +| Windows, patched Hivemind 1.1.12, Python 3.12.9, pytest 6.2.5 | 26 passed in 9.50 seconds | +| Black 22.3, isort 5.10, diff whitespace check | Passed for both modified source files | +| Independent read-only review | Passed after adding the completed-reset negative regression | + +The nine new cases cover normal and reset closure after the timeout, early release, counter contamination, an already-completed reset before producer release, an unrelated exception, and resets during admission/malformed probes. Their proof-ordering fixtures use a deterministic clock and health snapshots. The existing real-handler and actual loopback TLS cases remain, including the backend allocator that raises if invoked and the missing-peer-cap regression. + +Run the focused suite with: + +```text +python -m pytest tests/test_gate16_live_rpc.py -q -p no:cacheprovider --disable-warnings +``` + +The Linux runs used a disposable container limited to two CPUs, 2 GiB memory, 128 processes and a 128 MiB temporary filesystem, with networking disabled except container loopback. The repository, existing environment volume and root filesystem were read-only; no GPU device or model-cache volume was attached. All owned test containers were removed. CI used Python 3.12.3 and pytest 6.2.5, so a fresh CI pass remains separate evidence from this local reproduction. + +The [original 24-pass preparation record](gate16-20260908-driver-preparation.json) and its source hashes remain unchanged. The failed CI and before-fix logs, the initial 25-case runs, and the final 26-case runs are separately hashed in the companion record. Raw logs remain private. + +The timing proof is client-observed and health is sampled: it establishes timeout-consistent release, not the remote transport's exact close cause. Health-to-peer pairing still requires the operator because health exposes no peer identity. No model, GUI, public worker, external route, credential or cloud resource was changed. The actual live inference, disclosure, catalog withdrawal/restore and other observations in the [Gate 16 runbook](../GATE16_CANARY.md) remain outstanding. diff --git a/docs/evidence/gateq38-20260903-a-fp8-loader-checkpoint.md b/docs/evidence/gateq38-20260903-a-fp8-loader-checkpoint.md new file mode 100644 index 000000000..ef3421450 --- /dev/null +++ b/docs/evidence/gateq38-20260903-a-fp8-loader-checkpoint.md @@ -0,0 +1,85 @@ +# Gate Q3.8 FP8 loader checkpoint — 2026-09-03 + +## Scope + +This is an implementation checkpoint, not Qwen3.8 release qualification. It binds the +first executable Qwen3.8 FP8 product-path work to pushed source +`de15d9cf21b946fd9b916ca6048bed9b188ef888` without claiming official artifact +verification, a real Qwen layer execution, a complete route, stock parity, recovery, +packaged acquisition, or consumer-GPU measurements. + +## Exact candidate + +- Official source: `Qwen/Qwen3.8-27B-FP8` +- Immutable revision: `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` +- Manifest: `manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json` +- Manifest digest: + `sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4` +- Declared inventory: 73 artifacts and 30,889,967,831 bytes +- Product execution profile: 64 text blocks, BF16 execution, eager attention, and + explicit `fp8_dequant` source handling +- Stock reference: `Qwen/Qwen3.8-27B` at + `1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` + +The official model card describes fine-grained FP8 quantization with 128-by-128 blocks; +the source config declares `quant_method=fp8`. Manifest/config validation now requires +that source declaration and the `fp8_dequant` runtime profile in both directions. + +## Implemented boundary + +- Preserve the outer checkpoint's quantization method through + `AutoDistributedConfig`. +- Dequantize every fine-grained FP8 weight with its matching + `weight_scale_inv` grid into the manifested BF16/FP16 execution dtype. +- Reject missing scales, orphan FP8 tensors, incompatible shapes, a prequantized source + without an explicit compatible profile, and an `fp8_dequant` profile without an FP8 + source. +- Carry the exact quantization profile through server advertisement, block loading, + memory accounting, throughput labeling, CLI resolution, and the local product-path + qualification worker. +- Pin both the official FP8 candidate and BF16 stock-reference inventories. +- Exercise an actual Transformers Qwen3.5-family block through the production + config-dispatch, FP8 dequantization, load, and forward path using a small synthetic + checkpoint. This does not represent a downloaded Qwen3.8 layer. + +## Verification + +An independent tester verified the frozen 18-path Git-index snapshot: + +- primary offline matrix: 158 passed; +- independent offline regression subset: 146 passed; +- Black, isort, Python compilation, and `git diff --cached --check`: passed; +- bidirectional source/profile validation, existing INT8/NF4 behavior, memory sizing, + and worker advertisement/effective-profile agreement: passed. + +A clean detached checkout of the pushed source ran: + +`python scripts/qualify_model_manifest.py manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json --manifest-only --machine-id local-windows-metadata --source-commit de15d9cf21b946fd9b916ca6048bed9b188ef888` + +The report passed manifest structure only and explicitly recorded +`complete_release_qualification=false`, `artifacts_verified=false`, no local parity, +and no failover. + +## Real attempt and limits + +An exploratory one-block CUDA product smoke first exposed that the qualification worker +hardcoded the unquantized profile. After the worker was corrected, the retry advanced to +official-source materialization and stopped at byte zero of `tokenizer.json` because the +Hugging Face CDN connection timed out. The attempt was not retained as qualification +evidence because it preceded the clean commit and did not verify the artifact inventory +or load a Qwen3.8 block. + +The local Windows host is not representative release hardware and cannot hold the full +dequantized 64-block route. Read-only GCP preflight found native authentication healthy, +the single global GPU allowance unused, and only the protected bootstrap running. No +cloud resource, reservation, credit, or macOS work was used; checkpoint spend is USD 0. +The current USD 100 epoch still carries the prior conservative USD 56 Gate 13 maximum, +so no new paid run was authorized from the remaining USD 44. + +## Next required outcome + +Build and verify a source-bound split-worker execution plan that can acquire only each +worker's declared layer artifacts, then run the complete 64-block route, stock parity, +selected-worker interruption, same-session recovery, packaged cold acquisition/cache +reuse, and representative RTX 30/40/50 measurements under a separately recorded +authorization that keeps combined cloud exposure at or below USD 100. diff --git a/docs/evidence/gateq38-20260903-b-span-artifact-planning-checkpoint.md b/docs/evidence/gateq38-20260903-b-span-artifact-planning-checkpoint.md new file mode 100644 index 000000000..5039f9697 --- /dev/null +++ b/docs/evidence/gateq38-20260903-b-span-artifact-planning-checkpoint.md @@ -0,0 +1,117 @@ +# Gate Q3.8 exact span-artifact planning checkpoint — 2026-09-03 + +## Scope + +This is a no-spend implementation checkpoint, not Qwen3.8 release qualification. It +binds an exact per-worker artifact-selection and admission boundary to a candidate +Git-index snapshot based on +`a48ce320fc076de6470f422ef0250f5b3e6c3cd2`. It does not claim that a Qwen3.8 +weight shard was downloaded, that a real Qwen3.8 block executed, or that a complete +route, parity, recovery, packaged acquisition, or hardware measurement passed. + +## Exact candidate + +- Official source: `Qwen/Qwen3.8-27B-FP8` +- Immutable revision: `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` +- Manifest: `manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json` +- Manifest digest: + `sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4` +- Declared inventory: 73 artifacts and 30,889,967,831 bytes +- Exact weight index: 137,335 bytes, + SHA-256 `f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2` + +A read-only official-index audit confirmed that the 64 text layers use the +`model.language_model.layers.` prefix and that each layer maps only to its +declared layer shard. No model-weight artifact was acquired by this checkpoint. + +## Implemented boundary + +- Parse the manifested checkpoint index once through an exact size- and digest-checked + verifier, reject duplicate keys and unsafe or unknown shard paths, and use that + in-memory map for both planning and manifested sharded loading. +- Build each contiguous worker span from the union of the startup config, checkpoint + index, and exact assigned shards. Shared shards are counted once; tokenizer, chat + template, MTP, outside-layer, and other-worker artifacts are excluded. +- Install an exact config/index-only allowlist before bootstrap metadata access, then + expand it only through the digest-derived worker plan before any shard is resolved, + partially inspected, or loaded. +- Filter automatic-placement candidates against the exact span byte count and bind a + canonical private artifact-set digest into planner hysteresis and lease reuse. +- Preserve the public signed-intent v1 privacy shape. Its four resource fields remain + schema version, selected artifact bytes, block count, and normalized throughput; no + private path or artifact-set digest is published. +- Bind cached intent reuse to the exact current proposal, normalized signed claims, + configured identity path, freshly loaded cryptographic key ID, and a finite unexpired + lease. Throughput changes, same-path key rotation, expired leases, and budget-driven + proposal changes force republication or fail closed. +- Use the worker cache root for both planning and launch when configured, otherwise use + the model cache root. + +## Qwen3.8 declared-set result + +With the exact pinned manifest and index metadata, four 16-block plans declare: + +- blocks 0:16 — 6,095,829,165 bytes; +- blocks 16:32 — 6,095,829,389 bytes; +- blocks 32:48 — 6,095,829,389 bytes; and +- blocks 48:64 — 6,095,829,389 bytes. + +The unique union of their weights plus one copy of startup metadata is +24,382,751,277 bytes. The remaining 6,507,216,554 declared bytes are outside-layer, +MTP, tokenizer, or chat artifacts and are not selected for a block worker. These are +manifest/index accounting results, not downloaded-cache measurements and not a hard +filesystem quota for an arbitrary custom cache. + +## Candidate source binding + +SHA-256 over the staged candidate blobs: + +- `14ba2b3ee51334df2b9d77cbddd7f5cbde1f05a9afbc0c0c61ed892498a21489` + — `src/drift/model_manifest.py` +- `10614fc1cccbea663cffc5c66ddbf023cb3da060d4f5e655b2ba517f114fc944` + — `src/drift/node/contribution_planner.py` +- `15d88815dbc7f3268cd558dc28afbe53b8a3b614e5baaa0b49075680d5cc9897` + — `src/drift/cli/run_node.py` +- `34749c2cc832cc12dd6c01e7bd65705e359c111f109b25512f22d0cf2684b6c1` + — `src/drift/server/server.py` +- `5c0d8fdaf8f2eb24c085037c285ce4afebe878eb1311ba249ad36aba387c4af5` + — `src/drift/server/from_pretrained.py` +- `5b0029e8f847580feb7035f05b0db3ecd129cb4362159414264d34b62a05da1e` + — `tests/test_model_manifest.py` +- `39302a4f9530a2477c58f682a8f567974aa28adba7f62dae39f739e35f8f83fb` + — `tests/test_contribution_planner.py` +- `653981a601e1068f380e841e8bb633ca70548a880d16af6816969879d8811f5e` + — `tests/test_node_config.py` + +## Verification + +The final local candidate passed: + +- 142 focused manifest/planner/node/identity tests; +- 132 adjacent automatic-placement, discovery, node, acquisition, Qwen loading, and + server tests; +- 1,564 offline unit tests with 10 expected skips; +- Black, isort, Python compilation, and `git diff --check`; and +- independent adversarial review of cache precedence, selection/deduplication, + allowlist enforcement, strict index consumption, v1 privacy, budget/hysteresis, + throughput and identity binding, proposal mismatch, and lease expiry. + +The broad offline command deliberately excluded legacy peer-dependent integration files +that require external `INITIAL_PEERS`, plus the environment-specific optional +bitsandbytes probe whose installed `peft` imports an unusable fake bitsandbytes module. +The selected unit matrix itself completed without failures. + +## Spend and release status + +No provider mutation, reservation, cloud resource, credit, macOS work, or model-weight +download occurred; checkpoint spend is USD 0. The current USD 100 epoch still carries +the prior conservative USD 56 Gate 13 maximum, leaving USD 44 unreserved but not +authorized for this checkpoint. Gate Q3.8 remains `IN PROGRESS`. + +## Next required outcome + +Use this source-bound span plan in the actual split-worker acquisition/execution bridge, +then prove the complete 64-block route, stock parity, selected-worker interruption and +same-session recovery, packaged cold acquisition/cache reuse, and representative RTX +30/40/50 measurements. Any paid attempt still requires fresh authentication, inventory, +quota, price, and combined-ledger validation plus an explicit bounded reservation. diff --git a/docs/evidence/gateq38-20260903-c-worker-plan-execution-binding-checkpoint.md b/docs/evidence/gateq38-20260903-c-worker-plan-execution-binding-checkpoint.md new file mode 100644 index 000000000..691a8a83d --- /dev/null +++ b/docs/evidence/gateq38-20260903-c-worker-plan-execution-binding-checkpoint.md @@ -0,0 +1,89 @@ +# Gate Q3.8 worker plan execution binding checkpoint — 2026-09-03 + +## Scope + +This is a no-spend implementation checkpoint, not Qwen3.8 release qualification. It +binds the exact per-worker span plan from automatic placement into the real server +subprocess and validates it again before the worker can announce or access weights. The +candidate is based on `93daa9f2d5c25d489ac041b1e800d04b24ee7150`. + +It does not claim that a Qwen3.8 weight shard was downloaded, that a real Qwen3.8 +block executed, or that a complete route, parity, recovery, packaged acquisition, or +hardware measurement passed. + +## Implemented boundary + +- An acknowledged automatic placement now carries five inseparable private claims: + exact manifest digest, canonical block span, exact artifact byte count, artifact-set + digest, and canonical absolute cache root. +- The generated source and frozen server commands carry both the actual and expected + span/cache values. The immutable `WorkerLaunch` validates every bound flag exactly + once, rejects inline or duplicate claim forms and `--num_blocks`, and requires the + current node executable plus the canonical source or frozen server entrypoint. +- Placement-bound commands reject explicit configuration files, custom modules, + training RPCs, and credential flags. Any internal claim selects a parser with no + ambient `config.yml` and no `-c`/`--config`; `server_from_args` rejects claims + that did not come from that parser. +- The server independently compares the loaded manifested identity and canonical + span/cache, derives the config/index/shard plan from verified metadata, and compares + exact bytes and artifact-set digest before constructing the join announcer or + resolving a weight. +- Different spans that happen to share the same physical shard set remain distinct: + an acknowledged `0:1` cannot admit an actual `1:2` even when their selected + bytes and artifact-set digest are identical. +- The artifact-set digest and cache path stay out of supervisor snapshots, public + health, signed announcements, and DHT records. + +## Candidate source binding + +SHA-256 over the candidate source and tests: + +- `83edb3fe91dae83e393a39151d5cd24e6feb307145f6629ff1447ce9b9201c40` + — `src/drift/cli/run_node.py` +- `0ac20e620e0d94a13dd81f19dec679e926a99d6f46b163f00e218774763749d3` + — `src/drift/cli/run_server.py` +- `a39688a4f736d6c4aa1cd97ae384f4f4137411578edaf12c1691936d72e62822` + — `src/drift/node/worker_supervisor.py` +- `6dd709d3f38e089510d741b2eaa1375ead69aab12c6be1059a25f01558f4d535` + — `src/drift/server/server.py` +- `618ee282b04ede56a06baac2748c33df273083be940e47d1aeaf94289656d6b8` + — `tests/test_model_manifest.py` +- `efcc215e116027931b25e43ad3873e75f568b365660d78d968cb518515c3e8f8` + — `tests/test_node_config.py` +- `8477307bc202d84cb4921f9ad3357adc4569db3e387af46386dd91355c51a2c7` + — `tests/test_worker_supervisor.py` + +## Verification + +The final local candidate passed: + +- 147 focused manifest/node/supervisor/packaged-dispatch tests; +- 259 related planner, identity, automatic-placement, privacy, server-admission, + memory-budget, registry, and packaged-dispatch regressions; +- 1,568 offline unit tests with 10 expected skips; +- Black, isort, Python compilation, and `git diff --check`; and +- independent adversarial review of shared-shard span substitution, canonical cache + binding, exact command/executable identity, ambient and explicit configuration + injection, unsafe server options, pre-announcement validation, exact allowlisting, + and public-state privacy. + +The broad offline command excluded the legacy peer-dependent integration files that +require external `INITIAL_PEERS`, plus the environment-specific optional bitsandbytes +probe whose installed dependency graph is unusable. The selected unit matrix completed +without failures. + +## Spend and release status + +No provider mutation, reservation, cloud resource, credit, macOS work, model download, +or external endpoint was used; checkpoint spend is USD 0. Under the owner-specified +USD 100 ceiling, the current epoch still carries the prior conservative USD 56 maximum, +leaving USD 44 unreserved. Gate Q3.8 remains `IN PROGRESS`. + +## Next required outcome + +Use this bound command for a fresh official-source single-span acquisition and real +Qwen3.8 block execution, then build the complete 64-block route and prove stock parity, +selected-worker interruption with same-session recovery, packaged cold acquisition and +cache reuse, and representative RTX 30/40/50 measurements. Any paid attempt still +requires fresh authentication, inventory, quota, pricing, and combined-ledger validation +plus an explicit bounded reservation. diff --git a/docs/evidence/gateq38-20260903-d-fresh-single-span-execution-checkpoint.md b/docs/evidence/gateq38-20260903-d-fresh-single-span-execution-checkpoint.md new file mode 100644 index 000000000..b737ce02b --- /dev/null +++ b/docs/evidence/gateq38-20260903-d-fresh-single-span-execution-checkpoint.md @@ -0,0 +1,151 @@ +# Gate Q3.8 fresh single-span execution checkpoint — 2026-09-03 + +## Scope + +This is the first real Qwen3.8 outcome checkpoint, not complete Qwen3.8 release +qualification. The fresh run retained its launchers and acquisition/execution logs but did +not emit a Git source attestation at process start. A later network-disabled cache-reuse +replay checked the relevant worker and client source bytes against pushed commit +`af7d887a471c295bd593a6feb4f47f34056eb3e3` and tree +`4c064b2a60b57bc3136db89b17f2ad8cbf96353f` before launch and repeated the same +deterministic block result. These are reported as separate outcomes; this checkpoint does +not retroactively claim that the original fresh process cryptographically attested its +source tree. + +Pre-existing unrelated catalog, documentation, and test changes remained dirty and outside +the execution and evidence scope. The exact placement-bound automatic-worker command +contract from the preceding execution-binding checkpoint acquired and executed one +official Qwen3.8 block. The transient coordinator supplied the already-validated placement +fields; this run does not repeat the separate intent-publication or +remote-acknowledgement proof. + +This checkpoint does not claim a complete 64-block route, stock parity, selected-worker +recovery, packaged acquisition or restart/cache reuse, or the required RTX 30/40/50 +measurements. + +## Exact input and post-run environment audit + +- Official source: `Qwen/Qwen3.8-27B-FP8` at immutable revision + `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`. +- Manifest: `sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4`. +- Selected span: `model.language_model.layers.0`, canonical block range `0:1`. +- Exact selected files: `config.json`, `layers-0.safetensors`, and + `model.safetensors.index.json`. +- Exact selected bytes: `384054133`; selected-set SHA-256: + `43d8b1d59667b556e77b0ff7febcbb44d1831608f30f47f82c0eba0f3bf87aca`. +- The isolated run root was new and empty before its metadata-only exact plan was + resolved. The 383,865,448-byte layer shard was absent before the bound worker began. +- The source-bound replay preflight recorded Windows 10 `10.0.19045` with an + NVIDIA GeForce RTX 2070 SUPER (8,589,606,912 reported bytes). +- The same preflight recorded Python `3.12.9`, DRIFT `2.3.0.dev2`, PyTorch + `2.6.0+cu124`, CUDA `12.4`, Transformers `5.13.1`, and Hivemind `1.1.12`. + +The plan verifier used `token=False`; the server command had no credential flag, and +its launch environment was configured with Hugging Face token variables empty, implicit +Hub tokens disabled, the official `https://huggingface.co` endpoint, and uppercase +HTTP/HTTPS/all-proxy variables cleared. No packet capture or child-environment snapshot +was retained, so this is a launch-configuration claim rather than an independently +observed network-path claim. Exact post-run artifact hashes bind the selected bytes to the +pinned official revision. The command bound the exact manifest/span/bytes/set/cache claims, +limited the worker cache to 2 GiB and CUDA allocation to 6 GiB, and opened only a +loopback `--new_swarm` listener. + +## Real execution result + +All timestamps below are host-local UTC-05. + +- At 03:07:44 the bound source worker accepted the exact manifest plan, started its + loopback swarm, and began the manifested FP8-to-BF16 load. +- At 03:13:04 it reported `Loaded Qwen/Qwen3.8-27B-FP8 block 0`; at 03:13:06 its + one connection handler and runtime were ready. The observed startup-to-load interval + was about 320 seconds, including official-source transfer, verification, dequantization, + and device load; it is not presented as a download-only benchmark. +- An offline client joined only + `QmPDLtRoofeyrLKXP2yJT5Yy7ZQQDDjMz8krZbDmj9H3TV`, required the exact manifest + digest/runtime profile, and restricted routing to that PeerID and span `0:1`. +- A real `rpc_inference` session transformed one deterministic + `torch.bfloat16` hidden-state tensor of shape `[1, 1, 5120]` in 0.363 seconds. + The output retained the exact shape and dtype, was finite, and differed from the + input. +- Input SHA-256: + `87c62484d2e6c3ce38e94a9064871fff63c1f7f940b7975f588dd96c61996871`. + Output SHA-256: + `877302b713404bb60ccab8d72160156d360066828a71e0de2977cc048dafe631`. +- The server independently logged authenticated `rpc_inference.open`, allocation, + and `rpc_inference.close` for blocks `0:1`. +- The original client cleanup paths and supervisor shutdown returned, and the original + worker log ends with `worker_stopped`. The original run did not retain a contemporaneous + PID/listener audit, so no stronger original-process cleanup claim is made. + +A separate offline post-run verification rehashed all three selected files against the +manifest, recomputed the same `384054133` bytes and selected-set digest, and performed +no network acquisition. + +## Source-bound offline replay and cleanup + +A second run reused only that verified cache with `HF_HUB_OFFLINE=1`, token variables +empty, and upper- and lowercase proxy variables cleared. Before the worker started, its +launcher required HEAD to equal the tracked origin at `af7d887`, recorded tree +`4c064b2a60b57bc3136db89b17f2ad8cbf96353f`, and captured Git blob plus working-byte +hashes for 19 relevant manifest, CLI, supervisor, server, model, and client paths. The +client repeated its own eight-path source check immediately before the RPC. + +The replay loaded block `0` from cache, served the same exact peer and `0:1` span, and +returned the same deterministic input/output hashes as the fresh run in 0.206 seconds. +Both commands exited zero. The retained cleanup audit then found worker PID `31612` +absent, zero listeners on port `50593`, and zero replay-script-tagged process command +lines. This proves those exact cleanup observations, not an all-descendant forensic audit. + +The durable +[source/runtime/cleanup audit](gateq38-20260903-d-source-runtime-cleanup-audit.json) +contains the source identities, runtime/device versions, bounded launch/result fields, +cleanup record, original fresh-run limitation, and hashes for all nine ignored raw assets. +It is 9,487 bytes with SHA-256 +`d24e4ceb49354eb57924182174c2b9225618b05e5157c2be2d9d4598295311aa`. + +## Retained evidence binding + +The raw run assets remain ignored because the worker log contains an absolute local cache +path. They were present for independent review and are bound here by byte count and +SHA-256 so any later local copy can be checked exactly; they are not committed repository +artifacts. + +| Local ignored artifact | Bytes | SHA-256 | +| --- | ---: | --- | +| `bound-worker.log` | 6,307 | `768568e2e661c0d051fd6a3792fbc19152d6023e2bd8af51bcd34a2b9e9cd8bd` | +| `bound-rpc.log` | 742 | `9bd39827d34af0f607faea3c52b408bd02360f5cb9192c4dc5d4ecf50d695948` | +| `run_bound_worker.py` | 3,750 | `9ba92da0da0d45676fa7cfe2d689c4bb490b03ac5af512af5cc1ee8a0d079ab1` | +| `run_bound_rpc.py` | 4,061 | `4f529e40d5c01671d56a30d6f716473a81562cfb9bcbb8df67059ea5dfc6d16d` | + +The fresh worker log records the manifested load, handler readiness, authenticated +`rpc_inference` open/allocate/close events, and supervisor return. The fresh RPC log +records the exact peer/span, credential-environment check, tensor contract, elapsed time, +and input/output digests. The committed JSON audit binds those files separately from the +later source-bound cache-reuse logs and exact replay cleanup record. + +## Source and review binding + +The network-disabled replay is explicitly bound to the pushed +[worker-plan execution-binding checkpoint](gateq38-20260903-c-worker-plan-execution-binding-checkpoint.md). +That candidate had already passed 147 focused tests, 259 related regressions, 1,568 +offline tests with 10 expected skips, formatting/import/compile/diff checks, and +independent adversarial review. The fresh run adds the official-artifact acquisition and +first hardware result; the source-bound replay repeats that hardware result from the +verified cache. Neither reinterprets unit results as hardware evidence. + +## Spend and release status + +No provider mutation, reservation, cloud resource, credit, or macOS work occurred. +The layer transfer used the local Windows host, so checkpoint cloud spend is USD 0. +Under the owner-specified USD 100 ceiling, the current epoch retains the prior +conservative USD 56 maximum and USD 44 remains unreserved. Gate Q3.8 stays +`IN PROGRESS`. + +## Next required outcome + +Build the exact complete 64-block route across independent workers and prove stock +parity, then interrupt one selected worker and prove same-session recovery. After that, +prove packaged anonymous cold acquisition, restart/cache reuse, and bounded RTX +30/40/50 measurements. Any paid attempt still requires fresh authentication, inventory, +quota, pricing, and combined-ledger validation plus an explicit reservation within the +remaining USD 44. diff --git a/docs/evidence/gateq38-20260903-d-source-runtime-cleanup-audit.json b/docs/evidence/gateq38-20260903-d-source-runtime-cleanup-audit.json new file mode 100644 index 000000000..92a9b8730 --- /dev/null +++ b/docs/evidence/gateq38-20260903-d-source-runtime-cleanup-audit.json @@ -0,0 +1,222 @@ +{ + "schema": "communityai.gateq38.source-runtime-cleanup-audit.v1", + "audit_kind": "source-bound offline cache-reuse replay after the fresh single-span run", + "checkpoint_date": "2026-09-03", + "fresh_run": { + "source_attestation_at_launch": false, + "claim_boundary": "The retained fresh-run assets prove new-cache acquisition, manifested block load, exact-peer inference, and supervisor return. They do not independently attest the Git tree at original process start.", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "block_indices": "0:1", + "artifact_bytes": 384054133, + "artifact_set_sha256": "43d8b1d59667b556e77b0ff7febcbb44d1831608f30f47f82c0eba0f3bf87aca", + "inference_elapsed_seconds": 0.363, + "raw_assets": [ + { + "name": "bound-worker.log", + "bytes": 6307, + "sha256": "768568e2e661c0d051fd6a3792fbc19152d6023e2bd8af51bcd34a2b9e9cd8bd" + }, + { + "name": "bound-rpc.log", + "bytes": 742, + "sha256": "9bd39827d34af0f607faea3c52b408bd02360f5cb9192c4dc5d4ecf50d695948" + }, + { + "name": "run_bound_worker.py", + "bytes": 3750, + "sha256": "9ba92da0da0d45676fa7cfe2d689c4bb490b03ac5af512af5cc1ee8a0d079ab1" + }, + { + "name": "run_bound_rpc.py", + "bytes": 4061, + "sha256": "4f529e40d5c01671d56a30d6f716473a81562cfb9bcbb8df67059ea5dfc6d16d" + } + ] + }, + "source_bound_replay": { + "network_mode": "HF_HUB_OFFLINE=1 with token variables empty and upper/lowercase proxy variables cleared", + "cache_mode": "reuse of the exact selected files rehashed after the fresh run", + "repository": { + "commit": "af7d887a471c295bd593a6feb4f47f34056eb3e3", + "tree": "4c064b2a60b57bc3136db89b17f2ad8cbf96353f", + "tracked_origin_commit": "af7d887a471c295bd593a6feb4f47f34056eb3e3" + }, + "production_sources": [ + { + "path": "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json", + "git_blob_sha1": "2759bf08038dea06fc65a9abe1072b99706c8176", + "worktree_sha256": "0591317317adf4752425725658732997597c51150a7b165bcd674cbc319bf2a1" + }, + { + "path": "src/drift/__init__.py", + "git_blob_sha1": "4cbbacdd285285416f975a2a8541b226aa96516b", + "worktree_sha256": "25413cb38a8cbd8d9903301d2da628e39cc91769d564b6c1838f6252a7f3de6e" + }, + { + "path": "src/drift/client/__init__.py", + "git_blob_sha1": "1613ad1e17f04e1b518e8b763da7a4bbf1b4b8aa", + "worktree_sha256": "51c204cabf56fddb8bbab110041df34f9c5df9b09dabfd958705b2d389332d97" + }, + { + "path": "src/drift/client/config.py", + "git_blob_sha1": "3e7370b159c07d29e71708d12e1083bc370fea55", + "worktree_sha256": "b05f476f59bf4bd7ec9d89d9facda83b85c677eda8adecad6cd1a0b5038c79a5" + }, + { + "path": "src/drift/client/remote_sequential.py", + "git_blob_sha1": "e3f88fee23a9619b866552045e58fc10609ee7b2", + "worktree_sha256": "40d4042db011531a15953e16c6cb685d7bfdbeb6667a28f26e6593ae1dc9af82" + }, + { + "path": "src/drift/client/routing/sequence_manager.py", + "git_blob_sha1": "3643fb498b261f8f5b1fc1b3b96fa066d3e8a3fe", + "worktree_sha256": "080e4798bdb21a9c1941c9f2b926169b84b363b0f8cdb31e07cb1bf819b979ea" + }, + { + "path": "src/drift/cli/__init__.py", + "git_blob_sha1": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "worktree_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/drift/cli/__main__.py", + "git_blob_sha1": "89fa53db4a4f5b144ab9aa99a36c635f2b944ac3", + "worktree_sha256": "995060738b41c25af633a81e4819a3c01b85af143d8878e06c69100dc44d2323" + }, + { + "path": "src/drift/cli/run_server.py", + "git_blob_sha1": "f730fdb5bd3b6d3db6a0f5d2f53ea6c9d6f51244", + "worktree_sha256": "0ac20e620e0d94a13dd81f19dec679e926a99d6f46b163f00e218774763749d3" + }, + { + "path": "src/drift/model_manifest.py", + "git_blob_sha1": "e462b37a7703a7f3e0259686f28b7a6212b446c4", + "worktree_sha256": "b4684eeb9b83395b352275048c4959d9513c9e83768f75d489aec1eba5ff6913" + }, + { + "path": "src/drift/models/__init__.py", + "git_blob_sha1": "5eba9d6d101055da50d3624fb08bfc85b812a68f", + "worktree_sha256": "ece5852c7907a00d6f6921b3d8b868e42b19f5a474c131d25e5cd7ebc8231fef" + }, + { + "path": "src/drift/models/qwen3_5/__init__.py", + "git_blob_sha1": "d3700385f72e5ee09dfdf147e19fd55769c14652", + "worktree_sha256": "7a5c3bc89c45a882832adced740b8d19c2dded51218a5d31fccaf098fc7408a5" + }, + { + "path": "src/drift/models/qwen3_5/block.py", + "git_blob_sha1": "745e9bc2793fca9c12f9c09f2b7e123140bbfb63", + "worktree_sha256": "cb506f75ad664086ee5e750cfaa0cfd8ebf3b1b289efa210039bb38bf549165f" + }, + { + "path": "src/drift/models/qwen3_5/config.py", + "git_blob_sha1": "23331aff0566b3beeeff886f4bbaf9953dbf29ce", + "worktree_sha256": "6c467fc6948a2f06e2a7068864b473fa00b7d5cc6c5824420693a183d8a5a726" + }, + { + "path": "src/drift/models/qwen3_5/model.py", + "git_blob_sha1": "56a0c6d655ecb09fd5229c3a6572680b8f9a1cc2", + "worktree_sha256": "b9092f62eb2a26881e2b2452b74f192bc6bf4e70f9091116cb6f7ac5a0a0dffb" + }, + { + "path": "src/drift/node/worker_supervisor.py", + "git_blob_sha1": "42ea6224003fce4060ce4c2f9fda50052a7ed543", + "worktree_sha256": "ac80b3759831bed4f7ce018b9592d65f0acc876ba521cb51ec8bfbc048493020" + }, + { + "path": "src/drift/server/backend.py", + "git_blob_sha1": "6560fe7f29a95b9bc6dabb5bb90b5195436378dc", + "worktree_sha256": "38858453c0bd28c9941a0f9f9a88d1dfb0c04b363b19ef0cf1fac30776aa70da" + }, + { + "path": "src/drift/server/server.py", + "git_blob_sha1": "4ab4a43814ba29938188f98cbfa4aacfcbb27fbb", + "worktree_sha256": "6dd709d3f38e089510d741b2eaa1375ead69aab12c6be1059a25f01558f4d535" + }, + { + "path": "src/drift/utils/dht.py", + "git_blob_sha1": "d022cc6fa8c528529589dc4f42b2a57b59122fc3", + "worktree_sha256": "6b142a0333af466873c14a451047503f49e68f272fd4d5c48d4aa3ee69b73d14" + } + ], + "runtime": { + "platform": "Windows-10-10.0.19045-SP0", + "python": "3.12.9", + "drift": "2.3.0.dev2", + "torch": "2.6.0+cu124", + "cuda": "12.4", + "transformers": "5.13.1", + "hivemind": "1.1.12", + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_memory_bytes": 8589606912 + }, + "launch_contract": { + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "block_indices": "0:1", + "artifact_bytes": 384054133, + "artifact_set_sha256": "43d8b1d59667b556e77b0ff7febcbb44d1831608f30f47f82c0eba0f3bf87aca", + "device": "cuda:0", + "max_device_memory": "6GiB", + "max_disk_space": "2GiB", + "listener": "loopback only", + "source_preflight_before_worker_launch": true, + "client_source_preflight_before_rpc": true + }, + "result": { + "worker_exit_code": 0, + "rpc_exit_code": 0, + "server_dtype": "bfloat16", + "server_quant_type": "fp8_dequant", + "shape": [1, 1, 5120], + "input_sha256": "87c62484d2e6c3ce38e94a9064871fff63c1f7f940b7975f588dd96c61996871", + "output_sha256": "877302b713404bb60ccab8d72160156d360066828a71e0de2977cc048dafe631", + "finite": true, + "changed": true, + "elapsed_seconds": 0.206 + }, + "cleanup": { + "audit_utc": "2026-09-03T09:07:59.4135315Z", + "retained_worker_pid": 31612, + "pid_present": false, + "loopback_port": 50593, + "listener_count": 0, + "tagged_process_count": 0, + "worker_session_exit_code": 0, + "rpc_exit_code": 0, + "claim_boundary": "This proves the recorded worker PID, exact listener, and replay-script-tagged command lines were absent. It is not an all-descendant forensic claim." + }, + "raw_assets": [ + { + "name": "run_bound_worker_replay.py", + "bytes": 6770, + "sha256": "93ef83d640ff26815081aac26cd4899f9801bccd9cf2640fd1d99fce950e3d34" + }, + { + "name": "run_bound_rpc_replay.py", + "bytes": 5825, + "sha256": "4ffb94dd6bb7272b729be6a8b790b414b9e6acfe78ccc7f8a48b8645ee82d7c8" + }, + { + "name": "source-bound-worker-replay.log", + "bytes": 9422, + "sha256": "c536cb3e735e2396c78f4aaa8e797fda693419df9544683d3540ff4cf7e33c5b" + }, + { + "name": "source-bound-rpc-replay.log", + "bytes": 2521, + "sha256": "7fc6716bbf25c2a94700675885f6fe76ac7232dc5b554332f0dfaaeab12afe09" + }, + { + "name": "source-bound-cleanup-audit.json", + "bytes": 206, + "sha256": "e085da5fb97effc7f4627de99329115f2b736dbe492b5d58b21da710bef6df5d" + } + ] + }, + "limitations": [ + "The replay source list is a curated production-path binding, not a complete Python import trace.", + "The replay proves source-bound cache reuse, not another cold acquisition.", + "The original fresh run did not emit a source attestation at launch, so the fresh acquisition and source-bound replay are reported as separate outcomes.", + "No complete route, stock parity, selected-worker recovery, packaged acquisition, RTX 30/40/50 qualification, or cloud execution is claimed." + ], + "spend_usd": 0 +} diff --git a/docs/evidence/gateq38-20260903-e-complete-route-controller-checkpoint.md b/docs/evidence/gateq38-20260903-e-complete-route-controller-checkpoint.md new file mode 100644 index 000000000..3a7ef4d49 --- /dev/null +++ b/docs/evidence/gateq38-20260903-e-complete-route-controller-checkpoint.md @@ -0,0 +1,129 @@ +# Gate Q3.8 complete-route controller checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 controller contract; Gate Q3.8 remains IN PROGRESS +Base HEAD and upstream before this checkpoint: `870e97ee8e01dc95829001a681149ec88c459725` + +## Scope + +This checkpoint adds a durable, provider-neutral controller for one exact Qwen3.8-27B FP8 +complete-route attempt. The controller does not call GCP, Fly.io, or any other provider. It +opens a bounded plan and observation, rederives the production artifact plan, advances one +persistent state machine, and emits at most one allowlisted action for a later provider +adapter. + +The exact route contains four independent workers and no interchangeable spans: + +| Worker span | Selected bytes | Artifact-set SHA-256 | +| --- | ---: | --- | +| `0:16` | 6,095,829,165 | `70c0c950845c0c53dc0269d525c755bc72e661cf4ded8a78a7b5f99d8d195d89` | +| `16:32` | 6,095,829,389 | `01d4ca6e77a9564e6896343b0c8558619fcda78819eeafb0d49393a955460866` | +| `32:48` | 6,095,829,389 | `4b3ac15527d87d2dbd089fc4ba4ab0dec4610a5e9870df1401473159b55138e5` | +| `48:64` | 6,095,829,389 | `2e779c52ab2eb5156aa3cfba60e5d08b4dd691e0302101cbc1a39c24d45745e1` | + +The controller binds the official model revision +`017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`, manifest +`c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4`, +index `f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2`, +and the production `ManifestArtifactVerifier` source identity. It recomputes all four +span selections from the strict official `model.language_model.layers` index keys before +start or collection; it does not trust caller-supplied byte or digest claims. + +## Paid-start boundary + +A paid start requires all of the following at the first genuine issuance point: + +- an exact, source-bound line in the checked-in readiness ledger naming the run, + reservation ID, maximum USD amount, and deadline; +- controller-protected reservation and preflight files whose digests and sizes are + carried by the plan and independently bind its stable digest, exact source set, + execution inventory, worker plan, pricing horizon, and ledger scope; +- the reset-epoch arithmetic `USD 56.00 + at most USD 44.00 <= USD 100.00`; +- fresh native authentication, inventory, pricing, and capacity attestations; +- four unused GPU slots and the protected bootstrap still running; +- an exact eleven-resource inventory with cost lines bound to the canonical launch + specification digest. + +Every emitted start action carries the same canonical GCP specification: project +`community-ai-506321`, region `us-central1`, zone `us-central1-b`, four +`g2-standard-8` workers with one `nvidia-l4` each, one CPU-only +`e2-standard-2` bootstrap, the pinned CUDA image, five 50 GiB balanced disks, +the fixed network/subnet, and an 11-hour maximum lifetime. Every non-firewall +resource must have a positive price, quantity `1.00`, and exactly 11 priced hours. +The complete cost must equal the protected reservation maximum. + +The checked-in readiness ledger contains no `Q38_ROUTE_RESERVATION` line for this +controller. Therefore the live repository state authorizes no paid start. This run did +not create, reserve, modify, or delete any cloud resource and consumed USD 0. + +## Evidence and recovery boundary + +The state machine covers `ABSENT`, `STARTING`, `READY`, `COLLECTING`, +`CLEANING`, `CLEANED_PASS`, and `CLEANED_FAILURE`. Action IDs are deterministic +from the run, plan, and action. An issuance journal is durably written before the first +start decision. Loss of state after issuance cannot emit a second paid start; a completed +journal reconstructs its terminal result. Cleanup bypasses expired start authorization +and stale production inputs, but still requires exact run-scoped absence and survival of +the protected bootstrap. + +A passed route cannot be inferred from the observation's embedded JSON. The controller +requires a protected exact evidence directory containing one terminal record, one RPC +record, and four worker records, with no extras. It reopens every child by no-follow +identity, checks the recorded byte digest, and binds the exact run, job, action, plan, +source, model, worker, machine, peer, span, artifact set, cache, and session before +preserving a pass through cleanup. + +State, decision, journal, and lock paths must be distinct from every input and input root. +The controller serializes invocations with a native exclusive lock and writes a neutral +decision, state, and final decision atomically. Cleanup remains possible after reservation +expiry, deadline expiry, partial inventory, or protected-bootstrap loss. + +## Verification + +All checks ran on Windows with the repository's CUDA test environment: + +- `102 passed` in `tests/test_gateq38_route_controller.py`; +- `17 passed` in the final independent security-focused regression subset; +- `423 passed, 1 skipped` across the controller, model-manifest, and all Gate 14 + contract tests; +- `1,651 passed, 10 skipped` in the repository offline unit matrix; +- Black, isort, Python compilation, and Git whitespace checks passed. + +The offline matrix excludes the documented tests that require an externally provisioned +`INITIAL_PEERS` swarm and the unavailable optional bitsandbytes/PEFT runtime probes: +`test_aux_functions.py`, `test_block_exact_match.py`, +`test_chained_calls.py`, `test_deepseek_v3.py`, `test_dtype.py`, +`test_full_model.py`, `test_gemma4_block.py`, +`test_remote_sequential.py`, `test_sequence_manager.py`, +`test_server_stats.py`, `test_speculative_generation.py`, +`test_startup_guard.py`, `test_tensor_parallel.py`, `test_utils.py`, +`test_optional_bitsandbytes_runtime.py`, and `test_peft.py`. + +Independent review reproduced the focused and security subsets and directly proved that +a 24-hour lifetime priced for 11 hours and a fully rebound +`e2-micro` / bogus-GPU / bogus-image substitution both fail closed. It reproduced the +four official-index spans from `model.language_model.layers`, proved worker-plan or source +substitution changes both the stable plan and execution-inventory digests, and confirmed +that prior protected authorization is rejected after either substitution. + +## Canonical candidate blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 88,635 | `0599dc50ff5649693e8498ce33dd1f7b5dc44a21289bb18f9b78b73c823d8ac5` | +| `tests/test_gateq38_route_controller.py` | 70,797 | `2023a6bab707b11ea50b67d1d2f876a774af2f97ab25b5984099e54ec95703b2` | + +These are SHA-256 digests of the canonical Git-index blobs staged with this evidence. + +## Explicitly not proven + +This checkpoint does not include a provider adapter, a protected live plan or reservation, +a cloud create, a four-worker route, any new model download, stock-output parity, +same-session selected-worker recovery, packaged acquisition/cache reuse, or representative +RTX 30/40/50 qualification. It does not pass Gate Q3.8. + +The next unblocked USD 0 step is a source-bound provider adapter that consumes the exact +action specification, produces the strict observation/evidence inventory, and implements +idempotent cleanup. A paid attempt remains blocked until the readiness ledger contains a +fresh exact reservation and native preflight proves four available accelerator slots +inside the remaining USD 44 ceiling. diff --git a/docs/evidence/gateq38-20260903-f-gcp-adapter-checkpoint.md b/docs/evidence/gateq38-20260903-f-gcp-adapter-checkpoint.md new file mode 100644 index 000000000..df8bf130d --- /dev/null +++ b/docs/evidence/gateq38-20260903-f-gcp-adapter-checkpoint.md @@ -0,0 +1,97 @@ +# Gate Q3.8 GCP adapter checkpoint — 2026-09-03 + +Status: **PARTIAL / USD 0** + +This checkpoint adds the source-bound GCP adapter for the durable Qwen3.8 +complete-route controller. It deliberately does **not** authorize or execute a paid +route. The checked-in readiness ledger still contains no exact Q3.8 reservation, and +the adapter rejects both `start_route` and `collect_route` before authentication, +inventory, SSH, or any provider mutation because the protected Qwen3.8 host runtime +and status/evidence transport are not yet plan-bound. + +Base source commit: + +- `22b544b1b3b36c2c7997234e800ed67e53816fb1` + +## What this checkpoint proves + +The controller source boundary now includes +`scripts/gateq38_gcp_adapter.py`. A plan whose adapter bytes, controller bytes, +worker plan, ledger binding, or other required source changes cannot be loaded as the +same execution plan. + +The adapter compiles, without executing, an exact eleven-command start specification: + +- five image-backed 50 GB `pd-balanced` disks; +- one internal route firewall; +- four `g2-standard-8` / L4 workers and one `e2-standard-2` bootstrap; +- the pinned public deep-learning image and its explicit image project; +- private IPv4-only interfaces, no external IPv4 or IPv6, no service account, and no + IP forwarding; +- one digest-derived network tag unique to the run and plan rather than a tag shared + by every Q3.8 attempt; +- standard provisioning, restart-on-failure, terminate-on-maintenance, an 11-hour + maximum lifetime, and automatic instance deletion; +- boot-disk auto-deletion for the unattended maximum-lifetime path. Controlled + cleanup retains disks while deleting instances, then validates and deletes each + disk explicitly. + +Plan resource names must satisfy GCE's RFC1035 constraints and remain under the exact +run prefix. Read-only inventory enumerates that prefix for instances, disks, and the +firewall, rejecting missing, extra, foreign, malformed, publicly reachable, +privileged, deletion-protected, or shape-changed resources. The protected +`communityai-bootstrap-1` instance is observed only for its running health invariant +and is never included in compiled or cleanup mutations. + +Until the host runtime exists, observation accepts only the canonical all-absent host +status. A static file therefore cannot promote a provider instance to `ready` or a +route to `passed`. A future runtime checkpoint must replace that blank-only rule +with fresh, protected, instance-generation-bound status and exact evidence transfer. + +Cleanup dispatches directly from an exact source-bound controller decision rather +than requiring aggregate inventory to pass first. Every deletion re-describes and +validates the immutable resource binding independently, cleanup continues after +foreign resources and transient failures, terminal `TERMINATED` instances and +`FAILED` disks remain deletable, and a strict final inventory must prove exact +absence while the protected bootstrap remains running. A failed cleanup remains +retryable; it cannot be reported as complete while any run-scoped resource survives. + +## Verification + +All executable checks ran locally on Windows with injected provider responses. They +made no real GCP, Fly.io, or GitHub provider call and created no cloud resource: + +- `137 passed` in + `tests/test_gateq38_gcp_adapter.py tests/test_gateq38_route_controller.py`; +- `168 passed` across the adapter/controller plus the adjacent Gate 14 GCP + executor, Gate 13 GCP provider, and multi-machine qualification tests; +- `1,686 passed, 10 skipped` in the repository offline unit matrix; +- Black, isort, Python compilation, and Git whitespace checks passed; +- independent adversarial review reproduced the 137 focused tests and returned PASS + on the frozen source hashes. + +The offline matrix uses the same exclusions as the preceding complete-route +controller checkpoint: tests that require an externally provisioned +`INITIAL_PEERS` swarm and unavailable optional bitsandbytes/PEFT runtime probes. + +Canonical staged source bindings (SHA-256 over the Git-index blob bytes): + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_gcp_adapter.py` | 39,785 | `820d62a9a9d6c737d576fa659e3bbcdfb090f0f534494af6b8dab699c3d8db8b` | +| `scripts/gateq38_route_controller.py` | 88,955 | `bd972409e2c7932edad04c7bd452cc04db32347df650e5a82b47358ea167e7cb` | +| `tests/test_gateq38_gcp_adapter.py` | 25,877 | `14b37602ea24fdbb454642a1a3b108a83eba0b4a220485a103a4a77959cae740` | +| `tests/test_gateq38_route_controller.py` | 71,863 | `9adb7d33aff86345eb90f2cda0a735e68e4164fd0d4a14d2708a3bc32bcd8985` | + +## Explicitly not proved + +This checkpoint does not prove a Q3.8 reservation, four-GPU availability, live +provider creation, host bootstrap, protected host status, evidence collection, a +complete 64-block route, stock parity, same-session recovery, packaged cold +acquisition/cache reuse, or RTX 30/40/50 qualification. + +The next unblocked no-spend gate is the exact source-bound Qwen3.8 host +runtime/staging package and protected instance-generation-bound status/evidence +transport. Only after that gate, an exact readiness reservation, and a fresh native +four-GPU capacity/pricing proof fit the remaining USD 44 exposure may a paid route +start be considered. diff --git a/docs/evidence/gateq38-20260903-g-linux-runtime-package-validator-checkpoint.md b/docs/evidence/gateq38-20260903-g-linux-runtime-package-validator-checkpoint.md new file mode 100644 index 000000000..836d68c12 --- /dev/null +++ b/docs/evidence/gateq38-20260903-g-linux-runtime-package-validator-checkpoint.md @@ -0,0 +1,105 @@ +# Gate Q3.8 Linux runtime-package validator checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 source/validator contract; Gate Q3.8 remains IN PROGRESS +Source commit: `4264d33a3376945cc4ad0270be88c020d93966bc` +Source tree: `bcc06952e3addf1748b9c166510ccee3aad8065c` + +## Scope + +This checkpoint adds a standard-library-only controller-side validator for the exact +Qwen3.8 Linux production archive. It binds the packaged runtime to the route source +commit and tree, the exact Qwen3.8 manifest, both verifier sources, the release audit, +and the complete nested CommunityAI node onedir inventory before a future privileged +host stage may consume it. + +The controller's required source set now includes +`desktop/build_desktop.py` and `scripts/gateq38_stage_package.py`. +Changing either source therefore changes the route plan rather than leaving package +validation outside the execution boundary. + +This is a pre-host contract. It does not transfer, extract, protect, or execute the +multi-gigabyte production archive on a native Linux qualification host. + +## Fail-closed package contract + +The validator requires the exact Linux production archive and release-audit +companions. It: + +- validates the package platform, source commit/tree, archive digest/size, release + provenance, `SHA256SUMS`, metrics, and the physical and semantic Qwen3.8 + manifest identities; +- reads the manifest from one bounded payload and rechecks it after the release-tree + protection phase; +- enumerates and binds the complete packaged node onedir inventory, including the + exact `CommunityAI-Node` executable at mode `0755`; +- rejects missing, extra, changed, case-colliding, special, external-link, or + unsafe-mode runtime members, plus every bundled model-weight form; +- invokes a caller-supplied controller protection operation for every extracted + release file and directory, then compares complete before/after identity + snapshots so a protection-time mutation cannot become accepted input; +- reads the archive with a no-follow descriptor, bounded chunks, and pre/open/post + device, inode, type, size, and modification-time checks so pathname replacement + cannot substitute different bytes; and +- writes one bounded canonical package record atomically. + +Tests exercise synchronized archive replacement, release-verifier mutation, node +symlink confinement, unsafe modes, protection callback coverage/failure, weight-name +variants including `layers-0.safetensors`, source substitution, and atomic record +failure. + +## Verification + +Local Windows checks against the exact committed candidate passed: + +- `26 passed, 1 skipped` in `tests/test_gateq38_stage_package.py`; the skipped + case is the native POSIX external-node-symlink probe; +- `184 passed, 1 skipped` across the package validator, route controller, and + desktop release-builder matrix; +- `1,712 passed, 11 skipped` in the repository offline unit matrix; +- Black checked all 361 tracked and new Python files; isort, Python compilation, + and Git whitespace checks passed; and +- independent adversarial source review and an independent staged-snapshot test + review both returned PASS. + +Exact-source GitHub checks for `4264d33` also passed: + +- [Check style run 33759081363](https://github.com/flujo-app/CommunityAI/actions/runs/33759081363); +- [Tests run 33759081296](https://github.com/flujo-app/CommunityAI/actions/runs/33759081296). + +The generic Tests workflow does not select +`tests/test_gateq38_stage_package.py`, so its successful Ubuntu job is not claimed +as native Linux execution of the new validator. The production package result is +recorded below only as independent exact-source archive/build verification; that +workflow does not invoke this new stage-package validator. + +The Ubuntu job in [Production desktop run 33759081275](https://github.com/flujo-app/CommunityAI/actions/runs/33759081275) +completed successfully for exact source `4264d33`: production bundle build/smoke, +independent checksum and provenance verification, and both archive-bound uploads +passed. The Windows-only packaged native-credential/public-seed step was not +applicable to Ubuntu and was skipped. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 89,156 | `bae5461b3bf5d1e8ba367f9e08884d502cb421e9c9379bd27736a3df02d71337` | +| `scripts/gateq38_stage_package.py` | 23,565 | `1fae68a7a0302d9b1a2da7f7855287b3bc25b930d7a17584c3c05e17a743d69d` | +| `tests/test_gateq38_stage_package.py` | 17,067 | `1640fd6467272caad094f99d0a7cdd315445c6fac33c97b4c15fea39d7d471d6` | + +These are SHA-256 digests of the exact blobs in source commit `4264d33`. + +## Explicitly not proved + +This checkpoint does not prove native Linux execution of the new validator, a +host-owned extraction, qualification-user write denial, packaged Qwen3.8 preflight, +instance-generation-bound status/evidence collection, any cloud create, a complete +64-block route, stock parity, same-session recovery, packaged cold model acquisition +or cache reuse, or RTX 30/40/50 qualification. It used no provider resource and no +model download (USD 0). + +The next unblocked no-spend step is to embed this complete runtime-package record in +the route plan and start action without circularly requiring the final plan to create +its own package record. Then the privileged Linux host runtime and +instance-generation-bound transport can consume that exact identity. Paid route +actions remain blocked. diff --git a/docs/evidence/gateq38-20260903-h-runtime-package-plan-binding-checkpoint.md b/docs/evidence/gateq38-20260903-h-runtime-package-plan-binding-checkpoint.md new file mode 100644 index 000000000..de052d877 --- /dev/null +++ b/docs/evidence/gateq38-20260903-h-runtime-package-plan-binding-checkpoint.md @@ -0,0 +1,97 @@ +# Gate Q3.8 runtime-package plan-binding checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 controller/source contract; Gate Q3.8 remains IN PROGRESS +Source commit: `50dd0a3daf64de4a76afe0d51de2110f09804450` +Source tree: `406d11213abc2ce427eed4f1e130c7e86763b43d` + +## Scope + +This checkpoint removes the circular dependency in which creating the strict Linux +runtime-package record required the final route plan that was itself supposed to bind +that record. The package stage now consumes a narrow controller-protected source +context. The route controller independently validates the resulting complete package +record and carries it immutably through every paid-action identity. + +This is a controller/source-contract result. It does not stage or execute the +production package on a native Linux qualification host. + +## Non-circular source and package contract + +The source context has one exact schema and scope and binds the source commit, source +tree, and the controller's complete sorted required-source set. The stage command: + +- accepts `--source-context` and no longer accepts the final plan or a separate + source-tree claim; +- parses bounded JSON with duplicate-key rejection, checks the context parent and + file as controller-protected inputs, then requires an exact reread; +- verifies the imported package-stage and desktop-release verifier modules against + the exact source bindings before using them; +- emits a package record that includes the exact source-binding-set digest; and +- rejects an output path that aliases the manifest or source context, or resolves + beneath the source or extracted-release roots, before validation can modify it. + +The controller is the single validator for the complete package-record schema. It +requires exact platform, archive, node-root and executable identities; exact digest, +size, inventory, source-commit, manifest, and source-binding claims; and a canonical +self-digest computed over the record without that digest field using the package +domain's required trailing newline. That package digest remains deliberately distinct +from the route controller's stable-plan digest domain. + +## Route and authorization binding + +A validated package record and the source/authorization mappings are stored as +immutable mappings. The full record is included in: + +- the stable route-plan digest; +- the execution-inventory digest; +- every provider action record; +- every non-null action ID through the plan digest; and +- reservation and preflight comparisons through the plan and execution-inventory + digests. + +Changing any package or source-binding field, even with a correctly recomputed package +self-digest, therefore creates a distinct plan. Authorization or preflight evidence +for the former plan cannot be reused. + +## Verification + +Checks against the exact committed candidate passed: + +- `156 passed, 1 skipped` in the focused route-controller and package-stage suite; +- `184 passed, 1 skipped` across all `tests/test_gateq38_*.py`; +- `205 passed, 1 skipped` across the adjacent package, controller, GCP-adapter, and + desktop-builder matrix; +- `1,733 passed, 11 skipped` in the repository offline unit matrix; +- Black, isort, Python compilation, and Git whitespace checks; and +- independent adversarial source review plus an independent frozen-index security + and test review, both returning PASS. + +The native-POSIX skip is expected on this Windows verification host. No cloud +provider operation, archive or model download, reservation, or resource mutation was +performed. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 94,450 | `6c817c03e60f45216ab3d875fcb0f29ab48fc963997cbf0ac2471a53b7b07cb6` | +| `scripts/gateq38_stage_package.py` | 25,046 | `c08a630c8bd1f1eb40ebc92a1fb9ec1b7b2cff1c87cb4322819ade0ff56cb25d` | +| `tests/test_gateq38_route_controller.py` | 76,857 | `dc54bd8c20aeb10758296dc42cd7f689f44e6e46144b6cdbd2220e291147b7cc` | +| `tests/test_gateq38_stage_package.py` | 24,333 | `8b6dc421fee44a64e4faa780a48ebd0f7f57af2e0cba7e818f5ed16035a78da1` | + +These are SHA-256 digests of the exact blobs in source commit `50dd0a3`. + +## Explicitly not proved + +This checkpoint does not prove native Linux execution of the package validator, +privileged extraction or protection of the runtime tree, qualification-user write +denial, packaged `edge-acquire --help` or Qwen3.8 execution, instance-generation-bound +status/evidence transport, any cloud create, the complete 64-block route, stock +parity, same-session recovery, packaged cold acquisition/cache reuse, or RTX 30/40/50 +qualification. It used no provider resource and no model or archive bytes (USD 0). + +The next unblocked no-spend step is the privileged Qwen3.8 Linux host-runtime and +protected instance-generation-bound status/evidence bridge. Paid route actions remain +blocked until that gate, an exact checked-in reservation, and fresh capacity/pricing +evidence satisfy the remaining USD 44 ceiling. diff --git a/docs/evidence/gateq38-20260903-i-linux-host-runtime-preparation-checkpoint.md b/docs/evidence/gateq38-20260903-i-linux-host-runtime-preparation-checkpoint.md new file mode 100644 index 000000000..2f7c786b7 --- /dev/null +++ b/docs/evidence/gateq38-20260903-i-linux-host-runtime-preparation-checkpoint.md @@ -0,0 +1,98 @@ +# Gate Q3.8 Linux host-runtime preparation checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 source and local contract; Gate Q3.8 remains IN PROGRESS +Source commit: `caf9bc8f42f7d153433d0ae1f7a3fb40924c4284` +Source tree: `9ae262f9f2175168ff7ee320aa3ff9c641e8263b` + +## Scope + +This checkpoint adds the privileged Linux preparation and cleanup contract that a later +Qwen3.8 bootstrap may invoke. It consumes the controller-bound package record, verifies +the exact release companions and complete packaged-node inventory, extracts the exact +regular files and validated internal symlinks into a protected per-plan runtime, and +runs an offline +`edge-acquire --help` preflight as the exact unprivileged qualification identity. + +The implementation and tests were verified on Windows. The Linux-native ownership, +dropped-identity, lock, symlink, and open-file replacement probes remain skipped until +the exact candidate runs on a native Linux host. No package or model was downloaded and +no cloud resource was created. + +## Protected preparation contract + +The host runtime: + +- accepts only one strict protected plan/action schema and exact source bindings for the + controller and host-runtime implementation; +- validates the record-bound release manifest, provenance, checksums, metrics, and + complete archive/node inventory under controller-authoritative 16 MiB release- + attestation limits; +- keeps a no-follow archive descriptor open, applies bounded tar entry/archive/expanded + byte limits, and manually extracts the exact regular files plus validated internal + symlinks while rejecting traversal, hard links, unsafe or external symlinks, special + files, unsafe modes, extras, and case collisions; +- installs a root-owned read/execute runtime tree and a separate exact qualification- + user work leaf, then launches the verified Linux executable through + `/proc/self/fd` with offline stripped environment, empty supplementary groups, + resource limits, and a new process group; +- treats every exception after process start as cleanup-required, terminating, + reaping when necessary, and proving the process group absent before closing verified + inputs or removing work state; and +- serializes prepared-state publication and removal with a protected lock, recovers + only exact stale temporary state, publishes one digest-only prepared record with + no-replace hardlink semantics, and fsyncs the containing directory. + +The release-attestation regression includes a valid provenance document larger than the +real 1,241,883-byte production provenance, so the dedicated bounds no longer inherit the +unrelated 262,144-byte route-record limit. + +## Verification + +Checks against the exact committed candidate passed: + +- `55 passed, 3 skipped` in the Linux host-runtime suite; +- `240 passed, 4 skipped` across all `tests/test_gateq38_*.py`; +- `206 passed, 1 skipped` across the adjacent route-controller, package-stage, + GCP-adapter, and desktop-builder matrix; +- `1,789 passed, 14 skipped` in the established offline unit matrix; +- Black, isort, Python compilation, and Git whitespace checks; and +- independent adversarial review plus independent frozen-index verification, both + returning PASS. + +The native Linux skips are expected on this Windows verification host. They cover the +dropped-UID runtime and private work-root access, protected native lock, POSIX symlink +handling, and open-file replacement behavior; none is represented as passed here. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_linux_host_runtime.py` | 57,502 | `88785407462e65a524d02238b1b5a424f6a6665083698be6643805d8cb56e6f6` | +| `scripts/gateq38_route_controller.py` | 94,708 | `8c053e53e910625b562d153423d428e0cf1eab8a305001283812dfc4b87c04ce` | +| `scripts/gateq38_stage_package.py` | 25,251 | `a6da970ca8c3b873c3315ab3058ee54a27a00d6701853eafa9dd9d8435e0cfd4` | +| `tests/test_gateq38_linux_host_runtime.py` | 40,640 | `48b9c27bbb49aaf0e1656b17fccff81e39f3f848ba1212ccc1f5a6aff316e189` | +| `tests/test_gateq38_stage_package.py` | 25,342 | `d96841411ae3326d31c5dda09c568f77da2a867b200b0ae70b1654513c4f7870` | + +These are SHA-256 digests of the exact blobs in source commit `caf9bc8`. + +## Explicitly not proved + +This checkpoint does not prove native Linux package validation, extraction, ownership, +qualification-user access, or packaged preflight; provider bootstrap integration; +instance-generation-bound status/evidence transport; model acquisition or Qwen3.8 +execution; the complete 64-block route; stock parity; same-session recovery; packaged +cold acquisition/cache reuse; or RTX 30/40/50 qualification. + +The GCP adapter remains fail-closed before provider access. No reservation, provider +operation, archive/model download, or resource mutation occurred (USD 0). The checked-in +ledger still has no exact Q3.8 reservation and retains the prior conservative USD 56 +maximum within the combined USD 100 ceiling. + +## Next gate + +The next unblocked no-spend work is to verify this exact preparation contract on native +Linux and bind its prepared-record/status/evidence lifecycle to an exact GCP instance +generation and protected bootstrap handoff. Paid route start remains blocked until that +bridge, an exact checked-in reservation, and fresh four-GPU capacity/pricing evidence fit +the remaining USD 44 ceiling. diff --git a/docs/evidence/gateq38-20260903-j-instance-generation-latch-checkpoint.md b/docs/evidence/gateq38-20260903-j-instance-generation-latch-checkpoint.md new file mode 100644 index 000000000..cd0213fa1 --- /dev/null +++ b/docs/evidence/gateq38-20260903-j-instance-generation-latch-checkpoint.md @@ -0,0 +1,71 @@ +# Gate Q3.8 instance-generation latch checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 source and local control-plane contract; Gate Q3.8 remains IN PROGRESS +Source commit: `96bb6d1475ce01cdbd26da52ea9425dd6d2de8db` +Source tree: `ea30d8195b117bfdbca2ca10dac9620667896907` + +## Scope + +This checkpoint binds every observed GCP route instance to its immutable provider +generation before the Qwen3.8 controller may enter an active phase. The adapter now +carries the exact numeric instance ID and offset-bearing creation timestamp from the +provider inventory. The controller validates those values, derives a per-instance +digest bound to the exact project, zone, resource name, ID, and creation time, and +latches the canonical five-instance generation-set digest. + +The latch closes a same-name delete/recreate replay at the provider observation +boundary. Once set, any missing, replaced, or otherwise changed instance generation +forces the route directly to cleanup. Non-instance resources must expose no generation +metadata, terminal state retains the original latch, and resource reappearance after +terminal cleanup remains invalid. + +Paid `start_route` and `collect_route` stay disabled before provider authentication +or runner access. This checkpoint did not implement host bootstrap/status transport or +authorize a paid run. + +## Verification + +Checks against the exact committed candidate passed: + +- `170 passed` in the route-controller and GCP-adapter focused suite; +- `28 passed, 142 deselected` in the independent targeted generation, terminal, + stale-decision, and fail-closed start/collect subset; +- `264 passed, 4 skipped` across all `tests/test_gateq38_*.py`; +- Black, isort, Python compilation, and Git whitespace checks; and +- independent adversarial review plus independent frozen-candidate verification, + both returning PASS. + +The four skips are pre-existing native-platform probes unavailable on this Windows +verification host; none is represented as passed here. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 100,227 | `9726a3a4fe752bf2d6bfbe7af227471fc32c955a475b6134370c321fcececfb5` | +| `scripts/gateq38_gcp_adapter.py` | 40,863 | `ad1e4b614650be151126f3a930e3661e7be6e5e640cebc80e2033c7938c46c68` | +| `tests/test_gateq38_route_controller.py` | 80,961 | `16c1ff6650387d15adf675bb13be83ef5c3419b63757b1ee31f49a949d49d4cd` | +| `tests/test_gateq38_gcp_adapter.py` | 28,465 | `c4ca30ace264ceee11f225240a24b42f95e0a0b692c312839c976bf03e784729` | + +These are SHA-256 digests of the exact blobs in source commit `96bb6d1`. + +## Explicitly not proved + +This checkpoint does not prove native Linux host preparation, a protected bootstrap or +status/evidence transport, provider start/collection, model acquisition or Qwen3.8 +execution, the complete 64-block route, stock parity, same-session recovery, packaged +cold acquisition/cache reuse, or RTX 30/40/50 qualification. + +No reservation, provider command, cloud resource, model download, credit, or macOS work +was performed (USD 0). The checked-in ledger still contains no exact Q3.8 reservation +and retains the prior conservative USD 56 maximum within the user-specified combined +USD 100 ceiling. + +## Next gate + +The next unblocked no-spend work is the protected Linux bootstrap and bounded +instance-generation-bound status/evidence transport. It must bind host records to the +latched provider generation and exact plan/action before paid start or collection can be +enabled. A paid route still requires a separate exact checked-in reservation plus fresh +capacity and pricing evidence within the remaining USD 44 ceiling. diff --git a/docs/evidence/gateq38-20260903-k-authenticated-host-status-envelope-checkpoint.md b/docs/evidence/gateq38-20260903-k-authenticated-host-status-envelope-checkpoint.md new file mode 100644 index 000000000..4d3a5cce3 --- /dev/null +++ b/docs/evidence/gateq38-20260903-k-authenticated-host-status-envelope-checkpoint.md @@ -0,0 +1,78 @@ +# Gate Q3.8 authenticated host-status envelope checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 source and local transport contract; Gate Q3.8 remains IN PROGRESS +Source commit: `a0d548451b01894bcd363142751538f7a9965d3b` +Source tree: `573286350142b0ddad57416e6a98eff63c70ec55` + +## Scope + +This checkpoint adds the bounded authenticated envelope primitive required to carry +Qwen3.8 Linux host status across an untrusted byte transport. Controller-issued +instance contexts bind the exact source, stable route plan, execution inventory, +worker plan, start and collection actions, project, zone, resource identity, provider +instance ID, creation timestamp, generation digest, and validity window. + +Host envelopes bind one context to the current boot UUID, monotonic revision, +publication time, prepared-record digest, and a strict typed worker or route-job +payload. Canonical ASCII JSON, a 65,536-byte one-line limit, exact 32-byte per-instance +keys, domain-separated HMAC-SHA256, freshness bounds, expected resource/generation +checks, boot latching, and revision floors make duplicate, stale, replayed, substituted, +malformed, deeply nested, or oversized records fail closed. + +The serialized carrier is not a trust root. The HMAC and controller-issued context are +the prerequisite for a later protected host bootstrap and adapter transport; paid +`start_route` and `collect_route` remain disabled before provider access. + +## Verification + +Checks against the exact committed candidate passed: + +- `41 passed` in the authenticated Linux host-transport suite; +- `211 passed` across the transport, route-controller, and GCP-adapter suites; +- `305 passed, 4 skipped` across all `tests/test_gateq38_*.py`; +- Black, isort, Python compilation, and Git whitespace checks; and +- independent adversarial review plus independent frozen-index verification, both + returning PASS. + +The four skips are native-platform probes unavailable on this Windows verification +host; none is represented as passed here. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_linux_host_transport.py` | 19,204 | `f76df977e265836d2bc39ad30ecc9b08bac4330fd16a0bba962b1959b579b81f` | +| `scripts/gateq38_route_controller.py` | 100,351 | `c86d0b44ed76695da417e38d697342ff2a430c3a2e0c3e1d62caf231b9aef11f` | +| `tests/test_gateq38_linux_host_transport.py` | 14,366 | `b3442655ea1cf264f01d7c97c776e178684843fdbab1469eb3416d5812a7da6d` | + +These are SHA-256 digests of the exact blobs in source commit `a0d5484`. + +## Explicitly not proved + +This checkpoint does not implement or prove root-only per-instance key distribution or +storage, protected instance-context installation, equality between the authenticated +`prepared_record_digest` and a controller-known protected prepared record, guest- +attribute or other adapter consumption, metadata-server controls, systemd bootstrap, +terminal evidence indexing, or native Linux execution. The prepared-record digest is +strictly encoded and authenticated here, but the next integration must compare it with +the protected runtime record before accepting status. + +It also does not prove provider start/collection, model acquisition or Qwen3.8 +execution, the complete 64-block route, stock parity, same-session recovery, packaged +cold acquisition/cache reuse, or RTX 30/40/50 qualification. + +No reservation, provider command, cloud resource, model download, credit, or macOS work +was performed (USD 0). The checked-in ledger still contains no exact Q3.8 reservation +and retains the prior conservative USD 56 maximum within the user-specified combined +USD 100 ceiling. + +## Next gate + +The next unblocked no-spend work is to install the exact controller context and key +under the protected Linux host-runtime boundary, bind the prepared-record digest and +boot identity there, and make the GCP adapter consume the authenticated status between +generation-stable pre/post inventory reads. Paid start and collection must remain +disabled until that bootstrap/transport integration and native Linux probes pass. A +paid route still requires a separate exact checked-in reservation plus fresh capacity +and pricing evidence within the remaining USD 44 ceiling. diff --git a/docs/evidence/gateq38-20260903-l-protected-host-status-grounding-checkpoint.md b/docs/evidence/gateq38-20260903-l-protected-host-status-grounding-checkpoint.md new file mode 100644 index 000000000..c39d9362f --- /dev/null +++ b/docs/evidence/gateq38-20260903-l-protected-host-status-grounding-checkpoint.md @@ -0,0 +1,91 @@ +# Gate Q3.8 protected host-status grounding checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 source and local lifecycle contract; Gate Q3.8 remains IN PROGRESS +Source commit: `f3e70adfdf14885ed3e4e84866652a8136809bf7` +Source tree: `3af3e7418a2bdac56a47511fde062b5eaf00865a` + +## Scope + +This checkpoint grounds the authenticated Linux host-status envelope in the protected +runtime lifecycle. The host runtime opens the exact controller-issued instance context +and 32-byte transport key through root-private, no-follow, identity-checked handles, +authenticates the expected resource and provider generation, and binds the current boot +UUID into the prepared record. + +Prepared state now binds the context digest, resource name and kind, worker identity, +provider generation digest, boot UUID, and exact protected runtime result. Status is +derived from the reopened prepared record, so callers cannot substitute its digest. +Preparation re-samples publication time after the packaged preflight and reopens the +context, key, and boot identity before atomically publishing prepared state and the +initial authenticated status envelope. + +Preparation and cleanup use one root-private lifecycle lock. Newly installed runtime, +prepared state, and status roll back together on publication failure. Cleanup +authenticates the context and generation even when prepared state is absent, remains +available after context expiry, and publishes plus directory-fsyncs a +generation-bound terminal marker before deleting runtime or state. An interrupted +cleanup leaves that marker durable, blocks later preparation for the terminated +generation, and can be retried idempotently. + +The GCP adapter's paid `start_route` and `collect_route` paths remain blocked before +runner or provider access. + +## Verification + +Checks against the exact committed candidate passed: + +- `119 passed, 3 skipped` in the Linux host-runtime and host-transport suites; +- `328 passed, 4 skipped` across all `tests/test_gateq38_*.py`; +- `1,877 passed, 14 skipped` in the established offline unit matrix; +- Black, isort, Python compilation, and Git whitespace checks; +- independent adversarial review of the frozen working files; and +- independent staged-index verification, including 17 transactional race/recovery + tests and two paid-path fail-closed tests. + +The three focused skips are native POSIX/root probes unavailable on this Windows +verification host; none is represented as passed. The broad offline matrix excludes +the repository's documented live-peer and unavailable optional bitsandbytes/PEFT +probes. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_linux_host_runtime.py` | 82,630 | `9793ed31a3ceae31048877c24cdedadebbb763c646e87f340c00ab948aee1ca9` | +| `scripts/gateq38_linux_host_transport.py` | 21,874 | `b18a0f680596329cbb57fd39e2843164e12e45b98dd29a5382f03804bca768a2` | +| `tests/test_gateq38_linux_host_runtime.py` | 61,793 | `1f3b1a392da33c2e2293614c68544717f98ee719e9e9dd7e6c2d97433a112349` | +| `tests/test_gateq38_linux_host_transport.py` | 16,547 | `cd476a3e6acd325be653abeb821efe4c234cdf0880973e8140c7960e454af128` | + +These are SHA-256 digests of the exact blobs in source commit `f3e70ad`. + +## Explicitly not proved + +This checkpoint consumes already-protected context and key inputs; it does not deliver, +install, rotate, revoke, or remove those inputs. It does not publish status to an +external carrier, read GCP guest attributes, make the adapter consume host status, +or prove controller-to-host key transport. It also does not implement metadata or +systemd bootstrap. + +The lifecycle is structurally and behaviorally tested on Windows, with native POSIX +ownership, `flock`, dropped-UID, `/proc/self/fd`, process-group, boot-ID, and +root-private path execution still requiring native Linux verification. + +It does not prove provider start/collection, model acquisition or Qwen3.8 execution, +the complete 64-block route, stock parity, same-session recovery, packaged cold +acquisition/cache reuse, or RTX 30/40/50 qualification. + +No reservation, provider command, cloud resource, model download, credit, or macOS work +was performed (USD 0). The checked-in ledger still contains no exact Q3.8 reservation +and retains the prior conservative USD 56 maximum within the user-specified combined +USD 100 ceiling. + +## Next gate + +The next unblocked no-spend work is to bind controller-generated instance contexts and +per-instance keys into a protected delivery contract, publish the bounded authenticated +status through an explicitly untrusted carrier, and consume it only between +generation-stable pre/post GCP inventory reads. Paid start and collection must remain +disabled until that bridge and the native Linux probes pass. A paid route still +requires a separate exact checked-in reservation plus fresh capacity and pricing +evidence within the remaining USD 44 ceiling. diff --git a/docs/evidence/gateq38-20260903-n-protected-iap-and-status-consumer-checkpoint.md b/docs/evidence/gateq38-20260903-n-protected-iap-and-status-consumer-checkpoint.md new file mode 100644 index 000000000..168055be0 --- /dev/null +++ b/docs/evidence/gateq38-20260903-n-protected-iap-and-status-consumer-checkpoint.md @@ -0,0 +1,105 @@ +# Gate Q3.8 protected IAP and authenticated status-consumer checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 provider-plan and adapter-consumer contract; Gate Q3.8 remains IN PROGRESS +IAP source commit: `a61554497ddfc6a1776bcf83f82889bc971e7586` +IAP source tree: `8914821b1a2c4d02f6f0c374765a8b169aec79db` +Consumer source commit: `bd53c29fc726e89da6d5d3c6cff247954df45388` +Consumer source tree: `afcce2bae0c59884c22c0db933b43ccecd9d8382` + +## Scope + +This checkpoint closes two no-spend prerequisites in the Qwen3.8 protected-host +bridge. + +The route plan now contains a distinct twelfth resource for IAP SSH. Its exact +run-scoped firewall allows only TCP port 22 from Google's +`35.235.240.0/20` IAP TCP-forwarding range to the exact run target tag. The +controller, action identities, execution inventory, reservation, provider +observation, start command compilation, and retry-safe cleanup all bind this +resource separately from the route firewall. Missing, extra, substituted, +broadened, or foreign firewall state fails closed. + +The GCP adapter now has a fixed, bounded guest-attribute reader for +`communityai-q38/status-v1`. Authenticated consumption is available only when +both protected key and replay-checkpoint resolvers are supplied. Each present +carrier value is ASCII- and size-bounded, decoded through the canonical Linux +host transport, and validated against the exact source-bound plan, resource, +provider generation, HMAC key, boot checkpoint, and monotonic revision. +Instance ID and creation timestamp must agree with the provider observation. + +Every authenticated read is bracketed by complete provider inventories. The +adapter discards the whole result if the aggregate exact instance-generation +digest changes, if the protected bootstrap is not continuously running, or if +any carrier, resolver, context, envelope, payload, or checkpoint is malformed. +Cleanup does not consult guest attributes or protected key material. The +existing static status-file path remains unable to inject nonblank production +status. + +Paid `start_route` and `collect_route` remain blocked before provider +access. Resolver injection is a narrow consumption boundary for the forthcoming +controller vault/delivery implementation; this checkpoint does not manufacture +or deliver any key. + +## Verification + +Checks against the exact committed candidates passed: + +- `180 passed` in the focused route-controller and GCP-adapter suites for the + IAP-firewall candidate; +- `368 passed, 4 skipped` across all `tests/test_gateq38_*.py` for that + candidate; +- `101 passed` in the focused Linux-transport and GCP-adapter suites for the + authenticated-consumer candidate; +- `376 passed, 4 skipped` across all `tests/test_gateq38_*.py` for the + authenticated-consumer candidate; +- Python compilation and Git whitespace checks; and +- independent read-only verification of firewall isolation, exact inventory, + cleanup isolation, resolver pairing, wrong-key and replay rejection, + ambiguous carrier rejection, absent status, and provider-generation drift. + +The four complete-matrix skips are the existing native-platform probes +unavailable on this Windows verification host; none is represented as passed. +Black and isort are declared by the project but are not installed in the +available `.venv-cuda`, so this checkpoint does not claim those two checks. +A raw repository-wide `pytest` invocation is not the established offline +matrix: collection intentionally requires `INITIAL_PEERS` for live-peer tests +and also encounters the historical duplicate desktop spike test module. No +live-peer result is claimed. + +## Canonical committed blobs + +| Path | Bytes | SHA-256 | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 100,392 | `eeccb5afe300f4370013d215f715a6ca2ca086b6e121f36f82a4c7e6c0e872e4` | +| `scripts/gateq38_gcp_adapter.py` | 51,003 | `20a3cec0bd01293b5124e74ea39843c17ee15f018be4b7a262d107e1c8eb7f0a` | +| `tests/test_gateq38_route_controller.py` | 81,617 | `1a02f275708ba32f275fb098f442c8766aa97e0f8a973a154fb53e222d8c1a50` | +| `tests/test_gateq38_gcp_adapter.py` | 41,129 | `aecb2e6c05bdd19369d37bedb2dfc3ad6e1c113660690664cca4f4871a811287` | + +These are SHA-256 digests of the exact blobs in consumer source commit +`bd53c29`; the controller pair is unchanged from the IAP source commit. + +## Explicitly not proved + +No provider mutation, guest-attribute request, IAP connection, live metadata +publication, model download, or native Linux execution occurred. This +checkpoint does not generate, vault, deliver, install, rotate, revoke, or +remove per-instance key material or controller contexts. It does not prove +systemd bootstrap, terminal collection, provider start/collection, real +Qwen3.8 execution, the complete 64-block route, stock parity, same-session +recovery, packaged cold acquisition/cache reuse, or RTX 30/40/50 +qualification. + +No reservation, cloud resource, credit, or macOS work was performed (USD 0). +The checked-in ledger still has no exact Q3.8 reservation and retains the prior +conservative USD 56 maximum within the combined USD 100 ceiling. + +## Next gate + +The next unblocked no-spend work is the controller-owned secret and delivery +half of the bridge: generate and vault one key per exact instance generation, +compile protected IAP delivery without exposing key or private path material, +atomically install the context/key pair in the Linux lifecycle, bind delivery +receipts and replay checkpoints, and make rotation/revocation/cleanup +idempotent. Native Linux delivery, publication, and provider-read probes must +then pass before any reservation or paid route start. diff --git a/docs/evidence/gateq38-20260903-o-instance-key-vault-and-protected-delivery-checkpoint.md b/docs/evidence/gateq38-20260903-o-instance-key-vault-and-protected-delivery-checkpoint.md new file mode 100644 index 000000000..e51395c3b --- /dev/null +++ b/docs/evidence/gateq38-20260903-o-instance-key-vault-and-protected-delivery-checkpoint.md @@ -0,0 +1,106 @@ +# Gate Q3.8 instance-key vault and protected delivery checkpoint + +Date: 2026-09-03 +Result: PASS for the USD 0 key-vault and protected-delivery contract; Gate Q3.8 remains IN PROGRESS +Vault source commit: `b1505fc50bd4113afb2f3d72257690fdab779747` +Vault source tree: `1fd6b6b4a91f1e86bbe89961e54ca8923eb0dcfe` +Delivery source commit: `b807fff392f9b35acef1188677df780205d4b290` +Delivery source tree: `709ac70c6697af8a3a1d360507b1dd9325de0aa4` + +## Scope + +This checkpoint closes the remaining offline controller-secret and protected-delivery +prerequisites in the Qwen3.8 host-status bridge. + +The controller now owns one 32-byte transport key per exact run, resource, provider +instance generation, and epoch. Protected records bind source, plan, execution +inventory, start action, resource identity, provider ID and creation timestamp, +generation digest, expiry, key digest, predecessor record, and record digest. +Private vault directories and files are identity-checked, atomically persisted, and +validated with owner-only Windows ACLs or POSIX permissions. Same-generation ensure +reattaches only exact records; recreation, substitution, corruption, symlinks, +foreign files, or broad permissions fail closed. + +Rotation is serialized and predecessor-bound. Retries converge after interruption +before or after activation without advancing twice, old key bytes are removed only +after the replacement is durable, and revoked generations retain digest-only +tombstones. Interrupted initial creation prunes only orphaned material for the exact +generation before retry. Revocation and cleanup are idempotent and do not require +provider status. + +The transport now frames the controller context and key as one bounded canonical +authenticated delivery bundle. The Linux runtime validates the complete source, +plan, action, resource, generation, epoch, predecessor, expiry, key digest, context +digest, bundle digest, and HMAC before atomically replacing one root-private bundle +under the lifecycle lock. Preparation and publication reopen only that installed +bundle. Terminal cleanup removes it after publishing the durable generation marker, +and late or noncontiguous delivery fails closed. + +The GCP adapter compiles one fixed IAP SSH operation per exact planned instance and +streams the bundle through stdin. Key bytes, private paths, credentials, shell text, +provider output, and endpoints are absent from argv, environment, receipts, and +ordinary state. Delivery is accepted only when complete pre/post provider inventories +prove the exact instance generation set and protected bootstrap remained stable. +Receipts are canonical, HMAC-authenticated before timestamp semantics, bounded by the +same 300-second freshness window, and replay-checkpointed. + +Paid `start_route` and `collect_route` remain blocked before provider access. + +## Verification + +Checks against the exact committed candidates passed: + +- `147 passed, 1 skipped` in the controller vault suite; +- `224 passed, 3 skipped` in the focused transport/runtime/adapter delivery suites; +- `412 passed, 5 skipped` across all `tests/test_gateq38_*.py`; +- Black 22.3.0 check-only and isort 5.10.1 check-only on all six delivery files; +- in-memory Python compilation and Git whitespace checks; and +- independent read-only adversarial verification of interrupted initial creation, + both rotation interruption boundaries, Windows ACL rejection, tombstones, + generation isolation, atomic installation, IAP stdin isolation, cleanup retry, + provider-generation bracketing, paid-action blocking, and secret non-exposure. + +The five complete-matrix skips are existing native-platform probes unavailable on +this Windows verification host; none is represented as passed. Receipt-age +boundaries were exercised directly: an authentic age of 300 seconds and future skew +of 30 seconds pass, while ages of 301 and 900 seconds and future skew of 31 seconds +fail. Stale or future timestamp tampering fails receipt integrity/authentication +before time policy is interpreted. + +## Canonical committed blobs + +| Path | Bytes | Git blob | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 148,346 | `a02fc057d9af8f54765e42df5f3d72ebf4afa3fc` | +| `scripts/gateq38_gcp_adapter.py` | 56,842 | `ac5e94f6e2c4c6ab0f79ac2440886bc0f9b3f9b5` | +| `scripts/gateq38_linux_host_transport.py` | 38,206 | `0d31532762ecfe0dc349968bb9c7e51a0855e8a3` | +| `scripts/gateq38_linux_host_runtime.py` | 101,447 | `3880db57e2bebf809f5160b1c3ce06796041ce9e` | +| `tests/test_gateq38_route_controller.py` | 100,015 | `26d62c19c7c0cffe20036d12357a54b0df2547ff` | +| `tests/test_gateq38_gcp_adapter.py` | 47,046 | `2ce989018bc0002890ce72b18f6a24ae342e430c` | +| `tests/test_gateq38_linux_host_transport.py` | 22,930 | `0c0ed55e996d60f7a015fa7f96b164568f041743` | +| `tests/test_gateq38_linux_host_runtime.py` | 82,708 | `132cb43b841731f5a61dc209827178a30ab26a84` | + +The controller pair is the exact content carried into delivery source commit +`b807fff`; the four delivery implementation/test pairs were committed there. + +## Explicitly not proved + +No provider mutation, IAP connection, guest-attribute request, live metadata +publication, model download, native Linux execution, or paid route occurred. This +checkpoint does not prove a real root-owned install on Linux, systemd bootstrap, +terminal collection, provider start/collection, real Qwen3.8 execution, the complete +64-block route, stock parity, same-session recovery, packaged cold acquisition/cache +reuse, or RTX 30/40/50 qualification. + +No reservation, cloud resource, credit, or macOS work was performed (USD 0). The +checked-in ledger still has no exact Q3.8 reservation and retains the prior +conservative USD 56 maximum within the combined USD 100 ceiling. + +## Next gate + +The next unblocked work is native Linux execution of the delivery, preparation, +publication, cleanup, and provider-read probes. Those results must bind this exact +source and prove root ownership, process isolation, guest-attribute transport, stable +provider generations, receipt/replay behavior, and exact cleanup. Only after those +probes pass may a fresh exact reservation and four-GPU capacity/pricing proof fitting +the remaining USD 44 ceiling authorize a paid route start. diff --git a/docs/evidence/gateq38-20260903-p-native-linux-protected-host-probe.md b/docs/evidence/gateq38-20260903-p-native-linux-protected-host-probe.md new file mode 100644 index 000000000..3a98df83d --- /dev/null +++ b/docs/evidence/gateq38-20260903-p-native-linux-protected-host-probe.md @@ -0,0 +1,85 @@ +# Gate Q3.8 native Linux protected-host contract probe + +Date: 2026-09-03 +Result: PASS for the USD 0 native Linux protected-host contract; Gate Q3.8 remains IN PROGRESS +Verified source commit: `6a0f2ab1c26d44513e4e17052af0ca5153fe6133` +Verified source tree: `3b6314d42fbf27c4355ffc905f8163edfeeeadd5` +Linux image ID: `sha256:75590df11515662ce84f6eee67b710942c2b18f6e4c886423fe0488acb309777` + +## Scope + +This checkpoint executes the pushed Qwen3.8 protected-host contract as native Linux +root rather than representing Linux-only behavior through Windows skips. The exact +repository and dependency environment were mounted read-only into an ephemeral Linux +container, networking was disabled, pytest caching was disabled, and the process ran +as UID/GID `0:0` with Python 3.12.14 and pytest 6.2.5. + +The native matrix exercised the controller key vault, GCP adapter contract, canonical +host transport, privileged Linux runtime, and package staging suites together. It ran +the Linux-only root ownership and mode checks, nonroot traversal of isolated protected +parents, lifecycle-lock identity, directory-link and symlink rejection, POSIX release +link validation, atomic delivery and cleanup, terminal tombstones, receipt/replay +policy, and fake-provider generation bracketing. + +This is a native host-contract result, not a live provider result. Adapter tests used +their bounded fake runner; the container had no network and issued no GCP, IAP, +metadata, guest-attribute, reservation, model-download, or provider-capacity request. + +## Verification + +Independent read-only verification passed: + +- `415 passed, 2 skipped` in 17.56 seconds across + `tests/test_gateq38_route_controller.py`, + `tests/test_gateq38_gcp_adapter.py`, + `tests/test_gateq38_linux_host_transport.py`, + `tests/test_gateq38_linux_host_runtime.py`, and + `tests/test_gateq38_stage_package.py`; +- the exact pushed source and all ten implementation/test paths were clean before + and after the probe; +- the repository and dependency volume were read-only and the container used + `--network none`; +- protected stdin-only delivery, secret-free receipts, authenticated freshness, + terminal cleanup, exact provider-generation bracketing through the fake runner, + and the pre-provider blocks on paid `start_route` and `collect_route` passed. + +The only skips were the Windows-DACL controller contract and the Windows-host rejection +guard. Both are platform-specific and expected under native Linux; no Linux-native +contract remained skipped. + +## Canonical committed blobs + +| Path | Bytes | Git blob | +| --- | ---: | --- | +| `scripts/gateq38_route_controller.py` | 148,346 | `a02fc057d9af8f54765e42df5f3d72ebf4afa3fc` | +| `scripts/gateq38_gcp_adapter.py` | 56,842 | `ac5e94f6e2c4c6ab0f79ac2440886bc0f9b3f9b5` | +| `scripts/gateq38_linux_host_transport.py` | 38,206 | `0d31532762ecfe0dc349968bb9c7e51a0855e8a3` | +| `scripts/gateq38_linux_host_runtime.py` | 101,447 | `3880db57e2bebf809f5160b1c3ce06796041ce9e` | +| `tests/test_gateq38_route_controller.py` | 100,015 | `26d62c19c7c0cffe20036d12357a54b0df2547ff` | +| `tests/test_gateq38_gcp_adapter.py` | 47,046 | `2ce989018bc0002890ce72b18f6a24ae342e430c` | +| `tests/test_gateq38_linux_host_transport.py` | 22,930 | `0c0ed55e996d60f7a015fa7f96b164568f041743` | +| `tests/test_gateq38_linux_host_runtime.py` | 82,708 | `132cb43b841731f5a61dc209827178a30ab26a84` | +| `tests/test_gateq38_stage_package.py` | 25,342 | `1a21263e4a8ae75aae668b6870d4d6620e2604a0` | + +## Explicitly not proved + +No real IAP session, metadata publication, guest-attribute read, provider-generation +read, systemd boot, model download, reservation, or cloud mutation occurred. This +checkpoint does not prove the complete 64-block Qwen3.8 route, stock parity, +same-session selected-worker recovery, packaged cold acquisition/cache reuse, or +representative RTX 30/40/50 measurements. + +No cloud resource, credit, or macOS work was performed (USD 0). The checked-in ledger +still has no exact Q3.8 reservation and retains the prior conservative USD 56 maximum +within the combined USD 100 ceiling, leaving USD 44 unreserved and unauthorized. + +## Next gate + +Run a fresh read-only GCP authentication, protected-bootstrap, exact run-resource, +quota, accelerator, capacity, and pricing preflight against this pushed source. If and +only if the exact four-L4 plan has a conservative maximum no greater than the remaining +USD 44, commit a source-bound readiness reservation before any create. Then run the +durable controller through live start, protected IAP delivery, metadata publication, +authenticated generation-stable status collection, acceptance, and exact cleanup. +Any non-pass goes directly to cleanup; paid actions remain blocked until the reservation +and provider preflight are committed. diff --git a/docs/evidence/normalized-online-windows-download-failure-20260908.json b/docs/evidence/normalized-online-windows-download-failure-20260908.json new file mode 100644 index 000000000..d0759169f --- /dev/null +++ b/docs/evidence/normalized-online-windows-download-failure-20260908.json @@ -0,0 +1,179 @@ +{ + "schema_version": 1, + "result": "failed", + "phases": [ + { + "phase": "online-download-install", + "timeout_seconds": 8100, + "result": "failed", + "pid": 3960, + "exit_code": 1, + "seconds": 1260.312, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "started_at_utc": "2026-09-09T00:01:12.481268+00:00", + "online_sha256": "c8171619c835877f018dcf305bf70d7b4804535fd6ad548bd682971ecccdda5d", + "online_size_bytes": 2099037, + "offline_installer": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "size_bytes": 2462345104, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "version": "0.1.0-alpha.20260908.1" + }, + "manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "metadata_sha256": "1460eb03070e33889ffc783b0bd97eeb48dfb76f44b6ac4b2ef27e81a89a6f1d", + "ordinary_user": true, + "logical_cpu_limit": 2, + "priority": "BELOW_NORMAL", + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "owned_processes": [ + { + "pid": 3960, + "created_at": 1788912072.5575345, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 23136, + "created_at": 1788912072.8589523, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + } + ], + "updated_at_utc": "2026-09-09T00:22:12.841343+00:00", + "error": "RuntimeError: online-download-install exit or owned-process cleanup failed", + "installation_present": false, + "all_recorded_processes_stopped": true, + "scope": "Production Windows online download interrupted before offline installer handoff", + "transport_failure": { + "wininet_error": 12030, + "bytes_reported_before_failure": 2296254144, + "expected_bytes": 2462345104, + "offline_installer_launched": false, + "log_message": "Error reading data: (12030) Die Serververbindung wurde aufgrund eines Fehlers beendet", + "cause_attribution": "A connection termination was reported. This observation does not identify whether the origin, network path, client or an intermediary caused it." + }, + "cleanup_audit": { + "result": "passed-cleanup-after-failed-download", + "observed_at_utc": "2026-09-09T00:23:37.901382+00:00", + "raw_result_sha256": "033aecb1dbdb040a6cf1a8aaf22a63ed3a301acdac18fb88e522fec7da16498a", + "online_log_sha256": "6e7a6fc7b0429231f2133fddababa571e6bd163260b40d0663b497b9f817a460", + "raw_result_preserved": true, + "all_recorded_processes_stopped": true, + "recorded_process_count": 2, + "installation_directory_absent": true, + "offline_child_observed": false, + "remaining_temporary_entries": [], + "persisted_baseline_restored": true, + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "baseline_after_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "mutations_performed_by_audit": false + }, + "raw_evidence_sha256": { + ".gate13-runs/normalized-online-acceptance-20260908/result.json": "033aecb1dbdb040a6cf1a8aaf22a63ed3a301acdac18fb88e522fec7da16498a", + ".gate13-runs/normalized-online-acceptance-20260908/online.log": "6e7a6fc7b0429231f2133fddababa571e6bd163260b40d0663b497b9f817a460", + ".gate13-runs/normalized-online-acceptance-20260908/download-failure-cleanup-audit.json": "0f326dd1a1bc1a830c3a0d4ab91a6fe6725f4aea950da4ff6af3018b7f55219f", + ".gate13-runs/qualify-online-windows-20260908.py": "7f99358c019145eb66f323af52426ae23107b603a64cff57fa3591324c501c9b", + ".gate13-runs/qualify-normalized-windows-20260908.py": "ed88f48df908119f06a4d063214ede5cb64df094c6b09864f07b69ac26b333f5", + ".gate13-runs/audit-online-windows-failure-20260908.py": "6eb1fa5877c2bb8cf344c829e89f237da4636b23f454adacd27967fa014811d2", + ".gate13-runs/cloudflare-publication-20260908/windows-bounded-range-check.json": "917dec7700f241c512b65778b8107affc5e3989e858cee6b49000442bb8fefb4", + ".gate13-runs/append-online-failure-chronology-20260908.py": "d9d3f882eea9359667c5e9d2204b0a5507fee41542e5046db6efe706dd575762", + ".gate13-runs/cloudflare-publication-20260908/windows-cache-metadata-operator-record.json": "274c548af50ccff8e3d020b1543719e8935d28698fb6c28e68f57d280d42f376", + ".gate13-runs/bind-online-failure-operator-record-20260908.py": "2699cf7bcbcab690c2b47e814740121ba088ab1a62771cad4298381e15dd20d3", + ".gate13-runs/normalized-online-installers-20260908/communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe.json": "1460eb03070e33889ffc783b0bd97eeb48dfb76f44b6ac4b2ef27e81a89a6f1d" + }, + "positive_full_online_handoff_passed": false, + "limits": [ + "The partial bytes were not independently hashed; the byte count is the native downloader progress log.", + "No full offline installer, installed diagnostic, uninstall or product GUI was launched in this attempt.", + "Cleanup compares exact recorded process identities and persisted settings-file metadata, menu and native Run/uninstall registration snapshots." + ], + "concurrent_publication_chronology": { + "source": "Later operator transcription of observed CLI result chunkfa58bc; not a raw API response saved at operation time.", + "metadata_change_at_utc": "2026-09-09T00:01:55.375Z", + "download_started_at_utc": "2026-09-09T00:01:12.481268+00:00", + "seconds_after_download_started_approximately": 43, + "operation": "Server-side copy-object to the same key to correct Cache-Control metadata.", + "causal_conclusion": "Unconfirmed. A metadata update occurred during the active read; it is a recorded transport variable, not a demonstrated explanation of error12030.", + "retry_policy": "Preserve the failed attempt, use the exact same production wrapper with fresh run/install paths, wait for publication completion and serialize full downloads. No further object mutation during retry.", + "post_copy_size_and_declared_sha256_match_expected": true, + "copy_source_and_returned_version_ids_match": true, + "pre_copy_etag_persisted": false, + "etag_before_after_equality_verified": false, + "operator_record": { + "schema_version": 1, + "record_kind": "operator transcription of an observed CLI tool result", + "source_tool_result_chunk": "fa58bc", + "operation": "s3api copy-object from the object to the same key, metadata-directive REPLACE", + "bucket": "communityai-releases", + "key": "alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "reason": "The original unquoted PowerShell cache-control argument became a space-separated string; replace it with correctly comma-separated directives.", + "copy_response_last_modified_utc": "2026-09-09T00:01:55.375Z", + "copy_source_version_id": "7e5f7c9ee30c1badff3303cf08a30ef1", + "returned_version_id": "7e5f7c9ee30c1badff3303cf08a30ef1", + "returned_etag": "535fc3433bf995d4f9b8a2fbb1b2875c-147", + "post_copy_head": { + "content_length": 2462345104, + "cache_control": "public,max-age=31536000,immutable", + "declared_sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb" + }, + "concurrent_online_download_started_utc": "2026-09-09T00:01:12.994Z", + "limitations": [ + "This is a later transcription of the tool response, not a raw API response saved at operation time.", + "No pre-copy ETag was persisted, so ETag equality before and after cannot be independently asserted from this record.", + "The copy used the same source object and retained expected size and SHA metadata, but that is not independent whole-file download/hash verification.", + "A later transport termination occurred; causation by this concurrent metadata operation is unconfirmed." + ] + } + }, + "subsequent_bounded_origin_check": { + "recorded_at_utc": "2026-09-09T00:26:45.310165+00:00", + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "user_agent": "CommunityAI-Online-Installer/1", + "file_size_bytes": 2462345104, + "ranges": [ + { + "start": 0, + "end": 65535, + "bytes": 65536, + "http_status": 206, + "sha256": "3bebe436fae4d118223044d658128291f2137c5fd7e504b372558504725ecb53", + "matches_exact_local_file": true + }, + { + "start": 2462279568, + "end": 2462345103, + "bytes": 65536, + "http_status": 206, + "sha256": "a9d8f56aad78c72b9518f84b1a9bbb6a248dc4e23c89223a50679892da9d4ea7", + "matches_exact_local_file": true + } + ], + "complete_download_verified": false, + "purpose": "Origin range compatibility check using the real Linux installer user agent after transport failure; does not establish failure cause" + }, + "runtime_source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "source_commit_scope": "The inherited source_commit identifies only the packaged offline runtime.", + "online_source_identity": { + "scope": "Online builder, Inno script and any downloader helper are identified by the companion build hashes. They are working-tree files at online build time, not implied to exist in runtime_source_commit.", + "companion_metadata_sha256": "1460eb03070e33889ffc783b0bd97eeb48dfb76f44b6ac4b2ef27e81a89a6f1d", + "build_input_hashes": {} + } +} diff --git a/docs/evidence/normalized-online-windows-download-failure-20260908.md b/docs/evidence/normalized-online-windows-download-failure-20260908.md new file mode 100644 index 000000000..653d34136 --- /dev/null +++ b/docs/evidence/normalized-online-windows-download-failure-20260908.md @@ -0,0 +1,50 @@ +# Production Windows online download: interrupted transfer + +**The full download/install acceptance did not pass.** The production online +setup exited with code **1** after **1,260.312 seconds**, having reported +**2,296,254,144 of 2,462,345,104 bytes** downloaded (about 93%). Its native downloader +reported WinINet **12030**, a terminated connection. This record does not identify +which endpoint or part of the network caused the termination. + +The [sanitized record](normalized-online-windows-download-failure-20260908.json) +binds the exact production online setup, pinned R2 URL/full-installer hash, +unchanged failed result, progress log, executed helpers and subsequent read-only +cleanup audit. The attempt began at **2026-09-09 00:01:12 UTC** and ended at +**00:22:12 UTC**; filenames use the release's September 8 date. + +The online setup reported that no installer was launched. No offline child was +observed and no application installation was present. Both recorded process +identities stopped. The independent audit at **00:23:37 UTC** confirmed the owned +temporary directory was empty and the saved pre/post settings-file metadata, +Start Menu and native Run/uninstall registration snapshots matched exactly. +Their common SHA-256 is +`f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874`. + +This demonstrates that this interrupted transfer stopped safely before the +offline installer ran. It does **not** establish the full online handoff or +attribute the network failure to the package. The same full installer separately +passed its [installed native/CUDA and removal qualification](normalized-windows-installer-20260908.md). +Any retry must retain this failed attempt and use separate result/install paths. + +The release operator subsequently recorded a metadata change during the active +read: at **00:01:55 UTC**, about **43 seconds** after download launch, a server-side +copy to the same object key corrected its `Cache-Control` metadata. The operator +record is a later transcription of the observed CLI result. The copy response +returned matching source/returned VersionIds, and its subsequent HEAD retained +the expected size and SHA-256 metadata. No pre-copy ETag was persisted, so ETag +equality before/after is **not verified**. This is a concurrent transport variable; +it does **not** establish the cause of the later connection termination. The +original failed result and log remain unchanged. + +At **00:26:45 UTC**, separate first/last **64 KiB** requests using the real Linux +installer user agent returned HTTP **206** and matched the local file. These +bounded observations show those ranges were retrievable at that time. They do +not verify a whole transfer, Windows client behavior or the cause of this failure. +An unrelated default Python user-agent request was rejected; it is not evidence +that the production installer URL was unavailable. + +Source identity is component-specific: inherited `source_commit` and the explicit +`runtime_source_commit` identify the **offline packaged runtime only**. The online +builder, Inno script and any downloader helper are working-tree build inputs +bound by the companion metadata hashes in `online_source_identity`; they are not +implied to have existed in that runtime commit. Raw acceptance records are unchanged. diff --git a/docs/evidence/normalized-online-windows-download-retry-failure-20260908.json b/docs/evidence/normalized-online-windows-download-retry-failure-20260908.json new file mode 100644 index 000000000..547b62300 --- /dev/null +++ b/docs/evidence/normalized-online-windows-download-retry-failure-20260908.json @@ -0,0 +1,122 @@ +{ + "schema_version": 1, + "result": "failed", + "phases": [ + { + "phase": "online-download-install", + "timeout_seconds": 8100, + "result": "failed", + "pid": 18496, + "exit_code": 1, + "seconds": 479.484, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "started_at_utc": "2026-09-09T00:36:30.854722+00:00", + "online_sha256": "c8171619c835877f018dcf305bf70d7b4804535fd6ad548bd682971ecccdda5d", + "online_size_bytes": 2099037, + "offline_installer": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "size_bytes": 2462345104, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "version": "0.1.0-alpha.20260908.1" + }, + "manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "metadata_sha256": "1460eb03070e33889ffc783b0bd97eeb48dfb76f44b6ac4b2ef27e81a89a6f1d", + "ordinary_user": true, + "logical_cpu_limit": 2, + "priority": "BELOW_NORMAL", + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "owned_processes": [ + { + "pid": 18496, + "created_at": 1788914190.8650596, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 28424, + "created_at": 1788914191.2163317, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + } + ], + "updated_at_utc": "2026-09-09T00:44:30.364423+00:00", + "error": "RuntimeError: online-download-install exit or owned-process cleanup failed", + "installation_present": false, + "all_recorded_processes_stopped": true, + "scope": "Exact-wrapper Windows production HTTPS retry interrupted before offline handoff", + "transport_failure": { + "wininet_error": 12030, + "bytes_reported_before_failure": 1007128256, + "expected_bytes": 2462345104, + "offline_installer_launched": false, + "log_message": "Error reading data: (12030) Die Serververbindung wurde aufgrund eines Fehlers beendet", + "cause_attribution": "Unconfirmed. The native downloader reported a connection termination; origin, network path, client and intermediary causes are not distinguished by this result." + }, + "cleanup_audit": { + "result": "passed-cleanup-after-failed-download", + "observed_at_utc": "2026-09-09T00:51:18.820755+00:00", + "raw_result_sha256": "069ad1fb24fb8ebb9f8c0ea2d5a6d0488ab8ce2023fc0a2c0f3fcc4673b63d0c", + "online_log_sha256": "526dcf6002a2c7d9fc31d4005cb06ca37c71965da6afab83a6a80e88a136e76e", + "all_recorded_processes_stopped": true, + "recorded_process_count": 2, + "installation_directory_absent": true, + "offline_child_observed": false, + "remaining_temporary_entries": [], + "persisted_baseline_restored": true, + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "baseline_after_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "application_state_mutations_performed_by_audit": false, + "raw_result_preserved": true + }, + "prior_attempt": "normalized-online-windows-download-failure-20260908.json", + "retry_conditions": { + "source": "Release operator launch authorization and coordinated download ownership.", + "same_production_online_executable": true, + "same_pinned_offline_url_hash_and_size": true, + "fresh_run_and_install_paths": true, + "no_concurrent_full_release_upload_or_download": true, + "windows_object_metadata_unchanged_during_retry": true, + "last_reported_windows_object_metadata_change_utc": "2026-09-09T00:01:55.375Z", + "product_source_changed_for_retry": false + }, + "positive_full_online_handoff_passed": false, + "limits": [ + "The partial bytes were not independently hashed; progress comes from the native download log.", + "This attempt did not launch the offline setup, installed diagnostic, uninstall or product GUI.", + "The failed retry removes a concurrent metadata update as a variable for this attempt, but does not establish the cause of either connection termination." + ], + "raw_evidence_sha256": { + ".gate13-runs/normalized-online-acceptance-retry-20260908/result.json": "069ad1fb24fb8ebb9f8c0ea2d5a6d0488ab8ce2023fc0a2c0f3fcc4673b63d0c", + ".gate13-runs/normalized-online-acceptance-retry-20260908/online.log": "526dcf6002a2c7d9fc31d4005cb06ca37c71965da6afab83a6a80e88a136e76e", + ".gate13-runs/normalized-online-acceptance-retry-20260908/download-failure-cleanup-audit.json": "1a67ac604db3f84c5dbd5c2388d30ca43fed08e0a9b9ccda5285c5f322aac846", + ".gate13-runs/qualify-online-windows-retry-20260908.py": "6bcef4de294121c0109cb3548891fcfdf982b9a5b1112daeef145f93c0c04820", + ".gate13-runs/qualify-normalized-windows-20260908.py": "ed88f48df908119f06a4d063214ede5cb64df094c6b09864f07b69ac26b333f5", + ".gate13-runs/record-online-windows-retry-failure-20260908.py": "ce824afe2d1dc01ee2163ffcbab6014004e1dc65f9656d9a1115ad13cf17d2bb", + ".gate13-runs/normalized-online-installers-20260908/communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe.json": "1460eb03070e33889ffc783b0bd97eeb48dfb76f44b6ac4b2ef27e81a89a6f1d" + }, + "runtime_source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "source_commit_scope": "The inherited source_commit identifies only the packaged offline runtime.", + "online_source_identity": { + "scope": "Online builder, Inno script and any downloader helper are identified by the companion build hashes. They are working-tree files at online build time, not implied to exist in runtime_source_commit.", + "companion_metadata_sha256": "1460eb03070e33889ffc783b0bd97eeb48dfb76f44b6ac4b2ef27e81a89a6f1d", + "build_input_hashes": {} + } +} diff --git a/docs/evidence/normalized-online-windows-download-retry-failure-20260908.md b/docs/evidence/normalized-online-windows-download-retry-failure-20260908.md new file mode 100644 index 000000000..3d459890f --- /dev/null +++ b/docs/evidence/normalized-online-windows-download-retry-failure-20260908.md @@ -0,0 +1,37 @@ +# Windows online setup: isolated retry interrupted + +**The exact-wrapper retry did not complete the full online handoff.** It exited +with code **1** after **479.484 seconds**, reporting **1,007,128,256 of +2,462,345,104 bytes** downloaded (about **40.9%**), then WinINet **12030**. +Execution ran from **2026-09-09 00:36:30 UTC** to **00:44:30 UTC**. + +The [sanitized record](normalized-online-windows-download-retry-failure-20260908.json) +binds the exact production online setup, pinned full-installer URL/hash/size, +unchanged failed result, native progress log, executed helpers and cleanup audit. +The [first failed transfer](normalized-online-windows-download-failure-20260908.md) +remains separate and unchanged. + +This retry used fresh result/install paths after the Linux upload had finished. +The Linux full download was held until this retry ended. The release operator +reported no Windows object or metadata change during the retry; the last such +operation was at **00:01:55.375 UTC**. This removes the first attempt's concurrent +metadata update as a variable for this attempt. It does **not** identify whether +the origin, network path, client or an intermediary caused either termination. + +No full installer child was observed or application installation created. Both +recorded process identities stopped, the owned temporary directory was empty, +and the independently saved post-failure settings-file metadata, Start Menu and +Run/uninstall registration baseline matched the pre-launch snapshot exactly. +The baseline SHA-256 was +`f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874`. + +The wrapper stopped safely before running an incomplete installer. Its positive +production download-to-install acceptance remains **unverified**. The exact full +installer separately passed its +[installed CUDA/native and uninstall check](normalized-windows-installer-20260908.md). + +Source identity is component-specific: inherited `source_commit` and the explicit +`runtime_source_commit` identify the **offline packaged runtime only**. The online +builder, Inno script and any downloader helper are working-tree build inputs +bound by the companion metadata hashes in `online_source_identity`; they are not +implied to have existed in that runtime commit. Raw acceptance records are unchanged. diff --git a/docs/evidence/normalized-online-windows-installer-20260908.json b/docs/evidence/normalized-online-windows-installer-20260908.json new file mode 100644 index 000000000..34418d0e7 --- /dev/null +++ b/docs/evidence/normalized-online-windows-installer-20260908.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "result": "passed", + "phases": [ + { + "phase": "online-download-install", + "timeout_seconds": 8100, + "result": "passed", + "pid": 19524, + "exit_code": 0, + "seconds": 1365.187, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "phase": "installed-native-cpu", + "timeout_seconds": 180, + "result": "passed", + "pid": 27140, + "exit_code": 0, + "seconds": 2.813, + "all_recorded_processes_stopped": true, + "stdout_sha256": "a516c1787629cdf689b5a01aad52c83a7b963c31eb936c8ff305059f55fe2cca", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "phase": "uninstall", + "timeout_seconds": 300, + "result": "passed", + "pid": 4872, + "exit_code": 0, + "seconds": 3.688, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "started_at_utc": "2026-09-09T01:29:46.701131+00:00", + "online_sha256": "8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861", + "online_size_bytes": 2107751, + "offline_installer": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "size_bytes": 2462345104, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "version": "0.1.0-alpha.20260908.1" + }, + "manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "metadata_sha256": "c3fece282465df8c74907ff4c7c29fd48405b9dc579315a61d340929afe4584e", + "ordinary_user": true, + "logical_cpu_limit": 2, + "priority": "BELOW_NORMAL", + "runtime_source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "source_commit_scope": "The inherited source_commit identifies only the packaged offline runtime.", + "online_source_identity": { + "scope": "Online helper, Inno script and builder are working-tree build inputs bound by companion hashes, not implied to exist in the offline runtime commit.", + "companion_metadata_sha256": "c3fece282465df8c74907ff4c7c29fd48405b9dc579315a61d340929afe4584e", + "build_input_hashes": { + "builder_script_sha256": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "installer_script_sha256": "d975b6da3a1067e679d9ba55555fa5b6219ae644fce8050225486909600fbbb1", + "download_helper_source_sha256": "3bfa36e62f2a160b2e258af6d9d6816e815d8a9c5b918c29dd800eb6e6c20d8a", + "download_helper_sha256": "e96c0fce4b71f9236a77625b6af7af2924d5f2004d16d17ca8eb5527b43ec3f1" + } + }, + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "owned_processes": [ + { + "pid": 19524, + "created_at": 1788917386.7624488, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 27324, + "created_at": 1788917387.060789, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 28208, + "created_at": 1788917387.2605045, + "executable_name": "WindowsDownload.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 12868, + "created_at": 1788917387.2676601, + "executable_name": "conhost.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 28068, + "created_at": 1788918567.1587553, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 6428, + "created_at": 1788918567.4603271, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-setup.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 27140, + "created_at": 1788918753.505875, + "executable_name": "CommunityAI-Node.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 6972, + "created_at": 1788918753.5271788, + "executable_name": "conhost.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 4872, + "created_at": 1788918756.396247, + "executable_name": "unins000.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 27300, + "created_at": 1788918756.5527587, + "executable_name": "_unins.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 24172, + "created_at": 1788918756.6761663, + "executable_name": "CommunityAI.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + } + ], + "updated_at_utc": "2026-09-09T01:52:40.693693+00:00", + "download_helper": { + "pid": 28208, + "created_at": 1788917387.2605045, + "executable_name": "WindowsDownload.exe", + "size_bytes": 13824, + "sha256": "e96c0fce4b71f9236a77625b6af7af2924d5f2004d16d17ca8eb5527b43ec3f1", + "source_sha256": "3bfa36e62f2a160b2e258af6d9d6816e815d8a9c5b918c29dd800eb6e6c20d8a", + "observed_owned_helper_matches_companion_metadata": true + }, + "progress_publication_warning": "Skipped progress frame: IOException HRESULT 80070497", + "downloaded_offline_file": { + "size_bytes": 2462345104, + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "pid": 28068, + "created_at": 1788918567.1587543, + "verified_after_owned_child_launch": true, + "process_handle_identity_verified": true + }, + "offline_child_exit_code": 0, + "installed_payload_files": 4936, + "installed_payload_bytes": 4263859354, + "installed_identities": { + "CommunityAI.exe": "c4a500d7b4ca62f26029745e172c2a7fc33465146d7b3ab20b6702411f0f6fd2", + "node/CommunityAI-Node.exe": "5696046eff78d5ebf1badd0fcf7a02f5fcfde9f884072eb8c8f2fc5a3e527162", + "runtime-packaging.json": "ea551e5fe410260999804392d978600024db45a568770c385ea92eb9d9abf2c4", + "node/_internal/bitsandbytes/libbitsandbytes_cuda124.dll": "436aaa9499932fdbb4c447a29d71fcb5cfdadd2e972f932d6fad964eb7b3374f" + }, + "full_download_logged": true, + "child_install_success_logged": true, + "offline_child_log_path_verified": true, + "downloaded_package_and_parent_removed": true, + "native_cpu": { + "application": "CommunityAI-Native-Runtime", + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_required": false, + "cuda_test_performed": false, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "baseline_after_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "persisted_baseline_restored": true, + "installation_directory_removed": true, + "all_recorded_processes_stopped": true, + "qualification_credential_created": false, + "product_desktop_window_launched": false, + "models_loaded": false, + "model_network_joined": false, + "online_log_sha256": "49f93c4a367f3b376b3bd95e8624a255647c9d2a6887c2f3adbe5633aef70ab6", + "offline_log_sha256": "10b8d603bb25df45a9ea199848a315d01a14db08eab745ff2007ddfa46e73775", + "uninstall_log_sha256": "8e7a8210303c13d099fcc2971bee436c1866ccd69c28d9d28f0f39a9c5ce560b", + "offline_child_handle_closed": true, + "online_source_commit": "b6c8aad9cea208630785d890cfb966093f809e7e", + "online_source_commit_binding": { + "online_source_commit": "b6c8aad9cea208630785d890cfb966093f809e7e", + "all_three_git_blob_hashes_match_built_wrapper_companion": true, + "runtime_source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "online_wrapper_sha256": "8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861" + }, + "scope": "Resumable production HTTPS download, verified offline child install, installed CPU diagnostic and uninstall", + "online_filename": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "raw_evidence_sha256": { + ".gate13-runs/normalized-online-acceptance-resumable-v2-20260909/result.json": "19fc08496fb83eda3981d748f8461e54dac42fe1608c03ab3208a1b2b3fce514", + ".gate13-runs/normalized-online-acceptance-resumable-v2-20260909/online.log": "49f93c4a367f3b376b3bd95e8624a255647c9d2a6887c2f3adbe5633aef70ab6", + ".gate13-runs/normalized-online-acceptance-resumable-v2-20260909/offline.log": "10b8d603bb25df45a9ea199848a315d01a14db08eab745ff2007ddfa46e73775", + ".gate13-runs/normalized-online-acceptance-resumable-v2-20260909/uninstall.log": "8e7a8210303c13d099fcc2971bee436c1866ccd69c28d9d28f0f39a9c5ce560b", + ".gate13-runs/cloudflare-publication-20260908/online-source-commit-binding.json": "dc871c18c66b8ccfcb61304b9632d151cccb29cb623333055bef355e01419f94", + ".gate13-runs/qualify-online-windows-resumable-v2-20260909.py": "da83b28d2288e9ac7264181cd028a5adb2f9077ee24a8dae671d7a0a5636db20", + ".gate13-runs/qualify-normalized-windows-20260908.py": "ed88f48df908119f06a4d063214ede5cb64df094c6b09864f07b69ac26b333f5" + }, + "prior_attempts": [ + "normalized-online-windows-download-failure-20260908.json", + "normalized-online-windows-download-retry-failure-20260908.json", + "normalized-online-windows-resumable-progress-failure-20260909.json" + ], + "limits": [ + "This run uses the exact offline payload already qualified with installed CUDA/NF4 diagnostics; its repeated installed diagnostic is CPU-only.", + "No model loading, contribution worker, network join, product GUI or full Gate14/15 replay.", + "The offline setup uses its normal shortcut behavior; NOICONS does not suppress these entries because AllowNoIcons is not enabled. Both menu baselines match after uninstall.", + "State preservation compares persisted file metadata and native Run/uninstall registry snapshots. No credential was created or changed by the qualification.", + "R2 dev-domain delivery is one observed complete download, not a public-load or availability guarantee." + ] +} diff --git a/docs/evidence/normalized-online-windows-installer-20260908.md b/docs/evidence/normalized-online-windows-installer-20260908.md new file mode 100644 index 000000000..1e29e026d --- /dev/null +++ b/docs/evidence/normalized-online-windows-installer-20260908.md @@ -0,0 +1,48 @@ +# Production Windows online setup: full HTTPS handoff + +**Passed for one complete public HTTPS download/install/CPU diagnostic/uninstall.** +The [companion record](normalized-online-windows-installer-20260908.json) binds +the exact launchers, downloaded-file hash, retained child-process handle, logs, +installed payload and cleanup. Execution began at `2026-09-09T01:29:46.701131+00:00`. +The online source commit is `b6c8aad9cea208630785d890cfb966093f809e7e`; an independent +Git blob comparison matched the builder, Inno script and helper source to the +production companion hashes. The offline runtime source is separately +`84205f93fc73d3babd39e238944b97fab0d11b3e`. + +The online setup is **2,107,751 bytes**, SHA-256 +`8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861`. It downloaded the actual +**2,462,345,104-byte** offline installer from its +[embedded public R2 URL](https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe). The entire +downloaded temporary file was independently read with a bounded buffer after +the owned child launched; its SHA-256 was +`116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb`, matching the build-time pin. + +| Check | Result | +| --- | --- | +| Native online download and child install | Passed in 1365.187 seconds. Outer exit 0 and independently retained offline-child handle exit 0. | +| Separate logs | Online log records the pinned URL and all downloaded bytes; offline log identifies the downloaded setup and the requested isolated install/log paths. Both report successful installation. | +| Installed payload | All 4,936 attested file lengths matched, totaling 4,263,859,354 bytes. GUI/node executable, CUDA 12.4 bitsandbytes library and normalization-report hashes matched provenance. | +| Installed CPU diagnostic | Passed in 2.813 seconds; frozen Torch 2.6.0+cu124 completed the exact CPU operation without a CUDA test, model load or network join. | +| Silent uninstall | Passed in 3.688 seconds. The application directory and owned installer processes were absent; persisted pre/post user-state metadata, menu and native registration snapshots matched exactly. | +| Temporary-file/handle cleanup | The downloaded full setup and its online temporary directory were removed after the waited child exited. The retained child-process query handle closed successfully. | + +The run used an ordinary Windows user, hidden setup processes, two logical CPUs, +below-normal priority and separate online/offline installer logs. The pre-install +baseline was saved privately **before** launch; the post-uninstall baseline +matched it. Normal Start Menu +shortcuts were allowed and removed by the ordinary uninstaller; `/NOICONS` is not +a supported suppression option for this offline setup configuration. + +The qualification also retained this best-effort presentation diagnostic: +`Skipped progress frame: IOException HRESULT 80070497`. +Progress display does not authorize installation; the complete file size, SHA-256, +successful helper exit and independent Inno recheck still gate the handoff. + +This is the same offline payload that passed the +[installed required-CUDA/NF4 acceptance](normalized-windows-installer-20260908.md). +Earlier transport attempts remain separate: [first transfer](normalized-online-windows-download-failure-20260908.md), [isolated retry](normalized-online-windows-download-retry-failure-20260908.md). The earlier [progress-publication failure](normalized-online-windows-resumable-progress-failure-20260909.md) also remains separate. Repeating GPU checks was unnecessary. No product desktop window, model load, +network join, contribution worker or qualification credential was created. +Previous Gate 14/15 acceptance remains separate. The inherited source commit +identifies only the packaged offline runtime; online helper/Inno/builder working-tree +inputs are bound by companion hashes in the separate `online_source_identity`. This single transfer establishes +the observed production handoff, not broad network performance or availability. diff --git a/docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.json b/docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.json new file mode 100644 index 000000000..628b1e5ca --- /dev/null +++ b/docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.json @@ -0,0 +1,143 @@ +{ + "schema_version": 1, + "result": "failed", + "phases": [ + { + "phase": "online-download-install", + "timeout_seconds": 8100, + "result": "failed", + "pid": 15572, + "exit_code": 1, + "seconds": 263.343, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "started_at_utc": "2026-09-09T01:14:44.367520+00:00", + "online_sha256": "0d1c31206336b1e8c783a78a6cadbdc8f9e643e91d587b88d78581a85b5d9ef6", + "online_size_bytes": 2107444, + "offline_installer": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "size_bytes": 2462345104, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "version": "0.1.0-alpha.20260908.1" + }, + "manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "metadata_sha256": "6423e194ca120897cf25f20a49eb89896cf3e839c53e62c0cfba921f357b9e97", + "ordinary_user": true, + "logical_cpu_limit": 2, + "priority": "BELOW_NORMAL", + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "owned_processes": [ + { + "pid": 15572, + "created_at": 1788916484.4362707, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 23684, + "created_at": 1788916484.746788, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 23840, + "created_at": 1788916484.9901369, + "executable_name": "WindowsDownload.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 25804, + "created_at": 1788916484.9977367, + "executable_name": "conhost.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + } + ], + "updated_at_utc": "2026-09-09T01:19:07.803441+00:00", + "download_helper": { + "pid": 23840, + "created_at": 1788916484.9901369, + "executable_name": "WindowsDownload.exe", + "size_bytes": 13824, + "sha256": "ba76158d7762058cfc53f98054334ca0b6bc3a5d4e0f8e8f32eda87b824813df", + "source_sha256": "3df36f4f1273599a3680d3d4981f150f2eaf289d92926b19dbde6620fdad3094", + "observed_owned_helper_matches_companion_metadata": true + }, + "error": "RuntimeError: online-download-install exit or owned-process cleanup failed", + "installation_present": false, + "all_recorded_processes_stopped": true, + "scope": "First resumable production wrapper: progress-publication failure before installer handoff", + "failure": { + "component": "Local progress-file publication", + "helper_exit_code": 1, + "message": "Progress could not be persisted", + "reported_downloaded_bytes": 541341184, + "underlying_exception_type_or_native_error": "Not recorded by this helper version", + "network_failure_inferred": false, + "offline_installer_launched": false + }, + "cleanup_audit": { + "result": "passed-cleanup-after-progress-failure", + "observed_at_utc": "2026-09-09T01:21:11.532367+00:00", + "raw_result_sha256": "0c14be16ef030e9da56493b3a9ab2a8bf15db47f0ee78674dcd820004d307b8d", + "raw_result_preserved": true, + "recorded_process_count": 4, + "all_recorded_processes_stopped": true, + "installation_directory_absent": true, + "remaining_temporary_entries": [], + "persisted_baseline_restored": true, + "baseline_before_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "baseline_after_sha256": "f81e748eeefe3d3666ff20fe72d85b1e59cbc8dd8499437b71ea3c900b5f2874", + "application_state_mutations_performed_by_audit": false + }, + "positive_full_online_handoff_passed": false, + "prior_attempts": [ + "normalized-online-windows-download-failure-20260908.json", + "normalized-online-windows-download-retry-failure-20260908.json" + ], + "raw_evidence_sha256": { + ".gate13-runs/normalized-online-acceptance-resumable-20260909/result.json": "0c14be16ef030e9da56493b3a9ab2a8bf15db47f0ee78674dcd820004d307b8d", + ".gate13-runs/normalized-online-acceptance-resumable-20260909/online.log": "479d629702ff787c6901d67bb35280f21ff872f0ec2c59c88df4d0e23ac2edcc", + ".gate13-runs/normalized-online-acceptance-resumable-20260909/failure-cleanup-audit.json": "13e41959ec3242309a20502a401c8c43f31d1962968203c72d7608ea97615534", + ".gate13-runs/qualify-online-windows-resumable-20260909.py": "a1d7d1ee6be132042571bedc58ecce2d5dfd08d0e430749af61ec6d3dbd2020f", + ".gate13-runs/record-online-windows-resumable-failure-20260909.py": "9c5d7d6f409d5b98e7893d5c14b488eeb4666a9d087ef1e10c7c3b3fcfeec657", + ".gate13-runs/qualify-normalized-windows-20260908.py": "ed88f48df908119f06a4d063214ede5cb64df094c6b09864f07b69ac26b333f5", + ".gate13-runs/normalized-online-resumable-final-20260909/communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe.json": "6423e194ca120897cf25f20a49eb89896cf3e839c53e62c0cfba921f357b9e97" + }, + "runtime_source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "source_commit_scope": "The inherited source_commit identifies only the packaged offline runtime.", + "online_source_identity": { + "scope": "Online builder, Inno script and any downloader helper are identified by the companion build hashes. They are working-tree files at online build time, not implied to exist in runtime_source_commit.", + "companion_metadata_sha256": "6423e194ca120897cf25f20a49eb89896cf3e839c53e62c0cfba921f357b9e97", + "build_input_hashes": { + "builder_script_sha256": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "installer_script_sha256": "b6c680afa048ba581af6d01a596fa8f9f62837bb7d571979cb9d4dc9fbbf6633", + "download_helper_source_sha256": "3df36f4f1273599a3680d3d4981f150f2eaf289d92926b19dbde6620fdad3094", + "download_helper_sha256": "ba76158d7762058cfc53f98054334ca0b6bc3a5d4e0f8e8f32eda87b824813df" + } + } +} diff --git a/docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.md b/docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.md new file mode 100644 index 000000000..13f1ada3e --- /dev/null +++ b/docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.md @@ -0,0 +1,29 @@ +# Resumable Windows setup: progress-file failure + +**This production download-to-install attempt did not pass.** The new resumable +wrapper exited with code **1** after **263.343 seconds**, at **2026-09-09 01:19:07 UTC**. +The downloader reported **541,341,184 of 2,462,345,104 bytes** received, then failed +with **“Progress could not be persisted”**. This is a local progress-publication +failure, distinct from the earlier connection-termination attempts. This helper +version did not preserve the underlying I/O exception or native error code. + +The [sanitized record](normalized-online-windows-resumable-progress-failure-20260909.json) +binds the exact wrapper and observed helper SHA, unchanged failed result/log, +executed acceptance helpers and independent cleanup audit. No complete-package +SHA or successful online handoff is claimed. + +No offline installer child launched and no application installation was created. +All **four** recorded process identities stopped; the owned temporary directory +was empty. The persisted settings-file metadata, menu and Run/uninstall registry +baseline matched exactly before/after. The audit made no application-state changes. + +The two earlier transport failures remain separate: [first attempt](normalized-online-windows-download-failure-20260908.md) +and [isolated retry](normalized-online-windows-download-retry-failure-20260908.md). +The full offline installer's [native/CUDA and removal acceptance](normalized-windows-installer-20260908.md) +also remains separate. + +Source identity is component-specific: inherited `source_commit` and the explicit +`runtime_source_commit` identify the **offline packaged runtime only**. The online +builder, Inno script and any downloader helper are working-tree build inputs +bound by the companion metadata hashes in `online_source_identity`; they are not +implied to have existed in that runtime commit. Raw acceptance records are unchanged. diff --git a/docs/evidence/normalized-windows-installer-20260908.json b/docs/evidence/normalized-windows-installer-20260908.json new file mode 100644 index 000000000..71b9a0bd1 --- /dev/null +++ b/docs/evidence/normalized-windows-installer-20260908.json @@ -0,0 +1,223 @@ +{ + "schema_version": 1, + "source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "source_tree": "5bd44e4afb6b6911f5ddd1bb3d492dad438883e8", + "installer_filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "installer_sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "installer_bytes": 2462345104, + "provenance_sha256": "d5210329edf7236aaaadc0102c0f9363d3c0560f14ccb46ece3928f61b0ddaa1", + "app_identifier": "CommunityAI.Desktop", + "ordinary_user": true, + "installed_payload_files": 4936, + "installed_payload_bytes": 4263859354, + "installed_total_files": 4939, + "installed_total_bytes": 4270309816, + "installed_identities": { + "CommunityAI.exe": "c4a500d7b4ca62f26029745e172c2a7fc33465146d7b3ab20b6702411f0f6fd2", + "node/CommunityAI-Node.exe": "5696046eff78d5ebf1badd0fcf7a02f5fcfde9f884072eb8c8f2fc5a3e527162", + "runtime-packaging.json": "ea551e5fe410260999804392d978600024db45a568770c385ea92eb9d9abf2c4", + "node/_internal/bitsandbytes/libbitsandbytes_cuda124.dll": "436aaa9499932fdbb4c447a29d71fcb5cfdadd2e972f932d6fad964eb7b3374f" + }, + "native_runtime": { + "application": "CommunityAI-Native-Runtime", + "bitsandbytes_native_library": "libbitsandbytes_cuda124.dll", + "bitsandbytes_nf4_maximum_absolute_error": 0.14501953125, + "bitsandbytes_nf4_roundtrip_passed": true, + "cpu_matmul_passed": true, + "cuda_build": "12.4", + "cuda_linalg_passed": true, + "cuda_matmul_passed": true, + "cuda_required": true, + "cuda_test_performed": true, + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "schema_version": 1, + "torch": "2.6.0+cu124" + }, + "server_runtime": { + "application": "CommunityAI-Worker", + "entrypoint": "server", + "frozen": true, + "model_loading_performed": false, + "network_join_performed": false, + "process_lifetime_guard_armed": true, + "schema_version": 1, + "server_class": "Server", + "throughput_mode": "dry_run", + "training_rpcs_enabled": false + }, + "result": "passed-bounded-install-native-uninstall", + "recorded_at_utc": "2026-09-08T23:50:07.658876+00:00", + "environment": "Windows-10-10.0.19045-SP0", + "logical_cpu_limit": 2, + "priority": "BELOW_NORMAL", + "scope": "One new ordinary-user silent offline install, installed native CUDA/server contracts, and uninstall", + "phases": [ + { + "phase": "install", + "timeout_seconds": 900, + "result": "passed", + "pid": 26564, + "exit_code": 0, + "seconds": 202.937, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "phase": "installed-native-cuda", + "timeout_seconds": 180, + "result": "passed", + "pid": 10308, + "exit_code": 0, + "seconds": 5.187, + "all_recorded_processes_stopped": true, + "stdout_sha256": "7f8c4b8605f0b6421fc9b3e5e2de4de401fb604c0dd9703e5fcc0220fb02bd87", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "phase": "installed-server-contract", + "timeout_seconds": 180, + "result": "passed", + "pid": 23636, + "exit_code": 0, + "seconds": 7.391, + "all_recorded_processes_stopped": true, + "stdout_sha256": "3aa6b190fda9f84336b9a4a3ebd516f00e361169972d25a3d5f17f92412f2a40", + "stderr_sha256": "84eac7794ac271ba86fab9ca1ad642a7e92e9c7a228d5188f9db6a11c54ca4e1" + }, + { + "phase": "uninstall", + "timeout_seconds": 300, + "result": "passed", + "pid": 5784, + "exit_code": 0, + "seconds": 4.578, + "all_recorded_processes_stopped": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "cleanup": { + "installation_directory_removed": true, + "owned_menu_directory_removed": true, + "all_uninstall_registration_views_absent": true, + "common_menu_unchanged": true, + "app_state_metadata_unchanged_during_uninstall": true, + "login_registration_unchanged_during_uninstall": true, + "all_recorded_processes_stopped": true + }, + "owned_process_identities": [ + { + "pid": 26564, + "created_at": 1788910959.4571977, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 24092, + "created_at": 1788910959.8080733, + "executable_name": "communityai-0.1.0-alpha.20260908.1-windows-setup.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 10308, + "created_at": 1788911165.423862, + "executable_name": "CommunityAI-Node.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 27672, + "created_at": 1788911165.4386368, + "executable_name": "conhost.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 23636, + "created_at": 1788911170.5387845, + "executable_name": "CommunityAI-Node.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 26024, + "created_at": 1788911170.5689337, + "executable_name": "conhost.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 5784, + "created_at": 1788911404.1521368, + "executable_name": "unins000.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 10124, + "created_at": 1788911404.3143272, + "executable_name": "_unins.tmp", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + }, + { + "pid": 24288, + "created_at": 1788911404.445324, + "executable_name": "CommunityAI.exe", + "affinity": [ + 0, + 1 + ], + "priority": 16384 + } + ], + "initial_harness_result": "failed-after-successful-install-and-diagnostics", + "initial_harness_error": "AssertionError: Menu changed despite NOICONS", + "noicons_supported_by_offline_setup": false, + "shortcut_finding": "Both newly created shortcuts targeted this isolated install; uninstall removed both and their newly created directory.", + "qualification_credential_created": false, + "product_desktop_window_launched": false, + "raw_evidence_sha256": { + ".gate13-runs/normalized-windows-acceptance-20260908/result.json": "6e02ccaef85e9719c976da8f9232fc01ddc1930f070898b9f17feec3802e92c0", + ".gate13-runs/normalized-windows-acceptance-20260908/install.log": "5137e942e1480869e77d509c23b7ea5de340965b9fc76c7d84aa6a298410df9c", + ".gate13-runs/normalized-windows-acceptance-20260908/cleanup-followup/result.json": "cdb9be2f754a157c70e6600755a7ce82877f3d1cb9b3867dea97b654b051a420", + ".gate13-runs/normalized-windows-acceptance-20260908/cleanup-followup/uninstall.log": "b853f1d8992d47e63312554596c6d3deeabc523d61e41ef1d1409e7ce751b68e", + ".gate13-runs/qualify-normalized-windows-20260908.py": "ed88f48df908119f06a4d063214ede5cb64df094c6b09864f07b69ac26b333f5", + ".gate13-runs/cleanup-normalized-windows-20260908.py": "a9a2a5aafef34e31af4e2001197862866ea555a9136a93b9614be750cd7897ea" + }, + "limits": [ + "No full model load, network join, contribution worker, GUI or renewed Gate14/15 lifecycle acceptance.", + "The initial preinstall menu/state snapshots were not persisted before the harness assertion. The installer log records new menu creation; retained follow-up snapshots prove app-state metadata and Run preservation during uninstall only.", + "The offline installer does not enable AllowNoIcons; supplying NOICONS did not suppress its normal Start Menu shortcuts. This corrected harness assumption is retained, not presented as a product fix.", + "No real online-downloader handoff or public release is established by this offline acceptance." + ] +} diff --git a/docs/evidence/normalized-windows-installer-20260908.md b/docs/evidence/normalized-windows-installer-20260908.md new file mode 100644 index 000000000..834d40976 --- /dev/null +++ b/docs/evidence/normalized-windows-installer-20260908.md @@ -0,0 +1,43 @@ +# Normalized Windows installer: installed native runtime and removal + +**Passed for one bounded ordinary-user install/native diagnostic/uninstall, September 8, 2026.** +The [companion record](normalized-windows-installer-20260908.json) binds the exact +installer, installed identities, diagnostic results, process identities and raw +logs. It retains a corrected harness assumption described below. + +The unsigned `communityai-0.1.0-alpha.20260908.1-windows-setup.exe` is +2,462,345,104 bytes, SHA-256 +`116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb`. +Its runtime source is `84205f93fc73d3babd39e238944b97fab0d11b3e`. +Before installation the new target and stable `CommunityAI.Desktop` registration +were absent in both HKCU/HKLM registry views. All launches were hidden, with two +logical CPUs and below-normal priority; temporary and runtime-cache paths were +private to this qualification. + +| Check | Result | +| --- | --- | +| Silent ordinary-user installation | Passed in 202.937 seconds, exit 0. | +| Installed inventory | All 4,936 attested files and their lengths matched; 4,263,859,354 payload bytes. Including the installer marker/uninstaller, 4,939 files occupied 4,270,309,816 logical bytes. GUI/node executable, bitsandbytes CUDA library and normalization-report hashes matched provenance. | +| Installed required-CUDA native diagnostic | Passed in 5.187 seconds: exact CPU multiplication, CUDA multiplication, CUDA SVD reconstruction, and native CUDA 12.4 bitsandbytes NF4 roundtrip. Maximum NF4 absolute error was 0.14501953125. | +| Installed server contract | Passed in 7.391 seconds: frozen server entry point, disabled training RPCs and armed process-lifetime guard. No model load or network join. | +| Silent uninstall and cleanup | Passed in 4.578 seconds. Installation, newly created Start Menu folder/shortcuts and all uninstall registry views were absent. All nine recorded process identities had stopped. | + +The first harness stopped **after the successful install and diagnostics**, +before uninstall, because it expected `/NOICONS` to suppress Start Menu entries. +The offline setup does not enable `AllowNoIcons`, whose default is `no`; that +expectation was incorrect. The log records a newly created menu directory and +two shortcuts, both independently observed to target the isolated installation. +The initial failed result is unchanged. A separate authorized cleanup run +performed the normal uninstall and verified removal of those owned entries. +[Inno's documented setting](https://jrsoftware.org/ishelp/topic_setup_allownoicons.htm). + +The follow-up persisted its pre/post observations and confirmed unchanged +CommunityAI state-file metadata, login registration and common-menu state during +uninstall. The initial preinstall menu/state snapshots existed only in memory, +so they do not establish a complete persisted retention comparison across this +whole attempt. No qualification credential was created, no product desktop window +opened, and neither diagnostic loaded models or joined a network. + +This qualifies the new installer's installed native runtime and removal in the +stated Windows scope. Previous Gate 14/15 acceptance remains separate. A real +online HTTPS download and child-installer handoff are additional acceptance work. diff --git a/docs/evidence/owner-budget-authorization-20260831.json b/docs/evidence/owner-budget-authorization-20260831.json new file mode 100644 index 000000000..5bccd5661 --- /dev/null +++ b/docs/evidence/owner-budget-authorization-20260831.json @@ -0,0 +1,39 @@ +{ + "schema_version": 1, + "scope": "combined-cloud-budget-epoch-authorization", + "recorded_at": "2026-08-31", + "timezone": "America/Bogota", + "result": "authorized", + "owner_decision": { + "combined_cloud_ceiling_usd": "500.00", + "authorized_on": "2026-08-31", + "providers": [ + "GCP", + "Fly" + ], + "purpose": "CommunityAI public inference alpha critical-path infrastructure and qualification" + }, + "epoch_state_at_authorization": { + "prior_ceiling_usd": "100.00", + "committed_maximum_usd": "52.00", + "unreserved_maximum_usd": "448.00", + "committed_run": "gate13-20260831-a" + }, + "controls_unchanged": { + "exact_source_bound_authorization_per_paid_run": true, + "fresh_native_auth_and_fail_closed_preflight_before_create": true, + "exact_run_scoped_resources_only": true, + "cleanup_and_absence_proof_required": true, + "protected_bootstrap_must_remain": "communityai-bootstrap-1", + "observed_cost_tracking_required": true + }, + "prohibited": { + "credits_or_payments_work": true, + "macos_work": true + }, + "claims": { + "specific_provider_run_authorized_by_this_record": false, + "budget_is_observed_spend": false, + "prior_committed_maximum_released": false + } +} diff --git a/docs/evidence/qwen-c3-run-provenance-audit-20260906.md b/docs/evidence/qwen-c3-run-provenance-audit-20260906.md new file mode 100644 index 000000000..6c1c5d004 --- /dev/null +++ b/docs/evidence/qwen-c3-run-provenance-audit-20260906.md @@ -0,0 +1,132 @@ +# Audit of the last successful Qwen C3 run + +Audited run: `q38pm-20260906-091609-4351`, September 6, 2026, +09:16–10:03 UTC. This audit reads the execution history in +[Run full CPU swarm inference test](codex://threads/01a0730c-c3df-7b63-a20c-6d775e03cb40), +the retained scripts, source inventories, raw results and Git status. It does +not launch a new cloud test or change the old receipts. + +**Finding:** the passing inference/recovery/cache result is supported. It was +an agent-supervised run using an ignored Python wrapper and shared repository +scripts. It was not a run of either Qwen `.cmd` launcher, and it was not an +unchanged, fully unattended invocation from beginning to end. The fixes are +saved locally, but the complete runner is not preserved in Git yet. + +## What actually ran + +The chat records the invocation of +[run-public-c3.py](../../.gate13-runs/qwen-product-implementation/run-public-c3.py) +with the local qualification virtual environment. That wrapper calls: + +- [MixedProductRun](../../scripts/run_qwen_product_mixed.py), inheriting the + cloud operations in [MixedRun](../../scripts/run_qwen_mixed_inference.py) + and source-client recovery in [ProductRun](../../scripts/run_qwen_product_gcp.py). +- [qualify_qwen_remote_product.py](../../scripts/qualify_qwen_remote_product.py) + with the frozen Windows v9 executable, existing verified caches and + `--worker-recovery`. +- The owned worker-action helper and normal runner cleanup. + +`Run Qwen Mixed Inference.cmd` instead invokes `run_qwen_mixed_inference.py`'s +basic inference entry point. `Run Qwen Full Inference GCP.cmd` invokes the +separate CPU inference/recovery entry point. Neither launches the complete +packaged C3 suite above. + +The wrapper selected `c3-highmem-4` before provisioning. That support, including +quota accounting, balanced disks and gVNIC, is saved in the shared scripts. The +original E2 default remains. The wrapper also depends on a previous run's +configuration, fixed v9/cache paths and a fixed output directory; its existing +output-directory assertion deliberately prevents a second unchanged invocation. + +## Interventions found in the chat + +| When | Action | Effect and persistence | +| --- | --- | --- | +| Before launch | Added C3 support and its quota/provisioning tests; created the run-specific wrapper. | Shared changes are in `scripts/` and `tests/`; the wrapper is in the ignored run directory. | +| During cloud setup, before Windows launch | Added `HttpDownloadBlocker`, connected it to the offline phase, and ran its socket test. | Saved in `scripts/qwen_offline_http.py`, `scripts/qualify_qwen_remote_product.py` and `tests/test_qwen_offline_http.py`. This strengthened the cache test. | +| 09:27:27 UTC | Manually wrote the final four-file packaged-harness hash inventory, retaining the initial inventory separately. | The final inventory includes the blocker and matches current files. This inventory step is not automated by the retained wrapper. | +| During the run | Read hardware, setup logs, worker logs, process/disk/network state; saved CPU information locally. One diagnostic requested a nonexistent log. | The reviewed direct SSH commands did not edit remote code, restart services, change policy or repair the running swarm. The Ubuntu setup delay resolved without a repair command. | +| After runner completion | Fixed the summary recorder's attempt to read Azure metadata as GCP metadata by selecting the four actual GCP records. | Saved in ignored `record-public-c3-v9.py`. The raw runner/package results already said `passed`; this changed report generation, not those outcomes. | + +The source worker loss at 09:45:56 UTC, replacement with a new identity, +packaged worker stop/restart, and cleanup were performed by the test scripts. +The packaged-client handoff began at 09:49:22 UTC, after the final harness +inventory. The runner finished with verified cleanup at 10:02:55 UTC. +No manual runtime rescue or acceptance-threshold reduction was found in the +reviewed C3 execution history. Earlier failed attempts remain separate. + +## What the hash checks establish + +- All **197** files in the cloud source inventory match the current local files; + the retained source tarball matches its recorded SHA-256. +- All **four** final packaged-harness files match their recorded SHA-256. + The initial and final qualifier hashes differ, consistent with the documented + blocker change before the packaged phase. +- The retained Windows v9 executable matches the hash in the passing result. +- All **12** bindings in the C3 evidence JSON still match + the retained local receipts and inventories. +- Raw source and packaged results say `passed`; both packaged processes stopped; + the offline phase had HTTP downloads blocked and zero download attempts; + owned GCP instances/disks/firewall rules and the Azure group are absent. + +The main orchestration scripts and wrapper were not separately hashed at launch +by the retained wrapper. Their invocation/change history is supported by the chat +and command journal, rather than a complete launch-time source hash inventory. + +The Windows build snapshot also remains available. **187 of 191** recorded build +inputs match the current checkout. Four files have later edits: the desktop CI +workflow, `desktop/build_desktop.py`, `desktop/README.md`, and +`src/drift/catalog_release.py`. These concern Linux build dependencies/CUDA, +Qwen catalog publication inputs/documentation, and retrying Windows publication +directory replacement. The passing v9 binary is unchanged; this run does not +automatically qualify a fresh build containing those later edits. + +## Persistence and replay limitations + +At audit time, the principal Qwen runner, qualifier and recovery-helper files +are **untracked**, not committed. The wrapper and summary recorder are +**gitignored**. They are saved on this machine but absent from a fresh checkout. +Git HEAD is `14f11d0df5c93ebc4fc3870a07f80d26a462521b`; this is not the source +identity of the dirty-tree engineering package. + +A maintained replay still needs the wrapper/reporting/inventory steps integrated +into normal scripts, configurable verified package/cache inputs, fresh output +directories and a reviewed Git checkpoint. An unchanged run of that future +launcher must be qualified separately; it must not be claimed retrospectively +for this successful run. + +Primary evidence: [C3 result](qwen-packaged-recovery-v9-c3-20260906.json), +[raw runner result](../../.gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/result.json), +[final harness inventory](../../.gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/packaged-harness-final-source.json), +[source inventory](../../.gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/source-inventory.json), +[Windows build inputs](../../.gate13-runs/qwen-product-implementation/build-v9-source.json). + +## Persistence follow-up + +The audit's missing wrapper, reporting and inventory steps have now been moved +into maintained code: `Run Qwen Product Test.cmd`, +`scripts/run_qwen_product_test.py`, `scripts/qwen_product_provenance.py` and +`scripts/report_qwen_product.py`. [Replay instructions](../QWEN_PRODUCT_TEST.md) +describe explicit package/cache inputs, fresh output, source archives and checks, +bounded Windows handoff, automatic reporting, and inherited cloud cleanup. +The C3 setup, HTTP blocker, recovery helpers, runtime fixes and later desktop +build/catalog publication fixes are included in the same persistence checkpoint. + +Regression checks cover failed handoff/timeout, unchanged inputs, package +dependencies, malformed or mismatched receipts, recovery, and cleanup. The new +report reader also accepts the retained successful C3 receipts. This is a local +implementation/validation follow-up, not a new cloud run; the historical account +and old inventories above remain unchanged. An unchanged live run of the new +launcher and a new final-source package are still separate qualification work. + +Local validation for the persistence checkpoint: + +- 25 maintained-launcher tests passed, including a real owned process-tree stop + and rejection of a failed launcher's provenance during standalone reporting. +- The initial combined product/recovery/mixed/offline suite passed 53 tests + (including 19 earlier versions of those launcher tests). +- Runtime, catalog, cache, model-selection and desktop build/client checks: + 362 passed. Additional provider, route-fence and desktop lifecycle/client + checks: 74 passed; these groups overlap and are not a unique test total. +- `--validate-inputs` verified all 4,927 retained Windows v9 package files; + the report reader revalidated the historical C3 raw receipts without rewriting + them. This performed no cloud provisioning and no new inference. diff --git a/docs/evidence/qwen-catalog-desktop-20260907.json b/docs/evidence/qwen-catalog-desktop-20260907.json new file mode 100644 index 000000000..b87115cf1 --- /dev/null +++ b/docs/evidence/qwen-catalog-desktop-20260907.json @@ -0,0 +1,78 @@ +{ + "result": "passed", + "scope": "ordinary-user-frozen-Windows-desktop-signed-catalog-startup-migration", + "non_elevated": true, + "desktop_sha256": "abe961023597c70c095ebbabe067ec5077ba7b6451c4d68db18fc0144f587228", + "node_sha256": "158d4b8940b5e322a951819abbb31631a6cb059647a73e7a313a8c7f6e21955a", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_script_sha256": "c8c5623c5a6742948a38312a910c86f14ab5a5a504acb74104b9b5008b608806", + "launches": [ + { + "phase": "automatic-migration", + "seconds_to_visible_window_and_authenticated_status": 63.406, + "visible_native_window": true, + "model_ids": [ + "Gemma 4 E2B IT", + "Qwen3.5 2B", + "Qwen3.5-0.8B-Local", + "Qwen3.8 27B FP8 Dequant" + ], + "worker_states": [ + "paused" + ], + "owned_process_count": 4 + }, + { + "phase": "repeat-start", + "seconds_to_visible_window_and_authenticated_status": 46.781, + "visible_native_window": true, + "model_ids": [ + "Gemma 4 E2B IT", + "Qwen3.5 2B", + "Qwen3.5-0.8B-Local", + "Qwen3.8 27B FP8 Dequant" + ], + "worker_states": [ + "paused" + ], + "owned_process_count": 8 + } + ], + "legacy_install": { + "catalog_digest": "sha256:70937f0aa10aa73f158753386170c27a95c685b468c66945f05bb3929263f195", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 1, + "created": true, + "model_count": 2, + "schema_version": 1, + "source": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/catalog.signed.json" + }, + "activated_exact_published_sequence_2": true, + "preferences_workers_and_cache_preserved": true, + "repeat_start_config_and_native_credential_preserved": true, + "old_trust_root_rejected_without_config_change": true, + "recorded_runtime_trees_gone": true, + "native_qualification_credential_removed": true, + "limitations": [ + "Windows startup migration and unchanged-catalog restart, not a fresh Linux update observation.", + "The legacy state was installed from the actual signed online catalog, with private test preferences/cache marker.", + "No inference, worker execution, periodic newer-sequence activation, or active-generation drain was exercised.", + "The existing frozen bundle was launched directly; its installer lifecycle has separate evidence." + ], + "recorded_at_utc": "2026-09-08T00:55:17.549633+00:00", + "runtime_source_commit": "76b6d84fc52342af4fd2315926b187aaa36b1378", + "application_source_identical_to_reviewed_head": "e81103663c0c598cdaf43706726fe2a65b4357b6", + "replay_binding": { + "path": ".gate13-runs/gateq38-windows-catalog-startup-20260907/result.json", + "sha256": "5330a3734a53fe8883275f7bd49059d70e2694c2da1e3cc62449a5bbfa139f48", + "size_bytes": 2667 + }, + "fixture": { + "inference_mode": "local_only", + "max_vram": "37%", + "max_processing_percent": 43, + "max_disk_space": "8GiB", + "sharing_enabled": false + }, + "full_q38_gate": false +} diff --git a/docs/evidence/qwen-catalog-desktop-20260907.md b/docs/evidence/qwen-catalog-desktop-20260907.md new file mode 100644 index 000000000..1db1a5b87 --- /dev/null +++ b/docs/evidence/qwen-catalog-desktop-20260907.md @@ -0,0 +1,57 @@ +# Packaged Windows catalog startup migration + +**Passed**, September 7 local time (September 8 UTC), on ordinary non-elevated +Windows 10 with the qualified frozen desktop/node from `76b6d84`. The application, +node, model manifest and catalog source are unchanged through reviewed head +`e811036`. [Machine-readable evidence](qwen-catalog-desktop-20260907.json). + +The replay installed the real online signed sequence 1 into a new private state +directory, then started the normal frozen Qt desktop with its bundled bootstrap. +It observed the visible native CommunityAI window and authenticated node status. +Without a manual catalog command, startup activated the exact published signed +sequence 2 and its explicitly authorized replacement trust root. + +The desktop kept local-only mode, 37% VRAM, 43% processing, an 8 GiB storage limit, +worker preferences and a cache marker. Both Qwen entries became available; +legacy model entries remained explicit choices, while automatic selection used +the signed Qwen catalog priorities. Sharing remained paused. The first observed +window plus ready API took 63.406 seconds; repeat start took 46.781 seconds. +These are bounded startup observations, not isolated catalog-network timings. + +After shutdown, a second normal launch preserved the configuration bytes and +native control credential. Supplying the former application trust root to the +packaged bootstrap CLI was rejected without changing the migrated configuration. +Both recorded GUI/node process trees exited, and the private native credential +was deleted. No model inference or sharing worker was started. + +Replay: + +```powershell +python scripts/qualify_catalog_desktop.py ` + --desktop /CommunityAI.exe ` + --output --visible-ui +``` + +The script requires the desktop/client Python dependencies for orchestration and +uses the actual packaged executables for product behavior. It refuses elevation, +an existing GUI, an existing output directory or a reused native credential. +The maintained helper now defaults to offscreen Qt so an ordinary test invocation +does not open a desktop window. Explicit `--visible-ui` selects the native-window +acceptance shown above. This helper-only safety change came after the recorded +live replay; the evidence keeps that run's original script hash and visible-window +observations. An offscreen replay cannot claim visible-window acceptance. + +The maintained helper also now records ownership immediately and uses a bounded +PID/creation-time cleanup fallback if `--prepare-update` fails. The replay remains +failed when fallback is needed, and its private credential is retained if owned +process absence cannot be established. Three focused tests cover descendants, +PID reuse, unreadable ownership and the finite deadline. This later helper change +was validated without launching Qt; it does not change the original replay hash +or claim that the recorded live run exercised emergency cleanup. + +This closes the observed Windows **startup migration** slice. The later +[ordinary-user Linux startup replay](qwen-catalog-linux-startup-20260908.md) is +recorded separately. A newer catalog arriving during an active generation and +idle activation/drain remain separate observations. Installer replacement is +covered by the existing Gate 15 evidence. The complete Q3.8 gate is not closed by +this replay. diff --git a/docs/evidence/qwen-catalog-linux-startup-20260908.json b/docs/evidence/qwen-catalog-linux-startup-20260908.json new file mode 100644 index 000000000..bbaa278b5 --- /dev/null +++ b/docs/evidence/qwen-catalog-linux-startup-20260908.json @@ -0,0 +1,78 @@ +{ + "result": "passed", + "scope": "ordinary-user-frozen-Linux-desktop-signed-catalog-startup-migration", + "platform": "Linux", + "non_elevated": true, + "zombies_treated_as_non_executing": true, + "ui_mode": "offscreen", + "desktop_sha256": "41383b5e995a2f200f7c96b0fdb8764b7d9edae89ba8efbba0a93d80d883f0aa", + "node_sha256": "313f8cbe8b1329b0025e98e64cd12eb2abbadb5e0083f5de5360d997b23d4aa0", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_script_sha256": "68fc574a3b93e222da32a9836a2c09c20384a712d2f4274b06606c9f191844f2", + "launches": [ + { + "phase": "automatic-migration", + "seconds_to_authenticated_status": 22.835, + "visible_native_window": false, + "model_ids": [ + "Gemma 4 E2B IT", + "Qwen3.5 2B", + "Qwen3.5-0.8B-Local", + "Qwen3.8 27B FP8 Dequant" + ], + "worker_states": [ + "paused" + ], + "owned_process_count": 4 + }, + { + "phase": "repeat-start", + "seconds_to_authenticated_status": 24.48, + "visible_native_window": false, + "model_ids": [ + "Gemma 4 E2B IT", + "Qwen3.5 2B", + "Qwen3.5-0.8B-Local", + "Qwen3.8 27B FP8 Dequant" + ], + "worker_states": [ + "paused" + ], + "owned_process_count": 8 + } + ], + "legacy_install": { + "catalog_digest": "sha256:70937f0aa10aa73f158753386170c27a95c685b468c66945f05bb3929263f195", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 1, + "config_path": "/state/node-config.json", + "created": true, + "model_count": 2, + "schema_version": 1, + "source": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/catalog.signed.json" + }, + "activated_exact_published_sequence_2": true, + "preferences_workers_and_cache_preserved": true, + "repeat_start_config_and_native_credential_preserved": true, + "old_trust_root_rejected_without_config_change": true, + "recorded_runtime_trees_gone": true, + "native_qualification_credential_removed": true, + "limitations": [ + "Linux startup migration and unchanged-catalog restart only; other platforms require separate replay.", + "The legacy state was installed from the actual signed online catalog, with private test preferences/cache marker.", + "No inference, worker execution, periodic newer-sequence activation, or active-generation drain was exercised.", + "The existing frozen bundle was launched directly; its installer lifecycle has separate evidence." + ], + "raw_result_sha256": "3618f9eef066b1ba65e3e16250a0d3b969a0b56a7185069777249e105296d033", + "accepted_catalog_sequence": 2, + "accepted_catalog_sha256": "315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315", + "frozen_runtime_source_commit": "bf67f0d68df067f43797b4cd47a98f1fea49b2bc", + "environment": { + "distribution": "Ubuntu 22.04", + "uid": 1000, + "native_credential_backend": "gnome-keyring Secret Service in private DBus session" + }, + "recorded_at_utc": "2026-09-08T08:29:12.169616+00:00", + "prior_failed_attempt": "qwen-catalog-linux-startup-failed-20260908.json", + "prior_failure_cause": "Unconfirmed; original bootstrap child stderr not retained by GUI fallback" +} diff --git a/docs/evidence/qwen-catalog-linux-startup-20260908.md b/docs/evidence/qwen-catalog-linux-startup-20260908.md new file mode 100644 index 000000000..9e8f1ad2c --- /dev/null +++ b/docs/evidence/qwen-catalog-linux-startup-20260908.md @@ -0,0 +1,43 @@ +# Packaged Linux catalog startup migration + +**Passed on retry**, September 8 UTC, with the qualified frozen desktop/node +from `bf67f0d`, as UID 1000 in Ubuntu 22.04 with a private DBus session and native +gnome-keyring credentials. Qt was offscreen; no visible Linux window is claimed. +The [machine-readable result](qwen-catalog-linux-startup-20260908.json) binds the +actual executable, bootstrap and replay helper hashes. The +[first failed attempt](qwen-catalog-linux-startup-failed-20260908.md) remains +preserved separately; its bootstrap child failure cause is unconfirmed. + +The retry started with a new private fixture installed from the actual online +signed sequence 1. A normal frozen desktop launch, without a manual refresh of +that fixture, activated the exact published sequence-2 catalog and its explicitly +authorized replacement trust root. Both Qwen entries appeared in authenticated +node status. First startup took 22.835 seconds; the second took 24.480 seconds. +These timings include desktop, bootstrap and node readiness under concurrent +installer unpacking, rather than isolating catalog-network time. + +The replay preserved local-only mode, 37% VRAM, 43% processing, an 8 GiB storage +limit, worker preferences and a cache sentinel. The worker stayed paused and +sharing remained disabled. Restart preserved config bytes and the native +control credential. The old application root was then rejected by the frozen +bootstrap CLI without changing the migrated config. All recorded runtime trees +stopped, and the private credential was deleted. Defunct Linux processes are +treated as non-executing; PID and creation-time checks still protect unrelated +processes from cleanup. + +Replay, inside the existing ordinary-user DBus/unlocked-keyring session: + +```sh +PYTHONPATH=/repo/desktop/src /environment/venv/bin/python \ + /repo/scripts/qualify_catalog_desktop.py \ + --desktop /environment/release-linux-v2/CommunityAI/CommunityAI \ + --output /environment/qualification/qwen-linux-catalog-startup-20260908-retry \ + --node-url http://127.0.0.1:18104 +``` + +The output directory and private credential must be new. The helper refuses +root and an executing existing desktop. The frozen bundle was launched directly; +its installer lifecycle is separate evidence. No model inference, active worker, +periodic newer-sequence activation or active-generation drain ran here. This +closes the observed Linux startup-migration slice; the complete Q3.8 gate still +needs the separate [periodic live replay](qwen-catalog-periodic-source-20260908.md). diff --git a/docs/evidence/qwen-catalog-linux-startup-failed-20260908.json b/docs/evidence/qwen-catalog-linux-startup-failed-20260908.json new file mode 100644 index 000000000..372a57665 --- /dev/null +++ b/docs/evidence/qwen-catalog-linux-startup-failed-20260908.json @@ -0,0 +1,93 @@ +{ + "raw_result_sha256": "2437406ee38c448ecfb77d5435698ec70d83b7021495ac8d60f0664c991e183d", + "sanitized_raw_result": { + "result": "failed", + "scope": "ordinary-user-frozen-Linux-desktop-signed-catalog-startup-migration", + "platform": "Linux", + "non_elevated": true, + "zombies_treated_as_non_executing": true, + "ui_mode": "offscreen", + "desktop_sha256": "41383b5e995a2f200f7c96b0fdb8764b7d9edae89ba8efbba0a93d80d883f0aa", + "node_sha256": "313f8cbe8b1329b0025e98e64cd12eb2abbadb5e0083f5de5360d997b23d4aa0", + "bootstrap_sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "replay_script_sha256": "83d1013bc6bc8550d1656f0d26ef7a35832b5040347d9515378d378b1d767dbd", + "launches": [ + { + "phase": "automatic-migration", + "seconds_to_authenticated_status": 17.153, + "visible_native_window": false, + "model_ids": [ + "Gemma 4 E2B IT", + "Qwen3.5 2B" + ], + "worker_states": [ + "paused" + ], + "owned_process_count": 4 + } + ], + "legacy_install": { + "catalog_digest": "sha256:70937f0aa10aa73f158753386170c27a95c685b468c66945f05bb3929263f195", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 1, + "config_path": "/state/node-config.json", + "created": true, + "model_count": 2, + "schema_version": 1, + "source": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/catalog.signed.json" + }, + "error": "AssertionError: ", + "recorded_runtime_trees_gone": true, + "native_qualification_credential_removed": true, + "limitations": [ + "Linux startup migration and unchanged-catalog restart only; other platforms require separate replay.", + "The legacy state was installed from the actual signed online catalog, with private test preferences/cache marker.", + "No inference, worker execution, periodic newer-sequence activation, or active-generation drain was exercised.", + "The existing frozen bundle was launched directly; its installer lifecycle has separate evidence." + ] + }, + "observed_catalog_sequence": 1, + "observed_catalog_sha256": "54530f6ebdb3e27cab7d25762ad7db3ea0369236230a4f58a96870d83ca767c4", + "migration_log_sha256": "55cae40a3dbd4a66b39ace84114d099a08699c28fca720d071cc8b2a223a8aff", + "migration_log_message": "Catalog migration failed verification; starting the saved node configuration", + "metadata_only_diagnostics": [ + { + "exit_code": 0, + "seconds": 16.60634364798898, + "config_changed": true, + "copied_from_failed_state": true, + "original_unmodified": null, + "stderr_bytes": 0, + "stdout_bytes": 471, + "output": "/qwen-linux-catalog-diagnostic-20260908", + "accepted_catalog_sequence": 2, + "accepted_catalog_sha256": "315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315" + }, + { + "exit_code": 0, + "seconds": 13.704969879006967, + "config_changed": true, + "copied_from_failed_state": true, + "original_unmodified": true, + "stderr_bytes": 0, + "stdout_bytes": 479, + "output": "/qwen-linux-catalog-loader-environment-20260908", + "scope": "Metadata-only CLI with simulated frozen-GUI loader environment; no GUI or node server", + "injected_environment_names": [ + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "QT_QPA_PLATFORM", + "LD_LIBRARY_PATH" + ], + "accepted_catalog_sequence": 2, + "accepted_catalog_sha256": "315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315" + } + ], + "recorded_at_utc": "2026-09-08T08:27:30.752103+00:00", + "scope": "Failed ordinary-user frozen Linux GUI startup migration, preserved independently of retries", + "limitations": [ + "The frozen GUI did not retain the failing bootstrap child stderr; the original nonzero cause is unconfirmed.", + "Successful metadata-only CLI diagnostics do not establish normal GUI startup acceptance.", + "Private fixture config, credentials and raw stdout/stderr stay local; only sanitized result and file hashes are published." + ] +} diff --git a/docs/evidence/qwen-catalog-linux-startup-failed-20260908.md b/docs/evidence/qwen-catalog-linux-startup-failed-20260908.md new file mode 100644 index 000000000..89584e336 --- /dev/null +++ b/docs/evidence/qwen-catalog-linux-startup-failed-20260908.md @@ -0,0 +1,44 @@ +# Frozen Linux catalog startup: retained failed first attempt + +The first ordinary-user Linux desktop startup replay **failed** its exact +published sequence-2 assertion. This record remains separate from any later +retry. The [sanitized raw result](qwen-catalog-linux-startup-failed-20260908.json) +retains binary/helper hashes, the failure, cleanup results and both subsequent +metadata-only diagnostic outcomes. Private state and raw process logs stay local. +The later [fresh normal desktop retry](qwen-catalog-linux-startup-20260908.md) +passed and has its own record; it does not erase this first failure. + +The offscreen frozen desktop reached its authenticated API in 17.153 seconds, +with the two legacy models and a paused worker. The private configuration still +referenced signed catalog sequence 1; no sequence-2 catalog was staged. Its log +reported `Catalog migration failed verification; starting the saved node +configuration`. The helper then failed at the exact catalog assertion. All +recorded runtime processes stopped and the private native credential was removed. + +The existing-configuration branch in `NodeLifecycle._ensure_config` emits that +message when its bundled bootstrap child returns nonzero. Its timeout and +unavailable-executable branches emit different messages. This observation rules +out an early authenticated-status race as the explanation: the desktop had +deliberately started the saved configuration. It does not identify why the child +failed, because this branch does not retain the child's stderr. + +Two metadata-only invocations of the exact frozen node's `bootstrap +--refresh_if_needed` subsequently succeeded against separate, new copies of the +failed fixture. Both installed exact sequence-2 bytes and returned empty stderr: + +| Diagnostic | Result | Seconds | +| --- | --- | ---: | +| CLI with ordinary process environment | Exit 0 | 16.606 | +| CLI with simulated GUI `_internal` loader path and offscreen Qt | Exit 0 | 13.705 | + +Neither diagnostic started a GUI, node server, model or sharing worker. The +second explicitly checked that the original config hash stayed unchanged. The +first recorder did not populate its original-file comparison on success, so its +raw field remains `null`; a later inspection confirmed the original remained on +sequence 1. These diagnostics did not reproduce an invalid signed input or the +loader-path hypothesis. Transient transport failure versus other inherited +process state remains unconfirmed. They do not replace normal GUI acceptance. + +The [periodic refresh checkpoint](qwen-catalog-periodic-source-20260908.md) +describes the separate source coverage and still-required live newer-sequence +and active-generation replay. diff --git a/docs/evidence/qwen-catalog-online-20260906.json b/docs/evidence/qwen-catalog-online-20260906.json new file mode 100644 index 000000000..e64ae5ada --- /dev/null +++ b/docs/evidence/qwen-catalog-online-20260906.json @@ -0,0 +1,114 @@ +{ + "result": "passed", + "scope": "packaged-HTTPS-catalog-install-and-old-root-migration", + "files": [ + { + "path": "bundle.json", + "sha256": "a904ed1b8487762507fcd35c4a125f03f0f71ee6c164bcacb993fbab677e0be9" + }, + { + "path": "catalog-bootstrap.json", + "sha256": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410" + }, + { + "path": "catalog.signed.json", + "sha256": "315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315" + }, + { + "path": "manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json", + "sha256": "a2621aa34aa47f0c9074f5baa0254b17549424f1c85d828ae9b1bf6ad9e76bb3" + }, + { + "path": "manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json", + "sha256": "4536ac2bada7242b758db443b9ebb813a614dd167ab364643fc55c7eb657bb74" + }, + { + "path": "publication-preflight.json", + "sha256": "b80f6c4a8d0b87a1d365fc4be01e643429759cc52eda5c200dbf236f2073f538" + } + ], + "clean_install": { + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 2, + "created": true, + "model_count": 2, + "schema_version": 1, + "source": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/catalog.signed.json", + "seconds": 9.515999999995984 + }, + "legacy_install": { + "catalog_digest": "sha256:70937f0aa10aa73f158753386170c27a95c685b468c66945f05bb3929263f195", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 1, + "created": true, + "model_count": 2, + "schema_version": 1, + "source": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/catalog.signed.json", + "seconds": 9.796999999998661 + }, + "migration": { + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 2, + "created": true, + "model_count": 2, + "schema_version": 1, + "source": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/catalog.signed.json", + "seconds": 11.906000000002678 + }, + "preferences_workers_and_cache_marker_preserved": true, + "old_root_rejected": true, + "repeat_start": { + "catalog_digest": "", + "catalog_id": "communityai-public-alpha-v1", + "catalog_sequence": 0, + "created": false, + "model_count": 4, + "schema_version": 1, + "source": "existing-config", + "seconds": 8.734000000004016 + }, + "repeat_start_config_unchanged": true, + "node_sha256": "270e8a7482d4e5c689644e1d53c48afb82a75aa49547aa994f12cc424997ffa3", + "complete_gate15": false, + "limitations": [ + "Catalog/bootstrap CLI only; no ordinary-user installer lifecycle or full UI acceptance.", + "Migration fixture installed the actual online sequence 1, then added test preferences and a cache marker." + ], + "recorded_at": "2026-09-06", + "publication_commit": "2f79e6b774b599db5db1d87dbac8a25b847ab491", + "publication_branch": "codex/gate-v-auto-selection", + "publication_base_url": "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2", + "legacy_publication_unchanged": true, + "package": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "c95c1b1f94eba68a04ffc54b8dc17a49e425390eca053ef8f3996efdcc9dbf1d", + "size_bytes": 2692398749, + "binding": { + "path": ".gate13-runs/qwen-product-build-v6/desktop-metrics.json", + "sha256": "fe79b3477766ff6d28744553b7e49da45dda5190aebacdb6cb9378f47900a0b2" + }, + "real_local_result_binding": { + "path": ".gate13-runs/qwen-product-implementation/local-packaged-v6/result.json", + "sha256": "7d8379681c985e1ea3371806cfdce187d911e6552cf1eed68850fdef46da3102" + } + }, + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/catalog-online-v6/result.json", + "sha256": "2b514337209690399af69963ba989a6ff72d3448918b32ca3c2cce2a1ba76d18" + }, + { + "path": "scripts/qualify_catalog_online.py", + "sha256": "1ff07efe23e42afcba1349ccc861bb95a648d553bbb0a046149e383d4ebb78b6" + } + ] +} diff --git a/docs/evidence/qwen-catalog-periodic-source-20260908.json b/docs/evidence/qwen-catalog-periodic-source-20260908.json new file mode 100644 index 000000000..279e8db04 --- /dev/null +++ b/docs/evidence/qwen-catalog-periodic-source-20260908.json @@ -0,0 +1,161 @@ +{ + "recorded_at_utc": "2026-09-08T08:31:05.669933+00:00", + "result": "passed", + "scope": "source signed periodic refresh / manager integration and helper portability", + "reviewed_head": "23a1f99e81df5f98ee3260b7df2529332547ccca", + "source_tests": { + "passed": 59, + "failures": 0, + "errors": 0, + "seconds": 22.118, + "junit": { + "path": ".gate13-runs/release-audit-20260908/catalog-tests.xml", + "sha256": "70ebda7402f5183693921a20cbfe44dac6f4b5eaaeb55f5b4bbedf125bbe0302", + "size_bytes": 8493 + } + }, + "new_periodic_cases": 6, + "helper_cases": 12, + "no_model_or_gpu_load": true, + "no_frozen_periodic_acceptance_claim": true, + "source_bindings": [ + { + "path": "src/drift/node/catalog_refresh.py", + "sha256": "92412ad5915a812b28cff20aabf28e3d3e854cc88dcf7c90f0f93fdc53048bf3", + "size_bytes": 2241 + }, + { + "path": "src/drift/node/model_manager.py", + "sha256": "4182962cad854eaf934d29f942a8fc8901370295bdcce84949c910860ba05b61", + "size_bytes": 33071 + }, + { + "path": "src/drift/node/catalog_bootstrap.py", + "sha256": "8722bc8b8db4766abf9530425b0a4aa5ce0c95100f304bffb6d9d2bdfeac9be3", + "size_bytes": 38070 + }, + { + "path": "tests/test_catalog_refresh.py", + "sha256": "a6cf83e9b3af109ccef3ffec97c6883cbc2ff8addc6516876b7e00dbc146548e", + "size_bytes": 15930 + }, + { + "path": "scripts/qualify_catalog_desktop.py", + "sha256": "68fc574a3b93e222da32a9836a2c09c20384a712d2f4274b06606c9f191844f2", + "size_bytes": 17137 + }, + { + "path": "tests/test_qualify_catalog_desktop.py", + "sha256": "3224b209b72982497658f5e2b617805f2b0fc614e579676611e05acdd2a564bd", + "size_bytes": 2941 + }, + { + "path": "tests/test_catalog_desktop_cleanup.py", + "sha256": "0a90a3bff8482cd4618fbd0ff93a2308318f88c6d08351d9c66fdcf96571ee3c", + "size_bytes": 4040 + } + ], + "ci": { + "checks": [ + { + "link": "https://github.com/flujo-app/CommunityAI/runs/101904766214", + "name": "CodeQL", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175751429/job/101904686509", + "name": "run-tests (macos, 3.12)", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175750804/job/101904687184", + "name": "Analyze (actions)", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175751428/job/101904686239", + "name": "black", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175751433/job/101904686368", + "name": "package (ubuntu-22.04, linux, communityai-desktop-linux.tar.gz)", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175751429/job/101904686280", + "name": "run-tests (ubuntu, 3.12)", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175750804/job/101904687011", + "name": "Analyze (python)", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175751433/job/101904686451", + "name": "package (windows-latest, windows, communityai-desktop-windows.zip)", + "state": "SUCCESS" + }, + { + "link": "https://github.com/flujo-app/CommunityAI/actions/runs/34175751428/job/101904686157", + "name": "isort", + "state": "SUCCESS" + } + ], + "production_run_id": 34175751433, + "conclusion": "success", + "head_sha": "23a1f99e81df5f98ee3260b7df2529332547ccca", + "merge_source_commit": "1b528212a85a58081e6e80bd855573673b3710ef", + "merge_source_tree": "60aa51a940433aebe97a7b57318e22a496f9f4db", + "merge_delta_from_head": [ + "README.md" + ], + "audit_bundles": [ + { + "path": ".gate13-runs/release-audit-20260908/ci-windows/provenance.json", + "sha256": "6adf7b92b8d90cba3ccb41482f87d9733d5172469653902676a6c023d6b0c7a9", + "size_bytes": 1220945, + "platform": "windows", + "archive": { + "artifact_root": "CommunityAI", + "entry_count": 5851, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "6fd9ba2e8083b4b628a2da35799a426c204ca0fffe7b67a762d81ff21ba19d32", + "size_bytes": 2690878530 + }, + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80" + }, + { + "path": ".gate13-runs/release-audit-20260908/ci-linux/provenance.json", + "sha256": "4864c62f876478d38cb1027c05b755d5533b3783d1d3f85ec5f96676c4e3b8e8", + "size_bytes": 1283127, + "platform": "linux", + "archive": { + "artifact_root": "CommunityAI", + "entry_count": 6118, + "format": "tar.gz", + "path": "communityai-desktop-linux.tar.gz", + "platform": "Linux", + "preserves_executable_modes": true, + "preserves_internal_file_symlinks": true, + "schema_version": 1, + "sha256": "81cb850aa9b3ad0a9319b9e73fc61e17e819835a5a87b8400fa3ecb3dee2e1b9", + "size_bytes": 3109947825 + }, + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80" + } + ] + }, + "limitations": [ + "HTTPS document fetching and clock are substituted in source tests; signatures, installer, refresh thread and ModelManager are real.", + "Inert runtime objects exercise leases/loading without tokens or backend execution.", + "Frozen Linux startup replay and newer signed periodic activation remain separate live observations.", + "Only CI audit bundles were downloaded; CI multi-gigabyte runtime archives were not locally reverified or installed in this checkpoint." + ] +} diff --git a/docs/evidence/qwen-catalog-periodic-source-20260908.md b/docs/evidence/qwen-catalog-periodic-source-20260908.md new file mode 100644 index 000000000..e72f3c400 --- /dev/null +++ b/docs/evidence/qwen-catalog-periodic-source-20260908.md @@ -0,0 +1,109 @@ +# Signed periodic catalog refresh: source acceptance and frozen replay plan + +This checkpoint connects the real periodic `CatalogRefreshService`, +`CatalogBootstrapInstaller` signature/rollback handling, and `ModelManager` +admission/lease implementation. Only the HTTPS document transport and wall-clock +input are substituted. Test catalogs have disposable signing keys; model +runtimes are inert objects. No network request, model weight, GPU load, desktop +window, Docker start or production signing key is used by these tests. + +## Source observations + +- A signed withdrawal is staged while two leases hold the withdrawn manifest's + runtime. The original immutable catalog and active runtime remain usable; + restart waits until both leases have been released. +- At the restart callback boundary, the real manager has already closed request + admission. A manager configured from the accepted new catalog no longer + approves contribution for the withdrawn manifest. +- A loader that has not returned also holds off restart. When it returns, its + newly acquired lease continues to hold off restart until released. +- Periodic attempts with a tampered signature, rollback sequence or equivocation + keep the accepted configuration and rollback guard unchanged, do not request + restart, and leave request admission usable. +- Closing the refresh service while it waits for active work ends its thread + without interrupting that lease or claiming a restart. The staged immutable + update remains available for a later normal node startup. + +These are source-level threading and state-transition observations. They do not +claim generated tokens, an actual server restart, authenticated HTTPS transport, +frozen-binary behavior or complete Gate Q3.8 acceptance. The regression results +and source hashes are recorded in the companion +[checkpoint record](qwen-catalog-periodic-source-20260908.json). + +## Frozen Linux replay in two separate scopes + +The startup-migration helper now supports ordinary Linux users and the packaged +`node/CommunityAI-Node` name. It keeps Qt offscreen by default, rejects root, +uses no Windows-only process flags, and treats zombies as non-executing while +preserving PID/creation-time checks and failure on unreadable ownership. It does +not claim a visible Linux window. Run it only after the current installer test +has stopped its desktop and node, inside its existing ordinary-user DBus and +unlocked native keyring session: + +```sh +PYTHONPATH=/repo/desktop/src /environment/venv/bin/python \ + /repo/scripts/qualify_catalog_desktop.py \ + --desktop /environment/release-linux-v2/CommunityAI/CommunityAI \ + --output /environment/qualification/qwen-linux-catalog-startup-20260908 \ + --node-url http://127.0.0.1:18104 +``` + +The output directory must be new. This exercise reads the existing public signed +sequence 1 and 2, preserves private fixture settings/cache, restarts, rejects the +old root and verifies runtime/credential cleanup. It requires no inference or +sharing load. Only a passing actual result can close the Linux startup slice. +The [first Linux attempt](qwen-catalog-linux-startup-failed-20260908.md) failed +the sequence-2 assertion and is preserved with its successful cleanup and +separate metadata-only diagnostics; those diagnostics do not close this slice. +A [fresh normal frozen desktop retry](qwen-catalog-linux-startup-20260908.md) +subsequently passed startup migration, unchanged-catalog restart, old-root +rejection, preferences/cache/native-credential preservation and cleanup. + +The **periodic newer-sequence and active-generation** gap requires additional +inputs and a distinct result: + +1. Stage disposable signed catalogs and exact approved manifests at an owned + test HTTPS origin, with no changes to the production catalog or signing root. + The production bootstrap validates public HTTPS URLs and port 443, so a plain + localhost HTTP fixture cannot establish frozen transport acceptance. An + operator-controlled test origin or another authorized, isolated transport + environment must be arranged first; none is created by this checkpoint. +2. Start the verified frozen Linux node with private configuration, data, native + credentials and test catalog trust. Keep sharing disabled and use local-only + inference. For the no-load phase, publish a higher signed test sequence and + observe the real configured refresh timer, immutable config change, node + server restart and restored authenticated API. Recheck rejected signatures, + rollback and equivocation without changing accepted state. +3. For the drain phase, use the existing verified local Qwen cache on CPU with + one bounded request and one inference thread. Record an active request before + advancing the test catalog. The already admitted request must finish under + its original runtime; the server must not restart while it is active. Then + observe shutdown/restart, new catalog policy, resumed authenticated API and + preserved resource preferences/cache. This phase needs an explicit model + memory/time allowance; inert source leases cannot replace it. +4. Retain exact binary/catalog/manifest hashes, old/new catalog sequences, + request start/end and configuration/restart timestamps, output/token count, + rejection outcomes, process identities and final credential/cache checks. + Remove only the task's runtime, credential and test publication afterward. + +Use the existing catalog refresh interval or declare a shorter **test-only** +interval in the private config. Do not edit the production interval, disable +signature validation, patch the frozen process, or claim this two-phase plan +has run merely because the source tests passed. + +## Independent CI/source check + +PR #26 head `23a1f99e81df5f98ee3260b7df2529332547ccca` had passing style, tests, +CodeQL and both complete production packaging jobs. The +[packaging run](https://github.com/flujo-app/CommunityAI/actions/runs/34175751433) +completed successfully; its audit bundles identify merge commit +`1b528212a85a58081e6e80bd855573673b3710ef`, tree +`60aa51a940433aebe97a7b57318e22a496f9f4db`. GitHub's comparison reports only +`README.md` changed between the reviewed head and that merge commit. + +The application/catalog paths `src`, `desktop/src`, `public-alpha` and +`manifests` are unchanged between qualified Linux runtime source `bf67f0d` and +that reviewed head. CI archive provenance remains distinct from the previously +qualified installer bytes: only the small audit bundles were downloaded here, +not the multi-gigabyte archives. No installed acceptance is transferred to those +CI rebuilds by filename. diff --git a/docs/evidence/qwen-cpu-full-inference-20260905.json b/docs/evidence/qwen-cpu-full-inference-20260905.json new file mode 100644 index 000000000..3edf8055f --- /dev/null +++ b/docs/evidence/qwen-cpu-full-inference-20260905.json @@ -0,0 +1,171 @@ +{ + "schema_version": 1, + "run_id": "q38-20260905-205155-ed78", + "zone": "us-central1-b", + "result": "passed", + "topology": "four CPU workers, 16 blocks each, plus CPU client/coordinator", + "hosts": [ + { + "role": "c", + "machine_type": "e2-standard-4", + "original_instance_id": "407081139391512573", + "replacement_instance_id": null + }, + { + "role": "w0", + "machine_type": "e2-highmem-4", + "original_instance_id": "7246986295379050346", + "replacement_instance_id": null + }, + { + "role": "w1", + "machine_type": "e2-highmem-4", + "original_instance_id": "4767824729953353578", + "replacement_instance_id": "1182278691429890340" + }, + { + "role": "w2", + "machine_type": "e2-highmem-4", + "original_instance_id": "5533403870236065642", + "replacement_instance_id": null + }, + { + "role": "w3", + "machine_type": "e2-highmem-4", + "original_instance_id": "4036923224338022250", + "replacement_instance_id": null + } + ], + "source_bundle_sha256": "0f1751c96238a5ef183500aff032c04169217e6c43c9cbc4138c1a21c77a8eca", + "tested_host_script_sha256": "0da84c6e0bca09bc88c33ed966f028e564c3f33688a631c448cc33f5eadb6f90", + "tested_server_sha256": "550a0b9475cca3470510ff02a88c383827a41bb13028f599f66e04ce03861b4f", + "retained_client_result_sha256": "029a25c8e6340338c837d61b2e5ad5b091a73737dccba32138501b91981870b4", + "client_pid_before_and_during_resume": 4839, + "local_controller_resumed_after_scp_failure": true, + "client_evidence": { + "baseline": { + "route": [ + { + "end": 16, + "peer_id": "QmbHicR7BoXzyNsHGvLxfTTivukR1RUHqcn4jBeqk2qmoK", + "session_id": "8721fa9a-2e1a-42b9-80d0-da85dfc3c5bb", + "start": 0 + }, + { + "end": 32, + "peer_id": "QmemXP3tT2qbhQ9MvG8t2At6Dw6jH9ps82LvnX5jp1ojDk", + "session_id": "76c5fa63-1204-41f3-8059-7ef6efa88e00", + "start": 16 + }, + { + "end": 48, + "peer_id": "QmSfECfZKd7zpZJBmBrWpPsrTuvmXB3GejLwduna4cqGkq", + "session_id": "92309163-67af-4bd6-bf58-f78deb2199c0", + "start": 32 + }, + { + "end": 64, + "peer_id": "Qmar8Js57J2kT9d6C9CAdNKEYHQKAoMc8Ekn4mTLTy8UAj", + "session_id": "1eae609b-7b74-48d6-93f6-119a8e22fca4", + "start": 48 + } + ], + "seconds": 148.6120492399998, + "text": " Paris.\n", + "token_ids": [ + 11751, + 13, + 198 + ] + }, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", + "observed_at_unix": 1788644338.2040305, + "recovery": { + "after_route": [ + { + "end": 16, + "peer_id": "QmbHicR7BoXzyNsHGvLxfTTivukR1RUHqcn4jBeqk2qmoK", + "session_id": "c2b2b08a-77de-4c03-bc8b-febe6d12b8cc", + "start": 0 + }, + { + "end": 32, + "peer_id": "QmSgJ3yctUZgkp74NACwFsD2hTDnYEiTddW9kHH1oZh6jq", + "session_id": "28b0020e-d34e-4458-9998-cd708e4f43bd", + "start": 16 + }, + { + "end": 48, + "peer_id": "QmSfECfZKd7zpZJBmBrWpPsrTuvmXB3GejLwduna4cqGkq", + "session_id": "c71ad71a-95e2-44b9-8ec9-eec58b6105e0", + "start": 32 + }, + { + "end": 64, + "peer_id": "Qmar8Js57J2kT9d6C9CAdNKEYHQKAoMc8Ekn4mTLTy8UAj", + "session_id": "3083ed6a-239d-443e-919c-bdc1ed16339b", + "start": 48 + } + ], + "before_route": [ + { + "end": 16, + "peer_id": "QmbHicR7BoXzyNsHGvLxfTTivukR1RUHqcn4jBeqk2qmoK", + "session_id": "c2b2b08a-77de-4c03-bc8b-febe6d12b8cc", + "start": 0 + }, + { + "end": 32, + "peer_id": "QmemXP3tT2qbhQ9MvG8t2At6Dw6jH9ps82LvnX5jp1ojDk", + "session_id": "1f503fa0-5db5-4c12-a295-1a432669f48e", + "start": 16 + }, + { + "end": 48, + "peer_id": "QmSfECfZKd7zpZJBmBrWpPsrTuvmXB3GejLwduna4cqGkq", + "session_id": "c71ad71a-95e2-44b9-8ec9-eec58b6105e0", + "start": 32 + }, + { + "end": 64, + "peer_id": "Qmar8Js57J2kT9d6C9CAdNKEYHQKAoMc8Ekn4mTLTy8UAj", + "session_id": "3083ed6a-239d-443e-919c-bdc1ed16339b", + "start": 48 + } + ], + "matches_baseline": true, + "position_after": 7, + "position_before": 5, + "same_session": true, + "seconds": 307.51290894600015, + "text": " Paris.\n", + "token_ids": [ + 11751, + 13, + 198 + ] + }, + "result": "passed", + "versions": { + "hivemind": "1.1.12", + "python": "3.12.3", + "torch": "2.6.0+cpu", + "transformers": "5.13.1" + } + }, + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "limitations": [ + "Three generated tokens; no long-context or concurrency qualification.", + "Source runtime, not packaged desktop qualification.", + "No stock-Transformers logit parity or GPU performance claim." + ] +} diff --git a/docs/evidence/qwen-cpu-product-no-promotion-20260906.json b/docs/evidence/qwen-cpu-product-no-promotion-20260906.json new file mode 100644 index 000000000..5b7151d16 --- /dev/null +++ b/docs/evidence/qwen-cpu-product-no-promotion-20260906.json @@ -0,0 +1,65 @@ +{ + "result": "failed", + "reason": "Two retained probes failed the unchanged minimum one token/minute policy", + "measurements": [ + { + "first_token_seconds": 71.997, + "completion_tokens": 3, + "duration_seconds": 189.713 + }, + { + "first_token_seconds": 85.101, + "completion_tokens": 3, + "duration_seconds": 199.981 + } + ], + "local_fallback_retained": true, + "signed_policy_weakened": false, + "run_id": "q38p-20260906-002616-580c", + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "retry_source": { + "reason": "Live lightweight discovery reset eligibility on mixed signed-announcement generations", + "original_result": "local fallback passed; promotion did not complete", + "original_inventory": "ce538f6f6123231b3f3f3856601c947649154596badfc82666db12a21ee07db2", + "patched_files": { + "src/drift/utils/dht.py": "96b0cd8102cb9fc089d6f3497521752d6b686c77008d47ed4467f2380cca99c1", + "src/drift/node/model_selection.py": "f30f7360cdb1ffdf37a8edc5ba89aceac349a234ace93d8778e8eb8040f7905c" + }, + "changed_policy_thresholds": false, + "time": 1788657331.406534, + "installed_hashes": { + "drift.utils.dht": "96b0cd8102cb9fc089d6f3497521752d6b686c77008d47ed4467f2380cca99c1", + "drift.node.model_selection": "f30f7360cdb1ffdf37a8edc5ba89aceac349a234ace93d8778e8eb8040f7905c" + } + }, + "original_local_inference": { + "response": { + "id": "cmpl-486b6b2a7cde41a1b781d450", + "object": "text_completion", + "created": 1788654804, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 46.573782101999996, + "observed_at_unix": 1788654804.7331543 + } +} diff --git a/docs/evidence/qwen-desktop-v8-20260906.json b/docs/evidence/qwen-desktop-v8-20260906.json new file mode 100644 index 000000000..a529b25f7 --- /dev/null +++ b/docs/evidence/qwen-desktop-v8-20260906.json @@ -0,0 +1,93 @@ +{ + "result": "passed", + "scope": "Windows-v8-build-verification-and-real-local-GPU-chat", + "packaged": true, + "hardware": "NVIDIA RTX 2070 SUPER, 8 GB VRAM", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "fa4bb6aab41669bda6729041408e79d05ea17e94dee332ce1e37637bab3324c3", + "size_bytes": 2692401004 + }, + "source_commit": null, + "source_scope": "working-tree inputs recorded separately", + "ui_smoke_passed": true, + "onboarding_ui_smoke_passed": true, + "local_completion_seconds": 21.109000000004016, + "short_chat": { + "id": "chatcmpl-45e9644a1d8c499c9ea29d0d", + "object": "chat.completion", + "created": 1788672603, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "node_stopped": true, + "focused_tests": { + "selector_API_ranges_manifest_lifecycle": 109, + "desktop_builder": 23 + }, + "relocation": "Build completed on E:, moved to repository on C: at owner request; archive digest verified again", + "limits": [ + "Unsigned engineering package", + "Community inference remains unqualified", + "Source request-transition proof used an earlier recorded bundle", + "No Linux or unobserved RTX 30/40/50 hardware claim" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-storage-retry/qwen-product-build-v8/desktop-metrics.json", + "sha256": "f98b4c982a849e3cc8fc9ef4c5d05ff8eee96711683a789948be46cc9dccbfab" + }, + { + "path": ".gate13-runs/qwen-product-storage-retry/qwen-product-build-v8/CommunityAI/node/CommunityAI-Node.exe", + "sha256": "a5ddba56c681cce5c5e1769e5c30d3aac81d08a1209927280119d9c3f36c6db7" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v8-e-source.json", + "sha256": "ad12d9261ee2e5c5b7eaadb04f5366f051106f4bc199978d7e1963f2bca08315" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v8-e-succeeded.json", + "sha256": "1e91fe1050b9e65ace9fd6ee4e73b40d9e211a0ed1c514cbdf9116bba1a88cd8" + }, + { + "path": ".gate13-runs/qwen-product-implementation/local-packaged-v8/result.json", + "sha256": "4522107c8da52825807917aa54f884e7f195b1ed76c50abe05046926892ab37a" + }, + { + "path": ".gate13-runs/qwen-product-implementation/v8-focused-tests.log", + "sha256": "15d4550ee4c3b352d4dbdcc55f0b24e4482c134ef407b2cb59fe1fd41754414c" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-storage-tests.log", + "sha256": "81e61ab5d2d5a2ade66ba5d63599e18f9198091575eb9b2baf1f999ccfe65c26" + }, + { + "path": ".gate13-runs/qwen-product-implementation/long-generation-repro.log", + "sha256": "39d0a34e9f75ddcfe6789eee5e76322504dc1850a6866e6acf5b0df3236f906f" + } + ] +} diff --git a/docs/evidence/qwen-desktop-v9-20260906.json b/docs/evidence/qwen-desktop-v9-20260906.json new file mode 100644 index 000000000..86c202ddc --- /dev/null +++ b/docs/evidence/qwen-desktop-v9-20260906.json @@ -0,0 +1,161 @@ +{ + "result": "passed", + "scope": "Windows-v9-build-local-GPU-chat-and-resource-admission", + "packaged": true, + "hardware": "NVIDIA RTX 2070 SUPER, 8 GB VRAM", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "ddc74b7aef1e29615458b930a03c8393a90dd5ec36ebed19528df2f7081d28a3", + "size_bytes": 2692402624 + }, + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "source_commit": null, + "source_scope": "Exact build input inventory and tar snapshot retained; later working-tree changes excluded", + "ui_smoke_passed": true, + "onboarding_ui_smoke_passed": true, + "local_completion_seconds": 11.51600000000326, + "short_chat": { + "id": "chatcmpl-4d67d2824cfa4eefbe3e7bcf", + "object": "chat.completion", + "created": 1788676224, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "resource_admission": { + "schedule": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": false, + "schedule_reason": "outside the configured contribution schedule", + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + }, + "power": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "power usage 58.14 W exceeds the 1.00 W contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": 58.138, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + }, + "bandwidth": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "bandwidth usage 10.57 Mbps exceeds the 0.00 Mbps contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": 10.566464, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + }, + "storage": { + "state": "paused", + "pid": null, + "policy_admitted": false, + "policy_reason": "Qwen3.8 27B FP8 Dequant: exact block artifact planning failed: ManifestError", + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + } + }, + "node_stopped": true, + "cache_accounting": { + "included_in_package": true, + "source_tests_passed": 69, + "scope": "Owned manifest snapshots and partials plus legacy Hub cache; hardlinks deduplicated; only remaining transfer bytes reserved", + "packaged_cache_eviction_qualification": false + }, + "limits": [ + "Unsigned engineering package; no commit-bound release attestation", + "Community inference remains unqualified", + "Resource checks prove admission, not sustained load, overshoot, resumption or OS hard caps", + "No Linux or unobserved RTX 30/40/50 hardware claim" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-build-v9/desktop-metrics.json", + "sha256": "2660bad7d4a76e5582d50c138d4e463e0f49fa0710cc113051f66fe5370718ee" + }, + { + "path": ".gate13-runs/qwen-product-build-v9/CommunityAI/node/CommunityAI-Node.exe", + "sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.json", + "sha256": "4566fd0c2dfeb18d3ef4945705ca672d6ff6df492ed4880edda775d51367645a" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.tar.gz", + "sha256": "c76e426f65e946b1c69649797bdc05ba1879094ab9ea429aaa2e064db9ad4604" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-succeeded.json", + "sha256": "c2887b6b8a39c507f26e8f6828207e68e692cc6cbe47d00d41680dcbd3ce1f77" + }, + { + "path": ".gate13-runs/qwen-product-implementation/local-packaged-v9/result.json", + "sha256": "258608f2c0ac83009ab65db083abf44481ae114bff34e4849ccbeeba0072a788" + }, + { + "path": ".gate13-runs/qwen-product-implementation/resource-controls-v9/result.json", + "sha256": "1535ffd2deab9648178a04e3f2ed988d8722e8c05f3b8491bf862388c9580555" + }, + { + "path": ".gate13-runs/qwen-product-implementation/manifest-cache-budget-repro.log", + "sha256": "06290cc9c3cb46df7d566996f50872fbaaaf2fd9fcb0f4045954eb3494e1b677" + }, + { + "path": ".gate13-runs/qwen-product-implementation/manifest-cache-budget-tests-r2.log", + "sha256": "922b0c23e2a0e883e9402e914953c1670b4daf49ade0a2cabbb3b6d4127002e1" + } + ] +} diff --git a/docs/evidence/qwen-formation-checkpoint-20260907.json b/docs/evidence/qwen-formation-checkpoint-20260907.json new file mode 100644 index 000000000..a9bda0678 --- /dev/null +++ b/docs/evidence/qwen-formation-checkpoint-20260907.json @@ -0,0 +1,68 @@ +{ + "schema_version": 1, + "recorded_date_utc": "2026-09-07", + "scope": "automatic-formation-fixes-local-desktop-preflight-and-blocked-cloud-attempt", + "distributed_formation_result": "not_proved", + "cloud_attempt": { + "run_id": "q38af-20260907-050021-e76861", + "result": "failed", + "phase": "preflight", + "reason": "GCP token refresh requires owner reauthentication", + "cloud_resources_created": false, + "raw_result_sha256": "35bd7fba930cee91d2412b1b4a65d5f368e5eeaf67a0b2bc2fd6ea6c404bb5b6" + }, + "placement_regressions": { + "before_fix_four_16_block_allocations": [ + "44:60", + "2:18", + "22:38", + "29:45" + ], + "before_fix_covered_blocks": 54, + "before_fix_total_blocks": 64, + "complete_route_before_fix_moved_from": "0:16", + "complete_route_before_fix_moved_to": "21:37", + "fixes": [ + "Prefer equally useful ranges that leave fillable gaps", + "Retain sole-provider blocks on an already complete route", + "Disperse by persistent public identity instead of installation path" + ], + "regression_suite": { + "passed": 111, + "failed": 0, + "files": [ + "tests/test_qwen_formation_runner.py", + "tests/test_automatic_formation.py", + "tests/test_contribution_planner.py", + "tests/test_automatic_placement_convergence.py", + "tests/test_node_config.py" + ] + } + }, + "local_desktop_preflight": { + "run_id": "q38local-20260907-050558-e510c5", + "result": "passed", + "node": "hash-verified Windows v9 package", + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "ui": "production Qt source with real controls", + "fake_node": false, + "empty_local_dht": true, + "configured_local_device": "cuda:0", + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds_including_initial_model_load": 14.856259107589722, + "local_only_button_clicked_and_mode_verified": true, + "auto_button_clicked_and_mode_verified": true, + "local_process_cleanup_verified": true, + "raw_result_sha256": "93e955391645e35b57d9e54473609226a3b46cd9b6bdc8ed86c7548860b4d2d3", + "source_inventory_sha256": "d7c9c16d82c102b4e5903aa82407985e42ac2b9407fbcc342d6908baf635e27d", + "source_bundle_sha256": "ce1dcc58d8a33e77257b665fb2facac92b9303821b10dc4e0183d6623237e71b" + }, + "remaining": [ + "Run the full cloud formation/promotion/loss/rejoin experiment after native GCP auth works", + "This runner uses staggered source-node CPU contributors; simultaneous cold join and GPU contributor formation are separate", + "The Windows UI is real Qt source, not a freshly built frozen installer" + ], + "one_click": "Run Qwen Formation.cmd" +} diff --git a/docs/evidence/qwen-formation-desktop-recovered-20260907.png b/docs/evidence/qwen-formation-desktop-recovered-20260907.png new file mode 100644 index 000000000..6c2e0ed5b Binary files /dev/null and b/docs/evidence/qwen-formation-desktop-recovered-20260907.png differ diff --git a/docs/evidence/qwen-formation-discovery-startup-20260907.json b/docs/evidence/qwen-formation-discovery-startup-20260907.json new file mode 100644 index 000000000..4978c66fc --- /dev/null +++ b/docs/evidence/qwen-formation-discovery-startup-20260907.json @@ -0,0 +1,70 @@ +{ + "schema_version": 1, + "date_utc": "2026-09-07", + "run_id": "q38af-20260907-064802-b6b85b", + "result": "failed", + "entry_point": "Run Qwen Formation.cmd", + "wrapper_invoked": true, + "source_bundle_sha256": "3fdd2acb6e24ee0b7e6f4cfc3b6b072020175204721a2fb089fd47a95c9b5f88", + "catalog_sequence": 2, + "catalog_policy_modified": false, + "scope": "Five source Linux Qt desktops and nodes; one source Windows Qt desktop with retained v9 node; staggered CPU contributors", + "local_answers_passed": 6, + "automatic_joins": [ + { + "instance": "q38af-20260907-064802-b6b85b-w0", + "range": "48:64", + "observed_coverage": 16, + "remote_acknowledged": true + }, + { + "instance": "q38af-20260907-064802-b6b85b-w1", + "range": "0:16", + "observed_coverage": 32, + "remote_acknowledged": true + } + ], + "slow_growth_retention": { + "block_indices": "48:64", + "state": "running", + "seconds_since_worker_start": 915.4614226818085, + "remote_acknowledged": true, + "fix_commit": "f496b8d" + }, + "failure": "TypeError: '>=' not supported between instances of 'NoneType' and 'int'", + "diagnosis": "Third contributor discovery remained unknown. Live py-spy parent stack showed drift-coverage-0 waiting indefinitely in DHT.wait_until_ready -> MPFuture.result. P2P startup_timeout did not bound that parent future.", + "runtime_interventions_before_failure": [], + "diagnostics_after_failure": [ + "Installed py-spy on w2 and captured parent stack while cleanup ran; no application code or policy changed. Child stack capture was interrupted by VM deletion." + ], + "fix_commit": "17ffeb2", + "persisted_fixes": [ + "DHT startup has an explicit parent readiness timeout and cleans the failed child before retry.", + "Unknown coverage cannot crash numeric readiness checks.", + "Every client must show a fresh discovery observation before its initial local answer check; bounded to 180 seconds." + ], + "regression_tests": { + "failed_before_fix": 2, + "passed_after_fix": 72, + "files": [ + "tests/test_discovery.py", + "tests/test_qwen_formation_runner.py", + "tests/test_automatic_formation.py", + "tests/test_contribution_planner.py", + "tests/test_automatic_placement_convergence.py" + ] + }, + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "local_process_cleanup_verified": true, + "evidence_directory": ".gate13-runs/qwen-formation/q38af-20260907-064802-b6b85b", + "full_formation_passed": false, + "promotion_loss_recovery_executed": false +} diff --git a/docs/evidence/qwen-formation-passed-20260907.json b/docs/evidence/qwen-formation-passed-20260907.json new file mode 100644 index 000000000..ecde2c6bd --- /dev/null +++ b/docs/evidence/qwen-formation-passed-20260907.json @@ -0,0 +1,313 @@ +{ + "schema_version": 1, + "date_utc": "2026-09-07", + "run_id": "q38af-20260907-085527-0e997a", + "result": "passed", + "entry_point": "Run Qwen Formation.cmd", + "actual_wrapper_invoked": true, + "unattended_application_flow": true, + "operator_runtime_interventions": [], + "operator_observations": "Read-only logs, status, source hashes and process checks; no code, policy, range, service or catalog modifications during the passing run.", + "source_bundle_sha256": "b0edee241fb0b5dcf25946e4ab9ddf5b9838de3f333757f76539282c1c7e4298", + "catalog_sequence": 2, + "catalog_policy_modified": false, + "assigned_spans": false, + "provider": { + "project": "community-ai-506321", + "zone": "us-central1-b", + "contributors": "4 x n2-highmem-4 (4 vCPU, 32 GiB)", + "coordinator": "1 x e2-standard-4 (4 vCPU, 16 GiB)", + "quota_increase_requested": false + }, + "runtime_scope": { + "cloud": "Production source node and actual production Qt windows on Xvfb", + "windows": "Retained hash-verified v9 node and actual production Qt source window; RTX 2070 SUPER for local Qwen", + "seed": "Isolated test seed; standing bootstrap untouched", + "joining": "Staggered contributors, capacity 16 each, model auto, no assigned ranges", + "limits": [ + "Not simultaneous cold-join qualification", + "Not GPU contributor formation", + "Not fresh installer or fully frozen desktop UI qualification", + "New-request fallback/recovery, not in-flight session failover", + "Same VM/disk restart, not VM replacement" + ] + }, + "local_initial_answers": { + "desktop": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 32.13599920272827, + "observed_at_unix": 1788773085.330073 + }, + "q38af-20260907-085527-0e997a-c": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 80.69041204452515, + "observed_at_unix": 1788772610.166737 + }, + "q38af-20260907-085527-0e997a-w0": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 59.86133790016174, + "observed_at_unix": 1788772694.7006383 + }, + "q38af-20260907-085527-0e997a-w1": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 84.99434804916382, + "observed_at_unix": 1788772831.793035 + }, + "q38af-20260907-085527-0e997a-w2": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 53.66463661193848, + "observed_at_unix": 1788772935.8855278 + }, + "q38af-20260907-085527-0e997a-w3": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 51.75513482093811, + "observed_at_unix": 1788773037.4100401 + } + }, + "automatic_joins": [ + { + "participant": "q38af-20260907-085527-0e997a-w0", + "block_indices": "16:32", + "observed_coverage": 16, + "remote_acknowledged": true + }, + { + "participant": "q38af-20260907-085527-0e997a-w1", + "block_indices": "48:64", + "observed_coverage": 32, + "remote_acknowledged": true + }, + { + "participant": "q38af-20260907-085527-0e997a-w2", + "block_indices": "32:48", + "observed_coverage": 48, + "remote_acknowledged": true + }, + { + "participant": "q38af-20260907-085527-0e997a-w3", + "block_indices": "0:16", + "observed_coverage": 64, + "remote_acknowledged": true + } + ], + "gate13_ui_start_proofs": [ + "q38af-20260907-085527-0e997a-w0-desktop-response-c2832362245d00b008e6a55d.json", + "q38af-20260907-085527-0e997a-w1-desktop-response-6b507843ec2b252923804ea3.json", + "q38af-20260907-085527-0e997a-w2-desktop-response-1e89543c481d694e7841abfd.json", + "q38af-20260907-085527-0e997a-w3-desktop-response-1d05f7ca74b9938409492e94.json" + ], + "windows_recovered_screenshot": { + "path": "docs/evidence/qwen-formation-desktop-recovered-20260907.png", + "sha256": "2c348f586e59a42857b6f2a1f21ba126717d1544df305d2c183c428fd453ccef", + "observed_selection": { + "covered_blocks": 64, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "model": "Qwen3.8 27B FP8 Dequant", + "peer_count": 4, + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "source": "runtime", + "status": "selected", + "title": "auto selects Qwen3.8 27B FP8 Dequant", + "total_blocks": 64 + } + }, + "automatic_promotion_and_answers": { + "desktop": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 30.707194566726685, + "observed_at_unix": 1788775682.6931708 + }, + "q38af-20260907-085527-0e997a-c": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 33.64885878562927, + "observed_at_unix": 1788775416.9504063 + }, + "q38af-20260907-085527-0e997a-w0": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 23.747810125350952, + "observed_at_unix": 1788775485.0812094 + }, + "q38af-20260907-085527-0e997a-w1": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 23.98796534538269, + "observed_at_unix": 1788775537.5847306 + }, + "q38af-20260907-085527-0e997a-w2": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 24.080864191055298, + "observed_at_unix": 1788775593.781983 + }, + "q38af-20260907-085527-0e997a-w3": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 23.963600635528564, + "observed_at_unix": 1788775643.9234576 + } + }, + "windows_local_only_and_return_to_auto_passed": true, + "whole_participant_loss": { + "before": { + "observed_at_unix": 1788772486.0155964, + "packaged_node": false, + "pid": 4858, + "started": 1788772486.0155945 + }, + "instance": "q38af-20260907-085527-0e997a-w1", + "stopped_service": { + "q38-desktop": "MainPID=0\nActiveState=failed\n", + "q38-formation": "MainPID=0\nActiveState=failed\n" + } + }, + "survivor_local_answers": { + "desktop": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 0.6802542209625244, + "observed_at_unix": 1788776064.5467405 + }, + "q38af-20260907-085527-0e997a-c": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 8.603599309921265, + "observed_at_unix": 1788775946.6406784 + }, + "q38af-20260907-085527-0e997a-w0": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 0.8529844284057617, + "observed_at_unix": 1788775993.2734947 + }, + "q38af-20260907-085527-0e997a-w2": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 0.867504358291626, + "observed_at_unix": 1788776020.755811 + }, + "q38af-20260907-085527-0e997a-w3": { + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 0.861248254776001, + "observed_at_unix": 1788776046.784961 + } + }, + "automatic_recovery": { + "participant": "q38af-20260907-085527-0e997a-w1", + "restored_process": { + "pid": 10820, + "started": 1788776084.5410125, + "packaged_node": false, + "observed_at_unix": 1788776084.5410147 + }, + "saved_startup_enabled": true, + "saved_policy_enabled": true, + "same_peer_identity_verified": true, + "peer_identity_sha256": "96af5d44cf82a08a911876e5e3b2160f7e2c9419d49d9a21187ec6ece4d6ecac", + "automatically_selected_span": "48:64", + "fresh_community_answers": { + "desktop": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 29.94940209388733, + "observed_at_unix": 1788776569.7367244 + }, + "q38af-20260907-085527-0e997a-c": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 32.60596323013306, + "observed_at_unix": 1788776283.6489537 + }, + "q38af-20260907-085527-0e997a-w0": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 23.924932956695557, + "observed_at_unix": 1788776351.1140277 + }, + "q38af-20260907-085527-0e997a-w1": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 28.646197080612183, + "observed_at_unix": 1788776409.8571248 + }, + "q38af-20260907-085527-0e997a-w2": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 23.887675523757935, + "observed_at_unix": 1788776480.8266537 + }, + "q38af-20260907-085527-0e997a-w3": { + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "seconds": 23.911245584487915, + "observed_at_unix": 1788776533.9228458 + } + }, + "all_six_real_desktop_observations_passed": true + }, + "timing": { + "total_run_seconds": 5702.616530418396, + "first_validated_fallback_reply_after_kill_phase_seconds": 212.76526713371277, + "first_validated_recovery_reply_after_restart_phase_seconds": 204.50843834877014, + "interpretation": "Checks are sequential and include polling plus inference; these are checkpoint timings, not per-client detection latency. Fallback took minutes, not an instant switch." + }, + "persisted_fixes": [ + "f496b8d: retain unique coverage during slow growth", + "17ffeb2: bounded discovery startup and unknown-coverage checks", + "8d8fedf: Gate 13 policy-gated restart and literal UI normalization, durable checkpoints" + ], + "regressions": { + "latest_related_suite_passed": 60, + "discovery_and_formation_suite_preceding_ui_change_passed": 72 + }, + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "local_process_cleanup_verified": true, + "evidence_directory": ".gate13-runs/qwen-formation/q38af-20260907-085527-0e997a", + "retained_files": { + "result.json": "5ad24bf3c01b3f76dac276dc8fa996ca4e396dd5ececce57f3fa02de70041659", + "formation-checkpoints.json": "7cbbd77fc35f30502c221be74e71d8aa0b8c8f7e1831b015ab784c3acb9f6184", + "source-inventory.json": "b458c7aea0500b4ea551bf459d351eabbc1723a0f3da2bb34079db1ae46eb8aa", + "cleanup.json": "6f39cb931f6088fcf9968e8b8b7432fb4e72a1c366a21b15ebe937492ccfc395", + "command-journal.jsonl": "f8213615f3a8b3af27bd967ef6d2098935c7bc325df28d99cacb38cd812c20dc" + } +} diff --git a/docs/evidence/qwen-formation-restart-config-20260907.json b/docs/evidence/qwen-formation-restart-config-20260907.json new file mode 100644 index 000000000..64ef33c04 --- /dev/null +++ b/docs/evidence/qwen-formation-restart-config-20260907.json @@ -0,0 +1,184 @@ +{ + "schema_version": 1, + "date_utc": "2026-09-07", + "run_id": "q38af-20260907-073105-de0f20", + "run_result": "failed", + "entry_point": "Run Qwen Formation.cmd", + "wrapper_invoked": true, + "source_bundle_sha256": "697e6f133aa690fa47915f7f7988c9906e308d15a269a8f4cd76f16c145cb1a0", + "catalog_sequence": 2, + "catalog_policy_modified": false, + "assigned_spans": false, + "scope": "Staggered source Linux desktop/node CPU contributors; retained Windows v9 node and source Qt client; isolated seed", + "local_initial_answers_passed": 6, + "automatic_joins": [ + { + "participant": "q38af-20260907-073105-de0f20-w0", + "block_indices": "32:48", + "observed_coverage": 16, + "remote_acknowledged": true + }, + { + "participant": "q38af-20260907-073105-de0f20-w1", + "block_indices": "48:64", + "observed_coverage": 32, + "remote_acknowledged": true + }, + { + "participant": "q38af-20260907-073105-de0f20-w2", + "block_indices": "0:16", + "observed_coverage": 48, + "remote_acknowledged": true + }, + { + "participant": "q38af-20260907-073105-de0f20-w3", + "block_indices": "16:32", + "observed_coverage": 64, + "remote_acknowledged": true + } + ], + "automatic_formation_64_blocks_passed": true, + "automatic_promotion_and_community_answers": [ + { + "participant": "desktop", + "result": "passed", + "seconds": 31.216893434524536, + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770065.4468749 + }, + { + "participant": "q38af-20260907-073105-de0f20-c", + "result": "passed", + "seconds": 33.09199047088623, + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788769778.153126 + }, + { + "participant": "q38af-20260907-073105-de0f20-w0", + "result": "passed", + "seconds": 23.910712718963623, + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788769851.508694 + }, + { + "participant": "q38af-20260907-073105-de0f20-w1", + "result": "passed", + "seconds": 23.782187461853027, + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788769909.4130518 + }, + { + "participant": "q38af-20260907-073105-de0f20-w2", + "result": "passed", + "seconds": 23.636733770370483, + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788769963.3878212 + }, + { + "participant": "q38af-20260907-073105-de0f20-w3", + "result": "passed", + "seconds": 23.60848116874695, + "model": "Qwen3.8 27B FP8 Dequant", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770020.3227925 + } + ], + "windows_local_only_and_return_to_auto_passed": true, + "loss": { + "participant": "q38af-20260907-073105-de0f20-w1", + "mechanism": "SIGKILL both complete node and desktop systemd control groups", + "phase_started_at_unix": 1788770116.7822592, + "verification": "The source runner checked both units had MainPID=0 and inactive/failed state before advancing. This old version did not persist the raw stop output before its later failure; corrected in 8d8fedf.", + "survivor_local_answers": [ + { + "participant": "desktop", + "result": "passed", + "seconds": 0.6811134815216064, + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770576.6696255 + }, + { + "participant": "q38af-20260907-073105-de0f20-c", + "result": "passed", + "seconds": 5.073481321334839, + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770422.09496 + }, + { + "participant": "q38af-20260907-073105-de0f20-w0", + "result": "passed", + "seconds": 0.8178298473358154, + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770476.7966235 + }, + { + "participant": "q38af-20260907-073105-de0f20-w2", + "result": "passed", + "seconds": 0.7940897941589355, + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770504.5792828 + }, + { + "participant": "q38af-20260907-073105-de0f20-w3", + "result": "passed", + "seconds": 0.8545453548431396, + "model": "Qwen3.5-0.8B-Local", + "text": " Paris.\n", + "completion_tokens": 3, + "observed_at_unix": 1788770534.4977684 + } + ], + "timing_limit": "Replies were checked sequentially. The first validated fallback answer arrived about 305 seconds after the kill phase began; this is not a per-client detection-latency measurement or an instant-fallback claim." + }, + "restart_recovery_passed": false, + "restart_diagnosis": { + "saved_worker_enabled": false, + "saved_sharing_enabled": true, + "automatic_range_after_restart": "48:64", + "state_after_restart": "paused", + "cause": "Runner disabled automatic startup in the saved config to enforce a literal first Start click. This differed from Gate 13 and prevented unattended restart." + }, + "runtime_code_or_policy_changes": [], + "operator_intervention": "After observing the recovery mismatch, wrote an explicit error marker to stop waiting and trigger diagnostic capture and cleanup. Did not start the worker manually.", + "fix_commit": "8d8fedf", + "fix": "Restore policy-gated auto-start, normalize any post-Save automatic start through Gate 13 real per-model Pause then master Start controls, and persist checkpoints after each outcome.", + "tests": { + "passed": 60, + "files": [ + "tests/test_qwen_formation_runner.py", + "tests/test_automatic_formation.py", + "tests/test_contribution_planner.py", + "tests/test_automatic_placement_convergence.py" + ] + }, + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "local_process_cleanup_verified": true, + "evidence_directory": ".gate13-runs/qwen-formation/q38af-20260907-073105-de0f20" +} diff --git a/docs/evidence/qwen-formation-slow-growth-20260907.json b/docs/evidence/qwen-formation-slow-growth-20260907.json new file mode 100644 index 000000000..63409c472 --- /dev/null +++ b/docs/evidence/qwen-formation-slow-growth-20260907.json @@ -0,0 +1,192 @@ +{ + "reviewed_utc": "2026-09-07", + "run_id": "q38af-20260907-060309-6fc196", + "result": "failed-live-slow-growth", + "formation_pass": false, + "before": { + "automatic_worker": { + "auto_restart": true, + "automatic": true, + "block_indices": "0:16", + "current_bandwidth_mbps": null, + "current_power_watts": null, + "desired_running": true, + "id": "automatic", + "intent_published": true, + "last_error": null, + "last_exit_code": null, + "max_bandwidth_mbps": null, + "max_disk_bytes": 34359738368, + "max_power_watts": null, + "max_vram_bytes": null, + "model": "Qwen3.8 27B FP8 Dequant", + "operator_paused": false, + "pid": 5949, + "placement_reason": "selected 0:16 from fresh verified coverage; minimum replicas 0", + "policy_admitted": true, + "policy_reason": null, + "preferred": false, + "recent_logs": [ + "Sep 07 06:24:32.836 [INFO] p2pd daemons will now receive SIGKILL when the process that spawned them dies (PR_SET_PDEATHSIG)", + "Sep 07 06:24:32.839 [INFO] Running DRIFT-LLM 2.3.0.dev2", + "Sep 07 06:24:32.939 [INFO] Checking that identity from `/srv/q38/worker-identity.key` is not used by other peers", + "Sep 07 06:24:33.360 [INFO] This server is accessible directly", + "Sep 07 06:24:33.377 [INFO] Checking that identity from `/srv/q38/worker-identity.key` is not used by other peers", + "Sep 07 06:24:34.038 [INFO] Connecting to a swarm, initial peers: ['/ip4/34.55.24.53/tcp/31330/p2p/12D3KooWEm5F2sErckT9vdLSvv58uVt3eJgxGZXiX48zA18JcPTY']", + "Sep 07 06:24:34.039 [INFO] Running a server on ['/ip4/34.44.248.21/tcp/31330/p2p/QmcQaFnsJMg9Amdi5BSNW2vKhXCL4t8M2Wjni1oHxG3gGU']", + "Sep 07 06:24:34.039 [INFO] Model weights are loaded in bfloat16, loaded from fine-grained fp8 format", + "[transformers] The fast path is not available because one of the required library is not installed. Falling back to torch implementation. To install follow https://github.com/fla-org/flash-linear-attention#installation and https://github.com/Dao-AILab/causal-conv1d", + "Sep 07 06:24:34.221 [INFO] Attention cache for all blocks will consume up to 0.83 GiB", + "Sep 07 06:24:34.221 [INFO] Loading throughput info", + "Sep 07 06:24:34.222 [INFO] Measuring network and compute throughput. This takes about a minute and will be cached for future runs", + "Sep 07 06:24:42.308 [INFO] Inference throughput: 20.9 tokens/sec per block (1 tokens/batch, CPU, bfloat16, loaded from fine-grained fp8)", + "Sep 07 06:27:42.277 [INFO] Forward pass throughput: 63.8 tokens/sec per block (1024 tokens/batch, CPU, bfloat16, loaded from fine-grained fp8)", + "Sep 07 06:27:55.701 [INFO] Network throughput: 4663.1 tokens/sec (423.21 Mbit/s on download, 382.00 Mbit/s on upload)", + "Sep 07 06:27:55.702 [INFO] Reporting throughput: 7.5 tokens/sec for 16 blocks", + "Sep 07 06:27:55.765 [INFO] Announced that blocks range(0, 16) are joining", + "Sep 07 06:28:08.910 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 0", + "Sep 07 06:28:09.268 [WARN] [bitsandbytes.cextension.get_native_library:77] The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.", + "Sep 07 06:28:22.507 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 1", + "Sep 07 06:28:36.628 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 2", + "Sep 07 06:28:51.566 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 3", + "Sep 07 06:29:04.621 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 4", + "Sep 07 06:29:18.013 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 5", + "Sep 07 06:29:41.709 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 6", + "Sep 07 06:29:54.788 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 7", + "Sep 07 06:30:07.904 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 8", + "Sep 07 06:30:21.032 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 9", + "Sep 07 06:30:34.270 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 10", + "Sep 07 06:30:49.325 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 11", + "Sep 07 06:31:02.316 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 12", + "Sep 07 06:31:15.799 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 13", + "Sep 07 06:31:29.278 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 14", + "Sep 07 06:31:49.851 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 15", + "Sep 07 06:31:49.854 [INFO] Initialized backends for 16 blocks, merging inference pools", + "Sep 07 06:31:49.919 [INFO] Registering 1 connection handler(s) with the p2p daemon", + "Sep 07 06:31:50.141 [INFO] Connection handlers are ready, starting the runtime", + "Sep 07 06:31:50.154 [INFO] Started" + ], + "remote_acknowledged": true, + "resource_admitted": true, + "resource_reason": null, + "resource_suspended": false, + "restart_count": 0, + "schedule_admitted": true, + "schedule_reason": null, + "schedule_suspended": false, + "started_at": 1788762268.1110637, + "state": "running", + "vram_pool_bytes": null + }, + "instance": "q38af-20260907-060309-6fc196-w0", + "observed_coverage": 16 + }, + "after": { + "observed_at_unix": 1788763209.0815728, + "worker": { + "id": "automatic", + "model": "Qwen3.8 27B FP8 Dequant", + "state": "running", + "desired_running": true, + "operator_paused": false, + "auto_restart": true, + "policy_admitted": true, + "policy_reason": null, + "schedule_admitted": true, + "schedule_reason": null, + "schedule_suspended": false, + "resource_admitted": true, + "resource_reason": null, + "resource_suspended": false, + "preferred": false, + "automatic": true, + "block_indices": "16:32", + "placement_reason": "selected 16:32 from fresh verified coverage; minimum replicas 0", + "intent_published": true, + "remote_acknowledged": true, + "max_disk_bytes": 34359738368, + "max_vram_bytes": null, + "vram_pool_bytes": null, + "max_bandwidth_mbps": null, + "current_bandwidth_mbps": null, + "max_power_watts": null, + "current_power_watts": null, + "pid": 9605, + "started_at": 1788763170.622993, + "restart_count": 1, + "last_exit_code": null, + "last_error": null, + "recent_logs": [ + "Sep 07 06:28:22.507 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 1", + "Sep 07 06:28:36.628 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 2", + "Sep 07 06:28:51.566 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 3", + "Sep 07 06:29:04.621 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 4", + "Sep 07 06:29:18.013 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 5", + "Sep 07 06:29:41.709 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 6", + "Sep 07 06:29:54.788 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 7", + "Sep 07 06:30:07.904 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 8", + "Sep 07 06:30:21.032 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 9", + "Sep 07 06:30:34.270 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 10", + "Sep 07 06:30:49.325 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 11", + "Sep 07 06:31:02.316 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 12", + "Sep 07 06:31:15.799 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 13", + "Sep 07 06:31:29.278 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 14", + "Sep 07 06:31:49.851 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 15", + "Sep 07 06:31:49.854 [INFO] Initialized backends for 16 blocks, merging inference pools", + "Sep 07 06:31:49.919 [INFO] Registering 1 connection handler(s) with the p2p daemon", + "Sep 07 06:31:50.141 [INFO] Connection handlers are ready, starting the runtime", + "Sep 07 06:31:50.154 [INFO] Started", + "Sep 07 06:32:11.368 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:32:27.159 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:32:28.342 [INFO] reachability.rpc_check(remote_peer=...s8WTKz, check_peer=...s8WTKz) -> True", + "Sep 07 06:32:38.650 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:33:43.433 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:33:50.077 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:34:34.149 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:36:09.667 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:37:19.327 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:37:23.919 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:38:52.407 [INFO] Public admission health: active=0 tracked_peers=0 routes=0 pending_pushes=0 accepted=0 rejected=0 healthy=True", + "Sep 07 06:39:28.914 [INFO] Announced that blocks ['drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.0', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.1', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.2', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.3', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.4', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.5', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.6', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.7', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.8', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.9', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.10', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.11', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.12', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.13', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.14', 'drift-m1-c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.15'] are offline", + "Sep 07 06:39:29.075 [INFO] Shutting down", + "Sep 07 06:39:29.605 [INFO] Module container shut down successfully", + "Sep 07 06:39:29.776 [INFO] Caught shutdown signal, shutting down", + "Sep 07 06:39:35.314 [INFO] p2pd daemons will now receive SIGKILL when the process that spawned them dies (PR_SET_PDEATHSIG)", + "Sep 07 06:39:35.317 [INFO] Running DRIFT-LLM 2.3.0.dev2", + "Sep 07 06:39:35.416 [INFO] Checking that identity from `/srv/q38/worker-identity.key` is not used by other peers", + "Sep 07 06:39:35.875 [INFO] This server is accessible directly", + "Sep 07 06:39:35.889 [INFO] Checking that identity from `/srv/q38/worker-identity.key` is not used by other peers", + "Sep 07 06:39:36.401 [INFO] Connecting to a swarm, initial peers: ['/ip4/34.55.24.53/tcp/31330/p2p/12D3KooWEm5F2sErckT9vdLSvv58uVt3eJgxGZXiX48zA18JcPTY']", + "Sep 07 06:39:36.401 [INFO] Running a server on ['/ip4/34.44.248.21/tcp/31330/p2p/QmcQaFnsJMg9Amdi5BSNW2vKhXCL4t8M2Wjni1oHxG3gGU']", + "Sep 07 06:39:36.401 [INFO] Model weights are loaded in bfloat16, loaded from fine-grained fp8 format", + "[transformers] The fast path is not available because one of the required library is not installed. Falling back to torch implementation. To install follow https://github.com/fla-org/flash-linear-attention#installation and https://github.com/Dao-AILab/causal-conv1d", + "Sep 07 06:39:36.680 [INFO] Attention cache for all blocks will consume up to 0.83 GiB", + "Sep 07 06:39:36.681 [INFO] Loading throughput info", + "Sep 07 06:39:36.681 [INFO] Reporting throughput: 7.5 tokens/sec for 16 blocks", + "Sep 07 06:39:36.781 [INFO] Announced that blocks range(16, 32) are joining", + "Sep 07 06:39:51.078 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 16", + "Sep 07 06:39:51.424 [WARN] [bitsandbytes.cextension.get_native_library:77] The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.", + "Sep 07 06:40:04.493 [INFO] Loaded Qwen/Qwen3.8-27B-FP8 block 17" + ] + } + }, + "cause": "After the 15-minute residency, an incomplete route could lose its unique existing span to an equally sized missing span. This relocates a gap without adding coverage.", + "fix_commit": "f496b8d", + "fix": "When relocating would remove unique blocks, require a net increase in covered blocks. Redundant overlapping workers may still move to fill gaps.", + "validation": { + "regression_before_fix": "test_slow_growth_does_not_move_existing_unique_blocks_after_residency failed; 3 other tests passed", + "after_fix": "56 planner, convergence and formation-runner tests passed" + }, + "source_bundle_sha256": "6c99c75d9c1c614911b324934da6f813dae2cba037ee6c2a20b78e92cf59d3fc", + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "fresh_replay_required": true, + "local_process_cleanup_verified": true +} diff --git a/docs/evidence/qwen-goose-prefill-20260909.md b/docs/evidence/qwen-goose-prefill-20260909.md new file mode 100644 index 000000000..30a958377 --- /dev/null +++ b/docs/evidence/qwen-goose-prefill-20260909.md @@ -0,0 +1,122 @@ +# Qwen chat prompt transport repair — September 9, 2026 + +The reported Goose greeting includes a system prompt. Its chat template produced +533 tokens, corresponding to 5,457,920 bytes of BF16 hidden states before protocol +metadata. Public worker inference messages are bounded to Hivemind's 4 MiB limit. +The worker correctly rejected the oversized message; sending an ordinary prompt +as a single message was the client-side generation error. + +The text peer now uses 64-token prefill chunks, preserving the full prompt and +remote inference cache. It also admits one running generation and at most two +queued requests, so an answer and a background conversation title can coexist. +Queued requests remain cancellable and receive heartbeats. A forward pre-hook also +checks cancellation between prefill chunks, where Transformers does not check its +decoding stopping criteria. Thinking is disabled +by default on this text service and can be requested explicitly. + +The first live chunked request exposed a second issue: each Qwen block owns only +its own cache, but Transformers' causal-mask helper selected the first full +attention layer in the model-wide cache. At later full attention layers, that +entry was empty. The second prefill chunk then had 128 keys and a 64-key mask. +Mask creation now receives a view of the current block's cache. + +The first complete A100 worker loaded all 64 blocks but exited with SIGFPE while +constructing the input/output runtime. A separate CPU-only Modal subprocess +reproduced SIGFPE merely by importing `cpufeature`. The language-model head now +uses PyTorch's BF16 capability check, with a conservative fallback when the probe +is unavailable. Constructing that head on Modal then succeeded (exit code 0). + +Validation: + +- The eight-layer, 533-token regression reproduced the live 128-versus-64 error + before the mask fix. +- After the fix, chunked distributed generation produced the same generated + tokens as the reference model and retained every prompt token. +- Focused tests passed across text generation, Qwen block/cache behavior, + authenticated text transport, cancellation between prefill chunks, and head + construction without importing `cpufeature`. The transport case also checks + that a peer's bounded error message reaches the consumer. +- The original 4 MiB admission bound and signed model/peer checks remain active. + +Live rollout and handover evidence is retained in the private operator directory +`.gate13-runs/modal-live-20260909/`. Raw request/response logs are not published. + +The first GPU greeting began streaming after 17.11 seconds, but continued into an +invented conversation turn and reached the 128-token output cap. The tokenizer's +chat end token (`248046`, `<|im_end|>`) differed from the model's end-of-text token +(`248044`). Chat generation now stops on either token. A focused regression +checks that both stop IDs reach generation. This first live response did not +qualify for the cloud cutover; clean, naturally terminated replies are required. + +The post-mask-fix CPU run did not produce greeting text within the consumer's +900-second limit. The temporary CPU fleet's latency remains a separate limitation; +passing the small numerical tests is not evidence of usable CPU response time. + +The corrected Modal container's SHA-256 hashes match the local source overlays +for text generation, the Qwen block wrapper, and the language-model head. The +source repairs are committed as `540035f` and `40b9e53` on +`codex/gate14-20260902-b`. These are contributor/runtime changes; the installed +Windows binaries were not replaced, and the desktop window stayed open. + +## Verified deployment + +The complete 64-block model is running on one Modal A100 80 GB. The integrated +text role loaded its weights but failed to establish a usable discovery route. +Its underlying transport could ping the bootstrap successfully; the precise +cause of that role's discovery failure is not yet isolated. A separate text +process in the same container, seeded through the local block worker and pinned +to its signed identity, passed both real requests. This preserves the model +already loaded on the GPU. + +Moving the text process's input/output weights onto the A100 also removes the +CPU projection bottleneck. The generation engine now places input tokens on the +model's device. With this role, the original Goose greeting (497 prompt tokens, +thinking disabled) began returning text in 8.33 seconds and finished naturally +in 20.14 seconds with 60 output tokens. The queued title also ended naturally. +The worker log confirms an authenticated inference session covering blocks 0:64. + +With the temporary GCP workers stopped, the installed client's normal API and +`model="auto"` selected `Qwen3.8 27B FP8 Dequant`. Both requests returned HTTP 200, +streamed text, and ended with `finish_reason="stop"`. In that run, the title +finished in 12.78 seconds; the greeting finished in 25.70 seconds including its +wait behind the title. These checks validate text chat, not Goose tool calling. + +## Discovery cache failure during handover + +Stopping the CPU fleet exposed stale cached seed addresses in the installed +client. Its discovery process repeatedly failed to restart. Using the original +six-address set reproduced a 15.05-second startup timeout, while using the +configured bootstrap alone succeeded in 3.69 seconds. The new shared client-DHT +factory attempts seeds separately and closes failed attempts. With the same six +addresses, the repaired factory connected in 3.78 seconds. Focused regressions +also cover falling back when the first configured seed is down. + +For the running client, obsolete CPU peer hints were removed and only its owned +background node was restarted by the existing desktop supervisor. The desktop +window and installed version remained unchanged. The permanent factory repair +is source code for the next desktop update; the installed v3 binary does not yet +contain it. The final installed-client results above were recorded after this +reconnection. Focused discovery, transport, generation and desktop-contract tests +passed; one test initially imported an older editable desktop checkout and passed +when rerun with this checkout's desktop source on `PYTHONPATH`. + +## Timed cloud handover + +All four temporary GCP VMs were confirmed stopped. CPU0 was resized to +`e2-highmem-16` (16 vCPUs, 128 GiB), matching the combined CPU and RAM of the four +original workers, and its complete 73-artifact model cache is verified. Its +full-model text role is pinned to its own blocks. A native GCP schedule starts +only CPU0 at 2026-09-09 16:30 UTC. Full-model CPU response time still needs the +scheduled live check; the earlier four-worker CPU greeting exceeded 900 seconds. + +The Modal worker and both separate text processes use the original absolute +deadline, 2026-09-09 17:20:48 UTC. The account's paid-spending limit remains $0. +All four temporary GCP VMs retain native automatic deletion at +2026-09-10 05:57:40 UTC. The permanent bootstrap was inspected but not changed. +The handover automation checks both the block worker and the active `text_cuda` +role, verifies CPU recovery before GPU shutdown, and preserves those deadlines. + +At the final billing check, Modal reported $21.16 usage, all covered by credits, +and a $0 paid-spending cap. The CPU restart was advanced by 20 minutes because +the remaining monthly credits may be exhausted before the fixed GPU deadline. +The GPU may therefore stop earlier under the account's spending cap. diff --git a/docs/evidence/qwen-linux-v9-20260906.json b/docs/evidence/qwen-linux-v9-20260906.json new file mode 100644 index 000000000..1bcf33534 --- /dev/null +++ b/docs/evidence/qwen-linux-v9-20260906.json @@ -0,0 +1,121 @@ +{ + "result": "passed", + "scope": "Linux CUDA engineering package and offline local CPU inference", + "run_id": "q38pm-20260906-070127-a792", + "packaged": true, + "source_commit": null, + "source_archive_sha256": "c76e426f65e946b1c69649797bdc05ba1879094ab9ea429aaa2e064db9ad4604", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 6364, + "format": "tar.gz", + "path": "communityai-desktop-linux.tar.gz", + "platform": "Linux", + "preserves_executable_modes": true, + "preserves_internal_file_symlinks": true, + "schema_version": 1, + "sha256": "98fd7c2f9bef97915a9f915ccde1513c571c2c9aa03e0635c63ff82e9aaaadeb", + "size_bytes": 3241516842 + }, + "runtime": { + "application": "CommunityAI-Node", + "catalog_bootstrap_schema": 1, + "drift": "2.3.0.dev2", + "fastapi": "0.141.1", + "frozen": true, + "hivemind": "1.1.12", + "keyring": "25.7.0", + "p2pd": "p2pd", + "schema_version": 1, + "torch": "2.6.0+cu124", + "transformers": "5.13.1", + "uvicorn": "0.52.4" + }, + "ui_smoke_scope": "Qt offscreen on Ubuntu GCP VM; not native interactive desktop", + "ui_smoke_passed": true, + "onboarding_ui_smoke_passed": true, + "local_completion_seconds": 29.397944817999814, + "completion": { + "id": "cmpl-3b730e839d294904befcb9c3", + "object": "text_completion", + "created": 1788681356, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\nThe capital of France is", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 8, + "total_tokens": 13 + } + }, + "short_chat": { + "id": "chatcmpl-de9180aa09c44bc1b8f79dbc", + "object": "chat.completion", + "created": 1788681368, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "node_stopped": true, + "limits": [ + "Unsigned engineering artifact with fixed source snapshot; no commit-bound release attestation", + "GPU runtime bundled and verified; inference in this test ran on CPU", + "No native desktop install/upgrade/uninstall or Linux GPU sharing claim", + "Audit and checksums retained locally; the Linux install archive was not published" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/q38-linux-audit.tar.gz", + "sha256": "1ac692aacdeb8dc0dcdbc3c9438607548eed734627b9a63a8c69f9aa929730b7" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-070127-a792/linux-package-source.json", + "sha256": "d1dab1d48265c70460b9a489cce9da5932f2b30728eeb3ccd2661e76026d52f6" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.json", + "sha256": "4566fd0c2dfeb18d3ef4945705ca672d6ff6df492ed4880edda775d51367645a" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.tar.gz", + "sha256": "c76e426f65e946b1c69649797bdc05ba1879094ab9ea429aaa2e064db9ad4604" + }, + { + "path": ".gate13-runs/qwen-product-implementation/linux-cloud-v9-audit/output/desktop-metrics.json", + "sha256": "cd0ffd890d7ec31213a1d0ffb85a42712127113299e20cd459608c144005f402" + }, + { + "path": ".gate13-runs/qwen-product-implementation/linux-cloud-v9-audit/output/provenance.json", + "sha256": "9d07285c1db0de9fe6bba0369fafba3b543e39797eb8d7a898c97f864052a54a" + }, + { + "path": ".gate13-runs/qwen-product-implementation/linux-cloud-v9-audit/local-qualification/result.json", + "sha256": "db3b43844ce0e376802851a7587f496b8a5519ccd346eeceb6800f37ca3b1d27" + }, + { + "path": ".gate13-runs/qwen-product-implementation/linux-cloud-v9-audit/q38-linux-package.log", + "sha256": "c485cd35f9b02c117bc8f60b899e54250a7200f222769bc16c92b2253b5cec2d" + } + ] +} diff --git a/docs/evidence/qwen-local-product-20260906.json b/docs/evidence/qwen-local-product-20260906.json new file mode 100644 index 000000000..628944241 --- /dev/null +++ b/docs/evidence/qwen-local-product-20260906.json @@ -0,0 +1,213 @@ +{ + "recorded_at": "2026-09-06", + "scope": "Real standalone Qwen 0.8B through source and Windows packaged localhost API; no peers, verified cache, HF offline mode", + "hardware": "NVIDIA RTX 2070 SUPER, 8 GB", + "source_model_revision": "2fc06364715b967f1860aea9cf38778875588b17", + "runs": { + "local-gpu-source-1": { + "result": "passed", + "packaged": false, + "offline": true, + "device_requested": "cuda:0", + "completion_seconds": 7.85899999999674, + "completion": { + "id": "cmpl-7e4311ff3e6f4bf5b28ad005", + "object": "text_completion", + "created": 1788654335, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\nThe capital of France is", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 8, + "total_tokens": 13 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "node_stopped": true, + "memory": { + "budget_bytes": 3221225472, + "allocated_bytes": 1714515456, + "reserved_bytes": 1744830464, + "peak_allocated_bytes": 1735198720, + "peak_reserved_bytes": 1744830464 + }, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "peer_count": 0, + "unauthenticated_rejected": true + }, + "local-gpu-packaged-1": { + "result": "passed", + "packaged": true, + "offline": true, + "device_requested": "cuda:0", + "completion_seconds": 13.578999999997905, + "completion": { + "id": "cmpl-be95a6208f034d91b969fe2d", + "object": "text_completion", + "created": 1788654443, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\nThe capital of France is", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 8, + "total_tokens": 13 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "node_stopped": true, + "memory": { + "budget_bytes": 3221225472, + "allocated_bytes": 1714515456, + "reserved_bytes": 1744830464, + "peak_allocated_bytes": 1735198720, + "peak_reserved_bytes": 1744830464 + }, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "peer_count": 0, + "unauthenticated_rejected": true + }, + "local-packaged-v6": { + "result": "passed", + "packaged": true, + "offline": true, + "device_requested": "cuda:0", + "completion_seconds": 12.078000000001339, + "completion": { + "id": "cmpl-d61742741e99419b99271dcd", + "object": "text_completion", + "created": 1788667577, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\nThe capital of France is", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 8, + "total_tokens": 13 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "node_stopped": true, + "memory": { + "budget_bytes": 3221225472, + "allocated_bytes": 1714515456, + "reserved_bytes": 1744830464, + "peak_allocated_bytes": 1735198720, + "peak_reserved_bytes": 1744830464 + } + } + }, + "packaged_zip_sha256": "fa568d77cdb8c8a693beb33f63ee1f29508436d59bea7797aab55e540f980f45", + "new_catalog_clean_install_proved": false, + "qwen38_packaged_inference_proved": false, + "rtx_30_40_50_qualification": false, + "current_package": { + "archive": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "fdf9ba76ed6a5ea8dad326c9656bb4da38cca955b6574264fc925cddf75980d0", + "size_bytes": 2692390008 + }, + "catalog_digest": "sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80", + "catalog_sequence": 2, + "ui_smoke_passed": true, + "onboarding_ui_smoke_passed": true, + "runtime": { + "application": "CommunityAI-Node", + "catalog_bootstrap_schema": 1, + "drift": "2.3.0.dev2", + "fastapi": "0.141.1", + "frozen": true, + "hivemind": "1.1.12", + "keyring": "25.7.0", + "p2pd": "p2pd.exe", + "schema_version": 1, + "torch": "2.6.0+cu124", + "transformers": "5.13.1", + "uvicorn": "0.52.4" + }, + "local_inference": { + "result": "passed", + "completion_seconds": 10.953000000001339, + "completion": { + "id": "cmpl-58154c9bdd95437692617c86", + "object": "text_completion", + "created": 1788659442, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\nThe capital of France is", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 8, + "total_tokens": 13 + } + }, + "token_budget_rejected": true, + "local_only_persisted": true, + "stream_cancel_released_lease": true, + "node_stopped": true + }, + "memory": { + "budget_bytes": 3221225472, + "allocated_bytes": 1714515456, + "reserved_bytes": 1744830464, + "peak_allocated_bytes": 1735198720, + "peak_reserved_bytes": 1744830464 + }, + "source_commit": null, + "complete_release_qualification": false + }, + "current_windows_package": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "c95c1b1f94eba68a04ffc54b8dc17a49e425390eca053ef8f3996efdcc9dbf1d", + "size_bytes": 2692398749, + "binding": { + "path": ".gate13-runs/qwen-product-build-v6/desktop-metrics.json", + "sha256": "fe79b3477766ff6d28744553b7e49da45dda5190aebacdb6cb9378f47900a0b2" + }, + "real_local_result_binding": { + "path": ".gate13-runs/qwen-product-implementation/local-packaged-v6/result.json", + "sha256": "7d8379681c985e1ea3371806cfdce187d911e6552cf1eed68850fdef46da3102" + } + } +} diff --git a/docs/evidence/qwen-mixed-full-inference-20260905.json b/docs/evidence/qwen-mixed-full-inference-20260905.json new file mode 100644 index 000000000..e1d4871a8 --- /dev/null +++ b/docs/evidence/qwen-mixed-full-inference-20260905.json @@ -0,0 +1,202 @@ +{ + "schema_version": 1, + "run_id": "q38m-20260905-215111-8174", + "result": "passed", + "topology": "GCP L4 + Azure T4 + two GCP CPU workers", + "regions": { + "GCP": "us-central1-b", + "Azure": "eastus" + }, + "cpu_prerequisite": { + "result_sha256": "7e3c7a4bcc7dc6ca4d1ff861e040d444ee295db0ae79ed4483b335b48eb0808e", + "run_id": "q38-20260905-205155-ed78" + }, + "source_bundle_sha256": "c9550a4813e133f1e1e721364fcec737092d2cd2f2feb65dd6859342973377ab", + "tested_host_script_sha256": "19b458af4e69c3f2a0b58114f64185d49103d64d578d7909c4f88200b905eeae", + "tested_server_sha256": "550a0b9475cca3470510ff02a88c383827a41bb13028f599f66e04ce03861b4f", + "retained_client_result_sha256": "df61e318961fe3fdc8c3ddf14f32be3f477baec4ebcb22c9cb44e9552aeb0337", + "hosts": [ + { + "role": "c", + "provider": "GCP", + "machine_type": "e2-standard-4", + "instance_id": "1459408393892606281", + "runtime_inventory_sha256": "730cbd3e343e204e5218f077565317bd463f2122eb310cd523e9176f0d521603", + "gpu_probe": null + }, + { + "role": "w0", + "provider": "GCP", + "machine_type": "g2-standard-8", + "instance_id": "1395698738809352483", + "runtime_inventory_sha256": "0aa72014559a567c890a5d1a03dab0aaaa8420d3b5be0e240440ea780a854e89", + "gpu_probe": { + "result": "passed", + "gpu_name": "NVIDIA L4", + "memory_bytes": 23659151360, + "compute_capability": [ + 8, + 9 + ], + "torch": "2.6.0+cu124", + "cuda": "12.4", + "bf16_matmul": true, + "bf16_convolution": true, + "observed_at_unix": 1788646938.1362476 + } + }, + { + "role": "w1", + "provider": "Azure", + "machine_type": "Standard_NC4as_T4_v3", + "instance_id": "f6952d5a-1bee-4054-aed6-b3911f4f7d04", + "runtime_inventory_sha256": "ed51736c34a997be5cd0a4a2bb2ee5908fe8823c4327c8129ba06df0a4629cd5", + "gpu_probe": { + "result": "passed", + "gpu_name": "Tesla T4", + "memory_bytes": 16703356928, + "compute_capability": [ + 7, + 5 + ], + "torch": "2.6.0+cu124", + "cuda": "12.4", + "bf16_matmul": true, + "bf16_convolution": true, + "observed_at_unix": 1788646560.5139458 + } + }, + { + "role": "w2", + "provider": "GCP", + "machine_type": "e2-highmem-4", + "instance_id": "5589617213954928911", + "runtime_inventory_sha256": "e21b14a4fa03723804b2300dbdfd2657ce586dda6647c394ee2f1e73458d1791", + "gpu_probe": null + }, + { + "role": "w3", + "provider": "GCP", + "machine_type": "e2-highmem-4", + "instance_id": "4881313665882599700", + "runtime_inventory_sha256": "909a70470fd27324d1415f841353166f8c78ff448677be36fe1d2b1da73c1df9", + "gpu_probe": null + } + ], + "workers": [ + { + "span": "0:16", + "peer_id": "QmPD5cYHsmodEqmmyzPHmZ4CuDqw31sG5wCLe6okbMzeCn", + "hardware": { + "compute_capability": [ + 8, + 9 + ], + "device": "cuda:0", + "dtype": "torch.bfloat16", + "gpu_memory_bytes": 23659151360, + "gpu_name": "NVIDIA L4" + } + }, + { + "span": "16:32", + "peer_id": "QmViwwPXLNeDtgBxYRcsz6u2THLsoR6uumgN92Bhz2Bawo", + "hardware": { + "compute_capability": [ + 7, + 5 + ], + "device": "cuda:0", + "dtype": "torch.bfloat16", + "gpu_memory_bytes": 16703356928, + "gpu_name": "Tesla T4" + } + }, + { + "span": "32:48", + "peer_id": "QmWWG4KhhYfEerb13zGfXkfj3uTz87wwWZs2EC3TVQ6QCk", + "hardware": { + "device": "cpu", + "dtype": "torch.bfloat16" + } + }, + { + "span": "48:64", + "peer_id": "QmSu3aGY5zU4bsLkpjZbH53ht9Lyx8nEoXib35sF2CGQWU", + "hardware": { + "device": "cpu", + "dtype": "torch.bfloat16" + } + } + ], + "client_evidence": { + "baseline": { + "route": [ + { + "end": 16, + "peer_id": "QmPD5cYHsmodEqmmyzPHmZ4CuDqw31sG5wCLe6okbMzeCn", + "session_id": "8b340a88-a904-4cf3-aba8-f8f77472ced4", + "start": 0 + }, + { + "end": 32, + "peer_id": "QmViwwPXLNeDtgBxYRcsz6u2THLsoR6uumgN92Bhz2Bawo", + "session_id": "d5fdba2e-22da-4153-8416-8c62ade36d6d", + "start": 16 + }, + { + "end": 48, + "peer_id": "QmWWG4KhhYfEerb13zGfXkfj3uTz87wwWZs2EC3TVQ6QCk", + "session_id": "9132ba61-53c9-4256-bdc6-a9fb8d8618e6", + "start": 32 + }, + { + "end": 64, + "peer_id": "QmSu3aGY5zU4bsLkpjZbH53ht9Lyx8nEoXib35sF2CGQWU", + "session_id": "acfd2143-d578-4c84-a731-bb365f51a040", + "start": 48 + } + ], + "seconds": 75.69909206700004, + "text": " Paris.\n", + "token_ids": [ + 11751, + 13, + 198 + ] + }, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", + "observed_at_unix": 1788647585.565859, + "recovery": null, + "result": "passed", + "versions": { + "hivemind": "1.1.12", + "python": "3.12.3", + "torch": "2.6.0+cpu", + "transformers": "5.13.1" + } + }, + "same_tokens_as_cpu_baseline": true, + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "quota_increases_requested": false, + "persistent_configuration_change": "Microsoft.DevTestLab registered for VM auto-shutdown; remains enabled.", + "limitations": [ + "Three generated tokens; no long-context or concurrency qualification.", + "BF16 kernel execution does not establish native T4 BF16 tensor-core acceleration.", + "No stock-Transformers logit parity or GPU performance qualification." + ] +} diff --git a/docs/evidence/qwen-mixed-product-first-20260906.json b/docs/evidence/qwen-mixed-product-first-20260906.json new file mode 100644 index 000000000..f49e49a5f --- /dev/null +++ b/docs/evidence/qwen-mixed-product-first-20260906.json @@ -0,0 +1,103 @@ +{ + "run_id": "q38pm-20260906-015639-3c59", + "result": "failed", + "scope": "source-node-local-to-measured-community-promotion; transition harness failed before worker loss", + "promotion_passed": true, + "product_loss_rejoin_passed": false, + "packaged": false, + "local_before_growth": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788660227, + "id": "cmpl-9587d33e1c0444cdafa7aa2d", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 35.95786051499999 + }, + "qwen38_baseline": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788661360, + "id": "cmpl-b05d2083006a441f853ff3b9", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 64.27441845199996 + }, + "selected_manifest": { + "covered_blocks": 64, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "model": "Qwen3.8 27B FP8 Dequant", + "peer_count": 4, + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "selector": "auto", + "source": "runtime", + "status": "selected", + "total_blocks": 64 + }, + "failed_transition_actual_response": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788661363, + "id": "cmpl-94548333c95e4ee7be165bc2", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 3.6298966919998747 + }, + "failure_analysis": "The harness observed the previous completion lease before it drained, then switched mode before the newly submitted request acquired a model. That new request correctly used local Qwen. Corrected harness drains old leases and observes the new Qwen3.8 lease before changing mode.", + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "files": { + "result.json": "90a962f61509228b7a23c5de48dbab43dec44c413d300a5ddb66b30892383d10", + "product-result.json": "92d56a2cd8cee8d4375be3da19df1cec6328697735bec33e5c55b40f2283535e", + "product-node.log": "8e0e41b2722375d2a4916e3653a65cd6b3de7d0cb154ebdf755f0e387f49f163", + "source-inventory.json": "e1472f71b140b981c4df3cb47e409aa95475433667594a6a94bcabc9eacb1302" + } +} diff --git a/docs/evidence/qwen-mixed-product-fourth-20260906.json b/docs/evidence/qwen-mixed-product-fourth-20260906.json new file mode 100644 index 000000000..4a81bc783 --- /dev/null +++ b/docs/evidence/qwen-mixed-product-fourth-20260906.json @@ -0,0 +1,175 @@ +{ + "local_before_growth": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788668300, + "id": "cmpl-3ff87097eb7c4136821707fc", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 46.74294086499998 + }, + "qwen38_baseline": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788669438, + "id": "cmpl-687bf9c0314740adb503973f", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 61.91162192799993 + }, + "active_answer_after_mode_change": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788669554, + "id": "cmpl-67b0ea4f990645eea2d77b78", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 55.38500175699983 + }, + "local_only_next_request": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788669558, + "id": "cmpl-11ed943f986447f0b0884e93", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 4.035045044000071 + }, + "local_after_worker_loss": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788669677, + "id": "cmpl-be06a1eebb9449c4b126ab8c", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 3.677601690000074 + }, + "qwen38_after_replacement": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788669965, + "id": "cmpl-f174f8a1aa4d47498622de5c", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 62.824241735999976 + }, + "local_fallbacks_before_active_transition": [], + "manifest_digests": [ + "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + ], + "run_id": "q38pm-20260906-041033-fe7f", + "result": "passed", + "packaged": false, + "scope": "source-node-assigned-mixed-route-promotion-preference-loss-and-rejoin", + "catalog_scope": "ephemeral engineering trust root; not the public catalog", + "all_blocks": 64, + "source_bundle_sha256": "c035ea6924fead097fee3980973f584ae69a90221afc48f47f106711a58c7852", + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "limits": [ + "Assigned cloud spans; not autonomous desktop swarm formation", + "Source node; not packaged community acceptance", + "Worker process restarted with new peer identity; separate CPU run proved VM replacement", + "Bundle predates explicit chat option and independent readiness observer" + ], + "overall_runner_result": "failed", + "bindings": [ + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-041033-fe7f/product-result.json", + "sha256": "8024b3ad37fe8e3306ac580a62d7e7940c40d08b8ac5cb28da62680f4f3396df" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-041033-fe7f/source-inventory.json", + "sha256": "efbd8328a97719b34d720d26121dc511dd0f2c526bc54afb342d0e735beaf7a6" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-041033-fe7f/result.json", + "sha256": "08407a8360053e79fd9a3077163de2505a6a1ef63833dfb877ca510274264b9b" + } + ] +} diff --git a/docs/evidence/qwen-mixed-product-third-20260906.json b/docs/evidence/qwen-mixed-product-third-20260906.json new file mode 100644 index 000000000..e95998813 --- /dev/null +++ b/docs/evidence/qwen-mixed-product-third-20260906.json @@ -0,0 +1,90 @@ +{ + "run_id": "q38pm-20260906-032336-3c38", + "result": "failed", + "scope": "source-product-test-failed-before-worker-loss", + "promotion_passed": true, + "product_loss_rejoin_passed": false, + "packaged": false, + "probe": { + "first_token_seconds": 28.775, + "duration_seconds": 74.983, + "completion_tokens": 3, + "retained": true + }, + "local_before_growth": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788665681, + "id": "cmpl-b35b2992d01c4e4cafadfc87", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 60.02689081699998 + }, + "qwen38_baseline": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788667359, + "id": "cmpl-06ba80e33c5e43cbae882418", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 131.35887205200015 + }, + "error": "AssertionError: the new community generation was not active before the mode change", + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "failure_analysis": "The next automatic request completed before a community lease was observed. The prior long completion did not guarantee continued fresh eligibility. The harness now waits for fresh readiness and records intervening local fallback, bounded to ten minutes.", + "bindings": [ + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-032336-3c38/result.json", + "sha256": "1687b8e5ef27e0c2a4e06a904f420f5d94d378d7e09c52f2cd559b752146b662" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-032336-3c38/product-result.json", + "sha256": "1333ec45abe9b75bc932b2aa3ff7a92bf1602e7309c62be7e1aa0ac9e9e38622" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-032336-3c38/source-inventory.json", + "sha256": "eb40abd19695e5ae26e03f05ab7f93ac5787cee086de8f6bcc8c15638f3458ae" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-032336-3c38/product-node.log", + "sha256": "6f0bb63a6d0135fcd050945c57ef70ac3f3fe9db3ffa82b993c3701516cfacd5" + } + ] +} diff --git a/docs/evidence/qwen-modal-formation-blocker-20260907.json b/docs/evidence/qwen-modal-formation-blocker-20260907.json new file mode 100644 index 000000000..7c7213364 --- /dev/null +++ b/docs/evidence/qwen-modal-formation-blocker-20260907.json @@ -0,0 +1,558 @@ +{ + "reviewed_utc": "2026-09-07", + "result": "blocked-on-modal-filesystem", + "formation_pass": false, + "run_id": "q38mf-20260907-054716-44636d", + "result_sha256": "e4b7c6f8e3ff1f17962b9c644d0eb1a6fe273b1259c2b35310ae1e5ead1e0706", + "source_inventory": { + "bundle_sha256": "98c4e3b354c3cda7c6f7f4518d8304ac12ef37f5be45f0618131e306fb901f2c", + "files": { + "LICENSE": "d8576c8fbb92cdceedbee2d6c5bd62cc2fbcb188263b67c5b3edf45513c0988f", + "README.md": "3d4faa6f06ce5b8e2c39bb723efbf3a0fe4ea785594c81a9ba34a4db230714bc", + "config/qwen_formation.json": "6103e47844b894a5cd942591d1cb7a111c73a777261d554a16d3ae78fea0c93a", + "desktop/pyproject.toml": "b55984e8b2bda3ac943a763651f7a64b8f6a3edb1c29c5882602ed35c5addf26", + "desktop/src/communityai_desktop.egg-info/PKG-INFO": "effd2051c2765dc067b91aba3e65edf89ad20e4d21437c924a12fb9cacd5a537", + "desktop/src/communityai_desktop.egg-info/SOURCES.txt": "2d1381583b49fa03096aa9612bc7c7df57c8e3f59314bf0614c5649f8dac91a3", + "desktop/src/communityai_desktop.egg-info/dependency_links.txt": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "desktop/src/communityai_desktop.egg-info/entry_points.txt": "2b4446dfcc9f5a635692ad26edd88707d0eb7732cec570c1b9d0e74ebde654aa", + "desktop/src/communityai_desktop.egg-info/requires.txt": "1f7cc2e9e6166d70095eb76dbaef21faa9056aac993917b0dbe6fdf81753bab6", + "desktop/src/communityai_desktop.egg-info/top_level.txt": "c07d6499cb77347a40a3a73e1e4472305b23a38857a34f58b801ce0ae3e7b5a3", + "desktop/src/communityai_desktop/__init__.py": "c9facc4a758ef2b06881b8c3181f55791878633caa9498e7d2a5f92a5471e378", + "desktop/src/communityai_desktop/__main__.py": "67bfba4e3e7c93da8d6ea1e7ab05c5856809b2ba2057a2304c6efa417a176bc3", + "desktop/src/communityai_desktop/acceptance.py": "387a28a87d860dc422af8bb072d0d3c248d2678c2d344d9bb9e679b972d024a2", + "desktop/src/communityai_desktop/app.py": "8d37cb83685efa90b454f3f76976bd453ea90178752452edeabfc68958e70767", + "desktop/src/communityai_desktop/assets/communityai.ico": "0fcbedf7da4c5255670885ac4adbe910993d0b56e1fdee26157f6b1ff94633c5", + "desktop/src/communityai_desktop/client.py": "b7554fa84330c9d89d9903ceb5cc8c720ad01e29d46bf13804384f9cf4415639", + "desktop/src/communityai_desktop/controller.py": "72ea7b9825257a755a1c3ba7404642360274ac510c8e9f4de0a01a98155660e4", + "desktop/src/communityai_desktop/credentials.py": "2b5c7e53ddbbb4ac0703163dbb14fa3c13b1fb0a31deb6db48fbe3d2bc1ea066", + "desktop/src/communityai_desktop/gate13_playthrough.py": "73084cbb6abd0659a577c5423ad55f9d3df2fa8a7c0653861fd75e686a6ce593", + "desktop/src/communityai_desktop/lifecycle.py": "6fd21b14774e3b7a16ca97421d743a59fd971e3f28c11c674c7084360a748e13", + "desktop/src/communityai_desktop/pyside_shell.py": "b9a66717b8d1bffb8bac7dbd4c723b4fcda39c9d8bf44e68095c4c91340fa299", + "desktop/src/communityai_desktop/startup.py": "e3bf0370e05dcd978f0084ce3ce03ce80ce64f8f98d3c43438f8069347dc70d1", + "manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json": "31f3424423e1dc632a08dd029c72cd68b0952b06ec34ad70ed36d80e93840d5d", + "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json": "0591317317adf4752425725658732997597c51150a7b165bcd674cbc319bf2a1", + "public-alpha/catalog-qwen-v2/bundle.json": "a904ed1b8487762507fcd35c4a125f03f0f71ee6c164bcacb993fbab677e0be9", + "public-alpha/catalog-qwen-v2/catalog-bootstrap.json": "79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410", + "public-alpha/catalog-qwen-v2/catalog.signed.json": "315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315", + "public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json": "a2621aa34aa47f0c9074f5baa0254b17549424f1c85d828ae9b1bf6ad9e76bb3", + "public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json": "4536ac2bada7242b758db443b9ebb813a614dd167ab364643fc55c7eb657bb74", + "public-alpha/catalog-qwen-v2/publication-preflight.json": "b80f6c4a8d0b87a1d365fc4be01e643429759cc52eda5c200dbf236f2073f538", + "pyproject.toml": "082842d4be0720e96163014443ca992e72fcc68b2ac60a938de3959f9d3a2307", + "scripts/qualify_qwen_formation_local.py": "677d69ffc182db0cc7ef2ed24a769c209d8db48d897399b7717b2f008a4d42b2", + "scripts/qualify_qwen_modal_transport.py": "cda2bde6b094b72514c083fa03e4b1e10cd479b089e00c6d36675f6d5072b20d", + "scripts/qwen_formation_desktop.py": "9abd693997a3e1a5afd0f946b12c7a3b273a066344bd0fd62338a98d7314e449", + "scripts/qwen_formation_node.py": "0b64ef5c3de4981068e62d0283e27276b489dc703690c38cecf6c8c547393829", + "scripts/qwen_full_inference_host.py": "9b2f854ed8b1360c8947a679dfe0ad71261e53a686a2f71de785da475a936a25", + "scripts/qwen_modal_participant.py": "913023e9e563fb42bfe8962eed0ca9085306ae3c90268afd7f32e5f74af0c825", + "scripts/qwen_product_inference_host.py": "759be6507f6e74efd6760116d49bd2602666752b7f36c1799501fb84b4bd5150", + "scripts/qwen_product_recovery.py": "d1a51391234d423e984dc8270ad8c8e3690643e7c8e5a377a139d9950247800d", + "scripts/run_qwen_formation.py": "d8cba93541c4edc0ac4ab8746a42be6985525e4d868d9f82026f0f5d55fdedb9", + "scripts/run_qwen_formation_modal.py": "9d3e6359c87714c0644ccf5f76a5007ca36191b294c5025f672ce08c09f7e18f", + "src/drift/__init__.py": "25413cb38a8cbd8d9903301d2da628e39cc91769d564b6c1838f6252a7f3de6e", + "src/drift/api/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/drift/api/server.py": "3a996d1f94fd5a52835ff3eaab6d6c4ea06a3361be3469624f6febc60323d598", + "src/drift/catalog_release.py": "f4351c727b38b399eaac19b4543b6326b6c62646b3a12701317619fbc05b0d38", + "src/drift/cli/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/drift/cli/__main__.py": "995060738b41c25af633a81e4819a3c01b85af143d8878e06c69100dc44d2323", + "src/drift/cli/run_api.py": "ea5d2476066fce4c03e3d339523f809fcab871dc8c5204049bd8671c48e44fe5", + "src/drift/cli/run_bootstrap.py": "7c54cba9244a171d88d52614589ad43a635cf60c7f3647236911eb746f7dc996", + "src/drift/cli/run_catalog.py": "9c06e36ca010bfd81c240e77f0ed738a39219e7ca1bd44486202fc8f157cfefe", + "src/drift/cli/run_dht.py": "afc5f812e5a720177560b81ca6b2471f09c745399eff8b37ee9788675f35cbb3", + "src/drift/cli/run_down.py": "7ecda1be0dee49619d96de33677bedd83ce3d91e44de234a8124237fc6c91ea5", + "src/drift/cli/run_edge_acquisition.py": "dd8cedf3ef10e231e01aae3049bc542235d1f3ec39dea2ad3a01b130639359c2", + "src/drift/cli/run_edge_benchmark.py": "34acaa625f8e38caca0085dcfd8e47011303ddbe37c3c10cd22e9bdf6a18c0af", + "src/drift/cli/run_identity.py": "e22b266476399fe04a535971d410c5cf000b9bab1c2f3d7cfc4b8cc3e4d36346", + "src/drift/cli/run_manifest.py": "b0c462079debe002ff660bb861c5cb99bae124fe8bb47f1060575ea4fcc7fec9", + "src/drift/cli/run_node.py": "6af55b74674ad882c51d8d2d94975b936b787002c232e6653b9275ee013b0286", + "src/drift/cli/run_prod_server.sh": "a0f39efa6b101b4f05b4f941fe12785c56f416beb265d178110ac2c194bd7ed4", + "src/drift/cli/run_server.py": "0ac20e620e0d94a13dd81f19dec679e926a99d6f46b163f00e218774763749d3", + "src/drift/cli/run_up.py": "5dbcf8d17394903941341d5b56df5a01b5d327d55ce28efb89762375ff7ac8df", + "src/drift/client/__init__.py": "51c204cabf56fddb8bbab110041df34f9c5df9b09dabfd958705b2d389332d97", + "src/drift/client/config.py": "b05f476f59bf4bd7ec9d89d9facda83b85c677eda8adecad6cd1a0b5038c79a5", + "src/drift/client/from_pretrained.py": "c9ba0f108b16efa875a590d7dec2a7c6ebffc3be9f3efe4871de64cc5ad59f21", + "src/drift/client/inference_session.py": "57d1d11953a35f4c0c50e1078eee8c82e52e4103002bfa6f6dcc22fb0f48e749", + "src/drift/client/lm_head.py": "79bd28679b13863f7b2143414023be468263f3e69ddcbdd85742fa5a13f5feec", + "src/drift/client/ptune.py": "2d12ce9beac06ed076d8a41a6f6b3d85f69e5f37e0c842420bfbdf4d787fef3b", + "src/drift/client/remote_forward_backward.py": "93757c9b00c92d766d5edbdb6d729006f74baf34c97d9cdc0fb822d830d4c4a1", + "src/drift/client/remote_generation.py": "17eb8a822d14cb69995be5d5931e4f2a45dfb0ce789f602d239e9ce5cf933134", + "src/drift/client/remote_sequential.py": "40d4042db011531a15953e16c6cb685d7bfdbeb6667a28f26e6593ae1dc9af82", + "src/drift/client/routing/__init__.py": "e3ccf601afc13c57b3e7e7df31ff16673c7764b2af5a08dc654e4fca0750f378", + "src/drift/client/routing/sequence_info.py": "58e9bd1f312d73bbad6c234ada5fbcad888214e87d339cab249979077377d3b3", + "src/drift/client/routing/sequence_manager.py": "080e4798bdb21a9c1941c9f2b926169b84b363b0f8cdb31e07cb1bf819b979ea", + "src/drift/client/routing/spending_policy.py": "e5cd8570fe215e1441c5312fd7859b3600bf440a31a4c435c9406e638b6e262c", + "src/drift/client/sequential_autograd.py": "81e1729f258fc7830aa7cd77d0d58452618b8f4cb18362e4b848fb1a1ac70c90", + "src/drift/constants.py": "abf3737fe372622a178bf16b1b8e3e86f4a8e8c134ee0776398ccad7c272e89a", + "src/drift/data_structures.py": "ba4ee49792105f583143413a51f6077ebed10ef2143259f72e0afb0add1de405", + "src/drift/dht_utils.py": "16a02b934d144b46e6d824d265f267f8bb176c2a2d462d2d5add0f853cf4fa91", + "src/drift/model_catalog.py": "7ee7c631435dce02851e032d4a32844a26502e1d2333d0989868600950df19d0", + "src/drift/model_manifest.py": "2edf59430f9e3864f557b9c474e1775e73fab3a96d6aaa27b20fd889146454c5", + "src/drift/models/__init__.py": "ece5852c7907a00d6f6921b3d8b868e42b19f5a474c131d25e5cd7ebc8231fef", + "src/drift/models/_gemma_block.py": "60b53b5ac228a315446328f47adc9f20a43919fcda658cb8c51c529e4e9c42b0", + "src/drift/models/_gqa_block.py": "2c55a1e99a8eeba6528981d852209488bf3f25bb1423cdc4d90399d60c8b29d2", + "src/drift/models/bloom/__init__.py": "4ce43ff3770b4ede96eed7ffb2d11290fd036d57235198a1573aa199782c8612", + "src/drift/models/bloom/block.py": "df290e8ceb288cd26c3e11abb030b385692a4aa3504856defded7b5122742691", + "src/drift/models/bloom/config.py": "0861a6046afa9bc7e49123ed8cc113e2deb0c9ac74e2631c55745c6129a5c6ef", + "src/drift/models/bloom/model.py": "ecbb53aa163e1b714f35c78830a41f5391d85d5a36e6d0d44cc4711f57353d8d", + "src/drift/models/deepseek_v3/__init__.py": "6c9348e12520648e2ce4b01b25054ac1063062d82c2a74ad4ba5df3b680e1c14", + "src/drift/models/deepseek_v3/block.py": "2ece6b49dc12c5fa38c76501f90712b7c4e39b2251afe56ee0e602f52e567c29", + "src/drift/models/deepseek_v3/config.py": "a3bda63103eb2f74ba2f3ca6cf205b873ddbd522ce4a75ae800f1b3cf14a10f2", + "src/drift/models/deepseek_v3/model.py": "e9ceb1c6946dfeb9aa4625cc667c6aae73fc7aeebea7f7af85b66ef6d66f97e9", + "src/drift/models/falcon/__init__.py": "8aa9717c72ba2716637758532faff818f5a7364397007f937fe05226c721c628", + "src/drift/models/falcon/block.py": "0de3ecf8a7314ded90daf2f633fce3caea6d6f1dc468c2a5049e42b11de99d40", + "src/drift/models/falcon/config.py": "3cb1c5d1094980ebc27197d5d8983b2b0619c233151344d10b972c3079ae5619", + "src/drift/models/falcon/model.py": "4aa9e47c4427a82c61d058128aa56fa9722f0a47b57004e4d6a8705688dd9565", + "src/drift/models/gemma2/__init__.py": "3c86c608eca0bc82302254d978ef3f0853f43079f2288855fa4bda0849312f9b", + "src/drift/models/gemma2/block.py": "2bb0037d7838a65b4c9e20a8c2dfafbeddf6587fcff84d73e68ab0dbc341d548", + "src/drift/models/gemma2/config.py": "82bf127d782318ccc72247d2d4f7a984a575b7b640d0048962cd8c29563d6488", + "src/drift/models/gemma2/model.py": "35506cca302317b97563864d26416c450028985e71a41f124e3bd2db4f139522", + "src/drift/models/gemma3/__init__.py": "dd417bbd3146d83cefdd8e5061ce7b3ff6db4d223d7826120e9179f7244e0086", + "src/drift/models/gemma3/block.py": "cc1d53103ef2456deda030ca1a372e39d24d298dc23951aa86765ecc678a5e90", + "src/drift/models/gemma3/config.py": "75171fd0d6893db8a16e90b03edac09c6b34b21124dd4b7cdecd4f3a01c9cc39", + "src/drift/models/gemma3/model.py": "b9ef922d55ed379c4eff68f6449d246610958372c38ea729a561df0d19949ec1", + "src/drift/models/gemma4/__init__.py": "540a79396363ecdb585e8a6f5e4faa63f1cf25fa68d99651b6f87d14c481177d", + "src/drift/models/gemma4/block.py": "c60227d2ba31bb13632e4fc69f47497f5d51e3c51dfbcf71a8463ab954ff954a", + "src/drift/models/gemma4/cache.py": "2799a7141d62c3de37d162280bc985e389aff5121e00cb2df3c0ddc936adbe0e", + "src/drift/models/gemma4/config.py": "ae6bd6c756ba0f848682acc3f30595919d3b3fc0e8ac045b45e66e528b29c77f", + "src/drift/models/gemma4/model.py": "9931be0d59f68feb5501a2ef0a4f29ab703712cc484fd3d8ce2443a7695dad79", + "src/drift/models/gemma4_unified/__init__.py": "5454dadd3559a90a9ba6a16a0ea10d91fa1112febe8bcfa7a1abd5b2bb1cd038", + "src/drift/models/gemma4_unified/block.py": "aaafd573d3fb70b91eeb5d6887b64392a68f1911e0b2c366b381002d826fda82", + "src/drift/models/gemma4_unified/config.py": "94ec0bb04de04082f7ac630bf75986bdcfa4675b75ac798a595f2707b632beb0", + "src/drift/models/gemma4_unified/model.py": "a81e81be2fa5240783ace0da2dd53a70e9640e12b6adcd8da3e787891b788dbf", + "src/drift/models/llama/__init__.py": "e2dffa3d41153bd0c5cba79f8a48ef8dfbfc524ac53aca1321caadd4ddef5ce5", + "src/drift/models/llama/block.py": "40c455f5d801429839e6afc779d99d4a02dedec9f31ae3e7e88bf813d5175dac", + "src/drift/models/llama/config.py": "3e3492d577fd2798e1b017e6add6bb6efa791bf191d3a47a0f0638b38fffd6f4", + "src/drift/models/llama/model.py": "16c698ac15950345c2365eb0da9948c03338dbd226a885b3a19d37c82dd1eb1b", + "src/drift/models/llama/speculative_model.py": "fdc1973349065678615eeeed192fadd5036a915d4bb6e3571add1e6a0b9cf087", + "src/drift/models/mistral/__init__.py": "89794392643711595b9077e2c02580d07bb7c481dd79200ba09ab5a03d3a915c", + "src/drift/models/mistral/block.py": "008e07d94e0e00141f37acf1b50fbac900a40212a5fac267678320ab809f03d2", + "src/drift/models/mistral/config.py": "14b4893781d930a3024d6e9a0287acf6465fbb4c46576e786ef5fed47bc24a86", + "src/drift/models/mistral/model.py": "52d4a4ec95cf4f4af0f2a548c07ed70aea25c6951d1fd97606199d65468dccbb", + "src/drift/models/mixtral/__init__.py": "eac8ec3a91d60b9b1ec5d2ecf2d932b66ee1e63e962b75410245e78c38f402af", + "src/drift/models/mixtral/block.py": "d44709364ac009baba3ff5c3328fdc12a96995d5ee6cb10e8fee7bb3cd7e1063", + "src/drift/models/mixtral/config.py": "1e7f0481749ff27b5032f8d251af6b770ffd3563a1150409afcc8ff3df5e73dc", + "src/drift/models/mixtral/model.py": "eb28dced7bcc6c09941b9bcf52e1245eb8f7523f41e9ca9d3bc3e6f4df511582", + "src/drift/models/qwen2/__init__.py": "b270bfc8e3d7c77fceb485614638cafb5866410fd6abdc669f36cdb1cb7d2e0d", + "src/drift/models/qwen2/block.py": "5b44ba67bd9b9499aed2fd9f2ae7cd8e2972ee0b38adbf72f64f4a53b1279b1a", + "src/drift/models/qwen2/config.py": "80bcfb41558ac0c032bc10f0c300eae9410de685422ccd87d954f07d28075f7c", + "src/drift/models/qwen2/model.py": "5ba1423a919bc62e3c82fab5d7e9fb4082eb5bd93a5e2e1ac7720d8ae3780c1f", + "src/drift/models/qwen3/__init__.py": "3383af165a62159090d54359dadba7b1e22e4b2b899bda883e84b530766e18a7", + "src/drift/models/qwen3/block.py": "14eeda65e40022486f14648c065432362c6b1348b072355e7bd30476dd248c42", + "src/drift/models/qwen3/config.py": "e11179215162b48cc3f2ef8fde31a17dcb9c6aa76b2651eae3c3067741d5528a", + "src/drift/models/qwen3/model.py": "7e6b416a8d84e6688db21c8bba2538b4b734c553721c9b682447d6f35ff14bbf", + "src/drift/models/qwen3_5/__init__.py": "7a5c3bc89c45a882832adced740b8d19c2dded51218a5d31fccaf098fc7408a5", + "src/drift/models/qwen3_5/block.py": "cb506f75ad664086ee5e750cfaa0cfd8ebf3b1b289efa210039bb38bf549165f", + "src/drift/models/qwen3_5/cache.py": "e7455ea6d38d30c0ca7dbdc8486697351cbfe6173ed00fd1022533e4d9fdb196", + "src/drift/models/qwen3_5/config.py": "6c467fc6948a2f06e2a7068864b473fa00b7d5cc6c5824420693a183d8a5a726", + "src/drift/models/qwen3_5/model.py": "b9092f62eb2a26881e2b2452b74f192bc6bf4e70f9091116cb6f7ac5a0a0dffb", + "src/drift/node/__init__.py": "4224d8431ac378cad874c88f9002da5585438cc2e514e5915777628c8fbc90bc", + "src/drift/node/catalog_bootstrap.py": "af426b0cadc1f5ddf2949e42d100883c761874324e04e07254ca8b5cd423442d", + "src/drift/node/catalog_refresh.py": "92412ad5915a812b28cff20aabf28e3d3e854cc88dcf7c90f0f93fdc53048bf3", + "src/drift/node/config.py": "699e7cb7a91c5d77dfc122aba09fbe0c431eb8c0660458fcc61729e35b065535", + "src/drift/node/config_lock.py": "c5f894e433d5d8ad3812c76940ca407674203880b258ab2c8e721e6d61f84108", + "src/drift/node/contribution_planner.py": "89792399821a5ed01e4a1cec42c0bcfce949eb8df05f2f8033e0f57a2f78be7d", + "src/drift/node/discovery.py": "6b9b36e777eb13d2331855429d15b0a4885066b43bed76d44492ee2cdfc3da18", + "src/drift/node/edge_acquisition.py": "4390bcdbe6172ac1633c45ea8991c2369677685c3e5c8c70d3be2baa9aeaca2f", + "src/drift/node/edge_benchmark.py": "ceea4869c93706a5b0162b71652aab96aaca4add953b331cf0dab4e90d6f05e7", + "src/drift/node/edge_supervisor.py": "34fafac20cba1daad65c8e09fe55bd67635fc6238904e1d96df66a1243fc9ff9", + "src/drift/node/keys.py": "3bf8d6209b2e8994f86011d4230f2f88e0f051e3f133ab8790f6f3fd4ed6662d", + "src/drift/node/loading.py": "dba38d80902782edbf7432c4c40423f5a49e1a54acc647c9001400d9aec41327", + "src/drift/node/local_inference.py": "0866368b4eab9fbcecd3311b6760fedba9369d0b3b4aa4dbcb71caf40c65fa78", + "src/drift/node/model_manager.py": "bfb97b0bb5ff39d69b28c4379dcd251c536219ebae5ea359f350319b92aee0ce", + "src/drift/node/model_selection.py": "5eb6dec25116a675490209e3283b33d3eabf70efb4895ba230ceb844f7352f92", + "src/drift/node/native_credentials.py": "c34bc1d01783eb3fc933911abbb87c191a5c4e4f60f3395920d5d3cd03e4dbf8", + "src/drift/node/policy_store.py": "bd88c40b1b610349c3718ff1ce9dd12c1901b83af5bce97976858cfe7846839f", + "src/drift/node/route_health.py": "2d5f5b8d5deb8a363eff50412646a3252a02869e7ce725964d63b0ae30ea32ec", + "src/drift/node/route_metrics.py": "2280fa0affb938856e54dc613cd3cadc1c671fe03635c2213a4a4e2905de473c", + "src/drift/node/server.py": "f8715f41dffc2e312eda3d83fb5ac26cac866d3264a3a0b3dced38ce93f9c101", + "src/drift/node/worker_supervisor.py": "ee5348d3ca6e632cd844830f1263222ede41ed21d4f39f233eb10e627e81f2bf", + "src/drift/protocol_identity.py": "0f35b4729929250875649a835af6b7fb38b9006f8f6e71e494ccbb906488373d", + "src/drift/server/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/drift/server/admission.py": "228021a291ad46ae24eb2e9ee886c71ce263264baa790ddcd8b481808df7d904", + "src/drift/server/backend.py": "38858453c0bd28c9941a0f9f9a88d1dfb0c04b363b19ef0cf1fac30776aa70da", + "src/drift/server/block_functions.py": "b9663eed654f339517162a279556336c556352ed195c70d33a1f55c7e841377e", + "src/drift/server/block_selection.py": "3f30f47c234352f5e0b087f8ae5790c91ea490c429bcf10a056ab608d29a4dca", + "src/drift/server/block_utils.py": "8968d8461edd11eff34a2c4bb189ccc251eaeae4b5a160b4bb834186023b4dcf", + "src/drift/server/from_pretrained.py": "9333d3a6c83bbe9b48854bbdc1e5cb0ce0849968f049043c58bca6b8b9b71271", + "src/drift/server/handler.py": "9aa18b62a7aeaeeb0fda87b9b505e1e513de7dc47ad03c4f3b707bb064b1058d", + "src/drift/server/health.py": "5442ed30c17daa6d2755ccb99c63b1f50a2a68a06290bc9aef33e617b14aff32", + "src/drift/server/memory_cache.py": "213c16fa03663e5eca120977a565225a291ba150227c72286891e0ca02a3d280", + "src/drift/server/reachability.py": "bea5c2889c1c2ceaf1ebcca2c6a34b5c6ae3623e81ff1d849797f8200a5bdac8", + "src/drift/server/rejection_logging.py": "8b15251165ed65a563cf18e0a807e95d155e9d215fc093c0a282a43768a67f32", + "src/drift/server/server.py": "02acea42248c07ff379ae5cab6eb14c40ef422b885c306cb5ef8ca1fb0b1d810", + "src/drift/server/task_pool.py": "6961c24ee8ecabf789322dfb28fb2f43b52e9aea77833670b24e78c341138d64", + "src/drift/server/task_prioritizer.py": "333c620a317d7c88aab4ebb099681212f55d7e22536e18da41f27e467ed93669", + "src/drift/server/throughput.py": "82bea37188e4f51947eec0c00660179a83d9c6b461d9bdb2b9b22a337dabbfc9", + "src/drift/utils/__init__.py": "83f00a50e5ec190e2c75552dd6ab81523f18b8c121b1dbf19ac7d23cf847fb55", + "src/drift/utils/asyncio.py": "3f45ab9fcd9e84c82691b7ffe9b8958d6e32b3cf62247d9ec802abb29681181a", + "src/drift/utils/auto_config.py": "641c04c2692349ed606d15e53da63e3cf628a5497049bd3e28dc508c2d6be3ad", + "src/drift/utils/convert_block.py": "15622a0a4a8fb887d686fd4e1a340d5f5044cf84ed0a40b3d0f119365d0609bf", + "src/drift/utils/dht.py": "3da5c73f0c81609681ee94a60dee541ceffcce7c8ead37ef2e360012d83c190e", + "src/drift/utils/disk_cache.py": "b85ffab6973593ffc5a1b49541d77e6ec07d0efbeaaaa1a407fe7530ac6beb42", + "src/drift/utils/file_lock.py": "f8530419e2419fefd7d6f89cb3ea6afcc135a9c39a0b6d711bc3eb2467cd8ce9", + "src/drift/utils/hardware.py": "39da5961f43b562bd151f861389a26abf4aba806021058a441f163481b5a217f", + "src/drift/utils/hub_ranges.py": "c548c8402dbeebf003b47f8fad7d59fb195d0035808c239a449e2140c7f4efed", + "src/drift/utils/join_token.py": "dc826a66cce14e830a2c03d7cc273f9cd9cc4c2678eb54d83fe86947961457c1", + "src/drift/utils/kv_cache.py": "a0c6d8796ac67c78b652d6c83113f189a2c6cbd8d934206dc782786b9167b74e", + "src/drift/utils/logging.py": "7ba375161fcc21b4972a70d7a40a645e782bd69d25f76e7f3ec4d2c8ce6fda96", + "src/drift/utils/misc.py": "4c1975f3353ddf8ad3e69fc3b8b11ffe47664c5bfc7962fa647ca32b0a2aed3b", + "src/drift/utils/packaging.py": "f20d92f013c0436baa34c3785643e33a7e2fc0912ddb90369ff5558e116f9765", + "src/drift/utils/peft.py": "a584251e0434d11dbe442292b37b627a4cd1cc30a52eeab82aa460a8d9ce3e64", + "src/drift/utils/ping.py": "d07b84504473d1fe00c0242ab6c1ae7774959291654a52e67113bb796c15eff4", + "src/drift/utils/process_lifetime.py": "2c254546516e7b6ae70eeeddfd597831c47f66cfe5a255e03c842f319119d0cc", + "src/drift/utils/random.py": "0b8a704a57dfdb28ffbf6d13c13374d7eb9aa4d316fbe596d5d46df2bae420cf", + "src/drift/utils/reference_model.py": "f32974976022ce342f21d87ac1c2fea980191b47abc78217e3d87f44c4b2b56b", + "src/drift/utils/server_registry.py": "82540da14725c9800a0a7971dce612ee3b1dfd0eafc6d16298c0502c81aee20c", + "src/drift/utils/shared_kv.py": "f08aa9320e2c147fa00ad5f982b0f51c573ab9736db4c886101cd62a2a5ae0b1", + "src/drift/utils/tensor_parallel/__init__.py": "4e98d8d0943c6c7cb7de15ac559fd5a1cb47c9370923d18155a958337df26b29", + "src/drift/utils/tensor_parallel/communications.py": "17115399ced4da75871c137236dd335eb5a71bf0d5f0e62674bf8f173b8241b0", + "src/drift/utils/tensor_parallel/configs.py": "1471f543381b9f85ddba902a111ee9b50a3c0b3ea6a76329c8309c88d3e15ae4", + "src/drift/utils/tensor_parallel/cross_device_ops.py": "968c01a5d54b86cdbbd9fdae5a374b94de99aa9e0546c12012e5674417de868b", + "src/drift/utils/tensor_parallel/slicer.py": "61c5c420dc72b7231727cfeace8dcbf46c8d65f04689a4e8ac7c2f1fefd53c0a", + "src/drift/utils/tensor_parallel/utils.py": "12727f9c9ba09e6049bb0128178af9a2e7ac2a7e9225ac845bea076c6cecc9b1", + "src/drift/utils/tensor_parallel/wrapper.py": "0e5f0e228dd80461871d514bbfd9f01ca2ee34cdfddf92d8c537b086147f2c83", + "src/drift/utils/version.py": "8d9520a26fa93f0ff733f6735ab2615328b2581d490e36f9eb701a6733c08b9f", + "src/drift/utils/weight_conversion.py": "64aceff28cd038d1529cdad0f52633d4680c2e50be9710decbc3accee21d10b7" + }, + "scope": "Current source snapshot; source cloud nodes, retained frozen Windows node, production Qt source" + }, + "cause": "Actual production Qt sharing-policy Save returned HTTP 503. renameat2(RENAME_EXCHANGE) failed with errno 22 in /srv/q38, /tmp and /dev/shm. Atomic persistence was retained.", + "local_answers": { + "desktop-formation-response-3428add90658b9b9603b0f24.json": { + "id": "3428add90658b9b9603b0f24", + "observed_at_unix": 1788760476.7144344, + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788760466, + "id": "cmpl-8e5c190389f74101bbae702c", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "result": "passed", + "seconds": 15.453490972518921 + }, + "q38mf-20260907-054716-44636d-c-formation-response-b1c54863432e87784971ab1e.json": { + "id": "b1c54863432e87784971ab1e", + "observed_at_unix": 1788760210.7421823, + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788760204, + "id": "cmpl-02061afa863f49f69f878a0e", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "result": "passed", + "seconds": 35.318546533584595 + }, + "q38mf-20260907-054716-44636d-w0-formation-response-b44766e35726261346b0c095.json": { + "id": "b44766e35726261346b0c095", + "observed_at_unix": 1788760267.0684235, + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788760258, + "id": "cmpl-682876d42d654636a265b9a5", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "result": "passed", + "seconds": 39.64627528190613 + }, + "q38mf-20260907-054716-44636d-w1-formation-response-79bc13d256943c5b0c94b8ad.json": { + "id": "79bc13d256943c5b0c94b8ad", + "observed_at_unix": 1788760325.5024073, + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788760320, + "id": "cmpl-9c14eec7ebbb45ba80e96373", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "result": "passed", + "seconds": 47.743013858795166 + }, + "q38mf-20260907-054716-44636d-w2-formation-response-db9df9c907b1f93c7a2e74a9.json": { + "id": "db9df9c907b1f93c7a2e74a9", + "observed_at_unix": 1788760392.9267492, + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788760385, + "id": "cmpl-4c902c64df234f628f2b85bd", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "result": "passed", + "seconds": 48.40693664550781 + }, + "q38mf-20260907-054716-44636d-w3-formation-response-22f81a51de9a18a55bc49963.json": { + "id": "22f81a51de9a18a55bc49963", + "observed_at_unix": 1788760443.824954, + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788760436, + "id": "cmpl-736348ced70b4c0c8dcadf6b", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "result": "passed", + "seconds": 38.21051907539368 + } + }, + "desktop_observations": { + "desktop-desktop-response-b98a6f184237e3516fe59ce4.json": { + "button_clicked": false, + "detail": "Selected a verified standalone model on this computer.", + "fake_node": false, + "id": "b98a6f184237e3516fe59ce4", + "inference_mode": "auto", + "observed_at_unix": 1788760486.150406, + "real_window_visible": true, + "result": "passed", + "screenshot": "desktop-b98a6f184237e3516fe59ce4.png", + "selection": { + "covered_blocks": 24, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "model": "Qwen3.5-0.8B-Local", + "peer_count": 0, + "reason": "Selected a verified standalone model on this computer.", + "source": "local", + "status": "selected", + "title": "auto selects Qwen3.5-0.8B-Local", + "total_blocks": 24 + }, + "source": "local", + "title": "auto selects Qwen3.5-0.8B-Local", + "ui_runtime": "production Qt source" + }, + "q38mf-20260907-054716-44636d-c-desktop-response-0ff68cfef716ab4c915fbba0.json": { + "button_clicked": false, + "detail": "Selected a verified standalone model on this computer.", + "fake_node": false, + "id": "0ff68cfef716ab4c915fbba0", + "inference_mode": "auto", + "observed_at_unix": 1788760221.9401457, + "real_window_visible": true, + "result": "passed", + "screenshot": "desktop-0ff68cfef716ab4c915fbba0.png", + "selection": { + "covered_blocks": 24, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "model": "Qwen3.5-0.8B-Local", + "peer_count": 0, + "reason": "Selected a verified standalone model on this computer.", + "source": "local", + "status": "selected", + "title": "auto selects Qwen3.5-0.8B-Local", + "total_blocks": 24 + }, + "source": "local", + "title": "auto selects Qwen3.5-0.8B-Local", + "ui_runtime": "production Qt source" + }, + "q38mf-20260907-054716-44636d-w0-desktop-response-cdd4e7b6624da33b5dae86a1.json": { + "button_clicked": false, + "detail": "Selected a verified standalone model on this computer.", + "fake_node": false, + "id": "cdd4e7b6624da33b5dae86a1", + "inference_mode": "auto", + "observed_at_unix": 1788760272.7169282, + "real_window_visible": true, + "result": "passed", + "screenshot": "desktop-cdd4e7b6624da33b5dae86a1.png", + "selection": { + "covered_blocks": 24, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "model": "Qwen3.5-0.8B-Local", + "peer_count": 0, + "reason": "Selected a verified standalone model on this computer.", + "source": "local", + "status": "selected", + "title": "auto selects Qwen3.5-0.8B-Local", + "total_blocks": 24 + }, + "source": "local", + "title": "auto selects Qwen3.5-0.8B-Local", + "ui_runtime": "production Qt source" + }, + "q38mf-20260907-054716-44636d-w1-desktop-response-cb9a3d60f625989ef84eb1db.json": { + "button_clicked": false, + "detail": "Selected a verified standalone model on this computer.", + "fake_node": false, + "id": "cb9a3d60f625989ef84eb1db", + "inference_mode": "auto", + "observed_at_unix": 1788760335.9681635, + "real_window_visible": true, + "result": "passed", + "screenshot": "desktop-cb9a3d60f625989ef84eb1db.png", + "selection": { + "covered_blocks": 24, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "model": "Qwen3.5-0.8B-Local", + "peer_count": 0, + "reason": "Selected a verified standalone model on this computer.", + "source": "local", + "status": "selected", + "title": "auto selects Qwen3.5-0.8B-Local", + "total_blocks": 24 + }, + "source": "local", + "title": "auto selects Qwen3.5-0.8B-Local", + "ui_runtime": "production Qt source" + }, + "q38mf-20260907-054716-44636d-w2-desktop-response-3cec0a281cfd910866308ec1.json": { + "button_clicked": false, + "detail": "Selected a verified standalone model on this computer.", + "fake_node": false, + "id": "3cec0a281cfd910866308ec1", + "inference_mode": "auto", + "observed_at_unix": 1788760400.019935, + "real_window_visible": true, + "result": "passed", + "screenshot": "desktop-3cec0a281cfd910866308ec1.png", + "selection": { + "covered_blocks": 24, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "model": "Qwen3.5-0.8B-Local", + "peer_count": 0, + "reason": "Selected a verified standalone model on this computer.", + "source": "local", + "status": "selected", + "title": "auto selects Qwen3.5-0.8B-Local", + "total_blocks": 24 + }, + "source": "local", + "title": "auto selects Qwen3.5-0.8B-Local", + "ui_runtime": "production Qt source" + }, + "q38mf-20260907-054716-44636d-w3-desktop-response-0cb7dcded7ff0555c429193a.json": { + "button_clicked": false, + "detail": "Selected a verified standalone model on this computer.", + "fake_node": false, + "id": "0cb7dcded7ff0555c429193a", + "inference_mode": "auto", + "observed_at_unix": 1788760451.623794, + "real_window_visible": true, + "result": "passed", + "screenshot": "desktop-0cb7dcded7ff0555c429193a.png", + "selection": { + "covered_blocks": 24, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "model": "Qwen3.5-0.8B-Local", + "peer_count": 0, + "reason": "Selected a verified standalone model on this computer.", + "source": "local", + "status": "selected", + "title": "auto selects Qwen3.5-0.8B-Local", + "total_blocks": 24 + }, + "source": "local", + "title": "auto selects Qwen3.5-0.8B-Local", + "ui_runtime": "production Qt source" + } + }, + "cleanup": { + "sandboxes": { + "q38mf-20260907-054716-44636d-c": { + "exit_code": 137, + "sandbox_id": "sb-XpGl6rHTKRudxsBpcfKFX3" + }, + "q38mf-20260907-054716-44636d-w0": { + "exit_code": 137, + "sandbox_id": "sb-ttc0l5OLeuwjtEpjvyEWbR" + }, + "q38mf-20260907-054716-44636d-w1": { + "exit_code": 137, + "sandbox_id": "sb-Pxc6BxDXEDgKQXRv3UD8Gd" + }, + "q38mf-20260907-054716-44636d-w2": { + "exit_code": 137, + "sandbox_id": "sb-41vQhHWB3GwtcgvoPL3Ll3" + }, + "q38mf-20260907-054716-44636d-w3": { + "exit_code": 137, + "sandbox_id": "sb-E3FBDFNwVZf4EqE4YDC9pi" + } + }, + "verified": true + }, + "app_cleanup": { + "observations": [ + { + "app_id": "ap-jzX96NEvA2MzUDK0YCtBhe", + "created_at": "2026-09-07 00:47:16-05:00", + "description": "q38mf-20260907-054716-44636d", + "state": "stopped", + "stopped_at": "2026-09-07 01:01:40-05:00", + "tasks": "0" + } + ], + "verified": true + }, + "local_cleanup_verified": true, + "operator_intervention": "Read-only diagnostics identified the platform incompatibility; an explicit error marker aborted the attempt. No successful formation is claimed.", + "persisted_followup": "Real-window and atomic-settings-exchange setup probe; immediate capture of production Qt sharing-action errors; GCP runner now also uses five actual remote Qt windows and their Save/Start controls." +} diff --git a/docs/evidence/qwen-packaged-cold-acquisition-20260906.json b/docs/evidence/qwen-packaged-cold-acquisition-20260906.json new file mode 100644 index 000000000..064a014fe --- /dev/null +++ b/docs/evidence/qwen-packaged-cold-acquisition-20260906.json @@ -0,0 +1,184 @@ +{ + "result": "passed", + "scope": "packaged-direct-Hub-cold-client-artifact-acquisition", + "packaged": true, + "node_sha256": "80957b3569ed9c976273eca9d5061e1d604bb04d380cd65e2f578d41f67bc1f1", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "cfbba0ab2d5c1fed6216a049b979d75019e521eb10bdb93b63e938a5b0a39fac", + "size_bytes": 2692397285 + }, + "manifest_file_sha256": "0591317317adf4752425725658732997597c51150a7b165bcd674cbc319bf2a1", + "started_at_unix": 1788672920.1102881, + "maximum_seconds": 10800, + "acquisition": { + "acquired_at_unix": 1788680102, + "artifacts": [ + { + "elapsed_seconds": 0.6119755000036093, + "materialization_attempts": 1, + "path": "chat_template.jinja", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "chat_template", + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "size_bytes": 8952 + }, + { + "elapsed_seconds": 0.4899148000040441, + "materialization_attempts": 1, + "path": "config.json", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "config", + "sha256": "74227dd615bf1ea975aa676bdf355a0379858c12f394b5365cd9dfa5fc2c70bc", + "size_bytes": 51350 + }, + { + "elapsed_seconds": 1.6342221000013524, + "materialization_attempts": 1, + "path": "merges.txt", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "tokenizer", + "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "size_bytes": 3353259 + }, + { + "elapsed_seconds": 0.5074022999979206, + "materialization_attempts": 1, + "path": "model.safetensors.index.json", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "weight_index", + "sha256": "f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2", + "size_bytes": 137335 + }, + { + "elapsed_seconds": 7150.6574702, + "materialization_attempts": 4, + "path": "outside.safetensors", + "resumed_from_bytes": [ + 1098907648, + 1098907648, + 4362076160 + ], + "resumptions": 3, + "role": "weight", + "sha256": "ddff1d6665a2b39f2612fce0ef955e2436724c565bfbcbc127c7ffd078b698ff", + "size_bytes": 6007102112 + }, + { + "elapsed_seconds": 15.560392800005502, + "materialization_attempts": 1, + "path": "tokenizer.json", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "tokenizer", + "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3", + "size_bytes": 12809320 + }, + { + "elapsed_seconds": 0.5792560000045341, + "materialization_attempts": 1, + "path": "tokenizer_config.json", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "tokenizer", + "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27", + "size_bytes": 17928 + }, + { + "elapsed_seconds": 2.311231300001964, + "materialization_attempts": 1, + "path": "vocab.json", + "resumed_from_bytes": [], + "resumptions": 0, + "role": "tokenizer", + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + "size_bytes": 6722759 + } + ], + "model": { + "dtype": "bfloat16", + "id": "Qwen3.8 27B FP8 Dequant", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "repository": "Qwen/Qwen3.8-27B-FP8", + "revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a" + }, + "privacy": { + "credentials_retained": false, + "local_paths_retained": false, + "response_bodies_retained": false, + "urls_retained": false + }, + "runtime": { + "drift": "2.3.0.dev2", + "platform": "Windows-10-10.0.19045-SP0", + "python": "3.12.9" + }, + "schema_version": 1, + "selection": { + "artifact_bytes": 6030203015, + "artifact_count": 8, + "startup_artifact_paths": [ + "chat_template.jinja", + "config.json", + "merges.txt", + "model.safetensors.index.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ], + "weight_artifact_bytes": 6007102112, + "weight_artifact_paths": [ + "outside.safetensors" + ] + }, + "storage": { + "cache_bytes_after": 6030203015, + "cache_bytes_before": 0, + "cache_growth_bytes": 6030203015, + "cold_start": true, + "verified": true + }, + "transfer": { + "completed": true, + "direct_upstream_transfer": true, + "elapsed_seconds": 7172.387451099996, + "max_resumptions": 3, + "mirror_used": false, + "resumptions": 3, + "source_class_verified": true, + "transport_override_present": false + } + }, + "seconds": 7185.235951900482, + "node_stopped": true, + "limits": [ + "Acquisition only: no distributed generation or desktop GUI inference claim", + "Frozen Windows v7 edge-acquire; the reusable cache will be checked by the later v9 client", + "One observed connection; elapsed time is not a general bandwidth guarantee" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-client-acquisition-v7/result.json", + "sha256": "06e9b5ec058785bd7fe60f42b0bd1ccdbd9c9dfee112e6cb78eb2cec77c90a50" + }, + { + "path": ".gate13-runs/qwen-product-client-acquisition-v7/acquisition.json", + "sha256": "44b071d16344c3103d3868ffd800de191b14b9b25755f13d1bc62350bdd78c09" + }, + { + "path": ".gate13-runs/qwen-product-build-v7/CommunityAI/node/CommunityAI-Node.exe", + "sha256": "80957b3569ed9c976273eca9d5061e1d604bb04d380cd65e2f578d41f67bc1f1" + } + ] +} diff --git a/docs/evidence/qwen-packaged-community-v9-20260906.json b/docs/evidence/qwen-packaged-community-v9-20260906.json new file mode 100644 index 000000000..7462473ac --- /dev/null +++ b/docs/evidence/qwen-packaged-community-v9-20260906.json @@ -0,0 +1,216 @@ +{ + "result": "passed", + "scope": "Windows packaged Qwen3.8 automatic selection, completion and short chat", + "run_id": "q38pm-20260906-070127-a792", + "catalog_scope": "staged signed public sequence 2, explicit test configuration", + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "ddc74b7aef1e29615458b930a03c8393a90dd5ec36ebed19528df2f7081d28a3", + "size_bytes": 2692402624 + }, + "source_commit": null, + "community_completion": { + "response": { + "id": "cmpl-4c104ba4c45b4245a657cccd", + "object": "text_completion", + "created": 1788681628, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 56.04699999999866 + }, + "community_chat": { + "response": { + "id": "chatcmpl-54fe359476b04175bae5f07c", + "object": "chat.completion", + "created": 1788681913, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "seconds": 285.82800000000134 + }, + "peak_observed_process_tree_rss_bytes": 4280983552, + "cache_scope": "Reused the separately verified direct-Hub cold acquisition; this run does not claim an empty cache", + "node_stopped": true, + "whole_packaged_exercise_result": "failed", + "unfinished_reason": "RuntimeError: Owned worker did not stop", + "worker_stop_diagnostic": { + "instance": "q38pm-20260906-070127-a792-w2", + "action": "start", + "before": "MainPID=0\nActiveState=failed\n", + "after": "MainPID=5054\nActiveState=active\n", + "observed_at_unix": 1788681983.6545467 + }, + "source_public_policy": { + "local_before_growth": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788678603, + "id": "cmpl-f1c138ac3d004b09adab836a", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 44.78853917999999 + }, + "qwen38_baseline": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788679685, + "id": "cmpl-8467459213c4469894e5af46", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 55.82814912999993 + }, + "active_answer_after_mode_change": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788679740, + "id": "cmpl-f4cee06101ef48e486bacfef", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 55.094758717999866 + }, + "local_only_next_request": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788679744, + "id": "cmpl-19bef85c7eb148afa3a694cf", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 3.9012851410000167 + } + }, + "source_injected_recovery": { + "result": "not-qualified", + "reason": "Source receipt completed before the orchestrator injected worker loss during the authentication interruption", + "source_finished_utc": "2026-09-06T07:39:04.115577+00:00", + "orchestrator_kill_utc": "2026-09-06T07:55:19.316051+00:00" + }, + "limits": [ + "Four assigned cloud spans and explicit owned seed configuration; not autonomous desktop formation", + "Frozen Windows node and desktop controller, not an ordinary-user GUI/install lifecycle", + "Packaged outage/fallback/rejoin and offline-Hub restart remain unfinished in this run", + "Single short chat and one 8 GB RTX 2070 SUPER host; not broad latency/concurrency qualification" + ], + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "overall_runner_result": "failed", + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/remote-packaged-v9-public/result.json", + "sha256": "5af4c2ccc62e802fecc36ca970ce31ef559db6c0614a3bf6653879b91cd98df8" + }, + { + "path": ".gate13-runs/qwen-product-build-v9/desktop-metrics.json", + "sha256": "2660bad7d4a76e5582d50c138d4e463e0f49fa0710cc113051f66fe5370718ee" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.json", + "sha256": "4566fd0c2dfeb18d3ef4945705ca672d6ff6df492ed4880edda775d51367645a" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-070127-a792/source-inventory.json", + "sha256": "f8141b16f8a34d92a7b408c370a34b6084dea8661611bce331bab3ac063a8bc2" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-070127-a792/product-result.json", + "sha256": "7959bcab519943b9cf9417736385ac52c3ed55a169ec93273bf30e5eaaadd2ab" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-070127-a792/result.json", + "sha256": "411725bcce2ce4a0ecd8f0ef84cf75693342693fae3d260746a3cc11a7778ae1" + } + ] +} diff --git a/docs/evidence/qwen-packaged-recovery-v9-c3-20260906.json b/docs/evidence/qwen-packaged-recovery-v9-c3-20260906.json new file mode 100644 index 000000000..b9fde9e99 --- /dev/null +++ b/docs/evidence/qwen-packaged-recovery-v9-c3-20260906.json @@ -0,0 +1,623 @@ +{ + "result": "passed", + "scope": "Windows v9 packaged public-policy recovery, local preference and HTTP-blocked cache restart on a C3 remainder route", + "run_id": "q38pm-20260906-091609-4351", + "packaged": true, + "source_commit": null, + "cpu_remainder_machine_type": "c3-highmem-4", + "gcp_instances": [ + { + "name": "q38pm-20260906-091609-4351-c", + "machine_type": "e2-standard-4" + }, + { + "name": "q38pm-20260906-091609-4351-w0", + "machine_type": "g2-standard-8" + }, + { + "name": "q38pm-20260906-091609-4351-w2", + "machine_type": "c3-highmem-4" + }, + { + "name": "q38pm-20260906-091609-4351-w3", + "machine_type": "c3-highmem-4" + } + ], + "cpu_hardware": { + "lscpu": [ + { + "field": "Architecture:", + "data": "x86_64" + }, + { + "field": "CPU op-mode(s):", + "data": "32-bit, 64-bit" + }, + { + "field": "Address sizes:", + "data": "52 bits physical, 57 bits virtual" + }, + { + "field": "Byte Order:", + "data": "Little Endian" + }, + { + "field": "CPU(s):", + "data": "4" + }, + { + "field": "On-line CPU(s) list:", + "data": "0-3" + }, + { + "field": "Vendor ID:", + "data": "GenuineIntel" + }, + { + "field": "Model name:", + "data": "Intel(R) Xeon(R) Platinum 8481C CPU @ 2.70GHz" + }, + { + "field": "CPU family:", + "data": "6" + }, + { + "field": "Model:", + "data": "143" + }, + { + "field": "Thread(s) per core:", + "data": "2" + }, + { + "field": "Core(s) per socket:", + "data": "2" + }, + { + "field": "Socket(s):", + "data": "1" + }, + { + "field": "Stepping:", + "data": "8" + }, + { + "field": "BogoMIPS:", + "data": "5399.99" + }, + { + "field": "Flags:", + "data": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid rtm avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 arat avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq la57 rdpid cldemote movdiri movdir64b fsrm md_clear serialize tsxldtrk amx_bf16 avx512_fp16 amx_tile amx_int8 arch_capabilities" + }, + { + "field": "Hypervisor vendor:", + "data": "KVM" + }, + { + "field": "Virtualization type:", + "data": "full" + }, + { + "field": "L1d cache:", + "data": "96 KiB (2 instances)" + }, + { + "field": "L1i cache:", + "data": "64 KiB (2 instances)" + }, + { + "field": "L2 cache:", + "data": "4 MiB (2 instances)" + }, + { + "field": "L3 cache:", + "data": "105 MiB (1 instance)" + }, + { + "field": "NUMA node(s):", + "data": "1" + }, + { + "field": "NUMA node0 CPU(s):", + "data": "0-3" + }, + { + "field": "Vulnerability Gather data sampling:", + "data": "Not affected" + }, + { + "field": "Vulnerability Ghostwrite:", + "data": "Not affected" + }, + { + "field": "Vulnerability Indirect target selection:", + "data": "Not affected" + }, + { + "field": "Vulnerability Itlb multihit:", + "data": "Not affected" + }, + { + "field": "Vulnerability L1tf:", + "data": "Not affected" + }, + { + "field": "Vulnerability Mds:", + "data": "Not affected" + }, + { + "field": "Vulnerability Meltdown:", + "data": "Not affected" + }, + { + "field": "Vulnerability Mmio stale data:", + "data": "Not affected" + }, + { + "field": "Vulnerability Old microcode:", + "data": "Not affected" + }, + { + "field": "Vulnerability Reg file data sampling:", + "data": "Not affected" + }, + { + "field": "Vulnerability Retbleed:", + "data": "Not affected" + }, + { + "field": "Vulnerability Spec rstack overflow:", + "data": "Not affected" + }, + { + "field": "Vulnerability Spec store bypass:", + "data": "Mitigation; Speculative Store Bypass disabled via prctl" + }, + { + "field": "Vulnerability Spectre v1:", + "data": "Mitigation; usercopy/swapgs barriers and __user pointer sanitization" + }, + { + "field": "Vulnerability Spectre v2:", + "data": "Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI BHI_DIS_S" + }, + { + "field": "Vulnerability Srbds:", + "data": "Not affected" + }, + { + "field": "Vulnerability Tsa:", + "data": "Not affected" + }, + { + "field": "Vulnerability Tsx async abort:", + "data": "Not affected" + }, + { + "field": "Vulnerability Vmscape:", + "data": "Not affected" + } + ] + }, + "catalog_scope": "staged signed public sequence 2, explicit test configuration", + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "remote_cache_source": "explicitly seeded cache", + "cold_acquisition_evidence": "qwen-packaged-cold-acquisition-20260906.json", + "phases": [ + { + "hub_offline": false, + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "application_replaced": false, + "community_completion": { + "response": { + "id": "cmpl-ec6f81e2e62945819299931e", + "object": "text_completion", + "created": 1788688250, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 12.25 + }, + "community_chat": { + "response": { + "id": "chatcmpl-f43b08322cb840db9ae33718", + "object": "chat.completion", + "created": 1788688269, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "seconds": 19.35899999999674 + }, + "local_only_completion": { + "response": { + "id": "cmpl-4293a41fda5c411a9b2bf811", + "object": "text_completion", + "created": 1788688464, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 0.3909999999887077 + }, + "intervening_local_fallbacks": [], + "peak_observed_process_tree_rss_bytes": 4973985792, + "node_stopped": true, + "worker_outage": { + "scope": "CPU worker service stop and same-identity restart", + "stop": { + "instance": "q38pm-20260906-091609-4351-w2", + "action": "stop", + "before": "MainPID=4405\nKillMode=control-group\nActiveState=active\n", + "after": "MainPID=0\nKillMode=control-group\nActiveState=failed\n", + "observed_at_unix": 1788688318.311622 + }, + "local_completion": { + "response": { + "id": "cmpl-9f63c3db4ae548a494769c1a", + "object": "text_completion", + "created": 1788688343, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 21.0 + }, + "restart": { + "instance": "q38pm-20260906-091609-4351-w2", + "action": "start", + "before": "MainPID=0\nKillMode=control-group\nActiveState=failed\n", + "after": "MainPID=5222\nKillMode=control-group\nActiveState=active\n", + "observed_at_unix": 1788688360.6170838 + }, + "community_after_rejoin": { + "response": { + "id": "cmpl-fd780559bd7d439b8756c7e1", + "object": "text_completion", + "created": 1788688464, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 13.672000000005937 + }, + "selected_before_stop": { + "selector": "auto", + "status": "selected", + "model": "Qwen3.8 27B FP8 Dequant", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "covered_blocks": 64, + "total_blocks": 64, + "peer_count": 4, + "source": "runtime" + }, + "fallback": { + "selector": "auto", + "status": "selected", + "model": "Qwen3.5-0.8B-Local", + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "reason": "Selected a verified standalone model on this computer.", + "covered_blocks": 24, + "total_blocks": 24, + "peer_count": 0, + "source": "local" + }, + "reselected_after_rejoin": { + "selector": "auto", + "status": "selected", + "model": "Qwen3.8 27B FP8 Dequant", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "covered_blocks": 64, + "total_blocks": 64, + "peer_count": 4, + "source": "runtime" + } + } + }, + { + "hub_offline": true, + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "application_replaced": false, + "community_completion": { + "response": { + "id": "cmpl-bd9ec9a7ed0a487e9cd271cc", + "object": "text_completion", + "created": 1788688551, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 12.438000000009197 + }, + "community_chat": { + "response": { + "id": "chatcmpl-ee98be1744b74d0882f16052", + "object": "chat.completion", + "created": 1788688573, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "seconds": 21.625 + }, + "local_only_completion": { + "response": { + "id": "cmpl-76a778e75e9c4013b6a02dcc", + "object": "text_completion", + "created": 1788688584, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 10.719000000011874 + }, + "intervening_local_fallbacks": [], + "peak_observed_process_tree_rss_bytes": 4758171648, + "node_stopped": true, + "http_downloads_blocked": true, + "denied_http_requests": 0 + } + ], + "node_stopped": true, + "overall_runner_result": "passed", + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "source_product_result": "passed", + "source_product_observations": { + "all_blocks": 64, + "qwen38_baseline": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788687918, + "id": "cmpl-42f968502f1e4f188a43ff5b", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 14.249802970000019 + }, + "active_answer_after_mode_change": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788687932, + "id": "cmpl-8605c987da5d4b6784ade5ca", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 14.145367668000063 + }, + "local_after_worker_loss": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788688005, + "id": "cmpl-b5c3a7bbee374c61a4c5f273", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 4.1058418729999175 + }, + "qwen38_after_replacement": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788688140, + "id": "cmpl-d1d81417877f4e67abc3535d", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 13.707472945000063 + }, + "worker_stopped_acknowledgement": { + "observed_at_unix": 1788687963.1939743, + "peer_id": "QmVJ8ycdNxBLRATLzyTCSjXYUc134GMhVmBajueEdJhJ8W", + "recovery_nonce": "9836f8404554f429b220b3bca96a7dfd" + }, + "worker_replaced_acknowledgement": { + "observed_at_unix": 1788688057.1014302, + "peer_id": "QmVMkBdTZsvRYDegZqUV78A6Po2gR8DbkiwhHb6vsmYnq6", + "recovery_nonce": "9836f8404554f429b220b3bca96a7dfd" + } + }, + "source_bundle_sha256": "c4da1feb23d17fc4412d683d473548b547c3c739e75e5f40046095f06a9b2ea1", + "limits": [ + "Four assigned cloud spans and explicit owned bootstrap seeds; not autonomous desktop formation", + "Packaged recovery stops a CPU worker between requests and restarts its same identity", + "Active generation replay after VM replacement is separately evidenced by the original CPU test", + "Hub-offline restart keeps the swarm network online; it proves reuse of cached model artifacts", + "Unsigned engineering package, frozen node/controller checks; ordinary-user UI lifecycle remains open", + "Single bounded Windows hardware case, not RTX 30/40/50 or Linux GPU qualification" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/result.json", + "sha256": "b4cb4d2a5d96dd4f2d4bd6eaec8116ac36bc386ccf40416871059a39ccc367d8" + }, + { + "path": ".gate13-runs/qwen-product-implementation/remote-packaged-v9-c3/result.json", + "sha256": "8856a5c562b8129d26a9318e0de7e9b01e67ab6fbdd12dd72481720079c93665" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/source-inventory.json", + "sha256": "9326ac45612b789913dea8f2a262fbd878d4521a44336f6228bf647b42d90700" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/packaged-harness-final-source.json", + "sha256": "5a56822a18b6da373bf954ef23700e15c2c6549218277b4ac9a396025ef76a89" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.json", + "sha256": "4566fd0c2dfeb18d3ef4945705ca672d6ff6df492ed4880edda775d51367645a" + }, + { + "path": ".gate13-runs/qwen-product-build-v9/desktop-metrics.json", + "sha256": "2660bad7d4a76e5582d50c138d4e463e0f49fa0710cc113051f66fe5370718ee" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/provider-config.json", + "sha256": "b65bea86ea68b20fe6ec87e3dd9875ab1c6414168315462a9355e46fe386ab9d" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/c3-cpu-hardware.json", + "sha256": "68dc5d5175d0f739bfe510661892c9e226127521cc7052153bccc82c4ead92c5" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/q38pm-20260906-091609-4351-c-instance.json", + "sha256": "df3c2e22da74f0b2678bdeee60e559863b8f86d1ab14e1f4597ebebc65383922" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/q38pm-20260906-091609-4351-w0-instance.json", + "sha256": "5f3ce86339d0beede18f99d179062a1f08b75240728c8b8f21e9a3a801280a9e" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/q38pm-20260906-091609-4351-w2-instance.json", + "sha256": "2a046485fc70b540508333daa2f09b9ec970f3052b66acecdb3abcbb34febd08" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-091609-4351/q38pm-20260906-091609-4351-w3-instance.json", + "sha256": "e4df29c15e28956de4e7cd6de724251b0768180157fb54b1ef7960c106fa33bf" + } + ] +} diff --git a/docs/evidence/qwen-packaged-rejoin-timeout-v9-20260906.json b/docs/evidence/qwen-packaged-rejoin-timeout-v9-20260906.json new file mode 100644 index 000000000..2f5dc92e8 --- /dev/null +++ b/docs/evidence/qwen-packaged-rejoin-timeout-v9-20260906.json @@ -0,0 +1,158 @@ +{ + "run_id": "q38pm-20260906-081449-5fdb", + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "overall_runner_result": "failed", + "topology": "GCP L4 + Azure T4 + two E2 CPU workers; assigned 16-block spans", + "source_commit": null, + "result": "failed", + "scope": "Windows v9 packaged public-policy stop/fallback/rejoin and offline cache restart attempt", + "node_sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45", + "node_stopped": true, + "error": "TimeoutError: Public catalog route thresholds were not satisfied", + "community_completion": { + "response": { + "id": "cmpl-25fc5dee16354181967be1b8", + "object": "text_completion", + "created": 1788684763, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 78.01600000000326 + }, + "community_chat": { + "response": { + "id": "chatcmpl-e9cafbbd117e40c89acce0b1", + "object": "chat.completion", + "created": 1788684903, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Paris\n" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35 + } + }, + "seconds": 140.59399999999732 + }, + "peak_observed_process_tree_rss_bytes": 4962603008, + "worker_outage": { + "scope": "CPU worker service stop and same-identity restart", + "stop": { + "instance": "q38pm-20260906-081449-5fdb-w2", + "action": "stop", + "before": "MainPID=4324\nKillMode=control-group\nActiveState=active\n", + "after": "MainPID=0\nKillMode=control-group\nActiveState=failed\n", + "observed_at_unix": 1788684957.1369302 + }, + "local_completion": { + "response": { + "id": "cmpl-941fabdea1a04cfa9ec6b15f", + "object": "text_completion", + "created": 1788684967, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "seconds": 10.438000000009197 + }, + "restart": { + "instance": "q38pm-20260906-081449-5fdb-w2", + "action": "start", + "before": "MainPID=0\nKillMode=control-group\nActiveState=failed\n", + "after": "MainPID=5191\nKillMode=control-group\nActiveState=active\n", + "observed_at_unix": 1788684985.3282397 + } + }, + "selected_before_stop": { + "selector": "auto", + "status": "selected", + "model": "Qwen3.8 27B FP8 Dequant", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "covered_blocks": 64, + "total_blocks": 64, + "peer_count": 4, + "source": "runtime" + }, + "fallback": { + "selector": "auto", + "status": "selected", + "model": "Qwen3.5-0.8B-Local", + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "reason": "Selected a verified standalone model on this computer.", + "covered_blocks": 24, + "total_blocks": 24, + "peer_count": 0, + "source": "local" + }, + "passed_observations": [ + "Automatic Qwen selection, actual completion and chat", + "Confirmed worker stop followed by local selection and actual local tokens" + ], + "unfinished": [ + "Automatic return to Qwen timed out ten minutes after the restart action", + "Offline-Hub cache restart was not reached" + ], + "limits": [ + "Readiness rejection does not identify a unique root cause; no retained per-probe timing in the frozen v9 log", + "Reused independently verified client cache; no fresh acquisition claim", + "Assigned cloud spans and explicit owned seeds; not autonomous desktop formation" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/remote-packaged-v9-recovery-retry/result.json", + "sha256": "3978008ab1510c11a175cffd1d413e1d7c3b48fe7e40c91c7b227571019172b6" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-081449-5fdb/result.json", + "sha256": "21c01588085064e6c95ead6e4d49b30b26a4697bbda5c8a48bd99a302bee9104" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-081449-5fdb/packaged-harness-final-source.json", + "sha256": "dde8783396a7033334954254f19961ae3f3fd3d0d2de92d2b18796b64eb642bf" + } + ] +} diff --git a/docs/evidence/qwen-power-recovery-20260906.json b/docs/evidence/qwen-power-recovery-20260906.json new file mode 100644 index 000000000..c50ce8371 --- /dev/null +++ b/docs/evidence/qwen-power-recovery-20260906.json @@ -0,0 +1,91 @@ +{ + "result": "passed", + "scope": "Windows-v9-real-GPU-power-pause-automatic-resumption-and-local-inference", + "hardware": "NVIDIA RTX 2070 SUPER, 8 GB VRAM", + "install_archive": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "ddc74b7aef1e29615458b930a03c8393a90dd5ec36ebed19528df2f7081d28a3", + "size_bytes": 2692402624 + }, + "worker_block_indices": "59:60", + "threshold_watts": 120.0, + "load": { + "seconds": 25.0, + "matrix_products": 6115 + }, + "first_over_threshold_sample_seconds": 9.030999999995402, + "paused_sample_seconds": 9.23399999999674, + "observed_threshold_crossing_to_paused_seconds": 0.20300000000133878, + "peak_observed_watts_before_pause": 134.938, + "paused_process_tree_gone": true, + "resumed_without_policy_change_or_start_command": true, + "local_after_recovery": { + "seconds": 0.43800000000192085, + "response": { + "id": "cmpl-f034a356394c4711a7cb3a44", + "object": "text_completion", + "created": 1788677673, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "manual_pause_seconds": 0.125, + "manual_pause_tree_gone": true, + "manual_restart_tree_gone": true, + "node_stopped": true, + "complete_gate14": false, + "limits": [ + "25-second external CUDA load, one threshold crossing on one Windows GPU", + "Sampled aggregate device power; stopping sharing cannot stop unrelated GPU load", + "The peak covers samples through worker pause, not the entire load interval", + "No bandwidth resumption, native Linux, long-soak, or OS hard-cap claim" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/sharing-power-recovery-v9/result.json", + "sha256": "b25e3e26f2a2418e629a3e8881cb188c9d5d77ca6e1eece179aa866573054338" + }, + { + "path": ".gate13-runs/qwen-product-implementation/sharing-power-recovery-v9/power-recovery-samples.json", + "sha256": "45ece1773f782b8d07a297a303f00822a774df16f5c3d92d302aceccc1831c8f" + }, + { + "path": ".gate13-runs/qwen-product-implementation/sharing-power-recovery-v9/bounded-gpu-load.log", + "sha256": "1dadc8ec52abd7686b4f40ef817e68906210ec2c4a662f0fa6f4f99a2ed6f556" + }, + { + "path": "scripts/qualify_qwen_sharing_product.py", + "sha256": "06858471bfa883443b1f2170d6878959c4e4795a0090692cee97cb510c18e908" + }, + { + "path": "scripts/qwen_bounded_gpu_load.py", + "sha256": "4e56a64d9db22813042817ccbf89d1e13ce6bd311f64ca207c72c58f63915294" + }, + { + "path": ".gate13-runs/qwen-product-build-v9/CommunityAI/node/CommunityAI-Node.exe", + "sha256": "981e0a713deeb341ac3c7dfb2359c28b965935461659cc63942ca1613654bb45" + }, + { + "path": ".gate13-runs/qwen-product-implementation/build-v9-source.json", + "sha256": "4566fd0c2dfeb18d3ef4945705ca672d6ff6df492ed4880edda775d51367645a" + } + ] +} diff --git a/docs/evidence/qwen-product-download-failure-20260906.json b/docs/evidence/qwen-product-download-failure-20260906.json new file mode 100644 index 000000000..c1c0626e9 --- /dev/null +++ b/docs/evidence/qwen-product-download-failure-20260906.json @@ -0,0 +1,31 @@ +{ + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "duration_seconds": 1835.5582807064056, + "error": "RuntimeError: q38pm-20260906-025141-96fa-w3 worker failed: {'error': 'ManifestTransferInterrupted', 'message': 'Interrupted download of layers-60.safetensors at byte 0: ConnectionError', 'observed_at_unix': 1788664443.8187575}", + "hardware_qualification": false, + "packaged_qualification": false, + "result": "failed", + "run_id": "q38pm-20260906-025141-96fa", + "scope": "production-node-model-transitions", + "topology": "gcp-l4-azure-t4-cpu", + "failure_analysis": "A Hub TCP connection reset interrupted the first finite range of layers-60.safetensors. The run failed before full-route product inference; automatic cleanup was verified. Downloader now retries transient range transfers up to three times with backoff, retaining strict size/range/hash verification.", + "files": { + "source-inventory.json": "6c01d0d3ddc7db06b2bbae075ec363256882985312cd768b19b2c9acbe1eeeef", + "result.json": "85a959cda02568f85d63f004dfe760fecbc894d423f6fb6c67defee14c625d82", + "cleanup.json": "2590118322c53b1b505e856e38f36f70cf507af37873c18d5f058eb0e092d600", + "q38pm-20260906-025141-96fa-w3-diagnostics.txt": "fab86097d56b6b1ad6222f6ac6440fbcaf8e0cd307dc49ac4a58f0c5da8242d0" + } +} diff --git a/docs/evidence/qwen-reference-parity-20260906.json b/docs/evidence/qwen-reference-parity-20260906.json new file mode 100644 index 000000000..4c48b5a65 --- /dev/null +++ b/docs/evidence/qwen-reference-parity-20260906.json @@ -0,0 +1,158 @@ +{ + "cleanup": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "duration_seconds": 1343.6327471733093, + "evidence": { + "atol": 0.5, + "cached_decode_steps_per_prompt": 2, + "checks": [ + { + "actual_token": 11751, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.1328125, + "mean_absolute_error": 0.02063351683318615, + "prompt_index": 0, + "reference_token": 11751, + "seconds": 13.845619147000207, + "step": 0 + }, + { + "actual_token": 13, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.095703125, + "mean_absolute_error": 0.014627886936068535, + "prompt_index": 0, + "reference_token": 13, + "seconds": 4.654403945000013, + "step": 1 + }, + { + "actual_token": 198, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.125, + "mean_absolute_error": 0.01568158157169819, + "prompt_index": 0, + "reference_token": 198, + "seconds": 4.775757784999996, + "step": 2 + }, + { + "actual_token": 4097, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.15625, + "mean_absolute_error": 0.015451163984835148, + "prompt_index": 1, + "reference_token": 4097, + "seconds": 12.06427054400001, + "step": 0 + }, + { + "actual_token": 13, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.09375, + "mean_absolute_error": 0.015487470664083958, + "prompt_index": 1, + "reference_token": 13, + "seconds": 4.747066725999957, + "step": 1 + }, + { + "actual_token": 198, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.125, + "mean_absolute_error": 0.021709345281124115, + "prompt_index": 1, + "reference_token": 198, + "seconds": 4.669147470000098, + "step": 2 + }, + { + "actual_token": 271, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.25, + "mean_absolute_error": 0.027140162885189056, + "prompt_index": 2, + "reference_token": 271, + "seconds": 23.422649428999875, + "step": 0 + }, + { + "actual_token": 248068, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.25, + "mean_absolute_error": 0.027111373841762543, + "prompt_index": 2, + "reference_token": 248068, + "seconds": 4.702008863999936, + "step": 1 + }, + { + "actual_token": 198, + "all_vocabulary_logits_within_tolerance": true, + "finite": true, + "greedy_token_equal": true, + "max_absolute_error": 0.28125, + "mean_absolute_error": 0.03989838436245918, + "prompt_index": 2, + "reference_token": 198, + "seconds": 4.644652351999866, + "step": 2 + } + ], + "cross_host_qualification": false, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "prompts": [ + "The capital of France is", + "Two plus three equals", + "In one sentence, explain why the sky looks blue." + ], + "result": "passed", + "revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", + "rtol": 0.01, + "scope": "stock-versus-four-RPC-worker numerical reference on one CPU host", + "stock_class": "Qwen3_5ForConditionalGeneration", + "stock_dequantizer": "Transformers FineGrainedFP8Config(dequantize=True)", + "time": 1788659887.9600813, + "torch": "2.6.0+cpu", + "transformers": "5.13.1" + }, + "result": "passed", + "run_id": "q38r-20260906-013804-f9f3", + "scope": "stock-reference-numerical-qualification", + "recorded_at": "2026-09-06", + "source_bundle_sha256": "a05e07b49d02fe3ee95867f2b2a23d276b87a6224bdeb9c897bdb3636537cb83", + "source_files": { + "scripts/qwen_reference_inference_host.py": "82e73a1e6552e5e1c9c652f65b51c42df848c924ba845eed6d8d32c29f912258", + "src/drift/models/qwen3_5/model.py": "b9092f62eb2a26881e2b2452b74f192bc6bf4e70f9091116cb6f7ac5a0a0dffb", + "src/drift/models/qwen3_5/block.py": "cb506f75ad664086ee5e750cfaa0cfd8ebf3b1b289efa210039bb38bf549165f", + "src/drift/utils/dht.py": "efeb09884638c54118d3f62aada67a5fa5f1e32e7f2f43cf6e53570e6309e151" + }, + "earlier_failed_attempt": { + "run_id": "q38r-20260906-012725-8711", + "failure": "harness parallel first materialization raced snapshot root before inference", + "cleanup_verified": true + } +} diff --git a/docs/evidence/qwen-resource-controls-20260906.json b/docs/evidence/qwen-resource-controls-20260906.json new file mode 100644 index 000000000..612d1404f --- /dev/null +++ b/docs/evidence/qwen-resource-controls-20260906.json @@ -0,0 +1,109 @@ +{ + "result": "passed", + "packaged": true, + "scope": "resource-admission-guards", + "checks": { + "schedule": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": false, + "schedule_reason": "outside the configured contribution schedule", + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + }, + "power": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "power usage 55.24 W exceeds the 1.00 W contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": 55.241, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + }, + "bandwidth": { + "state": "paused", + "pid": null, + "policy_admitted": true, + "policy_reason": null, + "resource_admitted": false, + "resource_reason": "bandwidth usage 12.24 Mbps exceeds the 0.00 Mbps contribution budget", + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": 12.235487179277172, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + }, + "storage": { + "state": "paused", + "pid": null, + "policy_admitted": false, + "policy_reason": "Qwen3.8 27B FP8 Dequant: every 1-block artifact set exceeds the 1048576-byte disk budget", + "resource_admitted": true, + "resource_reason": null, + "schedule_admitted": true, + "schedule_reason": null, + "current_bandwidth_mbps": null, + "current_power_watts": null, + "max_disk_space_bytes": null, + "max_vram_bytes": 2147483648, + "local_tokens": 3 + } + }, + "node_sha256": "270e8a7482d4e5c689644e1d53c48afb82a75aa49547aa994f12cc424997ffa3", + "complete_gate14": false, + "limitations": [ + "Power and bandwidth are sampled host telemetry pause guards, not OS hard caps or traffic shapers.", + "Tiny thresholds prove blocked admission; sustained load, overshoot and automatic resumption are separate checks.", + "Storage checks declared manifested artifact admission, not total disk-cache quota.", + "Windows control API test; no Linux or literal desktop UI acceptance." + ], + "node_stopped": true, + "recorded_at": "2026-09-06", + "hardware": "Windows, NVIDIA RTX 2070 SUPER, 8 GB", + "other_admission_guards_open_for_each_checked_guard": true, + "package": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "c95c1b1f94eba68a04ffc54b8dc17a49e425390eca053ef8f3996efdcc9dbf1d", + "size_bytes": 2692398749, + "binding": { + "path": ".gate13-runs/qwen-product-build-v6/desktop-metrics.json", + "sha256": "fe79b3477766ff6d28744553b7e49da45dda5190aebacdb6cb9378f47900a0b2" + }, + "real_local_result_binding": { + "path": ".gate13-runs/qwen-product-implementation/local-packaged-v6/result.json", + "sha256": "7d8379681c985e1ea3371806cfdce187d911e6552cf1eed68850fdef46da3102" + } + }, + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/resource-controls-v6-final/result.json", + "sha256": "f31992f12391c32603329a0371e3776bba49d5d1bff0fc77dbdf444633aaf138" + }, + { + "path": "scripts/qualify_qwen_resource_controls.py", + "sha256": "d722ae1c6149230c67d3228aa3a7b9ea909eabc819475291c6e4ca848ab89f8e" + } + ] +} diff --git a/docs/evidence/qwen-sharing-packaged-20260906.json b/docs/evidence/qwen-sharing-packaged-20260906.json new file mode 100644 index 000000000..cccbb2421 --- /dev/null +++ b/docs/evidence/qwen-sharing-packaged-20260906.json @@ -0,0 +1,149 @@ +{ + "result": "passed", + "scope": "local-inference-and-one-automatic-contribution-worker", + "packaged": true, + "complete_gate14": false, + "worker_budget_bytes": 2147483648, + "local_budget_bytes": 3221225472, + "before_sharing": { + "seconds": 6.8439999999973224, + "response": { + "id": "cmpl-b69e106fd01042c7873d0061", + "object": "text_completion", + "created": 1788667698, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "while_sharing": { + "seconds": 0.5, + "response": { + "id": "cmpl-7e5332a53fb24b8e884a8a2b", + "object": "text_completion", + "created": 1788667738, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "after_pause": { + "seconds": 0.4219999999986612, + "response": { + "id": "cmpl-1c0f81410811480699e5bb42", + "object": "text_completion", + "created": 1788667739, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "after_restart": { + "seconds": 0.4530000000013388, + "response": { + "id": "cmpl-4e64fa9c752a478fa9ff6f53", + "object": "text_completion", + "created": 1788667764, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "pause_seconds": 0.11000000000058208, + "paused_worker_tree_gone": true, + "restarted_worker_tree_gone": true, + "node_stopped": true, + "recorded_at": "2026-09-06", + "hardware": "Windows, NVIDIA RTX 2070 SUPER, 8 GB", + "limitations": [ + "One automatic block using verified existing cache; not full consumer swarm formation.", + "Does not qualify bandwidth, power, schedules, Linux or all Gate 14 controls.", + "Explicit test configuration; not clean online catalog onboarding." + ], + "package": { + "artifact_root": "CommunityAI", + "entry_count": 5924, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "c95c1b1f94eba68a04ffc54b8dc17a49e425390eca053ef8f3996efdcc9dbf1d", + "size_bytes": 2692398749, + "binding": { + "path": ".gate13-runs/qwen-product-build-v6/desktop-metrics.json", + "sha256": "fe79b3477766ff6d28744553b7e49da45dda5190aebacdb6cb9378f47900a0b2" + }, + "real_local_result_binding": { + "path": ".gate13-runs/qwen-product-implementation/local-packaged-v6/result.json", + "sha256": "7d8379681c985e1ea3371806cfdce187d911e6552cf1eed68850fdef46da3102" + } + }, + "automatic_ready": { + "state": "running", + "model": "Qwen3.8 27B FP8 Dequant", + "block_indices": "59:60", + "remote_acknowledged": true + }, + "restarted": { + "state": "running", + "model": "Qwen3.8 27B FP8 Dequant", + "block_indices": "59:60", + "remote_acknowledged": true + }, + "bindings": [ + { + "path": ".gate13-runs/qwen-product-implementation/sharing-packaged-v6/result.json", + "sha256": "8eaa4867be54c142186d9b240fd2af150e890780e9deca373b3c8262b331aa96" + }, + { + "path": ".gate13-runs/qwen-product-implementation/sharing-packaged-v6/node.log", + "sha256": "2acbc1d93636af2400de8379dd290fd7e352995e2ec61f1f8c01501f2ca477c5" + }, + { + "path": ".gate13-runs/qwen-product-build-v6/desktop-metrics.json", + "sha256": "fe79b3477766ff6d28744553b7e49da45dda5190aebacdb6cb9378f47900a0b2" + } + ] +} diff --git a/docs/evidence/qwen-sharing-source-20260906.json b/docs/evidence/qwen-sharing-source-20260906.json new file mode 100644 index 000000000..903c1518f --- /dev/null +++ b/docs/evidence/qwen-sharing-source-20260906.json @@ -0,0 +1,158 @@ +{ + "result": "passed", + "scope": "local-inference-and-one-automatic-contribution-worker", + "packaged": false, + "complete_gate14": false, + "worker_budget_bytes": 2147483648, + "local_budget_bytes": 3221225472, + "before_sharing": { + "seconds": 10.43800000000192, + "response": { + "id": "cmpl-f3b5396328084a4782b4c871", + "object": "text_completion", + "created": 1788666739, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "while_sharing": { + "seconds": 0.375, + "response": { + "id": "cmpl-a71fa1d493634036aae38c42", + "object": "text_completion", + "created": 1788667607, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "after_pause": { + "seconds": 0.3440000000045984, + "response": { + "id": "cmpl-6b96700e13814263934cccfb", + "object": "text_completion", + "created": 1788667607, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "after_restart": { + "seconds": 0.46899999999732245, + "response": { + "id": "cmpl-c69b73bbe2d44da6895ee053", + "object": "text_completion", + "created": 1788667648, + "model": "Qwen3.5-0.8B-Local", + "choices": [ + { + "index": 0, + "text": " Paris.\n", + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + } + }, + "pause_seconds": 0.26599999999598367, + "paused_worker_tree_gone": true, + "restarted_worker_tree_gone": true, + "node_stopped": true, + "recorded_at": "2026-09-06", + "hardware": "Windows, NVIDIA RTX 2070 SUPER, 8 GB", + "limitations": [ + "One automatic block; does not establish full consumer swarm formation or all Gate 14 controls.", + "Bootstrap joins failed before automatic retries succeeded; cold selected block download required about ten minutes.", + "Prior cache contained block 59; this run selected block 6. Restart reused verified block 6.", + "Packaged result is recorded independently." + ], + "automatic_ready": { + "state": "running", + "model": "Qwen3.8 27B FP8 Dequant", + "block_indices": "6:7", + "remote_acknowledged": true + }, + "restarted": { + "state": "running", + "model": "Qwen3.8 27B FP8 Dequant", + "block_indices": "6:7", + "remote_acknowledged": true + }, + "bindings": [ + { + "path": "src/drift/cli/run_node.py", + "sha256": "13e35c335d36e719f70e87eeee436577c9604b753c1cc0c59dfd60584262a689" + }, + { + "path": "src/drift/model_manifest.py", + "sha256": "b3456cbc7da7ff368886d32ba407899cce3cc00cedd4b51aaa04e3a1ae8027f0" + }, + { + "path": "src/drift/utils/hub_ranges.py", + "sha256": "6e0b5a9a91fe70e11e3e4b35086236eedd8d1c57bd6eef027e1244c00cc36fe4" + }, + { + "path": "src/drift/utils/dht.py", + "sha256": "3da5c73f0c81609681ee94a60dee541ceffcce7c8ead37ef2e360012d83c190e" + }, + { + "path": "src/drift/node/discovery.py", + "sha256": "6b9b36e777eb13d2331855429d15b0a4885066b43bed76d44492ee2cdfc3da18" + }, + { + "path": "src/drift/server/server.py", + "sha256": "02acea42248c07ff379ae5cab6eb14c40ef422b885c306cb5ef8ca1fb0b1d810" + }, + { + "path": "src/drift/node/model_selection.py", + "sha256": "de2e2f4aa9d5a06e33afbde8277f780ab55b5c7bdc5d9ab68a7134d7fd0f3e0e" + }, + { + "path": "scripts/qualify_qwen_sharing_product.py", + "sha256": "366f322b046882a98126d0d106e5670760e22c576c213b817abd73aebdf9b991" + }, + { + "path": ".gate13-runs/qwen-product-implementation/sharing-source-release/result.json", + "sha256": "5c210598f5716d727b6198d7f09fb4195698cc89ae74bb6f6ed07aa2fa2ed546" + }, + { + "path": ".gate13-runs/qwen-product-implementation/sharing-source-release/node.log", + "sha256": "7157a5194c47efe0bc5a149d3a8e1c5b02572bfafc17aac173d9b143ff710891" + } + ] +} diff --git a/docs/evidence/qwen-source-public-recovery-20260906.json b/docs/evidence/qwen-source-public-recovery-20260906.json new file mode 100644 index 000000000..30404f802 --- /dev/null +++ b/docs/evidence/qwen-source-public-recovery-20260906.json @@ -0,0 +1,545 @@ +{ + "run_id": "q38pm-20260906-081449-5fdb", + "cleanup": { + "azure_group_absent": true, + "errors": [], + "gcp": { + "errors": [], + "remaining": { + "disks": [], + "firewall-rules": [], + "instances": [] + }, + "verified": true + }, + "verified": true + }, + "overall_runner_result": "failed", + "topology": "GCP L4 + Azure T4 + two E2 CPU workers; assigned 16-block spans", + "source_commit": null, + "result": "passed", + "scope": "Source node, signed public sequence 2: promotion, local preference and controlled new-peer recovery", + "evidence": { + "active_answer_after_mode_change": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788684186, + "id": "cmpl-ca236b868be8471482212708", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 70.55240217200003 + }, + "all_blocks": 64, + "catalog_scope": "staged signed public sequence 2; explicit owned test seed configuration", + "local_after_worker_loss": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788684241, + "id": "cmpl-b15b76e124ef443cbe325082", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 7.647138679999898 + }, + "local_before_growth": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788683035, + "id": "cmpl-2ff8051b73d8489487ede33a", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 70.13827751600002 + }, + "local_fallbacks_before_active_transition": [], + "local_only_next_request": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788684192, + "id": "cmpl-2c4aba86f32746daa4ee64d3", + "model": "Qwen3.5-0.8B-Local", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 5.893289684000138 + }, + "manifest_digests": [ + "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + ], + "observed_at_unix": 1788684541.5637813, + "packaged": false, + "promoted_status": { + "api_version": 1, + "auto_selection": { + "covered_blocks": 64, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "model": "Qwen3.8 27B FP8 Dequant", + "peer_count": 4, + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "selector": "auto", + "source": "runtime", + "status": "selected", + "total_blocks": 64 + }, + "contribution": { + "configured": true, + "editable": true, + "policy": { + "config_revision": "sha256:55af22e166beb11b7a689cc8eddaad64e0e8bc3aaa0f593d2b42f3a334036447", + "policy": { + "allowed_models": [], + "denied_models": [], + "max_bandwidth_mbps": null, + "max_disk_space": null, + "max_power_watts": null, + "max_vram": null, + "pause_timeout": 10.0, + "preferred_models": [], + "schedule": null, + "sharing_enabled": false + }, + "schema_version": 1 + }, + "schema_version": 3, + "workers": [] + }, + "inference_mode": "auto", + "inference_mode_editable": true, + "models": [ + { + "active_requests": 0, + "aliases": [ + "local-qwen", + "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0" + ], + "download": { + "schema_version": 1, + "selected_whole_shard_bytes": 1769904871 + }, + "id": "Qwen3.5-0.8B-Local", + "last_error": null, + "last_used_at": 1788683035.732954, + "loaded_at": 1788683031.4489868, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "repository": "Qwen/Qwen3.5-0.8B", + "route": { + "covered_blocks": 24, + "device": "cpu", + "last_updated_age": 0.0, + "peer_count": 0, + "source": "local", + "status": "complete", + "total_blocks": 24 + }, + "state": "ready" + }, + { + "active_requests": 0, + "aliases": [ + "qwen3.8-27b", + "qwen3.8-27b-fp8", + "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + ], + "download": { + "schema_version": 1, + "selected_whole_shard_bytes": null + }, + "id": "Qwen3.8 27B FP8 Dequant", + "last_error": null, + "last_used_at": 1788684040.266354, + "loaded_at": 1788683960.5176191, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "repository": "Qwen/Qwen3.8-27B-FP8", + "route": { + "coverage_fingerprint": "bf9da8e3f6d55ec05438014daf9f8447a5d09da03b56e6fe489caa1e7aba3996", + "covered_blocks": 64, + "independent_routes": 1, + "last_error": null, + "last_updated_age": 19.786563946999877, + "minimum_replicas": 1, + "missing_blocks": [], + "peer_count": 4, + "replica_counts": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "replicas_after_largest_peer_loss": 0, + "source": "runtime", + "status": "complete", + "total_blocks": 64 + }, + "state": "ready" + } + ], + "openai_base_url": "http://127.0.0.1:8080/v1", + "runtime_budget": { + "max_loaded_models": 2, + "resident_models": 2 + }, + "started_at": 1788682964, + "status": "running", + "workers": [] + }, + "qwen38_after_replacement": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788684540, + "id": "cmpl-23cbf3d822184f8b84dbf62b", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 103.85260538699981 + }, + "qwen38_baseline": { + "response": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "text": " Paris.\n" + } + ], + "created": 1788684115, + "id": "cmpl-8259351fac53465690cafa66", + "model": "Qwen3.8 27B FP8 Dequant", + "object": "text_completion", + "usage": { + "completion_tokens": 3, + "prompt_tokens": 5, + "total_tokens": 8 + } + }, + "seconds": 74.09326549599996 + }, + "recovered_status": { + "api_version": 1, + "auto_selection": { + "covered_blocks": 64, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "model": "Qwen3.8 27B FP8 Dequant", + "peer_count": 4, + "reason": "Selected catalog priority 1: live discovery reports a complete 64/64-block route from 4 verified peers.", + "selector": "auto", + "source": "runtime", + "status": "selected", + "total_blocks": 64 + }, + "contribution": { + "configured": true, + "editable": true, + "policy": { + "config_revision": "sha256:ffff58b75e962183b07f14a4d6ea30ed461d62d32e1cbc2d1cce605a43d5462a", + "policy": { + "allowed_models": [], + "denied_models": [], + "max_bandwidth_mbps": null, + "max_disk_space": null, + "max_power_watts": null, + "max_vram": null, + "pause_timeout": 10.0, + "preferred_models": [], + "schedule": null, + "sharing_enabled": false + }, + "schema_version": 1 + }, + "schema_version": 3, + "workers": [] + }, + "inference_mode": "auto", + "inference_mode_editable": true, + "models": [ + { + "active_requests": 0, + "aliases": [ + "local-qwen", + "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0" + ], + "download": { + "schema_version": 1, + "selected_whole_shard_bytes": 1769904871 + }, + "id": "Qwen3.5-0.8B-Local", + "last_error": null, + "last_used_at": 1788684241.140227, + "loaded_at": 1788683031.4489868, + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "repository": "Qwen/Qwen3.5-0.8B", + "route": { + "covered_blocks": 24, + "device": "cpu", + "last_updated_age": 0.0, + "peer_count": 0, + "source": "local", + "status": "complete", + "total_blocks": 24 + }, + "state": "ready" + }, + { + "active_requests": 0, + "aliases": [ + "qwen3.8-27b", + "qwen3.8-27b-fp8", + "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + ], + "download": { + "schema_version": 1, + "selected_whole_shard_bytes": null + }, + "id": "Qwen3.8 27B FP8 Dequant", + "last_error": null, + "last_used_at": 1788684434.2402947, + "loaded_at": 1788683960.5176191, + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "repository": "Qwen/Qwen3.8-27B-FP8", + "route": { + "coverage_fingerprint": "25d10f64d8cb92deccb7a153242033b9f8b60eb1c269bba3e047824dca1124c9", + "covered_blocks": 64, + "independent_routes": 1, + "last_error": null, + "last_updated_age": 50.24731808399997, + "minimum_replicas": 1, + "missing_blocks": [], + "peer_count": 4, + "replica_counts": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "replicas_after_largest_peer_loss": 0, + "source": "runtime", + "status": "complete", + "total_blocks": 64 + }, + "state": "ready" + } + ], + "openai_base_url": "http://127.0.0.1:8080/v1", + "runtime_budget": { + "max_loaded_models": 2, + "resident_models": 2 + }, + "started_at": 1788682964, + "status": "running", + "workers": [] + }, + "result": "passed", + "worker_replaced_acknowledgement": { + "observed_at_unix": 1788684268.2183309, + "peer_id": "QmaCwR4NFyCwv6BnTMF2wT14vS5RuWcRtQnfjfNhBup2GK", + "recovery_nonce": "8c3adcd409112258e03c5d80a922563c" + }, + "worker_stopped_acknowledgement": { + "observed_at_unix": 1788684204.9745443, + "peer_id": "QmfLeev8xgZwZ7P392q5VHgd89MrPTy9JQJgzHWYYrYjPh", + "recovery_nonce": "8c3adcd409112258e03c5d80a922563c" + } + }, + "replacement_peer_ids": { + "before": "QmfLeev8xgZwZ7P392q5VHgd89MrPTy9JQJgzHWYYrYjPh", + "after": "QmaCwR4NFyCwv6BnTMF2wT14vS5RuWcRtQnfjfNhBup2GK" + }, + "limits": [ + "Explicit owned bootstrap seeds and assigned spans; not autonomous desktop formation", + "Source node, not a Windows packaged recovery qualification", + "Bounded short synthetic prompts; no broader performance claim" + ], + "bindings": [ + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-081449-5fdb/product-result.json", + "sha256": "2a1eead36018c54a9666b2658b05a5f3a509dda2dbe343f35c67568d6b839739" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-081449-5fdb/result.json", + "sha256": "21c01588085064e6c95ead6e4d49b30b26a4697bbda5c8a48bd99a302bee9104" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-081449-5fdb/events.jsonl", + "sha256": "4e92a8b7445e1f419997b8416817d7d6cd1a1bededbce7d08f42fef47cdba153" + }, + { + "path": ".gate13-runs/qwen-product-mixed/q38pm-20260906-081449-5fdb/source-inventory.json", + "sha256": "bacb551ad5bcea4ec2e27fd4f5dbdcd597e54992191c7584f64c52db7f22b180" + } + ] +} diff --git a/docs/evidence/qwen-windows-installers-20260907.json b/docs/evidence/qwen-windows-installers-20260907.json new file mode 100644 index 000000000..527f42fef --- /dev/null +++ b/docs/evidence/qwen-windows-installers-20260907.json @@ -0,0 +1,157 @@ +{ + "schema_version": 1, + "date": "2026-09-07", + "result": "passed", + "scope": "bounded Windows packaged Qwen controls and Inno replacement/removal", + "complete_gate14": false, + "complete_gate15": false, + "public_release": false, + "source_commit": "0b875c19efd952342671371ddacebc09ca3a774c", + "source_tree": "840d9c570d0905cd317f77d723194f00d4fc0b07", + "platform": "Windows-10-10.0.19045-SP0", + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_memory_bytes": 8589606912, + "runtime": { + "application": "CommunityAI-Node", + "catalog_bootstrap_schema": 1, + "drift": "2.3.0.dev2", + "fastapi": "0.141.1", + "frozen": true, + "hivemind": "1.1.12", + "keyring": "25.7.0", + "p2pd": "p2pd.exe", + "schema_version": 1, + "torch": "2.6.0+cu124", + "transformers": "5.13.1", + "uvicorn": "0.52.4" + }, + "archive": { + "artifact_root": "CommunityAI", + "entry_count": 5948, + "format": "zip", + "path": "communityai-desktop-windows.zip", + "platform": "Windows", + "preserves_executable_modes": false, + "preserves_internal_file_symlinks": false, + "schema_version": 1, + "sha256": "c5d59f4ae8c057315cb50fb0dadc211e36ec3ea58e2952c4493f9e39c4ce29fb", + "size_bytes": 2693786190 + }, + "installer": { + "authenticode_status": "NotSigned", + "publisher": "Mario Andreschak", + "sha256": "95b2d70382ed91b61079581e3fed0c3b12364ebe70576a25eee3231460f8e4d8", + "settings_cache_policy": "Preserved on upgrade and uninstall", + "unsigned_engineering": true, + "version": "0.1.0-alpha.20260907", + "filename": "communityai-0.1.0-alpha.20260907-windows-setup.exe", + "size_bytes": 2518829949 + }, + "package_checks": { + "gui_smoke": true, + "onboarding_smoke": true, + "node_self_test": true, + "worker_self_test": true, + "archive_and_provenance_verified": true, + "bundle_bytes": 4486446226, + "file_count": 4942 + }, + "sharing": { + "controller": "source DesktopController calling full frozen node and worker", + "community_model": "Qwen3.8 27B FP8 Dequant", + "block_indices": "16:17", + "community_manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "local_manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "selected_artifact_bytes": 384054157, + "verified_artifact_bytes": 384054157, + "verified_files": 3, + "slider_changes": [ + { + "vram_percent": 20, + "processing_percent": 50, + "max_vram_bytes": 1717921382, + "old_process_tree_gone": true, + "policy_persisted": true, + "worker_processing_argument_verified": true, + "verified_download_bytes": 384054157, + "local_inference": { + "seconds": 0.5160000000032596, + "model": "Qwen3.5-0.8B-Local", + "completion_tokens": 3 + } + }, + { + "vram_percent": 25, + "processing_percent": 100, + "max_vram_bytes": 2147401728, + "old_process_tree_gone": true, + "policy_persisted": true, + "worker_processing_argument_verified": true, + "verified_download_bytes": 384054157, + "local_inference": { + "seconds": 0.42199999999138527, + "model": "Qwen3.5-0.8B-Local", + "completion_tokens": 3 + } + } + ], + "inference": { + "before_sharing": { + "seconds": 10.09399999998277, + "model": "Qwen3.5-0.8B-Local", + "completion_tokens": 3 + }, + "while_sharing": { + "seconds": 0.5160000000032596, + "model": "Qwen3.5-0.8B-Local", + "completion_tokens": 3 + }, + "after_pause": { + "seconds": 0.29699999999138527, + "model": "Qwen3.5-0.8B-Local", + "completion_tokens": 3 + }, + "after_restart": { + "seconds": 0.375, + "model": "Qwen3.5-0.8B-Local", + "completion_tokens": 3 + } + }, + "pause_seconds": 0.125, + "paused_worker_tree_gone": true, + "restarted_worker_tree_gone": true, + "node_stopped": true + }, + "installation": { + "result": "passed", + "packaged_node_worker": true, + "gui": "source Qt with production NodeLifecycleSupervisor", + "full_gate15": false, + "installer_sha256": "95b2d70382ed91b61079581e3fed0c3b12364ebe70576a25eee3231460f8e4d8", + "owned_node_worker_tree_gone": true, + "block_indices": "3:4", + "verified_download_bytes": 372502429, + "process_count": 7, + "settings_preserved_on_upgrade": true, + "uninstall_passed": true, + "external_cache_preserved": true, + "cleanup_reverified": { + "owned_process_tree_absent": true, + "probe_native_credential_absent": true, + "installer_registration_absent": true, + "installed_executable_absent": true + } + }, + "probe_scripts_sha256": { + "qualify-real-slider-probe.py": "2049481463b6506c14205ceaf7fbdcc1ca4515a9f91055c03112d91a6cdcba1f", + "test-full-windows-installer.py": "f9f08a21d40eb8d0998e6ed0fcf5f2e8a2c711827be8530ed1aa5b411ac2aa9a", + "full-installer-gui.py": "eb60dec6a152794d12ad1891a19b6bfa757a8389683b8f1385123f909f421a99" + }, + "limitations": [ + "Source Qt/controller drove the real frozen node; literal frozen-GUI slider interactions were not exercised.", + "A ready single Qwen contribution block does not measure processing duty cycle during remote Qwen inference load.", + "Windows only; Linux Qwen resource and full installer lifecycle acceptance remain open.", + "Installer is explicitly unsigned; signing inquiry is not provider acceptance.", + "No broader conversation, context, concurrency or consumer GPU profile acceptance is claimed." + ] +} diff --git a/docs/evidence/runtime-packaging-reduction-20260908.md b/docs/evidence/runtime-packaging-reduction-20260908.md new file mode 100644 index 000000000..b7e733380 --- /dev/null +++ b/docs/evidence/runtime-packaging-reduction-20260908.md @@ -0,0 +1,142 @@ +# Frozen runtime packaging reduction — source and fixture checks + +Date: 2026-09-08. Scope: fresh future Windows/Linux builds. Existing qualified +archives, installers and installed runtimes were not modified or launched. + +The builder now normalizes the assembled node before frozen self-tests and +attestation. It keeps bitsandbytes CPU libraries and the CUDA 12.4 variants for +the exact `torch==2.6.0+cu124` profile. Other bitsandbytes CUDA versions are +removed without consulting the build machine's GPU. A conflicting +`BNB_CUDA_VERSION` or missing package-local CUDA 12.4 library fails the build. +Source deployments are unaffected. + +On Linux, identical regular native libraries with equal size, SHA-256 and mode +share an inode through hardlinks. Every loader pathname remains present as an +ordinary file. This deliberately avoids symlinking Torch aliases: the installed +PyInstaller `hook-torch.py` suppresses those symlinks because older Torch wheels +could resolve their shared-library location incorrectly. Hardlinking is still a +runtime packaging change; fresh frozen CUDA and lifecycle qualification remains +required before publishing a normalized candidate. + +The deterministic Linux tar writer preserves hardlinks. All three archive +validators accept only direct backward links to previously verified, attested +regular files with identical hash, size and permissions. Forward references, +chains, symlink targets, missing or external targets and changed identities fail. +Both Linux extractors preserve the inode relationship; node-only extraction also +rejects targets outside the node inventory. The Debian staging copy preserves +hardlinks even across its copy fallback, and `Installed-Size` counts unique +payloads. Old archives containing separate regular files continue to verify. + +Existing release artifact records remain logical pathname/hash/mode inventories. +Existing `bundle_bytes` and node metrics retain their logical meanings. A new +attested `CommunityAI/runtime-packaging.json` records logical lengths, unique inode +content lengths, pruned variants and hardlink replacements. Unique content length +does not claim filesystem block allocation or a compressed installer size. + +## Inventory-only estimate + +The prior qualified Linux provenance contains 8,591,203,661 bytes of regular file +content. Applying the exact new rules to that inventory predicts: + +| Change | Content bytes | +| --- | ---: | +| Remove 9 bitsandbytes CUDA variants other than 12.4 | 224,905,160 | +| Share 22 duplicate native libraries after pruning | 3,205,188,328 | +| Total unique content reduction | 3,430,093,488 | +| Remaining logical file lengths | 8,366,298,501 | +| Remaining unique file content | 5,161,110,173 | + +This estimate excludes the small new normalization report. No binaries were +rehash-scanned, transformed or recompressed for this estimate. Source: +`.gate13-runs/installer-size-audit-20260908/linux-normalization-estimate.json`, +derived from the existing qualified provenance. Actual compressed installer size +and extracted hardlink preservation through `dpkg-deb` still require a fresh +Linux package build and installation. + +## Verification + +- New normalization/archive fixtures: 14 passed. They exercise the three real + validators, both extractors, old regular archives, new backward hardlinks, + physical versus logical byte accounting, BNB profile selection, changed source + detection, different modes, cross-device Debian staging, traversal, + noncanonical/absolute paths, missing/forward/chained/symlink targets, duplicate + members, tampered bytes, differing attested hashes/sizes/modes, and cleanup of + rejected extraction stages. +- Existing desktop builder tests: 23 passed. +- Existing Linux lifecycle and Q38 host runtime tests: 145 passed, 3 existing + platform skips. These ran on Windows with filesystem fixtures, not a frozen + Linux runtime. +- Black and isort checks passed for the changed Python files. + +Commands used the existing product/style virtual environments and did not start +a GUI, model, worker, container or build: + +```text +python -m unittest desktop.tests.test_runtime_packaging tests.test_runtime_archive_hardlinks -v +python -m unittest desktop.tests.test_build_desktop -q +python -m pytest --noconftest -q tests/test_gate13_linux_packaged_lifecycle.py tests/test_gateq38_linux_host_runtime.py +``` + +## Native diagnostic mode for the next frozen candidate + +The new optional node entry point provides finite native operations without a +model or network connection. Existing `--self-test` and `server --self-test` +contracts remain unchanged. Run each command under an owned process deadline +(180 seconds allows cold runtime imports): + +```text +CommunityAI-Node --native-self-test +CommunityAI-Node --native-self-test --require-cuda +CommunityAI-Node server --self-test +``` + +The first command checks an exact CPU matrix product without initializing CUDA. +The required-CUDA command also checks a CUDA matrix product, reconstruction from +a 4x4 CUDA SVD (exercising lazy native linalg loading), and quantization plus +dequantization of a 64-element NF4 tensor. It requires the exact package-local +CUDA 12.4 bitsandbytes native backend and rejects nonfinite or excessive roundtrip +error. Frozen module paths must remain under the actual PyInstaller runtime +directory. CUDA unavailability fails required mode rather than falling back. + +Eight deterministic fake-runtime regressions passed for dispatch, missing GPUs, +incorrect CPU/CUDA math and linalg, incompatible CUDA builds, NF4 shape/device or +accuracy failures, and native backend/path containment. These tests do not claim +real native execution. The next frozen build must run the commands above after +normalization and again from its extracted/installed runtime. No native test, +build, GUI, model or container was started while implementing this mode. + +## Follow-up: measured replacement builds + +Both replacement runtimes were built from clean commit +`84205f93fc73d3babd39e238944b97fab0d11b3e`. The later CI commit `fdd8d0b` +only fixes import ordering outside the packaged application; all nine CI checks +passed there. The following are actual build measurements, replacing the earlier +inventory-only predictions for these files: + +| Platform | Previous installer bytes | Replacement installer bytes | Regular-file runtime payload bytes | +| --- | ---: | ---: | ---: | +| Windows x64 | 2,519,046,440 | 2,462,345,104 | 4,263,859,354 | +| Linux amd64 | 3,781,591,484 | 2,302,428,788 | 5,161,115,250 | + +Windows saves 56,701,336 compressed bytes (2.25%); Linux saves 1,479,162,696 +compressed bytes (39.1%). Linux has 4,900 regular files and 35 internal symlinks; +the regular-file payload excludes repeated logical lengths of symlink targets. +These lengths do not claim physical filesystem block allocation. + +Each fresh node pruned nine unused bitsandbytes CUDA variants. The fresh Linux +collection already lacked the old large duplicate regular-library candidates, +so its normalization report records no additional hardlink replacements. The +measured end result is the smaller bundle above; the inventory prediction must +not be presented as an observed list of hardlinks created in this build. + +Small repeated content remains: 23,211,174 bytes in Windows and 55,221,669 bytes +in Linux, mainly libraries belonging to the separate desktop and node runtimes. +This change does not claim that every byte-identical file was removed. + +Both actual frozen runtimes passed CPU math, required-CUDA math and linalg, +loading their package-local CUDA 12.4 bitsandbytes backend, and an NF4 roundtrip +with maximum absolute error 0.14501953125. The +[Windows installed-runtime and removal check](normalized-windows-installer-20260908.md) +and [Linux installation/native check](alpha-normalized-linux-20260908.md) +also passed in their recorded scopes. Installer acceptance and real online handoff +are recorded separately from these build measurements. diff --git a/docs/evidence/text-mesh-release-20260909.json b/docs/evidence/text-mesh-release-20260909.json new file mode 100644 index 000000000..93f6bc429 --- /dev/null +++ b/docs/evidence/text-mesh-release-20260909.json @@ -0,0 +1,431 @@ +{ + "version": "0.1.0-alpha.20260909.3", + "source_commit": "d7f4333cc8ff74ce060076c1d6effb67fb4c6d2c", + "release_url": "https://github.com/flujo-app/CommunityAI/releases/tag/v0.1.0-alpha.20260909.3", + "installers": { + "linux-amd64": { + "filename": "communityai_0.1.0~alpha.20260909.3_amd64.deb", + "format": "deb", + "kind": "offline-installer", + "platform": "linux-amd64", + "publisher": "Mario Andreschak", + "sha256": "57361da76241997cb5437d9af973aeebc6410e082ad87bf1b2b6410415733a14", + "size_bytes": 2302529348, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai_0.1.0~alpha.20260909.3_amd64.deb", + "version": "0.1.0~alpha.20260909.3" + }, + "windows-x64": { + "filename": "communityai-0.1.0-alpha.20260909.3-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "5ba5d1a4890dd76227ee21ad0b2caf21c402d67f8fd617573c1942990682413b", + "size_bytes": 2465339019, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe", + "version": "0.1.0-alpha.20260909.3" + } + }, + "hosted_objects": [ + { + "anonymous_complete_body_verified": false, + "anonymous_range_samples_verified": true, + "key": "alpha/20260909.3/communityai_0.1.0~alpha.20260909.3_amd64.deb", + "samples": [ + { + "sha256": "f32ce2a227dfbc728621623bef7d468b0448248dc592cf088c1af8e241196ddc", + "size_bytes": 65536, + "start": 0 + }, + { + "sha256": "f55d71ad899e0a45755901b16cc28228029edf7a5456fbd3678c1a5dbc1fc7a0", + "size_bytes": 65536, + "start": 2302463812 + } + ], + "scope": "Uploaded metadata and first/last 64 KiB match. No complete public re-download; installers/updater verify complete downloads on the client.", + "sha256": "57361da76241997cb5437d9af973aeebc6410e082ad87bf1b2b6410415733a14", + "size_bytes": 2302529348, + "uploaded_size_and_checksum_metadata_verified": true, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai_0.1.0~alpha.20260909.3_amd64.deb", + "verified_at_utc": "2026-09-09T08:53:38.322004+00:00" + }, + { + "anonymous_complete_body_verified": false, + "anonymous_range_samples_verified": true, + "key": "alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe", + "samples": [ + { + "sha256": "3bebe436fae4d118223044d658128291f2137c5fd7e504b372558504725ecb53", + "size_bytes": 65536, + "start": 0 + }, + { + "sha256": "88548b150fca67e49034474974972343437ce61333a066740baccd71315278f0", + "size_bytes": 65536, + "start": 2465273483 + } + ], + "scope": "Uploaded metadata and first/last 64 KiB match. No complete public re-download; installers/updater verify complete downloads on the client.", + "sha256": "5ba5d1a4890dd76227ee21ad0b2caf21c402d67f8fd617573c1942990682413b", + "size_bytes": 2465339019, + "uploaded_size_and_checksum_metadata_verified": true, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe", + "verified_at_utc": "2026-09-09T08:53:42.133237+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-online-setup.exe", + "sha256": "c1b598dcc26c6ad7ddbe4694078c5b66506f271b447a4904f6780a3ebda3f520", + "size_bytes": 2107748, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-online-setup.exe", + "verified_at_utc": "2026-09-09T08:55:20.762253+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-linux-online.py", + "sha256": "94bb7b5fa8c9c7e8697a57bfb06d1700048058b7d19d11417da878b00058aa5f", + "size_bytes": 13900, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-linux-online.py", + "verified_at_utc": "2026-09-09T08:55:21.471689+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/release-downloads.json", + "sha256": "56ba09ba23d4335aa4d7b39bc4ba56d8bc698f5439420be60c981cd293063112", + "size_bytes": 1087, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/release-downloads.json", + "verified_at_utc": "2026-09-09T08:55:22.212066+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/source-checksums.json", + "sha256": "0fb659512e4f81deec9e2c2fc8ca57f48b062b09f422bc8b389f10f408d06399", + "size_bytes": 10724, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/source-checksums.json", + "verified_at_utc": "2026-09-09T08:55:22.966668+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/SOURCE-SHA256SUMS", + "sha256": "a40b4446af501e0db575d8047a88c5e20fb2f300d6416ffd0eb28668bfa40c1f", + "size_bytes": 3149, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/SOURCE-SHA256SUMS", + "verified_at_utc": "2026-09-09T08:55:23.724752+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/windows/provenance.json", + "sha256": "bb48a9f6b055076526471c0c042b7a4df9c41420377cd848c34b73bb96b86ea6", + "size_bytes": 1242642, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/windows/provenance.json", + "verified_at_utc": "2026-09-09T08:55:25.323968+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/windows/desktop-metrics.json", + "sha256": "2bba7e87a45132d939a8f7be8dd193ceffaf7d91647deea4c274d158219024b6", + "size_bytes": 3786, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/windows/desktop-metrics.json", + "verified_at_utc": "2026-09-09T08:55:26.041395+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/windows/release-metadata.json", + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "size_bytes": 872, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/windows/release-metadata.json", + "verified_at_utc": "2026-09-09T08:55:26.814023+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/windows/SHA256SUMS", + "sha256": "2b9d8a2b0881be4a880aec84a1b872a50936b7555749d3f034684adcc52715b7", + "size_bytes": 674959, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/windows/SHA256SUMS", + "verified_at_utc": "2026-09-09T08:55:28.033657+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/windows/communityai-0.1.0-alpha.20260909.3-windows-setup.exe.json", + "sha256": "f04a2c7d04e4fc3fafc691b31e1a09536297181583662b434da7de8676c820f0", + "size_bytes": 300, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/windows/communityai-0.1.0-alpha.20260909.3-windows-setup.exe.json", + "verified_at_utc": "2026-09-09T08:55:28.713534+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/windows/communityai-0.1.0-alpha.20260909.3-windows-online-setup.exe.json", + "sha256": "6a4a8bc929727bcee03de7eb1ec6953e2bfb060aa8556e147976031f39c19034", + "size_bytes": 1433, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/windows/communityai-0.1.0-alpha.20260909.3-windows-online-setup.exe.json", + "verified_at_utc": "2026-09-09T08:55:29.401713+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/linux/provenance.json", + "sha256": "45f4cba3e3c52d0467511e0d6c137b4b956b26606e258e5fe2f42db4a566eb41", + "size_bytes": 1244846, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/linux/provenance.json", + "verified_at_utc": "2026-09-09T08:55:31.755964+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/linux/desktop-metrics.json", + "sha256": "ce74dd8ba992465162e2a9a63fa2c8cde49b935831db734ebed0d431d1e60146", + "size_bytes": 3814, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/linux/desktop-metrics.json", + "verified_at_utc": "2026-09-09T08:55:32.437129+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/linux/release-metadata.json", + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "size_bytes": 872, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/linux/release-metadata.json", + "verified_at_utc": "2026-09-09T08:55:33.203263+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/linux/SHA256SUMS", + "sha256": "23e80e3ee4a228d27e7aa35e95bd203fe29affb4c35eae6e9ed6c38b27bad4c9", + "size_bytes": 675799, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/linux/SHA256SUMS", + "verified_at_utc": "2026-09-09T08:55:34.369533+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/linux/communityai_0.1.0~alpha.20260909.3_amd64.deb.json", + "sha256": "557c3f68bba0ec25b907d3ee858991cd0fbfd6f72444c122508600921105c0c6", + "size_bytes": 1170, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/linux/communityai_0.1.0~alpha.20260909.3_amd64.deb.json", + "verified_at_utc": "2026-09-09T08:55:35.133677+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/metadata/linux/communityai-0.1.0-alpha.20260909.3-linux-online.py.json", + "sha256": "1755f199e6c11ae54030a483915616539e42cc0798a4c0e5389d15959ce71c20", + "size_bytes": 915, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/metadata/linux/communityai-0.1.0-alpha.20260909.3-linux-online.py.json", + "verified_at_utc": "2026-09-09T08:55:35.838884+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/INSTALLER-SHA256SUMS", + "sha256": "0d7d58e279001d5a710649529d418184ac7ad034c81a9a1bc6e27e09e09e12cd", + "size_bytes": 473, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/INSTALLER-SHA256SUMS", + "verified_at_utc": "2026-09-09T08:55:36.531260+00:00" + }, + { + "anonymous_complete_body_verified": true, + "attempts": [ + { + "http_status": 200, + "offset": 0, + "request": 1, + "result": "complete" + } + ], + "download_copy_created": false, + "key": "alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-release-metadata.zip", + "sha256": "912ea4789f63676f0f231eed03b07a982662e876966b34028de348ba4541ce25", + "size_bytes": 997612, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-release-metadata.zip", + "verified_at_utc": "2026-09-09T08:55:37.841971+00:00" + } + ], + "feed_sha256": "7638ccc7f198f2ac48b016e4c3f7b740ebe497132824900367b7cd2dd0aad0f4", + "source_checks": { + "text_mesh_and_node_desktop_integration": "9 passed", + "desktop_status_parser": "16 passed", + "node_config": "72 passed", + "local_inference_hardware_resource_controls": "24 passed" + }, + "real_public_mesh_consumer": "Completion and chat passed with artifact downloads forbidden and an empty consumer cache; see text-only-mesh-consumer-20260909.md", + "consumer_timing_seconds": { + "completion": 106.937, + "chat": 157.656 + }, + "text_peer_placement": "Explicit operator text-peer role; automatic desktop placement of text peers is not implemented", + "new_installed_desktop_gpu_cloud_campaign": false, + "installed_linux_updater_handoff_tested": false, + "windows_postcompile_resume": "ISCC completed successfully; the original Windows PowerShell signature-module autoload failed. PowerShell 7 completed the unsigned-status, version and SHA-256 metadata checks on the unchanged installer. The initial failure record is retained locally.", + "portable_bundle_metadata": "The portable bundle's automatic_updates=false describes the uninstalled bundle. Installer-managed applications enable the updater.", + "publication_scope": "Fresh Windows and Linux builds; built-in packaging checks, uploaded size/checksum metadata and public start/end samples for large installers; complete hashes for small public files. Earlier installer/native-runtime evidence remains separate.", + "installed_windows_updater_observation": { + "from_version": "0.1.0-alpha.20260909.2", + "offered_version": "0.1.0-alpha.20260909.3", + "observed_at_utc": "2026-09-09T09:17:47.2607833Z", + "existing_desktop_process_preserved": true, + "check_triggered_through_actual_sidebar": true, + "download_completed": true, + "visible_button": "Restart to update", + "button_enabled": true, + "verification_basis": "The installed UpdateManager enters ready only after the complete installer matches the signed feed size and SHA-256.", + "installation_or_restart_triggered": false, + "scope": "Real installed Windows application detected, downloaded and offered the release. Installer handoff and the new application UI remain for the user to activate and review." + } +} diff --git a/docs/evidence/text-only-mesh-consumer-20260909.json b/docs/evidence/text-only-mesh-consumer-20260909.json new file mode 100644 index 000000000..2f4c9db72 --- /dev/null +++ b/docs/evidence/text-only-mesh-consumer-20260909.json @@ -0,0 +1,112 @@ +{ + "scope": "Two short real public-mesh requests from a fresh source client; not frozen installer qualification", + "base_commit": "c74efcfe913b75dded569e31ff943db2be813edf", + "provider_source_archive_sha256": "ee1a7c2cd795dcaffaead77845fbf2fc0e609c3dd743cbadee9120c3265ac497", + "results": { + "completion": { + "status_code": 200, + "response": { + "id": "cmpl-dae32d1af44f4f8c87e4052d", + "object": "text_completion", + "created": 1788938402, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "finish_reason": "length", + "text": " Paris.\n" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8 + } + }, + "elapsed_seconds": 106.93700000000536, + "consumer_cache_files": [], + "consumer_rss_bytes": 595922944, + "artifact_downloads_forbidden": true, + "gpu_visible": false, + "observed_at_unix": 1788938505.593864, + "covered_blocks": 64, + "text_peer_count": 1 + }, + "chat": { + "status_code": 200, + "response": { + "id": "chatcmpl-8b11eed2dc7e44989b91fe82", + "object": "chat.completion", + "created": 1788938604, + "model": "Qwen3.8 27B FP8 Dequant", + "choices": [ + { + "index": 0, + "finish_reason": "length", + "message": { + "role": "assistant", + "content": "4\n" + } + } + ], + "usage": { + "prompt_tokens": 26, + "completion_tokens": 3, + "total_tokens": 29 + } + }, + "elapsed_seconds": 157.6559999999954, + "consumer_cache_files": [], + "consumer_rss_bytes": 596209664, + "artifact_downloads_forbidden": true, + "gpu_visible": false, + "observed_at_unix": 1788938758.1669347, + "covered_blocks": 64, + "text_peer_count": 1 + } + }, + "focused_source_tests": { + "consumer_and_transport": 8, + "model_manager_discovery_desktop": 46, + "existing_api_node": 41, + "resource_controls": 12, + "fallback_budget_hardware_and_desktop": 24, + "node_configuration": 72 + }, + "resource_controls": "100% is the full physical GPU sharing budget. No permanent local fallback reservation. Fallback independently uses available GPU or CPU/RAM and rechecks after downloads.", + "gcp_deadline_utc": "2026-09-10T05:57:40Z", + "gcp_termination": "DELETE; attached boot disks auto-delete", + "limitations": [ + "No claim of minimum whole-system RAM qualification", + "One text service on an existing CPU block worker", + "Automatic desktop placement of text-serving roles is not implemented", + "No new installers built or published yet" + ], + "consumer_source_sha256": { + "desktop/src/communityai_desktop/controller.py": "1418d310f32cc553268d43dd6f1c3786ad15ad7c58c90c9aa1d322e880cae85a", + "desktop/src/communityai_desktop/model_health.py": "4e8c0b6f349159e682a7a118c011787e9096ad6b61db9e685eb448b0d2d9778d", + "desktop/src/communityai_desktop/presentation.py": "ef2146a3a648afe0abf3d48d5256d12f52bac088021d57a40e8ae27639d66b77", + "desktop/tests/test_simple_desktop.py": "2089612931f88fcfba36b1371f0a840cf98005afd3b828db49b3ad069458a8f2", + "src/drift/api/server.py": "8cc7fd031a4f8904e68403d35168ab9e44068f65a8e18f8d39aa624e8d42d78f", + "src/drift/api/text_response.py": "3b677512dda03d9b824771b755bb24a733628948e24722be369d828c37ec64cf", + "src/drift/cli/__main__.py": "38377950216ff8b037c1508c1ef6f26d017830da2a8b1b69a7d06014a9707d07", + "src/drift/cli/run_node.py": "7df0bfbe77404dcd668133baa212acd610cd521042c3293bdc222df3247ad12b", + "src/drift/cli/run_text_peer.py": "6d54477268adf038b5675d0073e4fd4ea2568dfa8eb25fb9dd8fe267702931af", + "src/drift/client/routing/sequence_manager.py": "7ed2e841eb3b105ad32f17f76da851a4f03ab4f6c59a78d69b7f56bf7a35b969", + "src/drift/node/discovery.py": "4ff7a95d7dbd01093af030128a6e8f05deba5151fd6c440cf2a2cbf39e7106fb", + "src/drift/node/loading.py": "370746f46ef15468b740ce3bd38663331954a88afa09b406298bf7415724d4eb", + "src/drift/node/model_manager.py": "b9f423ed10b32115d61412e3d2abfb5be341cd797ac21d5ec92e94e16762bef0", + "src/drift/server/text_generation.py": "cbab82e1568d229165c485eb3a77ad8c34d9099878f5038bdb94a07ffec760ad", + "src/drift/server/text_peer.py": "35c652fbc7cd64f592232fb9073f4b73a8245846a089a9b1cac9a4247a59f277", + "src/drift/text_mesh.py": "8e2d48f2ed2af0aa712593d96f0626242e6abe92b84837a7ef7664e2fee8224b", + "tests/test_text_mesh.py": "1d0d49e289e6a397186399ee39ebc71c37e74006333fa31ae9f9ca6f1cfaa5fe", + "desktop/src/communityai_desktop/resource_controls.py": "0427358a9a2f470f70e1bbf11587fb76cf66b19a95613b1d0cf7ea0780626ecf", + "desktop/tests/test_resource_controls.py": "5332dca166ca690c47e6bcc65aea4ced1f573aee056071d8259547cdba53ec11", + "src/drift/node/hardware_status.py": "ec9ed045722efa8863bcbc5ddfcd45db218e89ae630d651b61add2bb064ebc58", + "src/drift/node/local_inference.py": "07a4fad72f7e764cf3b4b0ff193294f24c03f11f5a6b36ebdd4ebeab9f32e32e", + "tests/test_node_hardware_status.py": "423e6553284a6c53a9a9c66eadc5a1c59c27c8871c86aa2390b12fe44c15a4e0", + "tests/test_local_inference.py": "3b8e568948015deea23f7804411c856f033c2a3f3dc1ba2e7505bdcc2b810951", + "tests/test_node_config.py": "eb9bb36a172fbecf1552995fa787e82a3db3db7ea36551243226243eee724532", + "desktop/tests/test_resource_playthrough.py": "8e3b69236837912c481eb02e85012522b302267f99f8b01b822f528cf74b93c6" + } +} diff --git a/docs/evidence/text-only-mesh-consumer-20260909.md b/docs/evidence/text-only-mesh-consumer-20260909.md new file mode 100644 index 000000000..32fef5528 --- /dev/null +++ b/docs/evidence/text-only-mesh-consumer-20260909.md @@ -0,0 +1,52 @@ +# Text-only community consumer: September 9, 2026 + +Two fresh Windows source clients completed real requests through the public +Qwen3.8 27B mesh without downloading model artifacts or loading a local tokenizer +or model. GPU visibility was disabled. The product node's actual model manager, +discovery, auto selection and OpenAI API were used; HTTP was exercised through +ASGI and peer inference used the real encrypted public network. + +| Request | Answer | Time | Client cache | +| --- | --- | --- | --- | +| Completion: `The capital of France is` | `Paris.` | 106.94 s | Empty | +| Chat: `What is 2 + 2? Reply with only the number.` | `4` | 157.66 s | Empty | + +Both requests used three generated tokens and the exact public manifest +`sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4`. +Client process RSS was approximately 569 MiB. This is not a measurement of total +system requirements or all child-process memory. CPU speed is not a performance +qualification. The chat log includes a transient DHT reconnect warning; the +request nevertheless completed through the authenticated provider. + +Input embeddings, output projection and tokenization ran on the existing +`qwen-live-0909-cpu0` contributor; all 64 transformer blocks remained distributed +over the four existing CPU workers. The public bootstrap was the only configured +consumer seed. No new paid machines, disks or firewall rules were created. +All four machines retain automatic deletion at **2026-09-10T05:57:40Z**, with +boot disks set to auto-delete. + +Focused source checks passed: eight consumer/protocol checks, 46 manager, +discovery and desktop checks, 41 existing API/node checks, and 12 resource-control +checks. These include fallback on incomplete or unavailable mesh, return to the +community, explicit local-only selection, identity/revocation checks, real +encrypted streaming/cancellation, and release of an abandoned API request. + +Two implementation failures were corrected before these results: cancellation +used an unsupported unary transport path on the desktop daemon; text-provider +discovery waited for the first inference request and could not advertise initial +readiness. Cancellation now uses the streaming transport and provider discovery +starts before requests are admitted. + +Following the owner's clarification, the permanent local-fallback deduction was +removed from hardware reporting and worker admission. A 100% sharing setting on +an 8 GiB GPU now produces an 8 GiB worker budget. Local execution independently +checks free memory and selects CPU/RAM when the GPU is occupied; it rechecks after +downloads before choosing the load device. This follow-up has separate focused +source checks; the public CPU inference results above did not exercise this GPU +budget change. + +This evidence covers source clients and the live provider, **not a new packaged +desktop release**. Automatic placement of text roles from the desktop sharing UI +is not implemented; the role is an explicit contributor service. + +[Machine-readable results](text-only-mesh-consumer-20260909.json). diff --git a/docs/evidence/windows-resumable-downloader-20260909.json b/docs/evidence/windows-resumable-downloader-20260909.json new file mode 100644 index 000000000..734323031 --- /dev/null +++ b/docs/evidence/windows-resumable-downloader-20260909.json @@ -0,0 +1,133 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-09T01:14:51.040079+00:00", + "result": "passed-bounded-helper-and-local-inno-integration", + "scope": "Resumable Windows online downloader source checks, tiny loopback transfers, real hidden Inno wrapper/harmless-child integration, and local publication metadata audit", + "environment": "Windows-10-10.0.19045-SP0", + "final_wrapper": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "version": "0.1.0-alpha.20260908.1", + "platform": "windows-x64", + "size_bytes": 2107444, + "sha256": "0d1c31206336b1e8c783a78a6cadbdc8f9e643e91d587b88d78581a85b5d9ef6", + "authenticode_status": "NotSigned", + "unsigned_alpha": true, + "live_download_verified": false, + "release_manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "download_helper_sha256": "ba76158d7762058cfc53f98054334ca0b6bc3a5d4e0f8e8f32eda87b824813df", + "download_helper_source_sha256": "3df36f4f1273599a3680d3d4981f150f2eaf289d92926b19dbde6620fdad3094", + "installer_script_sha256": "b6c680afa048ba581af6d01a596fa8f9f62837bb7d571979cb9d4dc9fbbf6633", + "builder_script_sha256": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893" + }, + "final_companion_sha256": "6423e194ca120897cf25f20a49eb89896cf3e839c53e62c0cfba921f357b9e97", + "offline_installer": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "size_bytes": 2462345104, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "version": "0.1.0-alpha.20260908.1" + }, + "source_identity": "Exact content hashes; the new downloader sources were working-tree changes at build time. No wrapper source commit is claimed.", + "source_sha256": { + "desktop/installers/WindowsDownload.cs": "3df36f4f1273599a3680d3d4981f150f2eaf289d92926b19dbde6620fdad3094", + "desktop/installers/communityai-online.iss": "b6c680afa048ba581af6d01a596fa8f9f62837bb7d571979cb9d4dc9fbbf6633", + "desktop/installers/build_windows_online_installer.ps1": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "tests/test_windows_download_helper.py": "af6146b674fc3f13a646375cdafba7574503c08a225511aa759e542e7b46109a", + "tests/test_windows_online_integration.py": "650d2013641f55c3ee876f0d9244af939e5ebe76bf6a1f2ba4f22cdc8062caab" + }, + "compiler": { + "inno_version": "6.7.3", + "setup_engine_machine": "IMAGE_FILE_MACHINE_I386 (0x014c)", + "setup_engine_optional_header": "PE32 (0x010b)", + "exact_version_preprocessor_guard_compiled": true, + "simulated_future_version_rejected": true, + "native_bridge_sizes_bytes": { + "startup_info": 68, + "process_information": 16, + "job_extended_limit_information": 112 + }, + "helper_runtime": ".NET Framework 4", + "additional_runtime_download_required": false + }, + "helper_tests": { + "passed": 21, + "failed": 0, + "seconds": 18.767, + "command": "python -m unittest discover -s tests -p test_windows_download_helper.py -v", + "test_source": "tests/test_windows_download_helper.py", + "log": ".gate13-runs/windows-download-helper-final-20260909.log", + "log_sha256": "9f30a84726357ce39c4fb1292ab0c244c13c2f44b3d7b98b89cd95355a186888", + "observation": "Execution reported by release_audit; saved final log and source hashes independently verified during this evidence export.", + "coverage": [ + "exact interrupted-response resume", + "wrong ranges and lengths, redirects, encoding and checksum mismatch rejected", + "finite retries and monotonic deadline", + "cancellation and exact-parent identity/loss", + "transient progress-reader lock recovery and bounded atomic updates", + "64-bit range validation above 2 GiB", + "production compile rejects loopback HTTP" + ] + }, + "real_inno_tests": { + "passed": 3, + "failed": 0, + "seconds": 8.894, + "command": "python -m unittest discover -s tests -p test_windows_online_integration.py -v", + "test_source": "tests/test_windows_online_integration.py", + "observation": "Directly observed final execution output in gate15_choices agent; no separate raw log was persisted.", + "fixtures": "Separately compiled WINDOWS_DOWNLOAD_TEST helper and harmless winexe child; actual maintained Inno script compiled with an explicit loopback URL.", + "coverage": [ + "interrupted small download resumed at observed exact Range offset; child received exact forwarded arguments plus independent /LOG", + "child file remained alive throughout its short execution, outer wrapper returned child exit 7, then Inno temporary directory disappeared", + "cancel file during held response produced nonzero outer exit, no child launch and no remaining downloaded file", + "terminating the exact engine handle matched the helper parent FILETIME, stopped the helper promptly and prevented child launch" + ], + "parent_death_acceptance": "Either job termination or cooperative parent watcher is valid; the test does not assert a timing-sensitive forced exit code.", + "all_recorded_owned_process_handles_stopped": true, + "owned_temporary_directories_cleaned": true, + "visible_windows_launched": false + }, + "style_checks": { + "black": "passed", + "isort_root_settings": "passed", + "git_diff_check": "passed", + "scope": "tests/test_windows_online_integration.py" + }, + "publication_staging": { + "audit_result": "passed", + "audit_sha256": "0f31f6cd09645fafb34eb53da3f430ef28467f8ee17dcc925e91bac790bf4d59", + "combined_manifest_sha256": "3ab88fd1dd7b5ccbb19cb226429c5b3646d241a676280bde55795fdc34c8df6a", + "windows_manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "installer_sha256sums_sha256": "2e646a9e81d9be66af8bfb4204bd17e24963dc5dfeda560edfb01d52cb679231", + "platform_metadata_files": { + "windows": 7, + "linux": 7 + }, + "raw_or_private_data_markers_found": false, + "other_three_installer_checksum_entries_unchanged": true, + "previous_windows_online_companion_preserved_privately": true, + "previous_windows_online_wrapper_sha256": "c8171619c835877f018dcf305bf70d7b4804535fd6ad548bd682971ecccdda5d", + "upload_performed_by_this_audit": false + }, + "retained_failures": [ + "docs/evidence/normalized-online-windows-download-failure-20260908.json", + "docs/evidence/normalized-online-windows-download-retry-failure-20260908.json" + ], + "qualification_boundaries": { + "complete_hosted_windows_download_passed": false, + "final_real_r2_installer_handoff_passed": false, + "product_installation_performed_by_these_tests": false, + "full_installer_files_rehashed_by_metadata_audit": false, + "model_loading_or_public_peer_contact": false, + "native_credential_access": false + }, + "notes": [ + "The prior native Inno downloader failed closed twice with error12030; these source and local fixture passes do not establish the cause or erase either failure.", + "The outer Inno shell retains independent final size/SHA checks and only then launches the unchanged offline installer.", + "The new production wrapper still requires its separate complete hosted download and actual installer handoff before online release readiness can be claimed." + ] +} diff --git a/docs/evidence/windows-resumable-downloader-20260909.md b/docs/evidence/windows-resumable-downloader-20260909.md new file mode 100644 index 000000000..2ee68515a --- /dev/null +++ b/docs/evidence/windows-resumable-downloader-20260909.md @@ -0,0 +1,85 @@ +# Windows resumable downloader — September 9, 2026 + +**Superseded build record.** A later full transfer with this wrapper stopped on +local progress publication after 541,341,184 bytes; no installer launched. The +[failure remains recorded](normalized-online-windows-resumable-progress-failure-20260909.md). +The [replacement progress fix](windows-resumable-progress-fix-20260909.md) has its +own artifact hashes and tests. The counts and hashes below retain their original +scope. + +**Passed for bounded helper and local Inno integration.** A complete R2 download +and real offline-installer handoff remain separate acceptance. The +[machine-readable record](windows-resumable-downloader-20260909.json) binds the +final wrapper, exact source hashes, tests and private publication staging audit. + +| Final Windows online artifact | Value | +| --- | --- | +| Filename | `communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe` | +| Bytes | 2,107,444 | +| SHA-256 | `0d1c31206336b1e8c783a78a6cadbdc8f9e643e91d587b88d78581a85b5d9ef6` | +| Embedded Windows-only manifest SHA-256 | `6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b` | +| Signing | Unsigned alpha | + +The wrapper still downloads the unchanged 2,462,345,104-byte offline setup, +SHA-256 `116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb`. +That installer has its own [installed native and removal acceptance](normalized-windows-installer-20260908.md). +The combined public manifest is a different file from the exact Windows-only +manifest embedded at build time; both bindings were checked independently. + +The small .NET Framework helper resumes interrupted responses using an exact byte +range and checks the response offset, end, total, length and final SHA-256. It +rejects redirects and unexpected encoding, bounds retries and elapsed time, and +observes cancellation and its exact parent process. Inno retains the wizard, +silent behavior, process containment and independent size/hash checks before +starting the offline setup. Atomic progress publication retries brief Windows +sharing conflicts with the wizard's reader. No additional runtime is downloaded. + +The Inno bridge compiled with version 6.7.3. The local Setup engine was verified +as PE32/x86; its 68-byte startup, 16-byte process and 112-byte job-limit records +compiled directly. The exact-version guard also rejected a simulated future +version. The helper is assigned to its job before executing download code. +Cancellation and cleanup retain the exact process handle through bounded waits. + +## Recorded checks + +- **21 helper tests passed in 18.767 seconds.** The saved log's SHA-256 is + `9f30a84726357ce39c4fb1292ab0c244c13c2f44b3d7b98b89cd95355a186888`. + Coverage includes real small HTTP interruptions/resume, malformed ranges, + lengths, redirects, encoding, hashes, finite retry/deadline behavior, parent + loss, cancellation, progress-reader locks and range arithmetic above 2 GiB. + A separately compiled production helper rejected loopback HTTP. +- **3 real Inno integration tests passed in 8.894 seconds.** The maintained + wrapper downloaded a tiny harmless executable from a loopback fixture after + an observed interrupted-response resume. The child received the exact + forwarded arguments, remained available until it exited, and its exit code 7 + reached the outer wrapper. Inno then removed its temporary directory. + Cancellation during a held response and termination of the exact parent + engine both stopped the helper without launching the child. +- All recorded fixture processes stopped and owned temporary directories were + cleaned. The fixtures used hidden executables and silent setup; no visible + windows, product installations, model loads or native credentials were involved. + Black, isort and whitespace checks passed for the integration test file. + +The helper tests are in [test_windows_download_helper.py](../../tests/test_windows_download_helper.py); +the real-wrapper tests are in +[test_windows_online_integration.py](../../tests/test_windows_online_integration.py). +The latter's final result was observed directly in agent execution output; no +separate raw integration log was saved. The JSON distinguishes that observation +from the independently checked saved helper-test log. + +## Publication boundary + +The private staging audit matched all four intended installer identities and +both platform provenance records. It changed only the Windows online companion +and its installer-checksum entry; the previous companion was preserved outside +the public metadata directories. Each platform folder contains seven release +records, with no raw logs or private data markers found. This audit uploaded +nothing and did not rehash the multi-gigabyte offline installers. + +The two earlier full-transfer failures remain recorded +[separately](normalized-online-windows-download-failure-20260908.json), +[including the unchanged-object retry](normalized-online-windows-download-retry-failure-20260908.json). +They failed closed with error 12030; these tests do not establish their cause. +The build companion retains `live_download_verified: false`. A subsequent full +hosted download and actual installer handoff must establish online release +readiness; neither is claimed by this evidence. diff --git a/docs/evidence/windows-resumable-progress-fix-20260909.json b/docs/evidence/windows-resumable-progress-fix-20260909.json new file mode 100644 index 000000000..0eddb4d8c --- /dev/null +++ b/docs/evidence/windows-resumable-progress-fix-20260909.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-09T01:33:02.760608+00:00", + "result": "passed-bounded-progress-regressions-and-publication-metadata-audit", + "scope": "Presentation-only progress fix; small local helper tests and actual hidden Inno integration. Full hosted handoff is separate.", + "wrapper": { + "version": "0.1.0-alpha.20260908.1", + "authenticode_status": "NotSigned", + "unsigned_alpha": true, + "release_manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "kind": "online-installer", + "live_download_verified": false, + "offline_installer": { + "filename": "communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "format": "exe", + "kind": "offline-installer", + "platform": "windows-x64", + "publisher": "Mario Andreschak", + "sha256": "116882e5d94e643e507efedebc4ec4b091275c5703f89bc646957f0f648d64bb", + "size_bytes": 2462345104, + "url": "https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260908.1/communityai-0.1.0-alpha.20260908.1-windows-setup.exe", + "version": "0.1.0-alpha.20260908.1" + }, + "download_helper_sha256": "e96c0fce4b71f9236a77625b6af7af2924d5f2004d16d17ca8eb5527b43ec3f1", + "installer_script_sha256": "d975b6da3a1067e679d9ba55555fa5b6219ae644fce8050225486909600fbbb1", + "builder_script_sha256": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "platform": "windows-x64", + "download_helper_source_sha256": "3bfa36e62f2a160b2e258af6d9d6816e815d8a9c5b918c29dd800eb6e6c20d8a", + "schema_version": 1, + "size_bytes": 2107751, + "filename": "communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe", + "sha256": "8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861" + }, + "companion_sha256": "c3fece282465df8c74907ff4c7c29fd48405b9dc579315a61d340929afe4584e", + "source_identity": "Exact working-tree file hashes at build/test time; no online-wrapper source commit is implied by the offline runtime commit.", + "source_sha256": { + "desktop/installers/WindowsDownload.cs": "3bfa36e62f2a160b2e258af6d9d6816e815d8a9c5b918c29dd800eb6e6c20d8a", + "desktop/installers/communityai-online.iss": "d975b6da3a1067e679d9ba55555fa5b6219ae644fce8050225486909600fbbb1", + "desktop/installers/build_windows_online_installer.ps1": "e5a33e969a0a4c110d960c4de240418be15d58288c3c4e43678a394f92dd5893", + "tests/test_windows_download_helper.py": "6f743863f91d79dacd871be96c2d7de7f8c890a4ab0c0f1bf725a3a312cb24d6", + "tests/test_windows_online_integration.py": "54bc2caf163ec8e295920976372aedbf8b277b340e0c477c99dab3670ebe6574" + }, + "behavior": [ + "Progress publication is best effort, including final frames; bounded sharing-conflict retry is 75ms.", + "I/O, access-denied and security exceptions skip the presentation frame and may record sanitized exception type/HRESULT; owned .new files are cleaned.", + "Cancellation, exact parent identity, transfer limits, final size and SHA-256 remain authoritative. Exit0 requires successful independent helper verification.", + "Inno may display actual partial-file length when the progress frame is stale. It independently verifies complete size and SHA-256 before logging completion or launching the offline setup." + ], + "helper_tests": { + "passed": 24, + "failed": 0, + "seconds": 22.299, + "command": "python -m unittest discover -s tests -p test_windows_download_helper.py -v", + "log": ".gate13-runs/windows-download-helper-progress-bounded-final-20260909.log", + "log_sha256": "3c6ecdb890a816c7d149ddd9af273519b615da3b304a6d23f6e86f5dc552ccdb", + "observation": "Execution reported by release_audit; saved final log and current source hashes independently verified for this record.", + "added_coverage": [ + "Reader denying delete sharing through helper exit", + "Read-only progress file causing actual access denial", + "Access-denied progress combined with bad SHA still fails" + ] + }, + "real_inno_tests": { + "passed": 4, + "failed": 0, + "seconds": 14.589, + "command": "python -m unittest discover -s tests -p test_windows_online_integration.py -v", + "log": ".gate13-runs/windows-online-integration-progress-bounded-final-20260909.log", + "log_sha256": "bdebf2ce6c711e3760bbdcf3c10473887d7406d7f5f030ab0a93772884ec4a67", + "observation": "Directly executed by gate15_choices; source SHA checked before and after the complete run.", + "coverage": [ + "Interrupted small download resumes at observed Range offset", + "Held response cancellation prevents child launch", + "Exact parent engine death stops owned helper and prevents child launch", + "Progress reader denies delete sharing for at least 1.4s and a skipped-publication warning is observed; valid child still launches" + ], + "handoff_assertions": [ + "Exact forwarded arguments", + "Outer exit7 equals harmless child exit7", + "Downloaded child exists until its exit", + "Owned processes stop and Inno temporary directory is removed" + ], + "fixtures": "Actual Inno6.7.3 script, separately compiled WINDOWS_DOWNLOAD_TEST loopback helper and tiny harmless winexe; hidden VERYSILENT invocation.", + "visible_windows_launched": false, + "product_installer_launched": false + }, + "style_checks": { + "black": "passed", + "isort_root_settings": "passed", + "git_diff_check": "passed", + "scope": "tests/test_windows_online_integration.py" + }, + "retained_real_failure": { + "evidence": "docs/evidence/normalized-online-windows-resumable-progress-failure-20260909.json", + "sha256": "f5003a51a4b4de1e64c3492d9b94d4d1b2769fb402fb27249ad56be869393781", + "wrapper_sha256": "0d1c31206336b1e8c783a78a6cadbdc8f9e643e91d587b88d78581a85b5d9ef6", + "reported_downloaded_bytes": 541341184, + "seconds": 263.343, + "message": "Progress could not be persisted", + "underlying_native_error": "Not recorded by the prior helper; no specific native cause inferred.", + "offline_installer_launched": false, + "cleanup_passed": true + }, + "publication_staging": { + "audit": ".gate13-runs/cloudflare-publication-private-20260909/progress-fix-v2-metadata-audit.json", + "audit_sha256": "c04d1c6e689442bf8a4341ce6c6b622b416e81f44d20f9d7c706fd119076ab92", + "result": "passed", + "combined_manifest_sha256": "3ab88fd1dd7b5ccbb19cb226429c5b3646d241a676280bde55795fdc34c8df6a", + "windows_manifest_sha256": "6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b", + "installer_sha256sums_sha256": "ba50a7b4502145303beb69b3f1c4a2b25bd5e297817b33d5c0f6d5428e284274", + "staged_windows_companion_sha256": "c3fece282465df8c74907ff4c7c29fd48405b9dc579315a61d340929afe4584e", + "other_three_installer_entries_unchanged": true, + "other_platform_metadata_and_manifests_unchanged": true, + "upload_performed": false, + "metadata_file_counts": { + "windows": 8, + "linux": 9 + } + }, + "qualification_boundaries": { + "complete_hosted_windows_download_passed": false, + "real_offline_installer_handoff_passed": false, + "full_transfer_performed_by_this_check": false, + "model_loading_or_peer_contact": false, + "native_credential_access": false + }, + "prior_bounded_evidence_preserved": "docs/evidence/windows-resumable-downloader-20260909.json", + "source_commit_binding_followup": { + "recorded_at_utc": "2026-09-09T01:36:10.134480+00:00", + "metadata_file": "metadata/windows/online-source-commit.json", + "metadata_sha256": "dc871c18c66b8ccfcb61304b9632d151cccb29cb623333055bef355e01419f94", + "git_blob_hashes_independently_verified": true, + "build_scope": "The wrapper was built from working-tree sources. This later commit contains matching bytes for all three production build inputs; it does not assert a clean committed checkout at original build time.", + "online_source_commit": "b6c8aad9cea208630785d890cfb966093f809e7e", + "all_three_git_blob_hashes_match_built_wrapper_companion": true, + "runtime_source_commit": "84205f93fc73d3babd39e238944b97fab0d11b3e", + "online_wrapper_sha256": "8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861" + } +} diff --git a/docs/evidence/windows-resumable-progress-fix-20260909.md b/docs/evidence/windows-resumable-progress-fix-20260909.md new file mode 100644 index 000000000..ceffb322a --- /dev/null +++ b/docs/evidence/windows-resumable-progress-fix-20260909.md @@ -0,0 +1,74 @@ +# Windows downloader progress fix — September 9, 2026 + +**24 helper tests and four real Inno tests passed.** A complete hosted Windows +download and actual offline-installer handoff remain separate acceptance. The +[JSON record](windows-resumable-progress-fix-20260909.json) binds this replacement +wrapper, exact source hashes, saved test logs and the private metadata audit. + +| Replacement Windows online artifact | Value | +| --- | --- | +| Filename | `communityai-0.1.0-alpha.20260908.1-windows-online-setup.exe` | +| Bytes | 2,107,751 | +| SHA-256 | `8ad0b7da83fdc7c32223902ed13d51f5f2946c89e00947e50aae742c1fc37861` | +| Embedded Windows-only manifest SHA-256 | `6e4423a2e0316eff9c19c18cf09b18224d8fcc769e6b0f9f10e514a37581355b` | +| Signing | Unsigned alpha | + +The previous resumable wrapper stopped after 263.343 seconds with 541,341,184 +bytes reported, because its progress record could not be published. No offline +installer launched; the recorded processes and temporary files were cleaned and +the persisted baseline was restored. The underlying exception was not recorded, +so no specific Windows error or network cause is inferred. That +[actual failed attempt](normalized-online-windows-resumable-progress-failure-20260909.md) +and the [earlier bounded checks](windows-resumable-downloader-20260909.md) remain +separate historical evidence. + +Progress publication is now best effort, including its final frame. Sharing +conflicts receive at most 75 ms of retries; other presentation I/O failures skip +the frame and may write a sanitized exception type and HRESULT. Inno can display +the actual partial-file length when the progress record is stale. Cancellation, +parent-process ownership, transfer limits and complete size/SHA-256 checks remain +authoritative. The helper must verify the download before returning success; +Inno independently verifies size and SHA-256 before logging completion or +launching the unchanged offline setup. + +The final helper suite passed **24 tests in 22.299 seconds**. Added cases keep a +reader open without delete sharing through helper exit, make the progress file +read-only to produce access denial, and combine that denial with an incorrect +download hash to confirm rejection. Existing response-range, retry, cancellation, +parent-loss, deadline and production-HTTPS checks remain covered. + +The actual Inno suite passed **four tests in 14.589 seconds**. The new case holds +the progress file without delete sharing for at least 1.4 seconds during a slow +local download and observes a skipped-publication warning. The verified harmless +child still receives the exact forwarded arguments, remains available until it +exits, and its exit code 7 reaches the wrapper. Inno then removes the temporary +directory. The prior interrupted-resume, cancellation and exact-parent-death +cases also passed. All recorded owned processes stopped; these tests used hidden +executables and silent setup, with no product installation or visible window. + +Both saved logs and the exact test/source hashes were checked for this record. +The helper source SHA-256 is +`3bfa36e62f2a160b2e258af6d9d6816e815d8a9c5b918c29dd800eb6e6c20d8a`; +the Inno script SHA-256 is +`d975b6da3a1067e679d9ba55555fa5b6219ae644fce8050225486909600fbbb1`. +These identify the working-tree build inputs; the offline runtime's source +commit does not identify the newer online wrapper. A subsequent source commit, +`b6c8aad9cea208630785d890cfb966093f809e7e`, contains independently verified matching +Git blobs for the helper, Inno script and builder. The staged +`metadata/windows/online-source-commit.json` records that binding; it does not +claim the original build used a clean committed checkout. Black, isort and +whitespace checks passed for the integration test file. + +The private publication audit matched all four installer identities, both +platform provenance records, the combined manifest and the distinct Windows-only +manifest. It updated only the Windows online companion and its checksum entry, +preserving the previous companion outside the public metadata folders. Eight +Windows records and nine Linux records matched the explicit metadata allowlists; +the two Linux acceptance records matched their sanitized documentation copies. +No raw logs or private-data markers were found in those metadata files. Linux +metadata and the other three installer checksum entries were unchanged. + +This check uploaded nothing, rehashed no multi-gigabyte offline installer, and +performed no full public transfer, model load, peer contact or credential access. +The replacement companion retains `live_download_verified: false`; these local +passes do not establish full online release readiness. diff --git a/docs/gate16-rpc-policy.example.json b/docs/gate16-rpc-policy.example.json new file mode 100644 index 000000000..2cf30fd25 --- /dev/null +++ b/docs/gate16-rpc-policy.example.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "admission": { + "max_active_sessions": 8, + "max_active_sessions_per_peer": 1, + "global_session_rate": 2.0, + "global_session_burst": 4, + "peer_session_rate": 0.25, + "peer_session_burst": 1, + "max_tracked_peers": 512, + "tracked_peer_ttl": 300.0, + "max_pending_pushes": 4, + "allow_training_rpcs": false + }, + "step_timeout": 30, + "session_timeout": 60 +} diff --git a/manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json b/manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json new file mode 100644 index 000000000..45cdec018 --- /dev/null +++ b/manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "name": "Qwen3.5-0.8B-Local", + "aliases": [ + "local-qwen" + ], + "source": { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17" + }, + "model": { + "architecture": "Qwen3_5ForConditionalGeneration", + "num_blocks": 24, + "context_length": 262144, + "license": "apache-2.0", + "gated": false + }, + "runtime": { + "implementation": "drift", + "minimum_version": "2.3.0.dev0", + "maximum_version_exclusive": "2.4.0", + "protocol_version": 1, + "tensor_schema": "hidden-states-v1", + "attention_implementation": "eager", + "dtype": "bfloat16", + "quantization": "none", + "adapter_profile": "none" + }, + "artifacts": [ + { + "role": "chat_template", + "path": "chat_template.jinja", + "sha256": "273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80", + "size": 7755 + }, + { + "role": "config", + "path": "config.json", + "sha256": "b90b86f35c8e6925ef74ee04d0e758f0a845c83a42089ad82bbaa948de9b4204", + "size": 2907 + }, + { + "role": "tokenizer", + "path": "merges.txt", + "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "size": 3353259 + }, + { + "role": "weight", + "path": "model.safetensors-00001-of-00001.safetensors", + "sha256": "04b1c301231dd422b8860db31311ab2721511346a32cb1e079c4c4e5f1fe4696", + "size": 1746942600 + }, + { + "role": "weight_index", + "path": "model.safetensors.index.json", + "sha256": "d8a08838a613b025eb7952ed9db11696213e57e76a375661ef5c12f9dd5dcf4e", + "size": 50900 + }, + { + "role": "tokenizer", + "path": "tokenizer.json", + "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + "size": 12807982 + }, + { + "role": "tokenizer", + "path": "tokenizer_config.json", + "sha256": "49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c", + "size": 16709 + }, + { + "role": "tokenizer", + "path": "vocab.json", + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + "size": 6722759 + } + ] +} diff --git a/manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json b/manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json new file mode 100644 index 000000000..2759bf080 --- /dev/null +++ b/manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json @@ -0,0 +1,470 @@ +{ + "schema_version": 1, + "name": "Qwen3.8 27B FP8 Dequant", + "aliases": [ + "qwen3.8-27b", + "qwen3.8-27b-fp8" + ], + "source": { + "repository": "Qwen/Qwen3.8-27B-FP8", + "revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a" + }, + "model": { + "architecture": "Qwen3_5ForConditionalGeneration", + "num_blocks": 64, + "context_length": 262144, + "license": "apache-2.0", + "gated": false + }, + "runtime": { + "implementation": "drift", + "minimum_version": "2.3.0.dev0", + "maximum_version_exclusive": "2.4.0", + "protocol_version": 1, + "tensor_schema": "hidden-states-v1", + "attention_implementation": "eager", + "dtype": "bfloat16", + "quantization": "fp8_dequant", + "adapter_profile": "none" + }, + "artifacts": [ + { + "role": "chat_template", + "path": "chat_template.jinja", + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "size": 8952 + }, + { + "role": "config", + "path": "config.json", + "sha256": "74227dd615bf1ea975aa676bdf355a0379858c12f394b5365cd9dfa5fc2c70bc", + "size": 51350 + }, + { + "role": "weight", + "path": "layers-0.safetensors", + "sha256": "07f700e293baeaf3cd4240c3df1a948c4403f16961ea7979e86c8d6a9f8fd466", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-1.safetensors", + "sha256": "35840b5d452c6e438d000e5c1d8d1bc793d257394404689b8f9424749eee8edc", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-10.safetensors", + "sha256": "ab5fbc076ccd6514aba3ab67a0ac1d6701ad395d40cccc0086cf0284ef5d68c9", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-11.safetensors", + "sha256": "076b4ba44e9311c8fb8fdc0045f8ce9fd0c75584ade4e95510056b7db4b7ec61", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-12.safetensors", + "sha256": "86d8ec0b8fc8e3ffbdfd667b97050533ddf5a4d273b4fe1190a4335542c52829", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-13.safetensors", + "sha256": "71dd986134f4338a7b8290ed24993ad34a0f77b76a4cc060fe6403e36dcdda5b", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-14.safetensors", + "sha256": "95b285915cba994917b527030874227382ea3421dc7f47d804c0ca5851f039fb", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-15.safetensors", + "sha256": "1e87d61e77d2b802f796ab8a049b1119648c9b70d2ba1067f456e5a04449cfc3", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-16.safetensors", + "sha256": "8bcdbf4e2c7a8dda3043c4453a476b30a582d58aabfaa49b2829a8b66a664acb", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-17.safetensors", + "sha256": "571cea44717878cc8cbbfbd544acfe2d155f5b963fbf5dc67b3fa96afe840a1b", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-18.safetensors", + "sha256": "d8700627ca2267fb7beb5d398d6a659a9bfeeb2af76f10e6827b1e4359d59286", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-19.safetensors", + "sha256": "2b24cfb752bbed959c614d7fd85412ee27ab2ac1f47deef5d09d0592781400e4", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-2.safetensors", + "sha256": "32e63c6455ebfa10b4c36af2878eefcb6f8ccf8e947a62575444dc412c38ac1a", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-20.safetensors", + "sha256": "52fd00c68d4f1df96ba701290c04237257ebb832879bb0b6a629525809d84a64", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-21.safetensors", + "sha256": "e3c29dd949808c13116d853ce4b0f96cb1238fc0cb6999ab76735deef6bda77a", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-22.safetensors", + "sha256": "63446546e3e09c2394fc51855f4e390cb47f896bb8b744f3a210c3159a3ccaf2", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-23.safetensors", + "sha256": "4166ed11d0329985f9813ba6a70474ea9f5aeff3d97db3d7b2e81185bd41a996", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-24.safetensors", + "sha256": "720bdcfe12b6627bed29feda9c96256e15c20602704669d05097668a9f03dd45", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-25.safetensors", + "sha256": "1b188e7e4e8ae6f5d753c597f61d86c1593495de4307c273d8dda9946ec48bb4", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-26.safetensors", + "sha256": "ad8c3e3d4b79dfedad674318ee4b150b2cd78b56747c7c8d88a65818109a545d", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-27.safetensors", + "sha256": "8d63f461430045960f27a22abf685bfc7445fffbb24764e4a7fa993ba7edac08", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-28.safetensors", + "sha256": "f78557cda66107ce25f8c5ecde2f83da874b3bf7c50381e062e14e7294a5b64d", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-29.safetensors", + "sha256": "09f264d6fdddab0fb5ecce18034a9a01944d15118147c97058fba101a60f8af1", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-3.safetensors", + "sha256": "302f9af90bb683a8be9e96d124b470a2eddee6612c95c39a8e93f26eb654563d", + "size": 372313744 + }, + { + "role": "weight", + "path": "layers-30.safetensors", + "sha256": "54afe4fc7262ee597d4d9b80d2d6fe8ae501846ff124334b575bed4c6850ed97", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-31.safetensors", + "sha256": "2779338203d0715ca323ac0b7fdbb0d450ce2bd1694be395a211a02ade3f34d9", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-32.safetensors", + "sha256": "f4af8893594458b6cdb074ef20f9accb708361fe904012465990d09e74994509", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-33.safetensors", + "sha256": "8f038505b9fc7d0b31ca976c728b56028ec28390a15ca0410f00df18d3abc1a4", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-34.safetensors", + "sha256": "0b28f43cc4d3ae50e37eb3c2ce12a6e181b1c7586a27d5a05c6a3367f08286c9", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-35.safetensors", + "sha256": "535b61b9eabbcda5de20e21d2d46fb52528cc9c1f8fac38920382ea2f1030837", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-36.safetensors", + "sha256": "2babd209cafbe12a6e375ff6e7a1029f36ee8226f2a9babf6646630f31dd3fb8", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-37.safetensors", + "sha256": "651cae579808983fefec954db6e2ac8c3f98bfb038ff1d50aef0505ba7394e65", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-38.safetensors", + "sha256": "1c60932dec650cc7679121eaa4d90b8d10a600bf58e23b0784588a339469dd0a", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-39.safetensors", + "sha256": "b0c7df4d51b99637ff4ea35eebf7eb134db592eb943e3ccde921729f33949058", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-4.safetensors", + "sha256": "b7f367125cdb4b3c3920d1d3ed1ba0d73e464c3e3a92dbd1c195658cb9200afd", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-40.safetensors", + "sha256": "6908f1ab1a3d7828a566099ecf393582bec988d0f82f6141848942d1494b0321", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-41.safetensors", + "sha256": "21ee5d9842074888bd9fdffa875d1f29f8c77f1f5ccbb8f53becb188e1749caa", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-42.safetensors", + "sha256": "565379d4291c06cdf9a4f66d5d21385848a1e5ab17b55fea6c1c8e544f8498f0", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-43.safetensors", + "sha256": "4bb8c0a4fadda1f1d66ef6f0dde5a5a94001d04af1842e42009833aff70bf416", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-44.safetensors", + "sha256": "14339304cd520a8673f179cb8352c36fbebe184f15309aee2a79afd2d4476fef", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-45.safetensors", + "sha256": "7a076aca542f937cc87dec92ab49fd782eae234642b40bc706a6108b3dff2c2a", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-46.safetensors", + "sha256": "d82d472cfc4792012934467b90c5e0242df4867a430dd651d13cdee72b7c6645", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-47.safetensors", + "sha256": "a83fc757fb11bc50ed3a0c9998b2b2f55c3b6a07598ed458bdbece4285ffaf1f", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-48.safetensors", + "sha256": "f6aa126d006c7d976c20b1330c118b42cf3bbe8a7d216bb14f2240d5c49bc72f", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-49.safetensors", + "sha256": "7ed54c375407c2baec8cfabe8336f8d3782246848298bb232ac2582113676c69", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-5.safetensors", + "sha256": "ebbc2c4bd2b98877caafa7073b6ed732a0284eb82269cf6d21710e245af5837a", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-50.safetensors", + "sha256": "73825e1f056c3de53276d3c65743e62b67bf3ca082cf7077ecba1ffef18e9bf1", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-51.safetensors", + "sha256": "b6abd8b62b5c4b7beca8cdccd306065be2192ef42491c587ca019d971ed86a1f", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-52.safetensors", + "sha256": "6251d51e6fbec6f463ca875fcbe852e07c5025145affe9af053a04fbdc37288f", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-53.safetensors", + "sha256": "a9f2f43b4978bba2f755c6d2eb39f8561965bc3c1b2fd1073bb363aff11a9b8c", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-54.safetensors", + "sha256": "f0246c9974a4b1207f8f5d05a09c54c50ea61fbfd2dc6480d2b76d3647ece292", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-55.safetensors", + "sha256": "649c400495616cf888b7c52e538113610a36265c531d84bff484f8149c157405", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-56.safetensors", + "sha256": "eb0a8139a36138639a37219c158b9415412b68d316235061a72e57da5509c188", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-57.safetensors", + "sha256": "d7616ad33f9256342573ff85b3f2a0e368dd120257c0c7158564587036119d95", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-58.safetensors", + "sha256": "a8dff0e37d0e3903cc29711b935c2865ccd38bd930e7487ae2f4bedbd18ff185", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-59.safetensors", + "sha256": "573ed65fd455ec9bd811e95946ea422e08ac36860fdff6968a6d108263a54175", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-6.safetensors", + "sha256": "2fea2aaa61d566ba8af27ee89d29d9ee9f1c7f6fbe692772d9ba61fc45109ef6", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-60.safetensors", + "sha256": "9a19c30b190bf552eb424af16103f610bcd944fb276b3275eb7c73849799b7de", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-61.safetensors", + "sha256": "ff744debd4dd0eef450ed24473df57680db404914302e7147a3b72154131f4a0", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-62.safetensors", + "sha256": "d9b3b9e472f78bcadf62c446084833bb549faff9e730b21219f1f80d583eff23", + "size": 383865472 + }, + { + "role": "weight", + "path": "layers-63.safetensors", + "sha256": "59ba4a3af5e6bc2008c2d6b5d9be8eeaf255cddc0aaf8a0d9583b496bb2e3ae0", + "size": 372313760 + }, + { + "role": "weight", + "path": "layers-7.safetensors", + "sha256": "f2e0137e878016a7afdb2314972e0cbbe2461a67b00eb1d691b4de8ea2665da8", + "size": 372313744 + }, + { + "role": "weight", + "path": "layers-8.safetensors", + "sha256": "1721b7bcd730891c224b70a7b6147e44a574a2d6c4d924760acf97728249c92c", + "size": 383865448 + }, + { + "role": "weight", + "path": "layers-9.safetensors", + "sha256": "0a21e07065b5bb04adc339300dea1872cc48ab8eab3b02df092f7cbc67c8d6f6", + "size": 383865448 + }, + { + "role": "tokenizer", + "path": "merges.txt", + "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "size": 3353259 + }, + { + "role": "weight_index", + "path": "model.safetensors.index.json", + "sha256": "f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2", + "size": 137335 + }, + { + "role": "weight", + "path": "mtp.safetensors", + "sha256": "e5e4464a3793cc261de536592830bca40e7f3af159ed038c358f5660917cf43b", + "size": 477202224 + }, + { + "role": "weight", + "path": "outside.safetensors", + "sha256": "ddff1d6665a2b39f2612fce0ef955e2436724c565bfbcbc127c7ffd078b698ff", + "size": 6007102112 + }, + { + "role": "tokenizer", + "path": "tokenizer.json", + "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3", + "size": 12809320 + }, + { + "role": "tokenizer", + "path": "tokenizer_config.json", + "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27", + "size": 17928 + }, + { + "role": "tokenizer", + "path": "vocab.json", + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + "size": 6722759 + } + ] +} diff --git a/manifests/reference/qwen3.8-27b-bfloat16-eager.json b/manifests/reference/qwen3.8-27b-bfloat16-eager.json new file mode 100644 index 000000000..34f24f958 --- /dev/null +++ b/manifests/reference/qwen3.8-27b-bfloat16-eager.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "name": "Qwen3.8 27B BF16 Reference", + "aliases": [ + "qwen3.8-27b-bf16-reference" + ], + "source": { + "repository": "Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + }, + "model": { + "architecture": "Qwen3_5ForConditionalGeneration", + "num_blocks": 64, + "context_length": 262144, + "license": "apache-2.0", + "gated": false + }, + "runtime": { + "implementation": "drift", + "minimum_version": "2.3.0.dev0", + "maximum_version_exclusive": "2.4.0", + "protocol_version": 1, + "tensor_schema": "hidden-states-v1", + "attention_implementation": "eager", + "dtype": "bfloat16", + "quantization": "none", + "adapter_profile": "none" + }, + "artifacts": [ + { + "role": "chat_template", + "path": "chat_template.jinja", + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "size": 8952 + }, + { + "role": "config", + "path": "config.json", + "sha256": "191e0af232104ed8b65258cf3fb2b842e288008baca7633c11b82a1ac7203aab", + "size": 4312 + }, + { + "role": "tokenizer", + "path": "merges.txt", + "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "size": 3353259 + }, + { + "role": "weight", + "path": "model-00001-of-00018.safetensors", + "sha256": "ba0ce20aae489ad196733da5064bcdf159a1fe84f53336648196e1ebb7751b1c", + "size": 3966730552 + }, + { + "role": "weight", + "path": "model-00002-of-00018.safetensors", + "sha256": "06a148c01bfbe3faa14a5f184a7ff29a706f7ae1c8b2705d2058e26d17a001fb", + "size": 3043080328 + }, + { + "role": "weight", + "path": "model-00003-of-00018.safetensors", + "sha256": "2e1bf62cbcd406eaa64b60d10353e1f0ef4039d0976e56f05cabe953454f9968", + "size": 2542796952 + }, + { + "role": "weight", + "path": "model-00004-of-00018.safetensors", + "sha256": "511e34063187882659753c4d93f3859f93c019fd438d8813071921c81d9a3f1a", + "size": 3988973152 + }, + { + "role": "weight", + "path": "model-00005-of-00018.safetensors", + "sha256": "635cb53446dc74f219740fc59e18b774f877b803b9722e289ca62575a6efa701", + "size": 2099339864 + }, + { + "role": "weight", + "path": "model-00006-of-00018.safetensors", + "sha256": "0bc5214fac607f0e6cc92eec3789d4b8559410ef9fce66621ba8158e8410dae0", + "size": 3979553696 + }, + { + "role": "weight", + "path": "model-00007-of-00018.safetensors", + "sha256": "80b0c49033e9a0d5762562aa12f4acdb7f54da586f3d0110f28c48d91cf07892", + "size": 2108759344 + }, + { + "role": "weight", + "path": "model-00008-of-00018.safetensors", + "sha256": "7192c5b66185d3592927daabee1cc19e6f6e0ce75988ee20e824b624765fda79", + "size": 3979553696 + }, + { + "role": "weight", + "path": "model-00009-of-00018.safetensors", + "sha256": "af3c48cc37af44f3db6ae0579baf019180d48d9c527caa0a1f03ff85813a56d8", + "size": 2108759344 + }, + { + "role": "weight", + "path": "model-00010-of-00018.safetensors", + "sha256": "163490a76f3bea3a40855b7efc04ce6d27afaf1a34f0bbde495b9491f76457c9", + "size": 3979553696 + }, + { + "role": "weight", + "path": "model-00011-of-00018.safetensors", + "sha256": "5f3ae1b948aeee39da77aec558e8236cd65fe4d7cb7686a76bb007acc563c6d8", + "size": 2108759344 + }, + { + "role": "weight", + "path": "model-00012-of-00018.safetensors", + "sha256": "a3de1c7114677a8f5ac5c4892c90e8238ea5c1e2038c80e757dfc87c3902ca55", + "size": 3979553696 + }, + { + "role": "weight", + "path": "model-00013-of-00018.safetensors", + "sha256": "06ab79a41f74c9c5cb734816feb0c7fc364104b227165ee7391231e1155aa02a", + "size": 2108759344 + }, + { + "role": "weight", + "path": "model-00014-of-00018.safetensors", + "sha256": "4138ed94603065ba884bbcadedb04d7718bb40117e85e6f5c6fc5b9c05b7a85b", + "size": 3979553696 + }, + { + "role": "weight", + "path": "model-00015-of-00018.safetensors", + "sha256": "69224e27b9de4e7dbf6fc936c6eaae08447bda3b80a6c31a871ab451173afd22", + "size": 2108759344 + }, + { + "role": "weight", + "path": "model-00016-of-00018.safetensors", + "sha256": "73cb9a1089fb6155cb648609478d6633be8a5c7d9ca5a05bc8925ce8a553cefe", + "size": 3979564040 + }, + { + "role": "weight", + "path": "model-00017-of-00018.safetensors", + "sha256": "beb51f01056142ac4984bd800507b0dd0fd18de57f8e9ef6ea41d1a3598983a8", + "size": 2108759344 + }, + { + "role": "weight", + "path": "model-00018-of-00018.safetensors", + "sha256": "1d3479509e21494658f9b64d317f5ea8e55c4025d28c702d6c4d0b356ce8ea06", + "size": 3392197344 + }, + { + "role": "weight_index", + "path": "model.safetensors.index.json", + "sha256": "77042094076611b69791a610065f28b7013b8c621795fa86ddccc8bac7d1b9df", + "size": 112216 + }, + { + "role": "tokenizer", + "path": "tokenizer.json", + "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3", + "size": 12809320 + }, + { + "role": "tokenizer", + "path": "tokenizer_config.json", + "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27", + "size": 17928 + }, + { + "role": "tokenizer", + "path": "vocab.json", + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + "size": 6722759 + } + ] +} diff --git a/public-alpha/catalog-qwen-candidate/catalog-bootstrap.json b/public-alpha/catalog-qwen-candidate/catalog-bootstrap.json new file mode 100644 index 000000000..1d58c0b98 --- /dev/null +++ b/public-alpha/catalog-qwen-candidate/catalog-bootstrap.json @@ -0,0 +1,22 @@ +{ + "catalog_mirrors": [ + "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/catalog.signed.json" + ], + "initial_peers": [ + "/dns4/bootstrap.communityai.flujo.com.co/tcp/31337/p2p/QmZhGcSVR6qPLZTq3TJPZEi734GbMkouv3kPxQLdDY2qUo" + ], + "max_loaded_models": 2, + "schema_version": 1, + "trust_root": { + "catalog_id": "communityai-public-alpha-v1", + "keys": [ + { + "algorithm": "ed25519", + "key_id": "sha256:a8fb23c4ac71c1f29cc9991d79d16743a2a931c1911b819da0fb5432d5ac8435", + "public_key": "MCowBQYDK2VwAyEALk4piTaDwzQzeAq2DxwOrognfuU+rdea1XkPVeFF7u4=" + } + ], + "schema_version": 1, + "threshold": 1 + } +} diff --git a/public-alpha/catalog-qwen-candidate/catalog.unsigned.json b/public-alpha/catalog-qwen-candidate/catalog.unsigned.json new file mode 100644 index 000000000..9fb602c9c --- /dev/null +++ b/public-alpha/catalog-qwen-candidate/catalog.unsigned.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "signatures": [], + "signed": { + "catalog_id": "communityai-public-alpha-v1", + "expires_at_ms": 1790624119369, + "issued_at_ms": 1788654808765, + "models": [ + { + "active_parameters": 800000000, + "execution": "local", + "manifest_digest": "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + "manifest_urls": [ + "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json" + ], + "role": "primary", + "rung": "local-qwen", + "total_parameters": 800000000, + "weight_bytes": 1746942600 + }, + { + "active_parameters": 27000000000, + "execution": "distributed", + "manifest_digest": "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "manifest_urls": [ + "https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-v1/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json" + ], + "role": "primary", + "rung": "community-qwen", + "total_parameters": 27000000000, + "weight_bytes": 30866866928 + } + ], + "rungs": [ + { + "id": "local-qwen", + "maximum_observation_age_seconds": 30, + "maximum_p95_first_token_ms": 60000, + "minimum_independent_routes": 1, + "minimum_replicas": 1, + "minimum_soak_seconds": 60, + "minimum_surviving_replicas": 0, + "minimum_tokens_per_minute": 1, + "order": 1 + }, + { + "id": "community-qwen", + "maximum_observation_age_seconds": 30, + "maximum_p95_first_token_ms": 60000, + "minimum_independent_routes": 1, + "minimum_replicas": 1, + "minimum_soak_seconds": 60, + "minimum_surviving_replicas": 0, + "minimum_tokens_per_minute": 1, + "order": 2 + } + ], + "sequence": 2 + } +} diff --git a/public-alpha/catalog-qwen-candidate/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json b/public-alpha/catalog-qwen-candidate/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json new file mode 100644 index 000000000..9a14a79fb --- /dev/null +++ b/public-alpha/catalog-qwen-candidate/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json @@ -0,0 +1 @@ +{"aliases":["qwen3.8-27b","qwen3.8-27b-fp8"],"artifacts":[{"path":"chat_template.jinja","role":"chat_template","sha256":"c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041","size":8952},{"path":"config.json","role":"config","sha256":"74227dd615bf1ea975aa676bdf355a0379858c12f394b5365cd9dfa5fc2c70bc","size":51350},{"path":"layers-0.safetensors","role":"weight","sha256":"07f700e293baeaf3cd4240c3df1a948c4403f16961ea7979e86c8d6a9f8fd466","size":383865448},{"path":"layers-1.safetensors","role":"weight","sha256":"35840b5d452c6e438d000e5c1d8d1bc793d257394404689b8f9424749eee8edc","size":383865448},{"path":"layers-10.safetensors","role":"weight","sha256":"ab5fbc076ccd6514aba3ab67a0ac1d6701ad395d40cccc0086cf0284ef5d68c9","size":383865472},{"path":"layers-11.safetensors","role":"weight","sha256":"076b4ba44e9311c8fb8fdc0045f8ce9fd0c75584ade4e95510056b7db4b7ec61","size":372313760},{"path":"layers-12.safetensors","role":"weight","sha256":"86d8ec0b8fc8e3ffbdfd667b97050533ddf5a4d273b4fe1190a4335542c52829","size":383865472},{"path":"layers-13.safetensors","role":"weight","sha256":"71dd986134f4338a7b8290ed24993ad34a0f77b76a4cc060fe6403e36dcdda5b","size":383865472},{"path":"layers-14.safetensors","role":"weight","sha256":"95b285915cba994917b527030874227382ea3421dc7f47d804c0ca5851f039fb","size":383865472},{"path":"layers-15.safetensors","role":"weight","sha256":"1e87d61e77d2b802f796ab8a049b1119648c9b70d2ba1067f456e5a04449cfc3","size":372313760},{"path":"layers-16.safetensors","role":"weight","sha256":"8bcdbf4e2c7a8dda3043c4453a476b30a582d58aabfaa49b2829a8b66a664acb","size":383865472},{"path":"layers-17.safetensors","role":"weight","sha256":"571cea44717878cc8cbbfbd544acfe2d155f5b963fbf5dc67b3fa96afe840a1b","size":383865472},{"path":"layers-18.safetensors","role":"weight","sha256":"d8700627ca2267fb7beb5d398d6a659a9bfeeb2af76f10e6827b1e4359d59286","size":383865472},{"path":"layers-19.safetensors","role":"weight","sha256":"2b24cfb752bbed959c614d7fd85412ee27ab2ac1f47deef5d09d0592781400e4","size":372313760},{"path":"layers-2.safetensors","role":"weight","sha256":"32e63c6455ebfa10b4c36af2878eefcb6f8ccf8e947a62575444dc412c38ac1a","size":383865448},{"path":"layers-20.safetensors","role":"weight","sha256":"52fd00c68d4f1df96ba701290c04237257ebb832879bb0b6a629525809d84a64","size":383865472},{"path":"layers-21.safetensors","role":"weight","sha256":"e3c29dd949808c13116d853ce4b0f96cb1238fc0cb6999ab76735deef6bda77a","size":383865472},{"path":"layers-22.safetensors","role":"weight","sha256":"63446546e3e09c2394fc51855f4e390cb47f896bb8b744f3a210c3159a3ccaf2","size":383865472},{"path":"layers-23.safetensors","role":"weight","sha256":"4166ed11d0329985f9813ba6a70474ea9f5aeff3d97db3d7b2e81185bd41a996","size":372313760},{"path":"layers-24.safetensors","role":"weight","sha256":"720bdcfe12b6627bed29feda9c96256e15c20602704669d05097668a9f03dd45","size":383865472},{"path":"layers-25.safetensors","role":"weight","sha256":"1b188e7e4e8ae6f5d753c597f61d86c1593495de4307c273d8dda9946ec48bb4","size":383865472},{"path":"layers-26.safetensors","role":"weight","sha256":"ad8c3e3d4b79dfedad674318ee4b150b2cd78b56747c7c8d88a65818109a545d","size":383865472},{"path":"layers-27.safetensors","role":"weight","sha256":"8d63f461430045960f27a22abf685bfc7445fffbb24764e4a7fa993ba7edac08","size":372313760},{"path":"layers-28.safetensors","role":"weight","sha256":"f78557cda66107ce25f8c5ecde2f83da874b3bf7c50381e062e14e7294a5b64d","size":383865472},{"path":"layers-29.safetensors","role":"weight","sha256":"09f264d6fdddab0fb5ecce18034a9a01944d15118147c97058fba101a60f8af1","size":383865472},{"path":"layers-3.safetensors","role":"weight","sha256":"302f9af90bb683a8be9e96d124b470a2eddee6612c95c39a8e93f26eb654563d","size":372313744},{"path":"layers-30.safetensors","role":"weight","sha256":"54afe4fc7262ee597d4d9b80d2d6fe8ae501846ff124334b575bed4c6850ed97","size":383865472},{"path":"layers-31.safetensors","role":"weight","sha256":"2779338203d0715ca323ac0b7fdbb0d450ce2bd1694be395a211a02ade3f34d9","size":372313760},{"path":"layers-32.safetensors","role":"weight","sha256":"f4af8893594458b6cdb074ef20f9accb708361fe904012465990d09e74994509","size":383865472},{"path":"layers-33.safetensors","role":"weight","sha256":"8f038505b9fc7d0b31ca976c728b56028ec28390a15ca0410f00df18d3abc1a4","size":383865472},{"path":"layers-34.safetensors","role":"weight","sha256":"0b28f43cc4d3ae50e37eb3c2ce12a6e181b1c7586a27d5a05c6a3367f08286c9","size":383865472},{"path":"layers-35.safetensors","role":"weight","sha256":"535b61b9eabbcda5de20e21d2d46fb52528cc9c1f8fac38920382ea2f1030837","size":372313760},{"path":"layers-36.safetensors","role":"weight","sha256":"2babd209cafbe12a6e375ff6e7a1029f36ee8226f2a9babf6646630f31dd3fb8","size":383865472},{"path":"layers-37.safetensors","role":"weight","sha256":"651cae579808983fefec954db6e2ac8c3f98bfb038ff1d50aef0505ba7394e65","size":383865472},{"path":"layers-38.safetensors","role":"weight","sha256":"1c60932dec650cc7679121eaa4d90b8d10a600bf58e23b0784588a339469dd0a","size":383865472},{"path":"layers-39.safetensors","role":"weight","sha256":"b0c7df4d51b99637ff4ea35eebf7eb134db592eb943e3ccde921729f33949058","size":372313760},{"path":"layers-4.safetensors","role":"weight","sha256":"b7f367125cdb4b3c3920d1d3ed1ba0d73e464c3e3a92dbd1c195658cb9200afd","size":383865448},{"path":"layers-40.safetensors","role":"weight","sha256":"6908f1ab1a3d7828a566099ecf393582bec988d0f82f6141848942d1494b0321","size":383865472},{"path":"layers-41.safetensors","role":"weight","sha256":"21ee5d9842074888bd9fdffa875d1f29f8c77f1f5ccbb8f53becb188e1749caa","size":383865472},{"path":"layers-42.safetensors","role":"weight","sha256":"565379d4291c06cdf9a4f66d5d21385848a1e5ab17b55fea6c1c8e544f8498f0","size":383865472},{"path":"layers-43.safetensors","role":"weight","sha256":"4bb8c0a4fadda1f1d66ef6f0dde5a5a94001d04af1842e42009833aff70bf416","size":372313760},{"path":"layers-44.safetensors","role":"weight","sha256":"14339304cd520a8673f179cb8352c36fbebe184f15309aee2a79afd2d4476fef","size":383865472},{"path":"layers-45.safetensors","role":"weight","sha256":"7a076aca542f937cc87dec92ab49fd782eae234642b40bc706a6108b3dff2c2a","size":383865472},{"path":"layers-46.safetensors","role":"weight","sha256":"d82d472cfc4792012934467b90c5e0242df4867a430dd651d13cdee72b7c6645","size":383865472},{"path":"layers-47.safetensors","role":"weight","sha256":"a83fc757fb11bc50ed3a0c9998b2b2f55c3b6a07598ed458bdbece4285ffaf1f","size":372313760},{"path":"layers-48.safetensors","role":"weight","sha256":"f6aa126d006c7d976c20b1330c118b42cf3bbe8a7d216bb14f2240d5c49bc72f","size":383865472},{"path":"layers-49.safetensors","role":"weight","sha256":"7ed54c375407c2baec8cfabe8336f8d3782246848298bb232ac2582113676c69","size":383865472},{"path":"layers-5.safetensors","role":"weight","sha256":"ebbc2c4bd2b98877caafa7073b6ed732a0284eb82269cf6d21710e245af5837a","size":383865448},{"path":"layers-50.safetensors","role":"weight","sha256":"73825e1f056c3de53276d3c65743e62b67bf3ca082cf7077ecba1ffef18e9bf1","size":383865472},{"path":"layers-51.safetensors","role":"weight","sha256":"b6abd8b62b5c4b7beca8cdccd306065be2192ef42491c587ca019d971ed86a1f","size":372313760},{"path":"layers-52.safetensors","role":"weight","sha256":"6251d51e6fbec6f463ca875fcbe852e07c5025145affe9af053a04fbdc37288f","size":383865472},{"path":"layers-53.safetensors","role":"weight","sha256":"a9f2f43b4978bba2f755c6d2eb39f8561965bc3c1b2fd1073bb363aff11a9b8c","size":383865472},{"path":"layers-54.safetensors","role":"weight","sha256":"f0246c9974a4b1207f8f5d05a09c54c50ea61fbfd2dc6480d2b76d3647ece292","size":383865472},{"path":"layers-55.safetensors","role":"weight","sha256":"649c400495616cf888b7c52e538113610a36265c531d84bff484f8149c157405","size":372313760},{"path":"layers-56.safetensors","role":"weight","sha256":"eb0a8139a36138639a37219c158b9415412b68d316235061a72e57da5509c188","size":383865472},{"path":"layers-57.safetensors","role":"weight","sha256":"d7616ad33f9256342573ff85b3f2a0e368dd120257c0c7158564587036119d95","size":383865472},{"path":"layers-58.safetensors","role":"weight","sha256":"a8dff0e37d0e3903cc29711b935c2865ccd38bd930e7487ae2f4bedbd18ff185","size":383865472},{"path":"layers-59.safetensors","role":"weight","sha256":"573ed65fd455ec9bd811e95946ea422e08ac36860fdff6968a6d108263a54175","size":372313760},{"path":"layers-6.safetensors","role":"weight","sha256":"2fea2aaa61d566ba8af27ee89d29d9ee9f1c7f6fbe692772d9ba61fc45109ef6","size":383865448},{"path":"layers-60.safetensors","role":"weight","sha256":"9a19c30b190bf552eb424af16103f610bcd944fb276b3275eb7c73849799b7de","size":383865472},{"path":"layers-61.safetensors","role":"weight","sha256":"ff744debd4dd0eef450ed24473df57680db404914302e7147a3b72154131f4a0","size":383865472},{"path":"layers-62.safetensors","role":"weight","sha256":"d9b3b9e472f78bcadf62c446084833bb549faff9e730b21219f1f80d583eff23","size":383865472},{"path":"layers-63.safetensors","role":"weight","sha256":"59ba4a3af5e6bc2008c2d6b5d9be8eeaf255cddc0aaf8a0d9583b496bb2e3ae0","size":372313760},{"path":"layers-7.safetensors","role":"weight","sha256":"f2e0137e878016a7afdb2314972e0cbbe2461a67b00eb1d691b4de8ea2665da8","size":372313744},{"path":"layers-8.safetensors","role":"weight","sha256":"1721b7bcd730891c224b70a7b6147e44a574a2d6c4d924760acf97728249c92c","size":383865448},{"path":"layers-9.safetensors","role":"weight","sha256":"0a21e07065b5bb04adc339300dea1872cc48ab8eab3b02df092f7cbc67c8d6f6","size":383865448},{"path":"merges.txt","role":"tokenizer","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size":3353259},{"path":"model.safetensors.index.json","role":"weight_index","sha256":"f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2","size":137335},{"path":"mtp.safetensors","role":"weight","sha256":"e5e4464a3793cc261de536592830bca40e7f3af159ed038c358f5660917cf43b","size":477202224},{"path":"outside.safetensors","role":"weight","sha256":"ddff1d6665a2b39f2612fce0ef955e2436724c565bfbcbc127c7ffd078b698ff","size":6007102112},{"path":"tokenizer.json","role":"tokenizer","sha256":"0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3","size":12809320},{"path":"tokenizer_config.json","role":"tokenizer","sha256":"b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27","size":17928},{"path":"vocab.json","role":"tokenizer","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size":6722759}],"model":{"architecture":"Qwen3_5ForConditionalGeneration","context_length":262144,"gated":false,"license":"apache-2.0","num_blocks":64},"name":"Qwen3.8 27B FP8 Dequant","runtime":{"adapter_profile":"none","attention_implementation":"eager","dtype":"bfloat16","implementation":"drift","maximum_version_exclusive":"2.4.0","minimum_version":"2.3.0.dev0","protocol_version":1,"quantization":"fp8_dequant","tensor_schema":"hidden-states-v1"},"schema_version":1,"source":{"repository":"Qwen/Qwen3.8-27B-FP8","revision":"017b9c7af6b5689d5dd426a76e0bc077eb5ca20a"}} diff --git a/public-alpha/catalog-qwen-candidate/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json b/public-alpha/catalog-qwen-candidate/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json new file mode 100644 index 000000000..0f4ef8988 --- /dev/null +++ b/public-alpha/catalog-qwen-candidate/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json @@ -0,0 +1 @@ +{"aliases":["local-qwen"],"artifacts":[{"path":"chat_template.jinja","role":"chat_template","sha256":"273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80","size":7755},{"path":"config.json","role":"config","sha256":"b90b86f35c8e6925ef74ee04d0e758f0a845c83a42089ad82bbaa948de9b4204","size":2907},{"path":"merges.txt","role":"tokenizer","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size":3353259},{"path":"model.safetensors-00001-of-00001.safetensors","role":"weight","sha256":"04b1c301231dd422b8860db31311ab2721511346a32cb1e079c4c4e5f1fe4696","size":1746942600},{"path":"model.safetensors.index.json","role":"weight_index","sha256":"d8a08838a613b025eb7952ed9db11696213e57e76a375661ef5c12f9dd5dcf4e","size":50900},{"path":"tokenizer.json","role":"tokenizer","sha256":"5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42","size":12807982},{"path":"tokenizer_config.json","role":"tokenizer","sha256":"49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c","size":16709},{"path":"vocab.json","role":"tokenizer","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size":6722759}],"model":{"architecture":"Qwen3_5ForConditionalGeneration","context_length":262144,"gated":false,"license":"apache-2.0","num_blocks":24},"name":"Qwen3.5-0.8B-Local","runtime":{"adapter_profile":"none","attention_implementation":"eager","dtype":"bfloat16","implementation":"drift","maximum_version_exclusive":"2.4.0","minimum_version":"2.3.0.dev0","protocol_version":1,"quantization":"none","tensor_schema":"hidden-states-v1"},"schema_version":1,"source":{"repository":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"}} diff --git a/public-alpha/catalog-qwen-candidate/review.json b/public-alpha/catalog-qwen-candidate/review.json new file mode 100644 index 000000000..ecd065c19 --- /dev/null +++ b/public-alpha/catalog-qwen-candidate/review.json @@ -0,0 +1,9 @@ +{ + "status": "unsigned-candidate", + "catalog_digest": "sha256:955ca1f547c9b881163de9d8d92b561199f17e8dbd30250e90705eda778c930d", + "requires_existing_trusted_key_ids": [ + "sha256:a8fb23c4ac71c1f29cc9991d79d16743a2a931c1911b819da0fb5432d5ac8435" + ], + "publication_performed": false, + "release_qualification_complete": false +} diff --git a/public-alpha/catalog-qwen-v2/bundle.json b/public-alpha/catalog-qwen-v2/bundle.json new file mode 100644 index 000000000..0e5492b74 --- /dev/null +++ b/public-alpha/catalog-qwen-v2/bundle.json @@ -0,0 +1 @@ +{"bootstrap_digest":"sha256:3e8b4a61f84f120b879dfdc51df771aaebdcd4cf74956261e4ef282aac2aec23","catalog_digest":"sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80","catalog_id":"communityai-public-alpha-v1","catalog_sequence":2,"complete_release_qualification":false,"files":[{"path":"catalog-bootstrap.json","sha256":"sha256:79a08b5a703283b75da89d7e1ac6406e1fc36ac1b76eecd81a7231f4f3ab5410","size":702},{"path":"catalog.signed.json","sha256":"sha256:315d7c3910fae3de0ba3aee6d1bdc5942bc2655294ab53b6566bbcfd14933315","size":1792},{"path":"manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json","sha256":"sha256:a2621aa34aa47f0c9074f5baa0254b17549424f1c85d828ae9b1bf6ad9e76bb3","size":10936},{"path":"manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json","sha256":"sha256:4536ac2bada7242b758db443b9ebb813a614dd167ab364643fc55c7eb657bb74","size":1711},{"path":"publication-preflight.json","sha256":"sha256:b80f6c4a8d0b87a1d365fc4be01e643429759cc52eda5c200dbf236f2073f538","size":737}],"schema_version":1,"scope":"catalog-publication-bundle"} diff --git a/public-alpha/catalog-qwen-v2/catalog-bootstrap.json b/public-alpha/catalog-qwen-v2/catalog-bootstrap.json new file mode 100644 index 000000000..a48232915 --- /dev/null +++ b/public-alpha/catalog-qwen-v2/catalog-bootstrap.json @@ -0,0 +1 @@ +{"catalog_mirrors":["https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/catalog.signed.json"],"initial_peers":["/dns4/bootstrap.communityai.flujo.com.co/tcp/31337/p2p/QmZhGcSVR6qPLZTq3TJPZEi734GbMkouv3kPxQLdDY2qUo"],"max_loaded_models":2,"replaces_trust_roots":["sha256:9388a51a4c3856256e9db2c53838045e6be34202c72f1c5abd84cd33391a6b31"],"schema_version":1,"trust_root":{"catalog_id":"communityai-public-alpha-v1","keys":[{"algorithm":"ed25519","key_id":"sha256:9505d3ac8ec996d4b794bd43d09dd84447acc5da8a0bb1eac1be4e9c9a34b14f","public_key":"MCowBQYDK2VwAyEAWoBIlT+fVxzjuMZi+rH1+CR11ZUZ/gonQuygpWAoQKU="}],"schema_version":1,"threshold":1}} diff --git a/public-alpha/catalog-qwen-v2/catalog.signed.json b/public-alpha/catalog-qwen-v2/catalog.signed.json new file mode 100644 index 000000000..08343f1c2 --- /dev/null +++ b/public-alpha/catalog-qwen-v2/catalog.signed.json @@ -0,0 +1 @@ +{"schema_version":1,"signatures":[{"algorithm":"ed25519","key_id":"sha256:9505d3ac8ec996d4b794bd43d09dd84447acc5da8a0bb1eac1be4e9c9a34b14f","signature":"nfXK6GanQcQ6mNBi00k5E0oJcvHGD/2TFyz2VSsX6QSDamX7hwQDMpNz4NkOwqNakbfjNdr2QpiAdvpRuTsFAA=="}],"signed":{"catalog_id":"communityai-public-alpha-v1","expires_at_ms":1790624119369,"issued_at_ms":1788654808765,"models":[{"active_parameters":800000000,"execution":"local","manifest_digest":"sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0","manifest_urls":["https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json"],"role":"primary","rung":"local-qwen","total_parameters":800000000,"weight_bytes":1746942600},{"active_parameters":27000000000,"execution":"distributed","manifest_digest":"sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4","manifest_urls":["https://raw.githubusercontent.com/flujo-app/CommunityAI/codex/gate-v-auto-selection/public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json"],"role":"primary","rung":"community-qwen","total_parameters":27000000000,"weight_bytes":30866866928}],"rungs":[{"id":"local-qwen","maximum_observation_age_seconds":30,"maximum_p95_first_token_ms":60000,"minimum_independent_routes":1,"minimum_replicas":1,"minimum_soak_seconds":60,"minimum_surviving_replicas":0,"minimum_tokens_per_minute":1,"order":1},{"id":"community-qwen","maximum_observation_age_seconds":30,"maximum_p95_first_token_ms":60000,"minimum_independent_routes":1,"minimum_replicas":1,"minimum_soak_seconds":60,"minimum_surviving_replicas":0,"minimum_tokens_per_minute":1,"order":2}],"sequence":2}} diff --git a/public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json b/public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json new file mode 100644 index 000000000..9a14a79fb --- /dev/null +++ b/public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json @@ -0,0 +1 @@ +{"aliases":["qwen3.8-27b","qwen3.8-27b-fp8"],"artifacts":[{"path":"chat_template.jinja","role":"chat_template","sha256":"c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041","size":8952},{"path":"config.json","role":"config","sha256":"74227dd615bf1ea975aa676bdf355a0379858c12f394b5365cd9dfa5fc2c70bc","size":51350},{"path":"layers-0.safetensors","role":"weight","sha256":"07f700e293baeaf3cd4240c3df1a948c4403f16961ea7979e86c8d6a9f8fd466","size":383865448},{"path":"layers-1.safetensors","role":"weight","sha256":"35840b5d452c6e438d000e5c1d8d1bc793d257394404689b8f9424749eee8edc","size":383865448},{"path":"layers-10.safetensors","role":"weight","sha256":"ab5fbc076ccd6514aba3ab67a0ac1d6701ad395d40cccc0086cf0284ef5d68c9","size":383865472},{"path":"layers-11.safetensors","role":"weight","sha256":"076b4ba44e9311c8fb8fdc0045f8ce9fd0c75584ade4e95510056b7db4b7ec61","size":372313760},{"path":"layers-12.safetensors","role":"weight","sha256":"86d8ec0b8fc8e3ffbdfd667b97050533ddf5a4d273b4fe1190a4335542c52829","size":383865472},{"path":"layers-13.safetensors","role":"weight","sha256":"71dd986134f4338a7b8290ed24993ad34a0f77b76a4cc060fe6403e36dcdda5b","size":383865472},{"path":"layers-14.safetensors","role":"weight","sha256":"95b285915cba994917b527030874227382ea3421dc7f47d804c0ca5851f039fb","size":383865472},{"path":"layers-15.safetensors","role":"weight","sha256":"1e87d61e77d2b802f796ab8a049b1119648c9b70d2ba1067f456e5a04449cfc3","size":372313760},{"path":"layers-16.safetensors","role":"weight","sha256":"8bcdbf4e2c7a8dda3043c4453a476b30a582d58aabfaa49b2829a8b66a664acb","size":383865472},{"path":"layers-17.safetensors","role":"weight","sha256":"571cea44717878cc8cbbfbd544acfe2d155f5b963fbf5dc67b3fa96afe840a1b","size":383865472},{"path":"layers-18.safetensors","role":"weight","sha256":"d8700627ca2267fb7beb5d398d6a659a9bfeeb2af76f10e6827b1e4359d59286","size":383865472},{"path":"layers-19.safetensors","role":"weight","sha256":"2b24cfb752bbed959c614d7fd85412ee27ab2ac1f47deef5d09d0592781400e4","size":372313760},{"path":"layers-2.safetensors","role":"weight","sha256":"32e63c6455ebfa10b4c36af2878eefcb6f8ccf8e947a62575444dc412c38ac1a","size":383865448},{"path":"layers-20.safetensors","role":"weight","sha256":"52fd00c68d4f1df96ba701290c04237257ebb832879bb0b6a629525809d84a64","size":383865472},{"path":"layers-21.safetensors","role":"weight","sha256":"e3c29dd949808c13116d853ce4b0f96cb1238fc0cb6999ab76735deef6bda77a","size":383865472},{"path":"layers-22.safetensors","role":"weight","sha256":"63446546e3e09c2394fc51855f4e390cb47f896bb8b744f3a210c3159a3ccaf2","size":383865472},{"path":"layers-23.safetensors","role":"weight","sha256":"4166ed11d0329985f9813ba6a70474ea9f5aeff3d97db3d7b2e81185bd41a996","size":372313760},{"path":"layers-24.safetensors","role":"weight","sha256":"720bdcfe12b6627bed29feda9c96256e15c20602704669d05097668a9f03dd45","size":383865472},{"path":"layers-25.safetensors","role":"weight","sha256":"1b188e7e4e8ae6f5d753c597f61d86c1593495de4307c273d8dda9946ec48bb4","size":383865472},{"path":"layers-26.safetensors","role":"weight","sha256":"ad8c3e3d4b79dfedad674318ee4b150b2cd78b56747c7c8d88a65818109a545d","size":383865472},{"path":"layers-27.safetensors","role":"weight","sha256":"8d63f461430045960f27a22abf685bfc7445fffbb24764e4a7fa993ba7edac08","size":372313760},{"path":"layers-28.safetensors","role":"weight","sha256":"f78557cda66107ce25f8c5ecde2f83da874b3bf7c50381e062e14e7294a5b64d","size":383865472},{"path":"layers-29.safetensors","role":"weight","sha256":"09f264d6fdddab0fb5ecce18034a9a01944d15118147c97058fba101a60f8af1","size":383865472},{"path":"layers-3.safetensors","role":"weight","sha256":"302f9af90bb683a8be9e96d124b470a2eddee6612c95c39a8e93f26eb654563d","size":372313744},{"path":"layers-30.safetensors","role":"weight","sha256":"54afe4fc7262ee597d4d9b80d2d6fe8ae501846ff124334b575bed4c6850ed97","size":383865472},{"path":"layers-31.safetensors","role":"weight","sha256":"2779338203d0715ca323ac0b7fdbb0d450ce2bd1694be395a211a02ade3f34d9","size":372313760},{"path":"layers-32.safetensors","role":"weight","sha256":"f4af8893594458b6cdb074ef20f9accb708361fe904012465990d09e74994509","size":383865472},{"path":"layers-33.safetensors","role":"weight","sha256":"8f038505b9fc7d0b31ca976c728b56028ec28390a15ca0410f00df18d3abc1a4","size":383865472},{"path":"layers-34.safetensors","role":"weight","sha256":"0b28f43cc4d3ae50e37eb3c2ce12a6e181b1c7586a27d5a05c6a3367f08286c9","size":383865472},{"path":"layers-35.safetensors","role":"weight","sha256":"535b61b9eabbcda5de20e21d2d46fb52528cc9c1f8fac38920382ea2f1030837","size":372313760},{"path":"layers-36.safetensors","role":"weight","sha256":"2babd209cafbe12a6e375ff6e7a1029f36ee8226f2a9babf6646630f31dd3fb8","size":383865472},{"path":"layers-37.safetensors","role":"weight","sha256":"651cae579808983fefec954db6e2ac8c3f98bfb038ff1d50aef0505ba7394e65","size":383865472},{"path":"layers-38.safetensors","role":"weight","sha256":"1c60932dec650cc7679121eaa4d90b8d10a600bf58e23b0784588a339469dd0a","size":383865472},{"path":"layers-39.safetensors","role":"weight","sha256":"b0c7df4d51b99637ff4ea35eebf7eb134db592eb943e3ccde921729f33949058","size":372313760},{"path":"layers-4.safetensors","role":"weight","sha256":"b7f367125cdb4b3c3920d1d3ed1ba0d73e464c3e3a92dbd1c195658cb9200afd","size":383865448},{"path":"layers-40.safetensors","role":"weight","sha256":"6908f1ab1a3d7828a566099ecf393582bec988d0f82f6141848942d1494b0321","size":383865472},{"path":"layers-41.safetensors","role":"weight","sha256":"21ee5d9842074888bd9fdffa875d1f29f8c77f1f5ccbb8f53becb188e1749caa","size":383865472},{"path":"layers-42.safetensors","role":"weight","sha256":"565379d4291c06cdf9a4f66d5d21385848a1e5ab17b55fea6c1c8e544f8498f0","size":383865472},{"path":"layers-43.safetensors","role":"weight","sha256":"4bb8c0a4fadda1f1d66ef6f0dde5a5a94001d04af1842e42009833aff70bf416","size":372313760},{"path":"layers-44.safetensors","role":"weight","sha256":"14339304cd520a8673f179cb8352c36fbebe184f15309aee2a79afd2d4476fef","size":383865472},{"path":"layers-45.safetensors","role":"weight","sha256":"7a076aca542f937cc87dec92ab49fd782eae234642b40bc706a6108b3dff2c2a","size":383865472},{"path":"layers-46.safetensors","role":"weight","sha256":"d82d472cfc4792012934467b90c5e0242df4867a430dd651d13cdee72b7c6645","size":383865472},{"path":"layers-47.safetensors","role":"weight","sha256":"a83fc757fb11bc50ed3a0c9998b2b2f55c3b6a07598ed458bdbece4285ffaf1f","size":372313760},{"path":"layers-48.safetensors","role":"weight","sha256":"f6aa126d006c7d976c20b1330c118b42cf3bbe8a7d216bb14f2240d5c49bc72f","size":383865472},{"path":"layers-49.safetensors","role":"weight","sha256":"7ed54c375407c2baec8cfabe8336f8d3782246848298bb232ac2582113676c69","size":383865472},{"path":"layers-5.safetensors","role":"weight","sha256":"ebbc2c4bd2b98877caafa7073b6ed732a0284eb82269cf6d21710e245af5837a","size":383865448},{"path":"layers-50.safetensors","role":"weight","sha256":"73825e1f056c3de53276d3c65743e62b67bf3ca082cf7077ecba1ffef18e9bf1","size":383865472},{"path":"layers-51.safetensors","role":"weight","sha256":"b6abd8b62b5c4b7beca8cdccd306065be2192ef42491c587ca019d971ed86a1f","size":372313760},{"path":"layers-52.safetensors","role":"weight","sha256":"6251d51e6fbec6f463ca875fcbe852e07c5025145affe9af053a04fbdc37288f","size":383865472},{"path":"layers-53.safetensors","role":"weight","sha256":"a9f2f43b4978bba2f755c6d2eb39f8561965bc3c1b2fd1073bb363aff11a9b8c","size":383865472},{"path":"layers-54.safetensors","role":"weight","sha256":"f0246c9974a4b1207f8f5d05a09c54c50ea61fbfd2dc6480d2b76d3647ece292","size":383865472},{"path":"layers-55.safetensors","role":"weight","sha256":"649c400495616cf888b7c52e538113610a36265c531d84bff484f8149c157405","size":372313760},{"path":"layers-56.safetensors","role":"weight","sha256":"eb0a8139a36138639a37219c158b9415412b68d316235061a72e57da5509c188","size":383865472},{"path":"layers-57.safetensors","role":"weight","sha256":"d7616ad33f9256342573ff85b3f2a0e368dd120257c0c7158564587036119d95","size":383865472},{"path":"layers-58.safetensors","role":"weight","sha256":"a8dff0e37d0e3903cc29711b935c2865ccd38bd930e7487ae2f4bedbd18ff185","size":383865472},{"path":"layers-59.safetensors","role":"weight","sha256":"573ed65fd455ec9bd811e95946ea422e08ac36860fdff6968a6d108263a54175","size":372313760},{"path":"layers-6.safetensors","role":"weight","sha256":"2fea2aaa61d566ba8af27ee89d29d9ee9f1c7f6fbe692772d9ba61fc45109ef6","size":383865448},{"path":"layers-60.safetensors","role":"weight","sha256":"9a19c30b190bf552eb424af16103f610bcd944fb276b3275eb7c73849799b7de","size":383865472},{"path":"layers-61.safetensors","role":"weight","sha256":"ff744debd4dd0eef450ed24473df57680db404914302e7147a3b72154131f4a0","size":383865472},{"path":"layers-62.safetensors","role":"weight","sha256":"d9b3b9e472f78bcadf62c446084833bb549faff9e730b21219f1f80d583eff23","size":383865472},{"path":"layers-63.safetensors","role":"weight","sha256":"59ba4a3af5e6bc2008c2d6b5d9be8eeaf255cddc0aaf8a0d9583b496bb2e3ae0","size":372313760},{"path":"layers-7.safetensors","role":"weight","sha256":"f2e0137e878016a7afdb2314972e0cbbe2461a67b00eb1d691b4de8ea2665da8","size":372313744},{"path":"layers-8.safetensors","role":"weight","sha256":"1721b7bcd730891c224b70a7b6147e44a574a2d6c4d924760acf97728249c92c","size":383865448},{"path":"layers-9.safetensors","role":"weight","sha256":"0a21e07065b5bb04adc339300dea1872cc48ab8eab3b02df092f7cbc67c8d6f6","size":383865448},{"path":"merges.txt","role":"tokenizer","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size":3353259},{"path":"model.safetensors.index.json","role":"weight_index","sha256":"f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2","size":137335},{"path":"mtp.safetensors","role":"weight","sha256":"e5e4464a3793cc261de536592830bca40e7f3af159ed038c358f5660917cf43b","size":477202224},{"path":"outside.safetensors","role":"weight","sha256":"ddff1d6665a2b39f2612fce0ef955e2436724c565bfbcbc127c7ffd078b698ff","size":6007102112},{"path":"tokenizer.json","role":"tokenizer","sha256":"0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3","size":12809320},{"path":"tokenizer_config.json","role":"tokenizer","sha256":"b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27","size":17928},{"path":"vocab.json","role":"tokenizer","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size":6722759}],"model":{"architecture":"Qwen3_5ForConditionalGeneration","context_length":262144,"gated":false,"license":"apache-2.0","num_blocks":64},"name":"Qwen3.8 27B FP8 Dequant","runtime":{"adapter_profile":"none","attention_implementation":"eager","dtype":"bfloat16","implementation":"drift","maximum_version_exclusive":"2.4.0","minimum_version":"2.3.0.dev0","protocol_version":1,"quantization":"fp8_dequant","tensor_schema":"hidden-states-v1"},"schema_version":1,"source":{"repository":"Qwen/Qwen3.8-27B-FP8","revision":"017b9c7af6b5689d5dd426a76e0bc077eb5ca20a"}} diff --git a/public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json b/public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json new file mode 100644 index 000000000..0f4ef8988 --- /dev/null +++ b/public-alpha/catalog-qwen-v2/manifests/e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0.json @@ -0,0 +1 @@ +{"aliases":["local-qwen"],"artifacts":[{"path":"chat_template.jinja","role":"chat_template","sha256":"273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80","size":7755},{"path":"config.json","role":"config","sha256":"b90b86f35c8e6925ef74ee04d0e758f0a845c83a42089ad82bbaa948de9b4204","size":2907},{"path":"merges.txt","role":"tokenizer","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size":3353259},{"path":"model.safetensors-00001-of-00001.safetensors","role":"weight","sha256":"04b1c301231dd422b8860db31311ab2721511346a32cb1e079c4c4e5f1fe4696","size":1746942600},{"path":"model.safetensors.index.json","role":"weight_index","sha256":"d8a08838a613b025eb7952ed9db11696213e57e76a375661ef5c12f9dd5dcf4e","size":50900},{"path":"tokenizer.json","role":"tokenizer","sha256":"5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42","size":12807982},{"path":"tokenizer_config.json","role":"tokenizer","sha256":"49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c","size":16709},{"path":"vocab.json","role":"tokenizer","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size":6722759}],"model":{"architecture":"Qwen3_5ForConditionalGeneration","context_length":262144,"gated":false,"license":"apache-2.0","num_blocks":24},"name":"Qwen3.5-0.8B-Local","runtime":{"adapter_profile":"none","attention_implementation":"eager","dtype":"bfloat16","implementation":"drift","maximum_version_exclusive":"2.4.0","minimum_version":"2.3.0.dev0","protocol_version":1,"quantization":"none","tensor_schema":"hidden-states-v1"},"schema_version":1,"source":{"repository":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"}} diff --git a/public-alpha/catalog-qwen-v2/publication-preflight.json b/public-alpha/catalog-qwen-v2/publication-preflight.json new file mode 100644 index 000000000..4743310a1 --- /dev/null +++ b/public-alpha/catalog-qwen-v2/publication-preflight.json @@ -0,0 +1 @@ +{"bootstrap_digest":"sha256:3e8b4a61f84f120b879dfdc51df771aaebdcd4cf74956261e4ef282aac2aec23","catalog_digest":"sha256:13c83590b7b47c86ae676c6e1a0e5277228fabbd2ba90c81babb6eaf430e5a80","catalog_id":"communityai-public-alpha-v1","catalog_mirror_count":1,"catalog_sequence":2,"complete_release_qualification":false,"distinct_seed_address_count":1,"distinct_seed_host_count":1,"distinct_seed_identity_count":1,"model_count":2,"not_covered":["cross-platform and multi-machine model qualification","mirror and seed redundancy or independent operator ownership","public-worker route redundancy and soak","packaged clean-install inference"],"result":"passed","rung_count":2,"schema_version":1,"scope":"catalog-publication-transport-preflight"} diff --git a/public-alpha/updates/alpha.json b/public-alpha/updates/alpha.json new file mode 100644 index 000000000..896f9edb7 --- /dev/null +++ b/public-alpha/updates/alpha.json @@ -0,0 +1 @@ +{"signature":"lC+K3wvBBtW4tGqo3gOp9kqjowL886JpMdNRgWeFYjmzifNRVdTOEl9l1XX3BIOuEXaKHP/kIrgQ+IXNbFy/CA==","signed":{"artifacts":{"linux-amd64":{"filename":"communityai_0.1.0~alpha.20260909.3_amd64.deb","sha256":"57361da76241997cb5437d9af973aeebc6410e082ad87bf1b2b6410415733a14","size_bytes":2302529348,"url":"https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai_0.1.0~alpha.20260909.3_amd64.deb"},"windows-x64":{"filename":"communityai-0.1.0-alpha.20260909.3-windows-setup.exe","sha256":"5ba5d1a4890dd76227ee21ad0b2caf21c402d67f8fd617573c1942990682413b","size_bytes":2465339019,"url":"https://pub-1f8764bf149e4e269735e087a4808e4c.r2.dev/alpha/20260909.3/communityai-0.1.0-alpha.20260909.3-windows-setup.exe"}},"channel":"alpha","expires_at":1796720139,"published_at":1788944139,"schema_version":1,"sequence":2026090903,"version":"0.1.0-alpha.20260909.3"}} diff --git a/scripts/catalog_key_backup.py b/scripts/catalog_key_backup.py new file mode 100644 index 000000000..56de0d003 --- /dev/null +++ b/scripts/catalog_key_backup.py @@ -0,0 +1,57 @@ +"""Verify the emergency Google Secret Manager backup without printing private material.""" + +import argparse +import json +import shutil +import subprocess + +from cryptography.hazmat.primitives import serialization + +from drift.model_catalog import CatalogSigningKey + +PROJECT = "community-ai-506321" +SECRET = "communityai-catalog-signer-20260906" +VERSION = "1" +KEY_ID = "sha256:9505d3ac8ec996d4b794bd43d09dd84447acc5da8a0bb1eac1be4e9c9a34b14f" + + +def load_online_backup(project=PROJECT, secret=SECRET, version=VERSION, expected_key_id=KEY_ID): + if not version.isdigit() or int(version) < 1: + raise ValueError("Recovery must pin a numeric secret version") + gcloud = shutil.which("gcloud") + if gcloud is None: + raise RuntimeError("Google Cloud CLI is required for emergency backup verification") + response = subprocess.run( + [gcloud, "secrets", "versions", "access", version, "--secret", secret, "--project", project, "--quiet"], + capture_output=True, + timeout=60, + check=False, + ) + if response.returncode: + # Neither stdout nor stderr is included in an exception or a test report. + raise RuntimeError("Could not retrieve the pinned emergency backup with the current Google identity") + recovered = CatalogSigningKey(serialization.load_pem_private_key(response.stdout, password=None)) + if recovered.key_id != expected_key_id: + raise ValueError("Emergency backup public identity does not match the release trust root") + return recovered + + +def verify_backup(): + key = load_online_backup() + challenge = b"CommunityAI emergency publisher backup recovery drill v1" + key.trusted_key.public_key_object.verify(key.sign(challenge), challenge) + return { + "result": "passed", + "project": PROJECT, + "secret": SECRET, + "version": VERSION, + "key_id": key.key_id, + "recovered_in_memory": True, + "signature_verified": True, + "private_material_printed": False, + } + + +if __name__ == "__main__": + argparse.ArgumentParser(description=__doc__).parse_args() + print(json.dumps(verify_backup(), sort_keys=True)) diff --git a/scripts/gate13_automated_playthrough.py b/scripts/gate13_automated_playthrough.py new file mode 100644 index 000000000..05e3ed609 --- /dev/null +++ b/scripts/gate13_automated_playthrough.py @@ -0,0 +1,708 @@ +"""Run the proven Gate 13 desktop playthrough without operator UI actions. + +Invoke this only after a production archive has been verified and unpacked on a +clean host. The frozen desktop opens its real window twice and replays the exact +platform-specific chronology accepted in the manual Gate 13 run. Windows performs +default-root inference before a full restart, then saves policy, starts, observes for +25 seconds, and pauses. Linux performs inference/policy/start before the restart, +then proves persisted intent, pauses, and performs post-restart inference. + +The script prints one bounded aggregate record. Private per-session files live +only in an exact run-scoped temporary root and are removed before success. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import stat +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +SCHEMA_VERSION = 2 +SCOPE = "gate13-automated-desktop-replay" +POLICY_PROFILE = "gate13-manual-cpu-v1" +SEQUENCE_PROFILES = { + "windows": "gate13-manual-windows-v1", + "linux": "gate13-manual-linux-v1", +} +MAX_CONFIG_BYTES = 65_536 +MAX_EVIDENCE_BYTES = 65_536 +MAX_PROGRESS_BYTES = 16_384 +_SESSION_FAILURE_CODES = {"playthrough_failed", "playthrough_timed_out", "inference_failed", "evidence_write_failed"} +_SESSION_FAILURE_PHASES = { + "wait_ready", + "wait_policy", + "wait_prestart_paused", + "wait_started_intent", + "wait_resumed", + "wait_paused_intent", + "after_initial_inference", + "after_restart_inference", + "editing_policy", +} +_SESSION_FAILURE_DETAILS = { + "bootstrap_failed", + "inference_failed", + "inference_rejected", + "inference_transport_failed", + "inference_timed_out", + "inference_response_invalid", + "inference_selection_changed", + "inference_key_baseline_missing", + "inference_key_baseline_dirty", + "inference_key_response_invalid", + "inference_model_mismatch", + "inference_token_count_invalid", + "inference_unexpected_error", + "inference_key_cleanup_failed", +} + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_MODEL_RE = re.compile(r"[ -~]{1,128}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_CONFIG_FIELDS = { + "schema_version", + "run_id", + "platform", + "source_commit", + "package_archive", + "package_sha256", + "package_bytes", + "desktop_executable", + "work_root", + "model_id", + "manifest_digest", + "total_blocks", + "policy", + "session_timeout_seconds", + "inference_timeout_seconds", +} +_POLICY_FIELDS = { + "sharing_enabled", + "allowed_models", + "preferred_models", + "denied_models", + "max_disk_space", + "max_vram", + "max_bandwidth_mbps", + "max_power_watts", + "pause_timeout", + "schedule", +} +_SESSION_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "stage", + "result", + "model_id", + "manifest_digest", + "duration_seconds", + "route", + "inference", + "ui", + "limits", + "timing", + "privacy", +} + + +def _manual_schedule() -> dict[str, Any]: + return { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + } + + +class ReplayError(ValueError): + """A replay input, session, or cleanup boundary failed closed.""" + + def __init__(self, message: str, *, diagnostics: Mapping[str, Any] | None = None) -> None: + super().__init__(message) + self.diagnostics = dict(diagnostics or {}) + + +def _reject_constant(_value: str) -> None: + raise ReplayError("JSON contains a non-finite value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ReplayError("JSON contains a duplicate field") + value[key] = item + return value + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + _regular_metadata(path, maximum) + path = Path(path) + try: + return path.read_bytes() + except OSError as exc: + raise ReplayError("required file is unreadable") from exc + + +def _regular_metadata(path: Path, maximum: int) -> os.stat_result: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise ReplayError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise ReplayError("required file is not a bounded regular file") + return metadata + + +def _json_file(path: Path, maximum: int) -> Mapping[str, Any]: + try: + value = json.loads( + _regular_bytes(path, maximum).decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ReplayError("JSON is invalid") from exc + if not isinstance(value, dict): + raise ReplayError("JSON root is invalid") + return value + + +def _number(value: Any, label: str, minimum: float, maximum: float) -> float: + if type(value) not in (int, float): + raise ReplayError(f"{label} is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise ReplayError(f"{label} is invalid") + return rendered + + +def _absolute_path(value: Any, label: str) -> Path: + if not isinstance(value, str) or not value or "\x00" in value: + raise ReplayError(f"{label} is invalid") + path = Path(value) + if not path.is_absolute(): + raise ReplayError(f"{label} must be absolute") + return path + + +@dataclass(frozen=True) +class ReplayConfig: + run_id: str + platform: str + source_commit: str + package_archive: Path + package_sha256: str + package_bytes: int + desktop_executable: Path + work_root: Path + model_id: str + manifest_digest: str + total_blocks: int + policy: Mapping[str, Any] + session_timeout_seconds: float + inference_timeout_seconds: float + + +def load_config(path: Path) -> ReplayConfig: + raw = _json_file(path, MAX_CONFIG_BYTES) + if set(raw) != _CONFIG_FIELDS or raw.get("schema_version") != SCHEMA_VERSION: + raise ReplayError("configuration schema is invalid") + run_id = raw["run_id"] + platform = raw["platform"] + source_commit = raw["source_commit"] + package_sha256 = raw["package_sha256"] + package_bytes = raw["package_bytes"] + model_id = raw["model_id"] + digest = raw["manifest_digest"] + blocks = raw["total_blocks"] + policy = raw["policy"] + if not isinstance(run_id, str) or _RUN_RE.fullmatch(run_id) is None: + raise ReplayError("run id is invalid") + if platform not in ("windows", "linux"): + raise ReplayError("platform is invalid") + if not isinstance(source_commit, str) or _COMMIT_RE.fullmatch(source_commit) is None: + raise ReplayError("source commit is invalid") + if not isinstance(package_sha256, str) or _DIGEST_RE.fullmatch(package_sha256) is None: + raise ReplayError("package digest is invalid") + if type(package_bytes) is not int or not 1 <= package_bytes <= 8 * 1024**3: + raise ReplayError("package size is invalid") + if not isinstance(model_id, str) or _MODEL_RE.fullmatch(model_id) is None or model_id != model_id.strip(): + raise ReplayError("model id is invalid") + if not isinstance(digest, str) or _DIGEST_RE.fullmatch(digest) is None: + raise ReplayError("manifest digest is invalid") + if type(blocks) is not int or not 1 <= blocks <= 512: + raise ReplayError("block count is invalid") + if not isinstance(policy, dict) or set(policy) != _POLICY_FIELDS: + raise ReplayError("policy schema is invalid") + if ( + policy["sharing_enabled"] is not True + or policy["allowed_models"] != [model_id] + or policy["preferred_models"] != [model_id] + or policy["denied_models"] != [] + or policy["schedule"] != _manual_schedule() + ): + raise ReplayError("policy does not match the proven manual replay") + if policy["max_disk_space"] != "32GB": + raise ReplayError("storage ceiling does not match the proven manual replay") + if policy["max_vram"] != "20GB": + raise ReplayError("memory ceiling does not match the proven manual replay") + if _number(policy["max_bandwidth_mbps"], "bandwidth ceiling", 0.001, 1_000_000) != 100.0: + raise ReplayError("bandwidth ceiling does not match the proven manual replay") + if policy["max_power_watts"] is not None: + raise ReplayError("the manual CPU-host replay requires an unset power ceiling") + if _number(policy["pause_timeout"], "pause timeout", 1, 300) != 120.0: + raise ReplayError("pause timeout does not match the proven manual replay") + executable = _absolute_path(raw["desktop_executable"], "desktop executable") + _regular_metadata(executable, 2 * 1024**3) + package_archive = _absolute_path(raw["package_archive"], "package archive") + if _regular_metadata(package_archive, 8 * 1024**3).st_size != package_bytes: + raise ReplayError("package size changed") + work_root = _absolute_path(raw["work_root"], "work root") + if work_root.name != f".gate13-playthrough-{run_id}" or work_root.exists() or not work_root.parent.is_dir(): + raise ReplayError("work root is not a fresh exact run root") + return ReplayConfig( + run_id=run_id, + platform=platform, + source_commit=source_commit, + package_archive=package_archive, + package_sha256=package_sha256, + package_bytes=package_bytes, + desktop_executable=executable, + work_root=work_root, + model_id=model_id, + manifest_digest=digest, + total_blocks=blocks, + policy=policy, + session_timeout_seconds=_number(raw["session_timeout_seconds"], "session timeout", 30, 3_600), + inference_timeout_seconds=_number(raw["inference_timeout_seconds"], "inference timeout", 10, 600), + ) + + +def _session_plan(config: ReplayConfig, stage: str) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_id": config.run_id, + "platform": config.platform, + "stage": stage, + "model_id": config.model_id, + "manifest_digest": config.manifest_digest, + "total_blocks": config.total_blocks, + "policy": dict(config.policy), + "timeout_seconds": config.session_timeout_seconds, + "inference_timeout_seconds": config.inference_timeout_seconds, + } + + +def _write_private_json(path: Path, value: Mapping[str, Any]) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + descriptor = os.open(path, flags, 0o600) + try: + payload = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + with os.fdopen(descriptor, "wb", closefd=False) as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + finally: + os.close(descriptor) + + +def _validate_session(path: Path, config: ReplayConfig, stage: str) -> Mapping[str, Any]: + value = _json_file(path, MAX_EVIDENCE_BYTES) + if set(value) != _SESSION_FIELDS: + raise ReplayError("session evidence schema is invalid") + if ( + value["schema_version"] != SCHEMA_VERSION + or value["scope"] != "gate13-packaged-desktop-playthrough" + or value["run_id"] != config.run_id + or value["platform"] != config.platform + or value["stage"] != stage + or value["result"] != "passed" + or value["model_id"] != config.model_id + or value["manifest_digest"] != config.manifest_digest + ): + raise ReplayError("session evidence identity is invalid") + _number(value["duration_seconds"], "session duration", 0, config.session_timeout_seconds + 30) + route = value["route"] + inference = value["inference"] + ui = value["ui"] + limits = value["limits"] + privacy = value["privacy"] + if route != { + "rendered_in_real_window": True, + "complete": True, + "covered_blocks": config.total_blocks, + "total_blocks": config.total_blocks, + }: + raise ReplayError("session route evidence is invalid") + inference_required = (config.platform, stage) in { + ("windows", "initial"), + ("linux", "initial"), + ("linux", "restart"), + } + if inference_required: + if ( + not isinstance(inference, dict) + or inference.get("passed") is not True + or inference.get("model_id") != config.model_id + or inference.get("manifest_digest") != config.manifest_digest + or inference.get("completion_count") != 1 + or inference.get("generated_token_count") != 1 + or inference.get("response_content_retained") is not False + or inference.get("token_identifiers_retained") is not False + or inference.get("temporary_key_removed") is not True + ): + raise ReplayError("session inference evidence is invalid") + elif inference is not None: + raise ReplayError("unexpected session inference evidence") + policy_session = (config.platform, stage) in { + ("windows", "restart"), + ("linux", "initial"), + } + start_session = policy_session + pause_session = stage == "restart" + resumed_session = config.platform == "linux" and stage == "restart" + expected_ui = { + "real_window_opened": True, + "policy_dialog_saved": policy_session, + "start_clicked": start_session, + "pause_control_observed": start_session or resumed_session, + "pause_clicked": pause_session, + "restart_resume_observed": resumed_session, + "sharing_intent_enabled_observed": start_session or resumed_session, + "sharing_intent_disabled_observed": pause_session, + } + expected_limits = { + "storage": policy_session, + "memory_or_vram": policy_session, + "bandwidth": policy_session, + "power": False, + "pause_timeout": policy_session, + "schedule": policy_session, + } + expected_privacy = { + "prompt_retained": False, + "response_content_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + } + expected_timing = { + "start_observation_seconds": 25.0 + if config.platform == "windows" and stage == "restart" + else (20.0 if config.platform == "linux" and stage == "initial" else 0.0), + "restart_observation_seconds": 15.0 if resumed_session else 0.0, + } + if ui != expected_ui or limits != expected_limits or value["timing"] != expected_timing: + raise ReplayError("session UI or limit evidence is invalid") + if privacy != expected_privacy: + raise ReplayError("session privacy evidence is invalid") + forbidden = ("prompt", "response", "secret", "credential", "endpoint", "path", "address") + rendered = json.dumps(value, sort_keys=True).lower() + for field in forbidden: + if f'"{field}"' in rendered: + raise ReplayError("session evidence retained a forbidden field") + return value + + +def _session_diagnostic(path: Path, config: ReplayConfig, stage: str) -> Mapping[str, Any]: + """Keep the bounded session outcome, never arbitrary child text or payloads.""" + value = _json_file(path, MAX_EVIDENCE_BYTES) + expected = { + "schema_version": SCHEMA_VERSION, + "scope": "gate13-packaged-desktop-playthrough", + "run_id": config.run_id, + "platform": config.platform, + "stage": stage, + "model_id": config.model_id, + "manifest_digest": config.manifest_digest, + } + if any(value.get(key) != item for key, item in expected.items()) or value.get("result") not in ("passed", "failed"): + raise ReplayError("session diagnostic identity is invalid") + duration = _number(value.get("duration_seconds"), "session duration", 0, config.session_timeout_seconds + 60) + diagnostic = {**expected, "result": value["result"], "duration_seconds": duration} + if value["result"] == "failed": + for field, allowed in (("failure_code", _SESSION_FAILURE_CODES), ("failure_phase", _SESSION_FAILURE_PHASES)): + item = value.get(field) + if isinstance(item, str) and item in allowed: + diagnostic[field] = item + detail = value.get("failure_detail") + if isinstance(detail, str) and ( + detail in _SESSION_FAILURE_DETAILS or re.fullmatch(r"inference_http_[1-5][0-9]{2}", detail) + ): + diagnostic["failure_detail"] = detail + # These are the existing structured observations. Ignore all unknown fields, + # even when the child adds them inside otherwise valid session evidence. + boolean_fields = { + "route": ("rendered_in_real_window", "complete"), + "inference": ("passed", "response_content_retained", "token_identifiers_retained", "temporary_key_removed"), + "ui": ( + "real_window_opened", + "policy_dialog_saved", + "start_clicked", + "pause_control_observed", + "pause_clicked", + "restart_resume_observed", + "sharing_intent_enabled_observed", + "sharing_intent_disabled_observed", + ), + "limits": ("storage", "memory_or_vram", "bandwidth", "power", "pause_timeout", "schedule"), + "privacy": ( + "prompt_retained", + "response_content_retained", + "token_identifiers_retained", + "credentials_retained", + "paths_retained", + "endpoints_retained", + ), + } + numeric_fields = { + "route": ("covered_blocks", "total_blocks"), + "inference": ("completion_count", "generated_token_count"), + "timing": ("start_observation_seconds", "restart_observation_seconds"), + } + for section in boolean_fields.keys() | numeric_fields.keys(): + source = value.get(section) + if not isinstance(source, dict): + continue + fields = {key: source[key] for key in boolean_fields.get(section, ()) if type(source.get(key)) is bool} + for key in numeric_fields.get(section, ()): + item = source.get(key) + if type(item) in (int, float) and 0 <= item <= 86_400 and math.isfinite(item): + fields[key] = item + if fields: + diagnostic[section] = fields + return diagnostic + + +def _run_desktop( + config: ReplayConfig, + arguments: Sequence[str], + *, + step: str, + timeout: float, + runner: Callable[..., subprocess.CompletedProcess], +) -> None: + diagnostics: dict[str, Any] = {"failed_step": step} + try: + result = runner( + [os.fspath(config.desktop_executable), *arguments], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=timeout, + close_fds=True, + ) + except subprocess.TimeoutExpired as exc: + diagnostics.update(error_category="process_timeout", timeout_seconds=timeout) + raise ReplayError("packaged desktop process timed out", diagnostics=diagnostics) from exc + except (OSError, subprocess.SubprocessError) as exc: + diagnostics["error_category"] = "process_launch_failed" if isinstance(exc, OSError) else "process_error" + raise ReplayError("packaged desktop process could not run", diagnostics=diagnostics) from exc + if result.returncode != 0: + diagnostics.update(error_category="process_exit", exit_code=result.returncode) + raise ReplayError("packaged desktop process exited unsuccessfully", diagnostics=diagnostics) + + +def _run_session( + config: ReplayConfig, + stage: str, + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, +) -> Mapping[str, Any]: + plan_path = config.work_root / f"{stage}-plan.json" + evidence_path = config.work_root / f"{stage}-evidence.json" + _write_private_json(plan_path, _session_plan(config, stage)) + try: + _run_desktop( + config, + [ + "--gate13-ui-playthrough", + os.fspath(plan_path), + "--gate13-ui-evidence", + os.fspath(evidence_path), + ], + step=f"{stage}_session", + timeout=config.session_timeout_seconds + 60, + runner=runner, + ) + return _validate_session(evidence_path, config, stage) + except ReplayError as exc: + exc.diagnostics.setdefault("failed_step", f"{stage}_session") + exc.diagnostics.setdefault("error_category", "session_evidence_invalid") + try: + progress = _regular_bytes(evidence_path.with_suffix(".log"), MAX_PROGRESS_BYTES) + # Preserve the desktop's bounded phase/bootstrap log in host stderr + # before run_replay removes the exact session temporary directory. + print(progress.decode("utf-8", errors="replace"), end="", file=sys.stderr, flush=True) + except Exception: + pass + try: + exc.diagnostics["session_evidence"] = {stage: _session_diagnostic(evidence_path, config, stage)} + except Exception as evidence_exc: + exc.diagnostics["session_evidence_error"] = ( + str(evidence_exc) if isinstance(evidence_exc, ReplayError) else "session diagnostic could not be read" + ) + raise + + +def _digest_file(path: Path) -> str: + _regular_metadata(path, 8 * 1024**3) + digest = hashlib.sha256() + try: + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + except OSError as exc: + raise ReplayError("package archive could not be hashed") from exc + return "sha256:" + digest.hexdigest() + + +def _run_package_self_tests( + config: ReplayConfig, + runner: Callable[..., subprocess.CompletedProcess], +) -> None: + for action in ("--check-runtime", "--self-test", "--ui-self-test", "--onboarding-ui-self-test"): + _run_desktop(config, [action], step=action, timeout=120, runner=runner) + + +def run_replay( + config: ReplayConfig, + *, + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, +) -> Mapping[str, Any]: + package_digest = _digest_file(config.package_archive) + executable_digest = _digest_file(config.desktop_executable) + if package_digest != config.package_sha256: + raise ReplayError("package digest changed") + _run_package_self_tests(config, runner) + config.work_root.mkdir(mode=0o700) + cleanup_passed = False + start: Mapping[str, Any] | None = None + resumed: Mapping[str, Any] | None = None + failure: BaseException | None = None + try: + start = _run_session(config, "initial", runner) + resumed = _run_session(config, "restart", runner) + except BaseException as exc: + failure = exc + if isinstance(exc, ReplayError) and start is not None: + try: + exc.diagnostics.setdefault("session_evidence", {})["initial"] = _session_diagnostic( + config.work_root / "initial-evidence.json", config, "initial" + ) + except Exception: + # Failure diagnostics must not replace the original failed step. + pass + raise + finally: + try: + resolved = config.work_root.resolve(strict=True) + parent = config.work_root.parent.resolve(strict=True) + if resolved.parent != parent or resolved.name != f".gate13-playthrough-{config.run_id}": + raise ReplayError("work-root cleanup target changed") + shutil.rmtree(resolved) + cleanup_passed = not config.work_root.exists() + except (OSError, ReplayError) as exc: + if failure is None: + raise ReplayError("qualification temporary cleanup failed") from exc + if isinstance(failure, ReplayError): + failure.diagnostics["cleanup_failure_code"] = "qualification_temporary_cleanup_failed" + finally: + if isinstance(failure, ReplayError): + failure.diagnostics["qualification_temporaries_removed"] = cleanup_passed + if start is None or resumed is None or not cleanup_passed: + raise ReplayError("automated replay did not complete") + if ( + _digest_file(config.package_archive) != package_digest + or _digest_file(config.desktop_executable) != executable_digest + ): + raise ReplayError("package inputs changed during the replay") + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "run_id": config.run_id, + "platform": config.platform, + "result": "passed", + "source_commit": config.source_commit, + "package": { + "sha256": config.package_sha256, + "bytes": config.package_bytes, + "verified_before_run": True, + "self_test_count": 4, + }, + "model_id": config.model_id, + "manifest_digest": config.manifest_digest, + "real_window_sessions": 2, + "localhost_inference_count": 1 if config.platform == "windows" else 2, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": config.platform == "linux", + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": POLICY_PROFILE, + "sequence_profile": SEQUENCE_PROFILES[config.platform], + "start_observation_seconds": 25.0 if config.platform == "windows" else 20.0, + "session_duration_seconds": { + "initial": start["duration_seconds"], + "restart": resumed["duration_seconds"], + }, + "privacy_safe": True, + "qualification_temporaries_removed": True, + } + + +def _failure(exc: BaseException) -> Mapping[str, Any]: + value: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "failed", + "failure_code": "automated_replay_failed", + } + if isinstance(exc, ReplayError): + # ReplayError messages are authored here; arbitrary exception/child text + # can contain paths, credentials, prompts, or generated output. + value["failure_reason"] = str(exc) + value.update(exc.diagnostics) + else: + value["error_category"] = "replay_interrupted" if isinstance(exc, KeyboardInterrupt) else "unexpected_error" + return value + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run the automated Gate 13 packaged desktop replay") + parser.add_argument("--config", type=Path, required=True) + args = parser.parse_args(argv) + try: + value = run_replay(load_config(args.config)) + except BaseException as exc: + value = _failure(exc) + print(json.dumps(value, sort_keys=True, separators=(",", ":"))) + return 0 if value.get("result") == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gate13_cloud_orchestrator.py b/scripts/gate13_cloud_orchestrator.py new file mode 100644 index 000000000..9b3a63c8a --- /dev/null +++ b/scripts/gate13_cloud_orchestrator.py @@ -0,0 +1,348 @@ +"""Provider-neutral orchestration for the one-click Gate 13 replay. + +The provider adapter owns cloud-specific resource operations. This module owns the +qualification order, durable local evidence, failure handling, and the invariant that +cleanup is attempted after every run which reaches cloud mutation. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Protocol + +SCHEMA_VERSION = 1 +PLATFORMS = ("windows", "linux") +TERMINAL_RESULTS = frozenset({"passed", "failed"}) + + +class Gate13CloudError(RuntimeError): + """The one-click qualification could not complete safely.""" + + +@dataclass(frozen=True) +class PackageArtifact: + """Public identity of one locally verified production package.""" + + platform: str + source_commit: str + workflow_run_id: int + artifact_id: int + artifact_name: str + wrapper_sha256: str + wrapper_bytes: int + archive_name: str + archive_sha256: str + archive_bytes: int + + def public_record(self) -> dict[str, Any]: + return asdict(self) + + +class PackageSource(Protocol): + """Build or resolve the exact production packages for one run.""" + + def prepare(self) -> Mapping[str, PackageArtifact]: + ... + + +class CloudProvider(Protocol): + """Cloud boundary used by the provider-neutral lifecycle.""" + + name: str + + def preflight(self) -> Mapping[str, Any]: + ... + + def create_route(self) -> None: + ... + + def prepare_route(self) -> Mapping[str, Any]: + ... + + def fence_route(self, platform: str) -> Mapping[str, Any]: + ... + + def create_client(self, platform: str, package: PackageArtifact) -> None: + ... + + def prepare_client(self, platform: str, package: PackageArtifact) -> Mapping[str, Any]: + ... + + def run_client(self, platform: str, package: PackageArtifact) -> bytes: + ... + + def delete_client(self, platform: str) -> None: + ... + + def delete_route(self) -> None: + ... + + def cleanup_all(self) -> Mapping[str, Any]: + ... + + def verify_cleanup(self) -> Mapping[str, Any]: + ... + + +def _canonical_json(value: Mapping[str, Any]) -> bytes: + return (json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8") + + +def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = _canonical_json(value) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def _atomic_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def _require_passed(value: Mapping[str, Any], label: str) -> dict[str, Any]: + result = dict(value) + if result.get("result") != "passed": + raise Gate13CloudError(f"{label} did not pass") + return result + + +def _validate_platforms(packages: Mapping[str, PackageArtifact]) -> dict[str, PackageArtifact]: + if set(packages) != set(PLATFORMS): + raise Gate13CloudError("package source did not return Windows and Linux") + result = dict(packages) + commits = {item.source_commit for item in result.values()} + if len(commits) != 1: + raise Gate13CloudError("production packages do not share one source commit") + for platform in PLATFORMS: + package = result[platform] + if package.platform != platform: + raise Gate13CloudError("production package platform changed") + if any( + len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest) + for digest in (package.wrapper_sha256, package.archive_sha256) + ): + raise Gate13CloudError("production package digest is invalid") + if package.wrapper_bytes <= 0 or package.archive_bytes <= 0: + raise Gate13CloudError("production package byte size is invalid") + return result + + +class RunRecorder: + """Small durable journal intended for a person, retry logic, and later evidence.""" + + def __init__(self, run_id: str, provider: str, output_root: Path, clock: Callable[[], float]) -> None: + self.run_id = run_id + self.provider = provider + self.output_root = output_root + self.clock = clock + self.started_at_unix = int(clock()) + self.current_phase = "INITIALIZING" + self.events: list[dict[str, Any]] = [] + self.package_records: dict[str, Any] = {} + self.route_fences: dict[str, Any] = {} + self.client_evidence: dict[str, Any] = {} + self.cleanup: dict[str, Any] | None = None + self.result: str | None = None + self.failure_code: str | None = None + self.failure_reason: str | None = None + self._persist() + + @property + def state_path(self) -> Path: + return self.output_root / "run-state.json" + + def phase(self, name: str, **details: Any) -> None: + self.current_phase = name + event = {"phase": name, "recorded_at_unix": int(self.clock())} + if details: + event["details"] = details + self.events.append(event) + self._persist() + + def _document(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "scope": "gate13-one-click-cloud-run-state", + "run_id": self.run_id, + "provider": self.provider, + "started_at_unix": self.started_at_unix, + "updated_at_unix": int(self.clock()), + "phase": self.current_phase, + "events": list(self.events), + "packages": dict(self.package_records), + "route_fences": dict(self.route_fences), + "clients": dict(self.client_evidence), + "cleanup": self.cleanup, + "result": self.result, + "failure_code": self.failure_code, + "failure_reason": self.failure_reason, + } + + def _persist(self) -> None: + _atomic_json(self.state_path, self._document()) + + def finish( + self, + result: str, + *, + failure_code: str | None = None, + failure_reason: str | None = None, + ) -> dict[str, Any]: + if result not in TERMINAL_RESULTS: + raise Gate13CloudError("terminal result is invalid") + self.result = result + self.failure_code = failure_code + self.failure_reason = failure_reason + self.current_phase = "COMPLETE" if result == "passed" else "FAILED" + document = self._document() + document["finished_at_unix"] = int(self.clock()) + document["duration_seconds"] = document["finished_at_unix"] - self.started_at_unix + _atomic_json(self.output_root / "result.json", document) + self._persist() + return document + + +class Gate13CloudOrchestrator: + """Run the exact route -> Windows -> Linux -> cleanup sequence.""" + + def __init__( + self, + *, + run_id: str, + package_source: PackageSource, + provider: CloudProvider, + output_root: Path, + evidence_validator: Callable[[str, bytes, PackageArtifact], Mapping[str, Any]], + clock: Callable[[], float] = time.time, + ) -> None: + self.run_id = run_id + self.package_source = package_source + self.provider = provider + self.output_root = output_root + self.evidence_validator = evidence_validator + self.clock = clock + + def run(self) -> Mapping[str, Any]: + recorder = RunRecorder(self.run_id, self.provider.name, self.output_root, self.clock) + cloud_mutated = False + failure_code: str | None = None + failure_reason: str | None = None + try: + recorder.phase("PREFLIGHT") + _require_passed(self.provider.preflight(), "cloud preflight") + recorder.phase("PACKAGES_RESOLVING") + packages = _validate_platforms(self.package_source.prepare()) + recorder.package_records = {platform: packages[platform].public_record() for platform in PLATFORMS} + + # From this point onward every exit path must execute provider cleanup. + cloud_mutated = True + recorder.phase("ROUTE_CREATING") + self.provider.create_route() + recorder.phase("ROUTE_PREPARING") + _require_passed(self.provider.prepare_route(), "route preparation") + + for platform in PLATFORMS: + recorder.phase("ROUTE_FENCING", platform=platform) + fence = _require_passed(self.provider.fence_route(platform), f"{platform} route fence") + recorder.route_fences[platform] = fence + recorder.phase("CLIENT_CREATING", platform=platform) + self.provider.create_client(platform, packages[platform]) + recorder.phase("CLIENT_PREPARING", platform=platform) + _require_passed( + self.provider.prepare_client(platform, packages[platform]), + f"{platform} client preparation", + ) + recorder.phase("CLIENT_RUNNING", platform=platform) + evidence_payload = self.provider.run_client(platform, packages[platform]) + validated = _require_passed( + self.evidence_validator(platform, evidence_payload, packages[platform]), + f"{platform} qualification", + ) + evidence_path = self.output_root / f"{platform}-evidence.json" + _atomic_bytes(evidence_path, evidence_payload) + recorder.client_evidence[platform] = { + "result": "passed", + "sha256": "sha256:" + hashlib.sha256(evidence_payload).hexdigest(), + "session_duration_seconds": validated.get("session_duration_seconds"), + } + recorder.phase("CLIENT_DELETING", platform=platform) + self.provider.delete_client(platform) + + recorder.phase("ROUTE_DELETING") + self.provider.delete_route() + except BaseException as exc: + failure_code = type(exc).__name__ + failure_reason = str(exc) or failure_code + failed_phase = recorder.current_phase + recorder.phase( + "FAILURE", + failed_phase=failed_phase, + failure_code=failure_code, + failure_reason=failure_reason, + ) + finally: + if cloud_mutated: + recorder.phase("CLEANUP") + try: + recorder.cleanup = dict(self.provider.cleanup_all()) + if recorder.cleanup.get("result") != "passed": + failure_code = failure_code or "CleanupError" + failure_reason = failure_reason or "cloud cleanup did not pass" + except BaseException as cleanup_exc: + recorder.cleanup = { + "result": "failed", + "failure_code": type(cleanup_exc).__name__, + } + failure_code = failure_code or "CleanupError" + failure_reason = failure_reason or str(cleanup_exc) or "CleanupError" + + try: + recorder.phase("CLEANUP_VERIFYING") + verified_cleanup = _require_passed(self.provider.verify_cleanup(), "cloud cleanup") + recorder.cleanup = verified_cleanup + except BaseException as cleanup_exc: + failure_code = failure_code or type(cleanup_exc).__name__ + failure_reason = failure_reason or str(cleanup_exc) or type(cleanup_exc).__name__ + recorder.cleanup = { + "result": "failed", + "failure_code": type(cleanup_exc).__name__, + } + + passed = ( + failure_code is None + and set(recorder.client_evidence) == set(PLATFORMS) + and isinstance(recorder.cleanup, dict) + and recorder.cleanup.get("result") == "passed" + ) + return recorder.finish( + "passed" if passed else "failed", + failure_code=failure_code, + failure_reason=failure_reason, + ) diff --git a/scripts/gate13_gcp_provider.py b/scripts/gate13_gcp_provider.py new file mode 100644 index 000000000..136f70d63 --- /dev/null +++ b/scripts/gate13_gcp_provider.py @@ -0,0 +1,2229 @@ +"""GCP and GitHub adapters for the one-click Gate 13 cloud replay.""" + +from __future__ import annotations + +import base64 +import hashlib +import ipaddress +import json +import os +import re +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from gate13_cloud_orchestrator import Gate13CloudError, PackageArtifact + +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"[0-9a-f]{64}") +_RUN_RE = re.compile(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?") +_GCLOUD_READ_ATTEMPTS = 5 +_GCLOUD_RETRY_SECONDS = 15 +_HOST_JOB_COMMAND_ATTEMPTS = 5 +_LINUX_STAGE_SUCCESS_RE = re.compile(r"\bGATE13_STAGE_RESULT[ \t]+result=passed[ \t]+ready=true\b") + + +class CommandError(Gate13CloudError): + """A bounded local or provider command failed.""" + + +def _command_for_subprocess(command: list[str]) -> list[str]: + if sys.platform == "win32" and command and command[0].casefold() == "gcloud": + launcher = shutil.which("gcloud.cmd") or shutil.which("gcloud") + if launcher is not None: + sdk_root = Path(launcher).resolve().parent.parent + python = sdk_root / "platform" / "bundledpython" / "python.exe" + entrypoint = sdk_root / "lib" / "gcloud.py" + if python.is_file() and entrypoint.is_file(): + return [os.fspath(python), "-S", os.fspath(entrypoint), *command[1:]] + raise CommandError("the Google Cloud SDK Python launcher is unavailable") + executable = shutil.which(command[0]) + if executable is not None: + command[0] = executable + return command + + +class LoggedRunner: + """Run argv-only commands and persist a privacy-safe action journal.""" + + def __init__(self, journal_path: Path, progress: Callable[[str], None] = print) -> None: + self.journal_path = journal_path + self.progress = progress + self.journal_path.parent.mkdir(parents=True, exist_ok=True) + + def _record(self, value: Mapping[str, Any]) -> None: + with self.journal_path.open("a", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") + stream.flush() + os.fsync(stream.fileno()) + + def run( + self, + argv: Sequence[str | os.PathLike[str]], + *, + action: str, + timeout: float = 300, + stdin: str | None = None, + check: bool = True, + sensitive_output: bool = False, + ) -> subprocess.CompletedProcess[str]: + command = _command_for_subprocess([os.fspath(item) for item in argv]) + started = time.time() + self.progress(action) + environment = dict(os.environ) + environment.update( + { + "CLOUDSDK_CORE_DISABLE_PROMPTS": "1", + "CLOUDSDK_SSH_PUTTY_FORCE_CONNECT": "1", + "GIT_TERMINAL_PROMPT": "0", + "PYTHONUTF8": "1", + } + ) + try: + result = subprocess.run( + command, + input=stdin, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + timeout=timeout, + env=environment, + shell=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + self._record( + { + "action": action, + "started_at_unix": int(started), + "finished_at_unix": int(time.time()), + "result": "failed", + "failure_code": type(exc).__name__, + } + ) + raise CommandError(f"{action} could not run") from exc + self._record( + { + "action": action, + "started_at_unix": int(started), + "finished_at_unix": int(time.time()), + "duration_seconds": round(time.time() - started, 3), + "exit_code": result.returncode, + "result": "passed" if result.returncode == 0 else "failed", + "output_retained": False, + "sensitive_output": sensitive_output, + } + ) + if check and result.returncode != 0: + raise CommandError(f"{action} failed with exit code {result.returncode}") + return result + + def json(self, argv: Sequence[str | os.PathLike[str]], *, action: str, timeout: float = 300) -> Any: + result = self.run(argv, action=action, timeout=timeout) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise CommandError(f"{action} returned invalid JSON") from exc + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _strict_object(payload: str, label: str) -> dict[str, Any]: + def unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + + try: + value = json.loads( + payload, object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ValueError()) + ) + except (json.JSONDecodeError, ValueError) as exc: + raise Gate13CloudError(f"{label} is not strict JSON") from exc + if not isinstance(value, dict): + raise Gate13CloudError(f"{label} is not an object") + return value + + +def _strict_terminal_object(payload: str, label: str) -> dict[str, Any]: + lines = [line for line in payload.splitlines() if line.strip()] + for line in reversed(lines): + try: + return _strict_object(line, label) + except Gate13CloudError: + pass + raise Gate13CloudError(f"{label} did not contain a strict JSON object") + + +class _RejectRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ARG002 + return None + + +class GitHubPackageSource: + """Resolve or build two exact workflow artifacts from the pushed HEAD.""" + + def __init__( + self, + *, + repository_root: Path, + output_root: Path, + repository: str, + workflow: str, + runner: LoggedRunner, + progress: Callable[[str], None] = print, + sleeper: Callable[[float], None] = time.sleep, + workflow_timeout_seconds: int = 7_200, + ) -> None: + self.repository_root = repository_root + self.output_root = output_root + self.repository = repository + self.workflow = workflow + self.runner = runner + self.progress = progress + self.sleeper = sleeper + self.workflow_timeout_seconds = workflow_timeout_seconds + self._artifacts_by_name: dict[str, Mapping[str, Any]] = {} + + def _git(self, *arguments: str, action: str) -> str: + return self.runner.run( + ["git", "-C", self.repository_root, *arguments], action=action, timeout=120 + ).stdout.strip() + + def _head_and_branch(self) -> tuple[str, str]: + head = self._git("rev-parse", "HEAD", action="Checking package source commit") + branch = self._git("symbolic-ref", "--short", "HEAD", action="Checking package source branch") + if not _COMMIT_RE.fullmatch(head) or not branch: + raise Gate13CloudError("package source is not a named Git branch") + remote_url = self._git("remote", "get-url", "origin", action="Checking canonical package source remote").rstrip( + "/" + ) + repository = self.repository.removesuffix(".git") + if remote_url not in { + f"https://github.com/{repository}", + f"https://github.com/{repository}.git", + f"git@github.com:{repository}", + f"git@github.com:{repository}.git", + f"ssh://git@github.com/{repository}", + f"ssh://git@github.com/{repository}.git", + }: + raise Gate13CloudError("origin is not the configured canonical GitHub repository") + remote = self._git( + "ls-remote", "origin", f"refs/heads/{branch}", action="Checking pushed package source" + ).split() + if not remote or remote[0] != head: + raise Gate13CloudError("current HEAD must be pushed to origin before the one-click run") + return head, branch + + def _workflow_runs(self, branch: str) -> list[Mapping[str, Any]]: + value = self.runner.json( + [ + "gh", + "api", + "--method", + "GET", + f"repos/{self.repository}/actions/workflows/{self.workflow}/runs", + "-f", + f"branch={branch}", + "-f", + "event=workflow_dispatch", + "-f", + "per_page=20", + ], + action="Inspecting production package workflow runs", + ) + runs = value.get("workflow_runs") if isinstance(value, dict) else None + if not isinstance(runs, list): + raise Gate13CloudError("GitHub workflow run listing is invalid") + return [item for item in runs if isinstance(item, dict)] + + def _artifacts(self, run_id: int) -> dict[str, Mapping[str, Any]]: + value = self.runner.json( + ["gh", "api", f"repos/{self.repository}/actions/runs/{run_id}/artifacts", "--paginate"], + action="Inspecting production package artifacts", + ) + items = value.get("artifacts") if isinstance(value, dict) else None + if not isinstance(items, list): + raise Gate13CloudError("GitHub artifact listing is invalid") + result = { + item["name"]: item + for item in items + if isinstance(item, dict) and isinstance(item.get("name"), str) and item.get("expired") is False + } + return result + + @staticmethod + def _has_required_artifacts(artifacts: Mapping[str, Any]) -> bool: + return all( + f"communityai-desktop-{kind}-{platform}" in artifacts + for kind in ("install", "audit") + for platform in ("windows", "linux") + ) + + def _select_or_build_run(self, head: str, branch: str) -> tuple[int, dict[str, Mapping[str, Any]]]: + prior_runs = self._workflow_runs(branch) + for run in prior_runs: + run_id = run.get("id") + if ( + run.get("head_sha") == head + and run.get("status") == "completed" + and run.get("conclusion") == "success" + and type(run_id) is int + and run_id > 0 + ): + artifacts = self._artifacts(run_id) + if self._has_required_artifacts(artifacts): + self.progress(f"Reusing successful production package run {run_id} for the pushed source commit") + return run_id, artifacts + prior_run_ids = {item.get("id") for item in prior_runs if isinstance(item.get("id"), int)} + self.runner.run( + ["gh", "workflow", "run", self.workflow, "--repo", self.repository, "--ref", branch], + action="Starting production package workflow", + ) + deadline = time.monotonic() + self.workflow_timeout_seconds + matching_run_id: int | None = None + while time.monotonic() < deadline: + for run in self._workflow_runs(branch): + run_id = run.get("id") + if run.get("head_sha") != head or not isinstance(run_id, int) or run_id in prior_run_ids: + continue + matching_run_id = run_id + if run.get("status") == "completed": + if run.get("conclusion") != "success": + raise Gate13CloudError(f"production package workflow {run_id} failed") + artifacts = self._artifacts(run_id) + if not self._has_required_artifacts(artifacts): + raise Gate13CloudError("production package workflow omitted required artifacts") + return run_id, artifacts + self.progress(f"Production package run {run_id} is {run.get('status')}; waiting") + break + else: + self.progress("Waiting for GitHub to create the production package run") + self.sleeper(30) + suffix = "" if matching_run_id is None else f" {matching_run_id}" + raise Gate13CloudError(f"production package workflow{suffix} exceeded its time bound") + + def _download_audit(self, run_id: int, platform: str) -> Path: + destination = self.output_root / "package-audit" / platform + destination.mkdir(parents=True, exist_ok=False) + self.runner.run( + [ + "gh", + "run", + "download", + str(run_id), + "--repo", + self.repository, + "--name", + f"communityai-desktop-audit-{platform}", + "--dir", + destination, + ], + action=f"Downloading {platform} package audit", + timeout=600, + ) + return destination + + def prepare(self) -> Mapping[str, PackageArtifact]: + head, branch = self._head_and_branch() + self.runner.run(["gh", "auth", "status"], action="Checking GitHub authentication", timeout=60) + run_id, artifacts = self._select_or_build_run(head, branch) + self._artifacts_by_name = dict(artifacts) + result: dict[str, PackageArtifact] = {} + for platform in ("windows", "linux"): + audit_root = self._download_audit(run_id, platform) + provenance_path = audit_root / "provenance.json" + provenance = _strict_object(provenance_path.read_text(encoding="utf-8"), "package provenance") + install = provenance.get("install_archive") + if provenance.get("source_commit") != head or not isinstance(install, dict): + raise Gate13CloudError("package provenance does not bind the pushed HEAD") + expected_platform = "Windows" if platform == "windows" else "Linux" + expected_archive = ( + "communityai-desktop-windows.zip" if platform == "windows" else "communityai-desktop-linux.tar.gz" + ) + digest = install.get("sha256") + byte_count = install.get("size_bytes") + if ( + install.get("platform") != expected_platform + or install.get("path") != expected_archive + or not isinstance(digest, str) + or not _DIGEST_RE.fullmatch(digest) + or not isinstance(byte_count, int) + or isinstance(byte_count, bool) + or byte_count <= 0 + ): + raise Gate13CloudError("package provenance install archive is invalid") + artifact_name = f"communityai-desktop-install-{platform}" + artifact = artifacts[artifact_name] + artifact_id = artifact.get("id") + wrapper_digest = artifact.get("digest") + wrapper_bytes = artifact.get("size_in_bytes") + if ( + not isinstance(artifact_id, int) + or not isinstance(wrapper_digest, str) + or not wrapper_digest.startswith("sha256:") + or not _DIGEST_RE.fullmatch(wrapper_digest.removeprefix("sha256:")) + or not isinstance(wrapper_bytes, int) + or isinstance(wrapper_bytes, bool) + or wrapper_bytes <= 0 + ): + raise Gate13CloudError("package artifact wrapper identity is invalid") + result[platform] = PackageArtifact( + platform=platform, + source_commit=head, + workflow_run_id=run_id, + artifact_id=artifact_id, + artifact_name=artifact_name, + wrapper_sha256=wrapper_digest.removeprefix("sha256:"), + wrapper_bytes=wrapper_bytes, + archive_name=expected_archive, + archive_sha256=digest, + archive_bytes=byte_count, + ) + return result + + def signed_download_url(self, artifact: PackageArtifact) -> str: + expected = self._artifacts_by_name.get(artifact.artifact_name) + if not isinstance(expected, Mapping) or expected.get("id") != artifact.artifact_id: + raise Gate13CloudError("package artifact was not prepared by this run") + token = self.runner.run( + ["gh", "auth", "token"], + action=f"Authorizing the {artifact.platform} route relay download", + timeout=60, + sensitive_output=True, + ).stdout.strip() + if not token or any(character.isspace() for character in token): + raise Gate13CloudError("GitHub token is unavailable") + request = urllib.request.Request( + f"https://api.github.com/repos/{self.repository}/actions/artifacts/{artifact.artifact_id}/zip", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "CommunityAI-Gate13/1", + }, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _RejectRedirects()) + try: + opener.open(request, timeout=30) + except urllib.error.HTTPError as exc: + if exc.code not in {301, 302, 303, 307, 308}: + raise Gate13CloudError("GitHub artifact authorization failed") from exc + location = exc.headers.get("Location") + except (OSError, TimeoutError) as exc: + raise Gate13CloudError("GitHub artifact authorization failed") from exc + else: + raise Gate13CloudError("GitHub artifact endpoint did not return a redirect") + finally: + token = "" + if not isinstance(location, str): + raise Gate13CloudError("GitHub artifact redirect is absent") + parsed = urllib.parse.urlsplit(location) + host = (parsed.hostname or "").lower() + if ( + parsed.scheme != "https" + or parsed.username is not None + or parsed.password is not None + or not host.startswith("productionresults") + or not host.endswith(".blob.core.windows.net") + ): + raise Gate13CloudError("GitHub artifact redirect host is not allowlisted") + return location + + +@dataclass(frozen=True) +class GcpConfig: + project: str + region: str + zone: str + network: str + subnet: str + protected_instance: str + protected_zone: str + route_machine_type: str + route_image: str + route_image_project: str + windows_machine_type: str + windows_image: str + windows_image_project: str + linux_machine_type: str + linux_image: str + linux_image_project: str + route_source_commit: str + catalog_source_commit: str + route_setup_commit: str + configure_helper_commit: str + acceptance_helper_commit: str + windows_startup_commit: str + linux_startup_commit: str + route_wheel_path: str + route_wheel_sha256: str + route_wheel_bytes: int + + @classmethod + def load(cls, path: Path) -> "GcpConfig": + value = _strict_object(path.read_text(encoding="utf-8"), "GCP one-click configuration") + expected = set(cls.__dataclass_fields__) + string_fields = expected - {"route_wheel_bytes"} + if ( + set(value) != expected + or not all(isinstance(value[field], str) and value[field] for field in string_fields) + or not isinstance(value["route_wheel_bytes"], int) + or isinstance(value["route_wheel_bytes"], bool) + or value["route_wheel_bytes"] <= 0 + ): + raise Gate13CloudError("GCP one-click configuration fields are invalid") + result = cls(**value) + for commit in ( + result.route_source_commit, + result.catalog_source_commit, + result.route_setup_commit, + result.configure_helper_commit, + result.acceptance_helper_commit, + result.windows_startup_commit, + result.linux_startup_commit, + ): + if not _COMMIT_RE.fullmatch(commit): + raise Gate13CloudError("GCP one-click configuration commit is invalid") + if not _DIGEST_RE.fullmatch(result.route_wheel_sha256): + raise Gate13CloudError("GCP route wheel digest is invalid") + return result + + +class GcpProvider: + """Exact GCP resource adapter; no Gate 13 sequencing lives here.""" + + name = "gcp" + + def __init__( + self, + *, + run_id: str, + repository_root: Path, + output_root: Path, + config: GcpConfig, + runner: LoggedRunner, + signed_url: Callable[[PackageArtifact], str], + progress: Callable[[str], None] = print, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + if not _RUN_RE.fullmatch(run_id): + raise Gate13CloudError("GCP run ID is invalid") + self.run_id = run_id + self.repository_root = repository_root + self.output_root = output_root + self.config = config + self.runner = runner + self.signed_url = signed_url + self.progress = progress + self.sleeper = sleeper + self.route = f"{run_id}-route" + self.clients = { + platform: f"{run_id}-{'win' if platform == 'windows' else 'linux'}" for platform in ("windows", "linux") + } + self.dht_firewall = f"{run_id}-dht" + self.iap_firewall = f"{run_id}-iap" + self.relay_firewall = f"{run_id}-relay" + self.client_tag = f"{run_id}-client" + for name in ( + self.route, + *self.clients.values(), + self.dht_firewall, + self.iap_firewall, + self.relay_firewall, + self.client_tag, + ): + if not _RUN_RE.fullmatch(name): + raise Gate13CloudError("derived GCP resource name is invalid") + + def _gcloud( + self, *arguments: str, action: str, timeout: float = 300, check: bool = True + ) -> subprocess.CompletedProcess[str]: + return self.runner.run(["gcloud", *arguments], action=action, timeout=timeout, check=check, stdin="n\n") + + def _gcloud_json(self, *arguments: str, action: str, timeout: float = 300) -> Any: + result = self._gcloud(*arguments, "--format=json", action=action, timeout=timeout) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise Gate13CloudError(f"{action} returned invalid JSON") from exc + + def _gcloud_json_with_retry(self, *arguments: str, action: str, timeout: float = 300) -> Any: + for attempt in range(1, _GCLOUD_READ_ATTEMPTS + 1): + try: + return self._gcloud_json(*arguments, action=action, timeout=timeout) + except CommandError: + if attempt == _GCLOUD_READ_ATTEMPTS: + raise + self.progress(f"{action} did not complete; trying again " f"({attempt + 1} of {_GCLOUD_READ_ATTEMPTS})") + self.sleeper(_GCLOUD_RETRY_SECONDS) + raise AssertionError("unreachable") + + def _ssh( + self, + instance: str, + command: str, + *, + action: str, + user: str | None = None, + timeout: float = 300, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + target = instance if user is None else f"{user}@{instance}" + return self._gcloud( + "compute", + "ssh", + target, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--tunnel-through-iap", + "--quiet", + "--command", + command, + action=action, + timeout=timeout, + check=check, + ) + + def _scp( + self, + sources: Sequence[str | Path], + destination: str, + *, + action: str, + timeout: float = 600, + ) -> None: + self._gcloud( + "compute", + "scp", + *[os.fspath(item) for item in sources], + destination, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--tunnel-through-iap", + "--quiet", + action=action, + timeout=timeout, + ) + + def _describe_instance(self, name: str, *, check: bool = True) -> Mapping[str, Any] | None: + action = f"Inspecting instance {name}" + attempts = 1 if check else _GCLOUD_READ_ATTEMPTS + for attempt in range(1, attempts + 1): + result = self._gcloud( + "compute", + "instances", + "describe", + name, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--format=json", + action=action, + timeout=60, + check=check, + ) + if result.returncode == 0: + break + output = f"{result.stdout}\n{result.stderr}".casefold() + if "was not found" in output: + return None + if attempt == attempts: + raise CommandError(f"{action} failed after {attempts} attempts") + self.progress(f"{action} did not complete; trying again ({attempt + 1} of {attempts})") + self.sleeper(_GCLOUD_RETRY_SECONDS) + try: + value = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise Gate13CloudError("GCP instance description is invalid") from exc + if not isinstance(value, dict): + raise Gate13CloudError("GCP instance description is invalid") + return value + + @staticmethod + def _basename(value: Any) -> str: + return value.rsplit("/", 1)[-1] if isinstance(value, str) else "" + + def _assert_owned_instance(self, name: str, value: Mapping[str, Any]) -> None: + labels = value.get("labels") + disks = value.get("disks") + if ( + value.get("name") != name + or not isinstance(labels, dict) + or labels.get("communityai_run") != self.run_id + or value.get("deletionProtection") is True + or not isinstance(disks, list) + or len(disks) != 1 + or not isinstance(disks[0], dict) + or disks[0].get("autoDelete") is not True + or self._basename(disks[0].get("source")) != name + ): + raise Gate13CloudError(f"refusing to mutate unbound instance {name}") + + def _wait_ssh( + self, + instance: str, + command: str, + *, + action: str, + user: str | None = None, + timeout_seconds: int = 1_800, + fatal_marker: str | None = None, + ) -> str: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + result = self._ssh( + instance, + command, + action=action, + user=user, + timeout=90, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + if fatal_marker is not None and fatal_marker in result.stdout: + raise Gate13CloudError(f"{action} reported a permanent startup failure") + self.sleeper(15) + raise Gate13CloudError(f"{action} exceeded its time bound") + + def _ensure_ssh_key(self) -> Path: + root = Path.home() / ".ssh" + private = root / "google_compute_engine" + public = private.with_suffix(".pub") + root.mkdir(mode=0o700, exist_ok=True) + if public.is_file() and 32 <= public.stat().st_size <= 16_384: + return public + if private.is_file(): + result = self.runner.run( + ["ssh-keygen", "-y", "-f", private], action="Deriving the GCP SSH public key", timeout=60 + ) + public.write_text(result.stdout.strip() + "\n", encoding="ascii", newline="\n") + else: + self.runner.run( + ["ssh-keygen", "-t", "rsa", "-b", "3072", "-N", "", "-f", private], + action="Creating the GCP SSH key", + timeout=120, + ) + if not public.is_file(): + raise Gate13CloudError("GCP SSH public key is unavailable") + return public + + def _resource_absence(self) -> tuple[list[str], list[str], list[str]]: + targets = {self.route, *self.clients.values()} + firewall_targets = { + self.dht_firewall, + self.iap_firewall, + self.relay_firewall, + } + instance_inventory = self._gcloud_json_with_retry( + "compute", + "instances", + "list", + "--project", + self.config.project, + action="Inventorying run-scoped GCP instances", + timeout=120, + ) + disk_inventory = self._gcloud_json_with_retry( + "compute", + "disks", + "list", + "--project", + self.config.project, + action="Inventorying run-scoped GCP disks", + timeout=120, + ) + firewall_inventory = self._gcloud_json_with_retry( + "compute", + "firewall-rules", + "list", + "--project", + self.config.project, + action="Inventorying run-scoped GCP firewalls", + timeout=120, + ) + if not all(isinstance(value, list) for value in (instance_inventory, disk_inventory, firewall_inventory)): + raise Gate13CloudError("GCP resource inventory is invalid") + instances = sorted( + item["name"] for item in instance_inventory if isinstance(item, dict) and item.get("name") in targets + ) + disks = sorted( + item["name"] for item in disk_inventory if isinstance(item, dict) and item.get("name") in targets + ) + firewalls = sorted( + item["name"] + for item in firewall_inventory + if isinstance(item, dict) and item.get("name") in firewall_targets + ) + return instances, disks, firewalls + + def _route_wheel(self) -> Path: + wheel = (self.repository_root / self.config.route_wheel_path).resolve() + if ( + not wheel.is_file() + or wheel.stat().st_size != self.config.route_wheel_bytes + or _sha256(wheel) != self.config.route_wheel_sha256 + ): + raise Gate13CloudError("the exact successful-run route wheel is absent or changed") + return wheel + + def _validate_immutable_sources(self) -> None: + objects = ( + (self.config.route_source_commit, None, "route runtime commit"), + ( + self.config.catalog_source_commit, + "public-alpha/catalog-v1", + "signed route catalog", + ), + ( + self.config.route_setup_commit, + "scripts/gate13_route_setup.sh", + "route setup", + ), + ( + self.config.configure_helper_commit, + "scripts/configure_product_route_node.py", + "route configuration helper", + ), + ( + self.config.acceptance_helper_commit, + "scripts/gate11_product_node_acceptance.py", + "route acceptance helper", + ), + ( + self.config.windows_startup_commit, + "scripts/gate13_windows_client_startup.ps1", + "Windows startup", + ), + ( + self.config.linux_startup_commit, + "scripts/gate13_linux_client_startup.sh", + "Linux startup", + ), + ) + for commit, path, label in objects: + object_name = f"{commit}:" + path if path is not None else f"{commit}^{{commit}}" + self.runner.run( + ["git", "-C", self.repository_root, "cat-file", "-e", object_name], + action=f"Checking immutable {label}", + timeout=120, + ) + self._route_wheel() + + def preflight(self) -> Mapping[str, Any]: + self._validate_immutable_sources() + self.runner.run(["gcloud", "--version"], action="Checking the gcloud CLI", timeout=60) + accounts = self._gcloud_json( + "auth", "list", "--filter=status:ACTIVE", action="Checking GCP authentication", timeout=60 + ) + if not isinstance(accounts, list) or len(accounts) != 1: + raise Gate13CloudError("exactly one active gcloud account is required") + credential = self.runner.run( + ["gcloud", "auth", "print-access-token"], + action="Checking reusable GCP credentials", + timeout=60, + sensitive_output=True, + ).stdout.strip() + if not credential or any(character.isspace() for character in credential): + raise Gate13CloudError("GCP access token is unavailable") + credential = "" + self._gcloud_json( + "compute", + "networks", + "describe", + self.config.network, + "--project", + self.config.project, + action="Checking the GCP network", + timeout=60, + ) + self._gcloud_json( + "compute", + "networks", + "subnets", + "describe", + self.config.subnet, + "--project", + self.config.project, + "--region", + self.config.region, + action="Checking the GCP subnet", + timeout=60, + ) + for machine_type in { + self.config.route_machine_type, + self.config.windows_machine_type, + self.config.linux_machine_type, + }: + self._gcloud_json( + "compute", + "machine-types", + "describe", + machine_type, + "--project", + self.config.project, + "--zone", + self.config.zone, + action=f"Checking GCP machine type {machine_type}", + timeout=60, + ) + for image, project in ( + (self.config.route_image, self.config.route_image_project), + (self.config.windows_image, self.config.windows_image_project), + (self.config.linux_image, self.config.linux_image_project), + ): + self._gcloud_json( + "compute", + "images", + "describe", + image, + "--project", + project, + action=f"Checking GCP image {image}", + timeout=60, + ) + bootstrap = self._gcloud_json( + "compute", + "instances", + "describe", + self.config.protected_instance, + "--project", + self.config.project, + "--zone", + self.config.protected_zone, + action="Checking the protected bootstrap", + timeout=60, + ) + if not isinstance(bootstrap, dict) or bootstrap.get("status") != "RUNNING": + raise Gate13CloudError("protected bootstrap is not running") + region = self._gcloud_json( + "compute", + "regions", + "describe", + self.config.region, + "--project", + self.config.project, + action="Checking GCP L4 quota", + timeout=60, + ) + quota = region.get("quotas") if isinstance(region, dict) else None + l4 = next( + (item for item in quota or [] if isinstance(item, dict) and item.get("metric") == "NVIDIA_L4_GPUS"), None + ) + if not isinstance(l4, dict) or float(l4.get("limit", 0)) - float(l4.get("usage", 0)) < 1: + raise Gate13CloudError("one free regional NVIDIA L4 is required") + self._ensure_ssh_key() + instances, disks, firewalls = self._resource_absence() + if instances or disks or firewalls: + raise Gate13CloudError("run-scoped GCP targets already exist") + return { + "result": "passed", + "project": self.config.project, + "zone": self.config.zone, + "one_l4_free": True, + "targets_absent": True, + "protected_bootstrap_running": True, + } + + def create_route(self) -> None: + labels = f"communityai_run={self.run_id},communityai_scope=gate13_one_click" + self._gcloud( + "compute", + "firewall-rules", + "create", + self.dht_firewall, + "--project", + self.config.project, + "--network", + self.config.network, + "--direction", + "INGRESS", + "--action", + "ALLOW", + "--rules", + "tcp:31337-31338", + "--source-ranges", + "0.0.0.0/0", + "--target-tags", + self.route, + action="Creating the run-scoped route firewall", + timeout=180, + ) + self._gcloud( + "compute", + "firewall-rules", + "create", + self.iap_firewall, + "--project", + self.config.project, + "--network", + self.config.network, + "--direction", + "INGRESS", + "--action", + "ALLOW", + "--rules", + "tcp:22", + "--source-ranges", + "35.235.240.0/20", + "--target-tags", + f"{self.route},{self.client_tag}", + action="Creating the run-scoped IAP firewall", + timeout=180, + ) + self._gcloud( + "compute", + "firewall-rules", + "create", + self.relay_firewall, + "--project", + self.config.project, + "--network", + self.config.network, + "--direction", + "INGRESS", + "--priority", + "1000", + "--action", + "ALLOW", + "--rules", + "tcp:38081", + "--source-tags", + self.client_tag, + "--target-tags", + self.route, + action="Creating the proven private package-relay firewall", + timeout=180, + ) + self._gcloud( + "compute", + "instances", + "create", + self.route, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--machine-type", + self.config.route_machine_type, + "--network", + self.config.network, + "--subnet", + self.config.subnet, + "--maintenance-policy", + "TERMINATE", + "--provisioning-model", + "STANDARD", + "--no-service-account", + "--no-scopes", + "--image", + self.config.route_image, + "--image-project", + self.config.route_image_project, + "--boot-disk-type", + "pd-balanced", + "--boot-disk-size", + "200GB", + "--boot-disk-device-name", + self.route, + "--boot-disk-auto-delete", + "--tags", + self.route, + "--labels", + labels, + "--max-run-duration", + "57600s", + "--instance-termination-action", + "DELETE", + action="Creating the GCP route VM", + timeout=900, + ) + value = self._describe_instance(self.route) + if value is None: + raise Gate13CloudError("route VM disappeared after creation") + self._assert_owned_instance(self.route, value) + + def _git_blob(self, commit: str, path: str, destination: Path) -> None: + result = self.runner.run( + ["git", "-C", self.repository_root, "show", f"{commit}:{path}"], + action=f"Extracting immutable route helper {Path(path).name}", + timeout=120, + ) + destination.write_text(result.stdout, encoding="utf-8", newline="\n") + + def _client_startup_script(self, platform: str) -> Path: + if platform == "windows": + name = "gate13_windows_client_startup.ps1" + commit = self.config.windows_startup_commit + elif platform == "linux": + name = "gate13_linux_client_startup.sh" + commit = self.config.linux_startup_commit + else: + raise Gate13CloudError("client platform is invalid") + root = self.output_root / "client-startup" + root.mkdir(exist_ok=True) + destination = root / name + self._git_blob(commit, f"scripts/{name}", destination) + return destination + + def _build_route_bundle(self) -> Path: + bundle = self.output_root / "route-bundle" + bundle.mkdir(parents=True, exist_ok=False) + shutil.copy2( + self._route_wheel(), + bundle / "drift-2.3.0.dev2-py3-none-any.whl", + ) + catalog = bundle / "catalog-v1.tar" + self.runner.run( + [ + "git", + "-C", + self.repository_root, + "archive", + "--format=tar", + f"--output={catalog}", + self.config.catalog_source_commit, + "public-alpha/catalog-v1", + ], + action="Archiving the exact successful signed catalog", + timeout=180, + ) + self._git_blob( + self.config.configure_helper_commit, + "scripts/configure_product_route_node.py", + bundle / "configure_product_route_node.py", + ) + self._git_blob( + self.config.acceptance_helper_commit, + "scripts/gate11_product_node_acceptance.py", + bundle / "gate11_product_node_acceptance.py", + ) + self._git_blob( + self.config.route_setup_commit, + "scripts/gate13_route_setup.sh", + bundle / "gate13_route_setup.sh", + ) + return bundle + + def _write_route_log(self, stage: str, name: str, text: str) -> Path: + root = self.output_root / "route-diagnostics" / stage + root.mkdir(parents=True, exist_ok=True) + # These are local diagnostic logs, not public evidence. Do not retain signed URLs or bearer tokens. + text = re.sub(r"https?://\S+", "", text) + text = re.sub(r"(?i)\bBearer\s+\S+", "Bearer ", text) + path = root / f"{name}.log" + with path.open("w", encoding="utf-8", newline="\n") as stream: + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) + return path + + def _logged_route_ssh( + self, stage: str, name: str, command: str, *, action: str, timeout: float + ) -> subprocess.CompletedProcess[str]: + output: Any = None + try: + output = self._ssh(self.route, command, action=action, timeout=timeout, check=False) + return output + except Exception as exc: + # LoggedRunner chains TimeoutExpired, which carries the partial stdout/stderr. + output = exc.__cause__ or exc + raise + finally: + if output is not None: + status = ( + f"{type(output).__name__}: {output}" + if isinstance(output, BaseException) + else f"exit_code={output.returncode}" + ) + parts = [status] + for channel in ("stdout", "stderr"): + value = getattr(output, channel, None) or "" + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + parts.append(f"{channel.upper()}:\n{value}") + try: + self._write_route_log(stage, name, "\n".join(parts)) + except OSError as log_error: + self.progress(f"Could not save route {name} output: {log_error}") + if not isinstance(output, BaseException): + raise + + def _capture_route_failure(self, stage: str, failure: Exception) -> str: + try: + path = self._write_route_log(stage, "failure", f"{type(failure).__name__}: {failure}\n") + errors = [] + commands = ( + ( + "services", + "sudo systemctl show communityai-qwen.service communityai-gemma.service " + "-p Id -p ActiveState -p SubState -p MainPID -p ExecMainStatus -p Result -p NRestarts", + ), + ( + "journal", + "sudo journalctl -b -u communityai-qwen.service -u communityai-gemma.service " + "-u google-startup-scripts.service --no-pager -o short-iso", + ), + ) + for name, command in commands: + try: + result = self._logged_route_ssh( + stage, name, command, action=f"Saving route failure {name}", timeout=90 + ) + if result.returncode != 0: + errors.append(f"{name}: exit code {result.returncode}") + except Exception as exc: + errors.append(f"{name}: {type(exc).__name__}: {exc}") + summary = "Collection incomplete: " + "; ".join(errors) if errors else "Collection completed" + self._write_route_log(stage, "collection", summary + "\n") + message = f"route diagnostics: {path.parent}; {summary}" + except Exception as exc: + # Diagnostics must not replace the original failure or prevent cloud cleanup. + message = f"route diagnostic collection failed ({type(exc).__name__}: {exc})" + self.progress(message) + return message + + def prepare_route(self) -> Mapping[str, Any]: + try: + return self._prepare_route() + except Exception as exc: + diagnostic = self._capture_route_failure("setup", exc) + raise Gate13CloudError(f"{exc}; {diagnostic}") from exc + + def _prepare_route(self) -> Mapping[str, Any]: + value = self._describe_instance(self.route) + if value is None: + raise Gate13CloudError("route VM is absent") + interfaces = value.get("networkInterfaces") + access = interfaces[0].get("accessConfigs") if isinstance(interfaces, list) and interfaces else None + public_ip = access[0].get("natIP") if isinstance(access, list) and access else None + if not isinstance(public_ip, str) or not public_ip: + raise Gate13CloudError("route VM has no public address") + bundle = self._build_route_bundle() + self._wait_ssh( + self.route, + "test -f /etc/os-release && " + 'test "$(cut -d. -f1 /proc/uptime)" -ge 300 && ' + "(! command -v fuser >/dev/null || " + "(! sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 && " + "! sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1 && " + "! sudo fuser /var/lib/apt/lists/lock >/dev/null 2>&1))", + action="Waiting for the new route machine to finish starting", + timeout_seconds=1_800, + ) + files = [path for path in bundle.iterdir() if path.is_file()] + self._ssh( + self.route, + "rm -rf -- /tmp/gate13-route && install -d -m 0700 /tmp/gate13-route", + action="Preparing the route staging directory", + ) + self._scp( + files, + f"{self.route}:/tmp/gate13-route/", + action="Staging the immutable route bundle", + timeout=1_800, + ) + setup = self._logged_route_ssh( + "setup", + "command", + "install -d -m 0755 /tmp/gate13-route/catalog-v1 && " + "tar -xf /tmp/gate13-route/catalog-v1.tar -C /tmp/gate13-route/catalog-v1 " + "--strip-components=2 && sudo bash /tmp/gate13-route/gate13_route_setup.sh", + action="Installing and starting the route services", + timeout=7_200, + ) + if setup.returncode != 0: + combined = "\n".join(part for part in (setup.stdout, setup.stderr) if part) + tail = "\n".join(combined.splitlines()[-30:])[-4_000:] + tail = re.sub(r"https?://\S+", "", tail) + if tail: + self.progress("Route setup failed; bounded redacted tail:\n" + tail) + raise CommandError( + "Installing and starting the route services failed with exit code " f"{setup.returncode}" + ) + fence = self.repository_root / "scripts" / "gate13_route_fence.py" + self._scp( + [fence], + f"{self.route}:/tmp/gate13_route_fence.py", + action="Staging the exact final route fence", + timeout=300, + ) + return { + "result": "passed", + "route_source_commit": self.config.route_source_commit, + "public_address_present": True, + "immutable_bundle_staged": True, + } + + def fence_route(self, platform: str) -> Mapping[str, Any]: + try: + return self._fence_route(platform) + except Exception as exc: + diagnostic = self._capture_route_failure(f"{platform}-fence", exc) + raise Gate13CloudError(f"{exc}; {diagnostic}") from exc + + def _fence_route(self, platform: str) -> Mapping[str, Any]: + result = self._logged_route_ssh( + f"{platform}-fence", + "command", + f"sudo /opt/communityai/venv/bin/python " + f"/tmp/gate13_route_fence.py --target {platform} " + "--timeout-seconds 1200 --settle-seconds 30", + action=f"Fencing the route for {platform}", + # Allow two 300s service actions, 1200s readiness, and SSH/settle overhead. + timeout=2_100, + ) + value = _strict_terminal_object(result.stdout, f"{platform} route fence") + if value.get("result") != "passed" or value.get("target") != platform: + failure_code = value.get("failure_code") + suffix = f": {failure_code}" if isinstance(failure_code, str) else "" + raise Gate13CloudError(f"{platform} route fence rejected{suffix}") + return value + + def _route_private_ip(self) -> str: + value = self._describe_instance(self.route) + interfaces = value.get("networkInterfaces") if isinstance(value, Mapping) else None + private_ip = ( + interfaces[0].get("networkIP") + if isinstance(interfaces, list) and len(interfaces) == 1 and isinstance(interfaces[0], dict) + else None + ) + try: + parsed = ipaddress.IPv4Address(private_ip) + except (ipaddress.AddressValueError, TypeError) as exc: + raise Gate13CloudError("route VM has no valid private IPv4 address") from exc + return str(parsed) + + def _relay_root(self, platform: str) -> str: + if platform not in self.clients: + raise Gate13CloudError("client platform is invalid") + return f"/tmp/{self.run_id}-{platform}-relay" + + def _relay_unit(self, platform: str) -> str: + if platform not in self.clients: + raise Gate13CloudError("client platform is invalid") + return f"{self.run_id}-{platform}-relay" + + def _relay_download_script(self, platform: str, package: PackageArtifact) -> Path: + stage = self.output_root / "route-relay" / platform + stage.mkdir(parents=True, exist_ok=False) + path = stage / f"route-download-{platform}.sh" + root = self._relay_root(platform) + content = f"""#!/usr/bin/env bash +set -euo pipefail +umask 077 + +root={root} +wrapper="$root/artifact-wrapper.zip" +archive="$root/{package.archive_name}" +install -d -m 0700 "$root" + +url="$(curl -fsS -H 'Metadata-Flavor: Google' \\ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/artifact-probe-url)" +curl -fL --retry 4 --retry-delay 3 --silent --show-error "$url" -o "$wrapper" +url= +test "$(stat -c %s "$wrapper")" = {package.wrapper_bytes} +test "$(sha256sum "$wrapper" | cut -d' ' -f1)" = {package.wrapper_sha256} + +/opt/communityai/venv/bin/python - "$wrapper" "$archive" <<'PY' +import pathlib +import shutil +import sys +import zipfile + +wrapper = pathlib.Path(sys.argv[1]) +target = pathlib.Path(sys.argv[2]) +expected = {package.archive_name!r} +with zipfile.ZipFile(wrapper) as bundle: + members = bundle.namelist() + if members != [expected]: + raise SystemExit("artifact wrapper inventory changed") + with bundle.open(members[0]) as source, target.open("xb") as output: + shutil.copyfileobj(source, output, length=1024 * 1024) +PY + +test "$(stat -c %s "$archive")" = {package.archive_bytes} +test "$(sha256sum "$archive" | cut -d' ' -f1)" = {package.archive_sha256} +printf '%s\\n' '{{"result":"passed","scope":"gate13-{platform}-artifact-relay","sha256":"{package.archive_sha256}","bytes":{package.archive_bytes}}}' +""" + path.write_text(content, encoding="utf-8", newline="\n") + return path + + def _prepare_route_relay(self, platform: str, package: PackageArtifact) -> str: + script = self._relay_download_script(platform, package) + remote_script = f"/tmp/{self.run_id}-route-download-{platform}.sh" + private_ip = self._route_private_ip() + url_path = self.output_root / f".{platform}-artifact-url" + with url_path.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(self.signed_url(package) + "\n") + stream.flush() + os.fsync(stream.fileno()) + metadata_added = False + try: + self._gcloud( + "compute", + "instances", + "add-metadata", + self.route, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--metadata-from-file", + f"artifact-probe-url={url_path}", + "--quiet", + action=f"Authorizing the proven {platform} route-relay download", + timeout=180, + ) + metadata_added = True + self._scp( + [script], + f"{self.route}:{remote_script}", + action=f"Staging the proven {platform} route-relay download", + timeout=300, + ) + downloaded = self._ssh( + self.route, + f"bash {remote_script}", + action=f"Downloading and verifying {platform} on the route relay", + timeout=1_800, + ) + finally: + url_path.unlink(missing_ok=True) + if metadata_added: + self._gcloud( + "compute", + "instances", + "remove-metadata", + self.route, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--keys", + "artifact-probe-url", + "--quiet", + action=f"Removing the {platform} signed URL from route metadata", + timeout=180, + ) + result = _strict_object(downloaded.stdout, f"{platform} route-relay download") + if ( + result.get("result") != "passed" + or result.get("sha256") != package.archive_sha256 + or result.get("bytes") != package.archive_bytes + ): + raise Gate13CloudError(f"{platform} route-relay verification rejected") + unit = self._relay_unit(platform) + root = self._relay_root(platform) + self._ssh( + self.route, + f"sudo systemd-run --unit={unit} --property=RuntimeMaxSec=3600 " + f"/opt/communityai/venv/bin/python -m http.server 38081 " + f"--bind {private_ip} --directory {root} && sleep 1 && " + f"systemctl is-active {unit}", + action=f"Starting the proven private {platform} package relay", + timeout=180, + ) + return f"http://{private_ip}:38081/artifact-wrapper.zip" + + def _cleanup_route_relay(self, platform: str) -> None: + root = self._relay_root(platform) + unit = self._relay_unit(platform) + remote_script = f"/tmp/{self.run_id}-route-download-{platform}.sh" + self._ssh( + self.route, + "set -euo pipefail; " + f'root={root}; test "$(realpath -e "$root")" = "$root"; ' + 'test ! -L "$root"; ' + f"sudo systemctl stop {unit}; " + 'rm -rf -- "$root"; ' + f"rm -f -- {remote_script}; " + 'test ! -e "$root"', + action=f"Removing the proven private {platform} package relay", + timeout=300, + ) + + def create_client(self, platform: str, package: PackageArtifact) -> None: + if platform not in self.clients: + raise Gate13CloudError("client platform is invalid") + name = self.clients[platform] + startup = self._client_startup_script(platform) + package_url = self._prepare_route_relay(platform, package) + public_key = self._ensure_ssh_key() + labels = f"communityai_run={self.run_id},communityai_scope=gate13_one_click" + metadata = ( + f"package-url={package_url}," + f"package-sha256={package.archive_sha256}," + f"package-bytes={package.archive_bytes}" + ) + metadata_files = [f"gate13-ssh-public-key={public_key}"] + if platform == "windows": + machine_type = self.config.windows_machine_type + image = self.config.windows_image + image_project = self.config.windows_image_project + metadata_files.append(f"windows-startup-script-ps1={startup}") + disk_size = "120GB" + else: + machine_type = self.config.linux_machine_type + image = self.config.linux_image + image_project = self.config.linux_image_project + metadata_files.append(f"startup-script={startup}") + disk_size = "120GB" + self._gcloud( + "compute", + "instances", + "create", + name, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--machine-type", + machine_type, + "--network", + self.config.network, + "--subnet", + self.config.subnet, + "--network-tier", + "PREMIUM", + "--maintenance-policy", + "MIGRATE", + "--provisioning-model", + "STANDARD", + "--no-service-account", + "--no-scopes", + "--image", + image, + "--image-project", + image_project, + "--boot-disk-type", + "pd-balanced", + "--boot-disk-size", + disk_size, + "--boot-disk-device-name", + name, + "--boot-disk-auto-delete", + "--tags", + self.client_tag, + "--labels", + labels, + "--metadata", + metadata, + "--metadata-from-file", + ",".join(metadata_files), + "--max-run-duration", + "21600s", + "--instance-termination-action", + "DELETE", + *(("--enable-display-device",) if platform == "windows" else ()), + action=f"Creating the clean {platform} client VM", + timeout=1_200, + ) + value = self._describe_instance(name) + if value is None: + raise Gate13CloudError(f"{platform} client disappeared after creation") + self._assert_owned_instance(name, value) + + @staticmethod + def _policy(model_id: str) -> Mapping[str, Any]: + return { + "sharing_enabled": True, + "allowed_models": [model_id], + "preferred_models": [model_id], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": None, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + }, + } + + def _lifecycle_config(self, platform: str, package: PackageArtifact) -> dict[str, Any]: + lifecycle_run_id = f"{self.run_id}-{platform}" + if platform == "windows": + archive = r"C:\Gate13Run\package\communityai-desktop-windows.zip" + executable = r"C:\Gate13Run\install\CommunityAI\CommunityAI.exe" + work_root = rf"C:\Gate13Run\.gate13-playthrough-{lifecycle_run_id}" + model_id = "Qwen3.5 2B" + manifest = "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33" + blocks = 24 + else: + archive = "/qualification/package/communityai-desktop-linux.tar.gz" + executable = "/qualification/install/CommunityAI/CommunityAI" + work_root = f"/qualification/.gate13-playthrough-{lifecycle_run_id}" + model_id = "Gemma 4 E2B IT" + manifest = "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd" + blocks = 35 + return { + "schema_version": 2, + "run_id": lifecycle_run_id, + "platform": platform, + "source_commit": package.source_commit, + "package_archive": archive, + "package_sha256": "sha256:" + package.archive_sha256, + "package_bytes": package.archive_bytes, + "desktop_executable": executable, + "work_root": work_root, + "model_id": model_id, + "manifest_digest": manifest, + "total_blocks": blocks, + "policy": self._policy(model_id), + "session_timeout_seconds": 3_600, + "inference_timeout_seconds": 600, + } + + def _host_config(self, platform: str, package: PackageArtifact, lifecycle_sha256: str) -> dict[str, Any]: + if platform == "windows": + root = r"C:\Gate13Run" + separator = "\\" + host_user = "M" + python = r"C:\Gate13Python\python.exe" + else: + root = "/qualification" + separator = "/" + host_user = "gate13" + python = "/usr/bin/python3" + + def remote(name: str) -> str: + return root + separator + name + + return { + "schema_version": 1, + "run_id": self.run_id, + "lifecycle_run_id": f"{self.run_id}-{platform}", + "platform": platform, + "attempt_ordinal": 1, + "source_commit": package.source_commit, + "job_name": f"communityai-gate13-{self.run_id}-{platform}", + "host_user": host_user, + "adapter_path": remote("gate13_host_job.py"), + "adapter_sha256": "sha256:" + _sha256(self.repository_root / "scripts" / "gate13_host_job.py"), + "config_path": remote("host-job.json"), + "entrypoint_path": remote("gate13_automated_playthrough.py"), + "entrypoint_sha256": "sha256:" + + _sha256(self.repository_root / "scripts" / "gate13_automated_playthrough.py"), + "lifecycle_config_path": remote(f"gate13-{platform}-run.json"), + "lifecycle_config_sha256": "sha256:" + lifecycle_sha256, + "evidence_path": remote("evidence.json"), + "stderr_path": remote("stderr.log"), + "status_path": remote("status.json"), + "terminal_path": remote("terminal.json"), + "working_directory": root, + "python_executable": python, + "max_run_seconds": 14_400, + } + + @staticmethod + def _write_json(path: Path, value: Mapping[str, Any]) -> None: + payload = json.dumps(value, allow_nan=False, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + path.write_text(payload, encoding="utf-8", newline="\n") + + def _build_client_stage(self, platform: str, package: PackageArtifact) -> tuple[Path, Path]: + stage = self.output_root / f"{platform}-stage" + stage.mkdir(parents=True, exist_ok=False) + lifecycle_path = stage / f"gate13-{platform}-run.json" + self._write_json(lifecycle_path, self._lifecycle_config(platform, package)) + host_path = stage / "host-job.json" + self._write_json(host_path, self._host_config(platform, package, _sha256(lifecycle_path))) + scripts = ( + "gate13_host_job.py", + "gate13_automated_playthrough.py", + "gate13_packaged_lifecycle.py", + ) + for name in scripts: + shutil.copy2(self.repository_root / "scripts" / name, stage / name) + if platform == "windows": + stage_script = self._windows_stage_script(stage, scripts, lifecycle_path, host_path) + else: + stage_script = self._linux_stage_script(stage, scripts, lifecycle_path, host_path, package) + return stage, stage_script + + def _windows_stage_script( + self, + stage: Path, + scripts: Sequence[str], + lifecycle_path: Path, + host_path: Path, + ) -> Path: + expected = {name: _sha256(stage / name) for name in (*scripts, lifecycle_path.name, host_path.name)} + entries = "\n".join(f' "{name}" = "{digest}"' for name, digest in expected.items()) + content = f"""$ErrorActionPreference = "Stop" +$root = "C:\\Gate13Run" +$expected = @{{ +{entries} +}} +$actual = @{{}} +foreach ($name in $expected.Keys) {{ + $digest = (Get-FileHash -LiteralPath "$root\\$name" -Algorithm SHA256).Hash.ToLowerInvariant() + if ($digest -cne $expected[$name]) {{ throw "digest mismatch for $name" }} + $actual[$name] = $digest +}} +$hostConfig = Get-Content -LiteralPath "$root\\host-job.json" -Raw | ConvertFrom-Json +if ($hostConfig.lifecycle_config_sha256 -cne "sha256:$($expected['{lifecycle_path.name}'])") {{ + throw "host config does not bind the staged lifecycle config" +}} +$explorer = @(Get-Process explorer -IncludeUserName -ErrorAction SilentlyContinue | + Where-Object {{ $_.UserName -like "*\\M" }}) +if ($explorer.Count -ne 1 -or $explorer[0].SessionId -lt 1) {{ + throw "ordinary M interactive session is not ready" +}} +[pscustomobject]@{{ + result = "passed" + ready = $true + hashes = $actual + host_user = $explorer[0].UserName + session_id = $explorer[0].SessionId +}} | ConvertTo-Json -Depth 5 -Compress +""" + path = stage / "stage.ps1" + path.write_text(content, encoding="utf-8-sig", newline="\r\n") + return path + + def _linux_stage_script( + self, + stage: Path, + scripts: Sequence[str], + lifecycle_path: Path, + host_path: Path, + package: PackageArtifact, + ) -> Path: + expected = {name: _sha256(stage / name) for name in (*scripts, lifecycle_path.name, host_path.name)} + installs = "\n".join( + f"install -o gate13 -g gate13 -m 0700 /tmp/{name} /qualification/{name}" for name in scripts + ) + checks = "\n".join( + f'test "$(sha256sum /qualification/{name} | cut -d\' \' -f1)" = "{digest}"' + for name, digest in expected.items() + ) + content = f"""#!/usr/bin/env bash +set -euo pipefail +umask 077 +{installs} +install -o gate13 -g gate13 -m 0600 /tmp/{lifecycle_path.name} /qualification/{lifecycle_path.name} +install -o gate13 -g gate13 -m 0600 /tmp/{host_path.name} /qualification/{host_path.name} +{checks} +test "$(sha256sum /qualification/package/{package.archive_name} | cut -d' ' -f1)" = "{package.archive_sha256}" +test "$(stat -c %s /qualification/package/{package.archive_name})" = "{package.archive_bytes}" +test -x /qualification/install/CommunityAI/CommunityAI +sudo -u gate13 env DISPLAY=:99 xdpyinfo >/dev/null +printf 'GATE13_STAGE_RESULT result=passed ready=true host_user=gate13 display=:99 hashes_verified=true\\n' +printf '%s\\n' '{{"result":"passed","ready":true,"host_user":"gate13","display":":99","hashes_verified":true}}' +""" + path = stage / "stage.sh" + path.write_text(content, encoding="utf-8", newline="\n") + return path + + def prepare_client(self, platform: str, package: PackageArtifact) -> Mapping[str, Any]: + name = self.clients[platform] + if platform == "windows": + user = "Gate13Admin" + ready_command = ( + "powershell.exe -NoLogo -NoProfile -NonInteractive -Command " + "\"if (!(Test-Path -LiteralPath 'C:\\Gate13Bootstrap\\ready.txt' " + "-PathType Leaf)) { exit 1 }; " + "$p=@(Get-Process explorer -IncludeUserName -ErrorAction SilentlyContinue | " + "Where-Object {$_.UserName -like '*\\M'}); " + 'if ($p.Count -ne 1 -or $p[0].SessionId -lt 1) { exit 1 }"' + ) + else: + user = None + ready_command = ( + "if test -f /var/lib/gate13-bootstrap-ready; then " + "sudo -u gate13 env DISPLAY=:99 xdpyinfo >/dev/null; " + "elif sudo grep -qx failed /var/lib/gate13-bootstrap-status 2>/dev/null; then " + "printf GATE13_BOOTSTRAP_FAILED; exit 2; else exit 1; fi" + ) + self._wait_ssh( + name, + ready_command, + user=user, + action=f"Waiting for the clean {platform} client", + timeout_seconds=5_400, + fatal_marker="GATE13_BOOTSTRAP_FAILED" if platform == "linux" else None, + ) + self._cleanup_route_relay(platform) + stage, stage_script = self._build_client_stage(platform, package) + files = [path for path in stage.iterdir() if path.is_file()] + if platform == "windows": + destination = f"Gate13Admin@{name}:C:/Gate13Run/" + else: + destination = f"{name}:/tmp/" + self._scp( + files, + destination, + action=f"Staging the {platform} qualification job", + timeout=900, + ) + if platform == "windows": + command = ( + "powershell.exe -NoLogo -NoProfile -NonInteractive " + "-ExecutionPolicy Bypass -File C:\\Gate13Run\\stage.ps1" + ) + else: + command = "sudo bash /tmp/stage.sh" + result = self._ssh( + name, + command, + user=user, + action=f"Validating the {platform} qualification stage", + timeout=300, + check=False, + ) + output_path = self.output_root / f"{platform}-stage-command-output.json" + self._write_json( + output_path, + { + "exit_code": result.returncode, + "stderr": result.stderr, + "stdout": result.stdout, + }, + ) + if result.returncode != 0: + raise Gate13CloudError( + f"{platform} qualification stage failed with exit code {result.returncode}; " + f"captured output: {output_path}" + ) + if platform == "linux": + if _LINUX_STAGE_SUCCESS_RE.search(result.stdout) is None: + raise Gate13CloudError( + "linux qualification stage success marker was not found; " f"captured output: {output_path}" + ) + else: + value = _strict_object(result.stdout, "windows qualification stage") + if value.get("result") != "passed" or value.get("ready") is not True: + raise Gate13CloudError(f"{platform} qualification stage rejected") + return { + "result": "passed", + "package_relay_verified": True, + "hashes_verified": True, + "ordinary_desktop_user_ready": True, + } + + def _host_command(self, platform: str, action: str) -> tuple[str, str | None]: + if platform == "windows": + return ( + f"C:\\Gate13Python\\python.exe C:\\Gate13Run\\gate13_host_job.py " + f"{action} --config C:\\Gate13Run\\host-job.json", + "Gate13Admin", + ) + return ( + f"sudo /usr/bin/python3 /qualification/gate13_host_job.py " + f"{action} --config /qualification/host-job.json", + None, + ) + + def _run_host_job_command( + self, + instance: str, + command: str, + *, + user: str | None, + action: str, + timeout: float, + ) -> subprocess.CompletedProcess[str]: + for attempt in range(1, _HOST_JOB_COMMAND_ATTEMPTS + 1): + result = self._ssh( + instance, + command, + user=user, + action=action, + timeout=timeout, + check=False, + ) + # Preserve the response before parsing or retrying it. Credentials and + # signed package URLs are never part of a host-job command response. + output_path = self.output_root / f"{instance}-host-job-command-output.json" + self._write_json( + output_path, + { + "action": action, + "attempt": attempt, + "exit_code": result.returncode, + "stderr": result.stderr, + "stdout": result.stdout, + }, + ) + if result.returncode == 0: + return result + if attempt == _HOST_JOB_COMMAND_ATTEMPTS: + raise CommandError( + f"{action} failed {_HOST_JOB_COMMAND_ATTEMPTS} times in a row; " f"captured output: {output_path}" + ) + self.progress( + f"{action} did not complete; trying again " f"({attempt + 1} of {_HOST_JOB_COMMAND_ATTEMPTS})" + ) + self.sleeper(30) + raise AssertionError("unreachable") + + def _capture_host_failure(self, platform: str, instance: str) -> Path: + output_path = self.output_root / f"{platform}-host-job-failure-output.json" + captured: dict[str, Any] = {} + for label, filename in ( + ("terminal", "terminal.json"), + ("stderr", "stderr.log"), + ("evidence", "evidence.json"), + ): + if platform == "windows": + script = ( + "$ErrorActionPreference = 'Stop'; " + "[Console]::Out.Write([IO.File]::ReadAllText(" + f"'C:\\Gate13Run\\{filename}'))" + ) + encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii") + command = f"powershell.exe -NoLogo -NoProfile -NonInteractive -EncodedCommand {encoded}" + user = "Gate13Admin" + else: + command = f"sudo cat /qualification/{filename}" + user = None + try: + result = self._ssh( + instance, + command, + user=user, + action=f"Collecting the failed {platform} host job {label}", + timeout=90, + check=False, + ) + captured[label] = { + "exit_code": result.returncode, + "stderr": result.stderr, + "stdout": result.stdout, + } + except Exception as exc: + # A missing file or failed SSH read must not hide the original + # qualification error or discard files already collected. + captured[label] = {"capture_error": type(exc).__name__} + self._write_json(output_path, captured) + for stream in ("stdout", "stderr"): + output = captured[label].get(stream, "").strip() + if output: + self.progress(f"Failed {platform} host job {label} ({stream}):\n{output}") + return output_path + + def run_client(self, platform: str, package: PackageArtifact) -> bytes: + try: + return self._run_client(platform, package) + except Exception as exc: + try: + output_path = self._capture_host_failure(platform, self.clients[platform]) + suffix = f"; captured output: {output_path}" + except Exception as capture_error: + suffix = f"; failure output collection failed ({type(capture_error).__name__})" + raise Gate13CloudError(f"{exc}{suffix}") from exc + + def _run_client(self, platform: str, package: PackageArtifact) -> bytes: + name = self.clients[platform] + start_command, user = self._host_command(platform, "start") + started = self._run_host_job_command( + name, + start_command, + user=user, + action=f"Starting the durable {platform} qualification job", + timeout=180, + ) + parser = _strict_terminal_object if platform == "linux" else _strict_object + start_value = parser(started.stdout, f"{platform} host-job start") + if start_value.get("job_state") not in {"starting", "running", "passed"}: + raise Gate13CloudError(f"{platform} host job did not start") + + status_command, _ = self._host_command(platform, "status") + deadline = time.monotonic() + 14_700 + while time.monotonic() < deadline: + observed = self._run_host_job_command( + name, + status_command, + user=user, + action=f"Checking the {platform} qualification job", + timeout=180, + ) + status = parser(observed.stdout, f"{platform} host-job status") + state = status.get("job_state") + if state == "passed": + break + if state in {"failed", "ambiguous", "absent"}: + raise Gate13CloudError(f"{platform} host job ended in state {state}") + if state not in {"starting", "running"}: + raise Gate13CloudError(f"{platform} host job returned an invalid state") + self.progress(f"{platform.capitalize()} qualification is {state}; waiting") + self.sleeper(30) + else: + raise Gate13CloudError(f"{platform} host job exceeded its time bound") + + collect_command, _ = self._host_command(platform, "collect") + collected = self._run_host_job_command( + name, + collect_command, + user=user, + action=f"Collecting the {platform} qualification evidence", + timeout=300, + ) + if platform == "linux": + lines = [line for line in collected.stdout.splitlines() if line.strip()] + if not lines: + raise Gate13CloudError("linux evidence is empty") + _strict_object(lines[-1], "linux qualification evidence") + payload = (lines[-1] + "\n").encode("utf-8") + else: + payload = collected.stdout.encode("utf-8") + cleanup_command, _ = self._host_command(platform, "cleanup") + self._run_host_job_command( + name, + cleanup_command, + user=user, + action=f"Removing the {platform} native host job", + timeout=180, + ) + if not payload: + raise Gate13CloudError(f"{platform} evidence is empty") + return payload + + def _expected_image(self, name: str) -> str: + if name == self.route: + return self.config.route_image + if name == self.clients["windows"]: + return self.config.windows_image + if name == self.clients["linux"]: + return self.config.linux_image + raise Gate13CloudError("instance target is outside this run") + + def _delete_orphan_disk(self, name: str) -> None: + inventory = self._gcloud_json_with_retry( + "compute", + "disks", + "list", + "--project", + self.config.project, + action=f"Checking orphan disk {name}", + timeout=120, + ) + present = ( + [item for item in inventory if isinstance(item, dict) and item.get("name") == name] + if isinstance(inventory, list) + else [] + ) + if not present: + return + if len(present) != 1: + raise Gate13CloudError(f"disk inventory for {name} is ambiguous") + disk = self._gcloud_json_with_retry( + "compute", + "disks", + "describe", + name, + "--project", + self.config.project, + "--zone", + self.config.zone, + action=f"Binding orphan disk {name}", + timeout=120, + ) + users = disk.get("users") if isinstance(disk, dict) else None + source_image = self._basename(disk.get("sourceImage")) if isinstance(disk, dict) else "" + if ( + not isinstance(disk, dict) + or disk.get("name") != name + or users not in (None, []) + or source_image != self._expected_image(name) + ): + raise Gate13CloudError(f"refusing to delete unbound disk {name}") + self._gcloud( + "compute", + "disks", + "delete", + name, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--quiet", + action=f"Deleting orphan disk {name}", + timeout=600, + ) + + def _delete_instance(self, name: str, *, label: str) -> None: + value = self._describe_instance(name, check=False) + if value is not None: + self._assert_owned_instance(name, value) + self._gcloud( + "compute", + "instances", + "delete", + name, + "--project", + self.config.project, + "--zone", + self.config.zone, + "--delete-disks", + "all", + "--quiet", + action=f"Deleting {label}", + timeout=900, + ) + self._delete_orphan_disk(name) + + def _delete_firewall(self, name: str) -> None: + bindings = { + self.dht_firewall: { + "source_ranges": ["0.0.0.0/0"], + "source_tags": [], + "target_tags": [self.route], + "ports": ["31337-31338"], + }, + self.iap_firewall: { + "source_ranges": ["35.235.240.0/20"], + "source_tags": [], + "target_tags": [self.client_tag, self.route], + "ports": ["22"], + }, + self.relay_firewall: { + "source_ranges": [], + "source_tags": [self.client_tag], + "target_tags": [self.route], + "ports": ["38081"], + }, + } + if name not in bindings: + raise Gate13CloudError(f"refusing to inspect unknown firewall {name}") + inventory = self._gcloud_json_with_retry( + "compute", + "firewall-rules", + "list", + "--project", + self.config.project, + action=f"Checking firewall {name}", + timeout=120, + ) + present = ( + [item for item in inventory if isinstance(item, dict) and item.get("name") == name] + if isinstance(inventory, list) + else [] + ) + if not present: + return + if len(present) != 1: + raise Gate13CloudError(f"firewall inventory for {name} is ambiguous") + firewall = self._gcloud_json_with_retry( + "compute", + "firewall-rules", + "describe", + name, + "--project", + self.config.project, + action=f"Binding firewall {name}", + timeout=120, + ) + expected = bindings[name] + allowed = firewall.get("allowed") if isinstance(firewall, dict) else None + first_allow = allowed[0] if isinstance(allowed, list) and len(allowed) == 1 else None + if ( + not isinstance(firewall, dict) + or firewall.get("name") != name + or self._basename(firewall.get("network")) != self.config.network + or firewall.get("direction") != "INGRESS" + or sorted(firewall.get("sourceRanges") or []) != sorted(expected["source_ranges"]) + or sorted(firewall.get("sourceTags") or []) != sorted(expected["source_tags"]) + or sorted(firewall.get("targetTags") or []) != sorted(expected["target_tags"]) + or not isinstance(first_allow, dict) + or first_allow.get("IPProtocol") != "tcp" + or first_allow.get("ports") != expected["ports"] + ): + raise Gate13CloudError(f"refusing to delete unbound firewall {name}") + self._gcloud( + "compute", + "firewall-rules", + "delete", + name, + "--project", + self.config.project, + "--quiet", + action=f"Deleting firewall {name}", + timeout=300, + ) + + def delete_client(self, platform: str) -> None: + if platform not in self.clients: + raise Gate13CloudError("client platform is invalid") + self._delete_instance(self.clients[platform], label=f"{platform} client") + + def delete_route(self) -> None: + self._delete_instance(self.route, label="route VM") + self._delete_firewall(self.dht_firewall) + self._delete_firewall(self.iap_firewall) + self._delete_firewall(self.relay_firewall) + + def cleanup_all(self) -> Mapping[str, Any]: + errors: list[str] = [] + for platform in ("windows", "linux"): + try: + self.delete_client(platform) + except BaseException as exc: + errors.append(f"{platform}:{type(exc).__name__}") + try: + self.delete_route() + except BaseException as exc: + errors.append(f"route:{type(exc).__name__}") + return { + "result": "failed" if errors else "passed", + "errors": errors, + "exact_targets_only": True, + } + + def verify_cleanup(self) -> Mapping[str, Any]: + instances, disks, firewalls = self._resource_absence() + bootstrap = self._gcloud_json_with_retry( + "compute", + "instances", + "describe", + self.config.protected_instance, + "--project", + self.config.project, + "--zone", + self.config.protected_zone, + action="Rechecking the protected bootstrap", + timeout=120, + ) + protected_running = isinstance(bootstrap, dict) and bootstrap.get("status") == "RUNNING" + passed = not instances and not disks and not firewalls and protected_running + return { + "result": "passed" if passed else "failed", + "instances_absent": not instances, + "disks_absent": not disks, + "firewalls_absent": not firewalls, + "remaining_instances": instances, + "remaining_disks": disks, + "remaining_firewalls": firewalls, + "protected_bootstrap_running": protected_running, + } diff --git a/scripts/gate13_host_job.py b/scripts/gate13_host_job.py new file mode 100644 index 000000000..181f71054 --- /dev/null +++ b/scripts/gate13_host_job.py @@ -0,0 +1,1292 @@ +"""Durable native host-job adapter for Gate 13 packaged lifecycle runs. + +A paid client attempt is launched exactly once under a native supervisor. The adapter +persists bounded status before starting the lifecycle, validates the canonical evidence, +and writes a digest-only terminal record. Re-entry never relaunches an attempt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import signal +import stat +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate13_packaged_lifecycle as lifecycle + +SCHEMA_VERSION = 1 +MAX_CONFIG_BYTES = 65_536 +MAX_STATE_BYTES = 262_144 +MAX_EVIDENCE_BYTES = lifecycle.MAX_INPUT_BYTES +MAX_STDERR_BYTES = 262_144 +MAX_SCRIPT_BYTES = 8 * 1024 * 1024 +MIN_RUN_SECONDS = 300 +MAX_RUN_SECONDS = 21_600 +SUPERVISOR_GRACE_SECONDS = 60 +READ_CHUNK_BYTES = 65_536 +POSIX_SIGTERM = getattr(signal, "SIGTERM", 15) +POSIX_SIGKILL = getattr(signal, "SIGKILL", 9) + +WINDOWS_RUNTIME_ENVIRONMENT = ( + "ALLUSERSPROFILE", + "APPDATA", + "COMMONPROGRAMFILES", + "COMMONPROGRAMFILES(X86)", + "COMMONPROGRAMW6432", + "COMSPEC", + "HOMEDRIVE", + "HOMEPATH", + "LOCALAPPDATA", + "OS", + "PATH", + "PATHEXT", + "PROGRAMDATA", + "PROGRAMFILES", + "PROGRAMFILES(X86)", + "PROGRAMW6432", + "PUBLIC", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TEMP", + "TMP", + "USERDOMAIN", + "USERNAME", + "USERPROFILE", + "WINDIR", +) +LINUX_RUNTIME_ENVIRONMENT = ( + "DBUS_SESSION_BUS_ADDRESS", + "DISPLAY", + "GNOME_KEYRING_CONTROL", + "HOME", + "LANG", + "LC_ALL", + "QT_QPA_PLATFORM", + "TMPDIR", + "XAUTHORITY", + "XDG_RUNTIME_DIR", +) + +HOST_ROOTS = { + "windows": Path(r"C:\Gate13Run"), + "linux": Path("/qualification"), +} +HOST_PYTHON = { + "windows": Path(r"C:\Gate13Python\python.exe"), + "linux": Path("/usr/bin/python3"), +} +ADAPTER_PATH = Path(__file__).resolve() +GATE_NAME = "gate13" +LINUX_HOST_USER = "gate13" +LINUX_HOME = "/home/gate13" +LINUX_RUNTIME_DIR = "/qualification/runtime" +LIFECYCLE_CONFIG_NAMES = { + "windows": "gate13-windows-run.json", + "linux": "gate13-linux-run.json", +} +LIFECYCLE_RUN_ID_BUILDER: Callable[[str, str], str] = lambda run_id, platform: f"{run_id}-{platform}" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_USER_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"(?:sha256:)?[0-9a-f]{64}") +_JOB_RE = re.compile(r"communityai-gate13-[a-z0-9-]{1,63}-(?:windows|linux)") +_CONFIG_FIELDS = { + "schema_version", + "run_id", + "lifecycle_run_id", + "platform", + "attempt_ordinal", + "source_commit", + "job_name", + "host_user", + "adapter_path", + "adapter_sha256", + "config_path", + "entrypoint_path", + "entrypoint_sha256", + "lifecycle_config_path", + "lifecycle_config_sha256", + "evidence_path", + "stderr_path", + "status_path", + "terminal_path", + "working_directory", + "python_executable", + "max_run_seconds", +} +_STATUS_FIELDS = { + "schema_version", + "run_id", + "platform", + "attempt_ordinal", + "state", + "started_at_unix", +} +_TERMINAL_FIELDS = { + "schema_version", + "run_id", + "platform", + "attempt_ordinal", + "result", + "failure_code", + "evidence_digest", + "exit_code", + "finished_at_unix", +} +_NATIVE_FIELDS = {"native_state", "binding_ok"} + + +class HostJobError(ValueError): + """The host job config, state, or native supervisor failed closed.""" + + +@dataclass(frozen=True) +class HostJobConfig: + run_id: str + lifecycle_run_id: str + platform: str + attempt_ordinal: int + source_commit: str + job_name: str + host_user: str + adapter_path: Path + adapter_sha256: str + config_path: Path + entrypoint_path: Path + entrypoint_sha256: str + lifecycle_config_path: Path + lifecycle_config_sha256: str + evidence_path: Path + stderr_path: Path + status_path: Path + terminal_path: Path + working_directory: Path + python_executable: Path + max_run_seconds: int + + +Runner = Callable[..., subprocess.CompletedProcess[str]] + + +def _reject_constant(_value: str) -> None: + raise HostJobError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise HostJobError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path, maximum: int, *, allow_empty: bool = False) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise HostJobError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + minimum = 0 if allow_empty else 1 + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not minimum <= metadata.st_size <= maximum: + raise HostJobError("required file is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise HostJobError("required file is unreadable") from exc + + +def _strict_json(payload: bytes, maximum: int) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= maximum: + raise HostJobError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HostJobError("invalid JSON") from exc + if not isinstance(value, dict): + raise HostJobError("JSON root is invalid") + return value + + +def _exact_mapping(value: Mapping[str, Any], fields: set[str], label: str) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise HostJobError(f"{label} schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str], label: str) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise HostJobError(f"{label} is invalid") + return value + + +def _integer(value: Any, label: str, minimum: int, maximum: int) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise HostJobError(f"{label} is invalid") + return value + + +def _digest_bytes(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _digest_file(path: Path) -> str: + return _digest_bytes(_regular_bytes(path, MAX_SCRIPT_BYTES)) + + +def _normalized_path(value: Any, label: str) -> Path: + if not isinstance(value, str) or not value or "\x00" in value: + raise HostJobError(f"{label} is invalid") + path = Path(value) + if not path.is_absolute(): + raise HostJobError(f"{label} is not absolute") + return Path(os.path.abspath(os.fspath(path))) + + +def _same_path(left: Path, right: Path) -> bool: + return os.path.normcase(os.path.abspath(os.fspath(left))) == os.path.normcase(os.path.abspath(os.fspath(right))) + + +def _inside(path: Path, root: Path) -> bool: + try: + return os.path.commonpath( + [os.path.normcase(os.path.abspath(os.fspath(path))), os.path.normcase(os.path.abspath(os.fspath(root)))] + ) == os.path.normcase(os.path.abspath(os.fspath(root))) + except ValueError: + return False + + +def _safe_existing_output(path: Path) -> None: + if not path.exists(): + return + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise HostJobError("output path is unsafe") + + +def load_config(path: Path) -> HostJobConfig: + config_path = Path(os.path.abspath(os.fspath(path))) + raw = _exact_mapping( + _strict_json(_regular_bytes(config_path, MAX_CONFIG_BYTES), MAX_CONFIG_BYTES), + _CONFIG_FIELDS, + "config", + ) + if raw["schema_version"] != SCHEMA_VERSION: + raise HostJobError("config version is invalid") + platform = raw["platform"] + if platform not in HOST_ROOTS: + raise HostJobError("platform is invalid") + run_id = _string(raw["run_id"], _RUN_RE, "run id") + lifecycle_run_id = raw["lifecycle_run_id"] + if lifecycle_run_id != LIFECYCLE_RUN_ID_BUILDER(run_id, platform): + raise HostJobError("lifecycle run id is invalid") + attempt = _integer(raw["attempt_ordinal"], "attempt ordinal", 1, 1) + source_commit = _string(raw["source_commit"], _COMMIT_RE, "source commit") + job_name = _string(raw["job_name"], _JOB_RE, "job name") + if job_name != f"communityai-{GATE_NAME}-{run_id}-{platform}": + raise HostJobError("job name is not source-bound") + host_user = _string(raw["host_user"], _USER_RE, "host user") + if (platform == "linux" and host_user != LINUX_HOST_USER) or host_user.casefold() in { + "system", + "local service", + "network service", + "administrator", + "root", + }: + raise HostJobError("host user is not an ordinary qualification user") + + values = { + field: _normalized_path(raw[field], field) + for field in ( + "adapter_path", + "config_path", + "entrypoint_path", + "lifecycle_config_path", + "evidence_path", + "stderr_path", + "status_path", + "terminal_path", + "working_directory", + "python_executable", + ) + } + root = Path(os.path.abspath(os.fspath(HOST_ROOTS[platform]))) + if not _same_path(values["working_directory"], root): + raise HostJobError("working directory changed") + for field in ( + "adapter_path", + "config_path", + "entrypoint_path", + "lifecycle_config_path", + "evidence_path", + "stderr_path", + "status_path", + "terminal_path", + ): + if not _inside(values[field], root): + raise HostJobError(f"{field} escapes the host root") + if not _same_path(config_path, values["config_path"]): + raise HostJobError("config path binding changed") + expected_lifecycle_name = LIFECYCLE_CONFIG_NAMES[platform] + if values["lifecycle_config_path"].name != expected_lifecycle_name: + raise HostJobError("lifecycle config path changed") + if platform == "windows" and not _same_path( + values["lifecycle_config_path"], + values["entrypoint_path"].parent / expected_lifecycle_name, + ): + raise HostJobError("Windows lifecycle config is not beside its entrypoint") + entrypoint_suffix = values["entrypoint_path"].suffix.casefold() + if (platform == "windows" and entrypoint_suffix not in {".ps1", ".py"}) or ( + platform == "linux" and entrypoint_suffix != ".py" + ): + raise HostJobError("entrypoint type is invalid") + if not _same_path(values["python_executable"], HOST_PYTHON[platform]): + raise HostJobError("Python executable changed") + if not _same_path(values["adapter_path"], ADAPTER_PATH): + raise HostJobError("adapter invocation changed") + root_metadata = root.lstat() + root_reparse = bool( + getattr(root_metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + if not root.is_dir() or root.is_symlink() or root_reparse: + raise HostJobError("host root is unsafe") + + outputs = [ + values["evidence_path"], + values["stderr_path"], + values["status_path"], + values["terminal_path"], + ] + bound_paths = [ + values["adapter_path"], + values["config_path"], + values["entrypoint_path"], + values["lifecycle_config_path"], + *outputs, + ] + if len({os.path.normcase(os.fspath(item)) for item in bound_paths}) != len(bound_paths): + raise HostJobError("bound paths overlap") + for output in outputs: + _safe_existing_output(output) + + adapter_sha = _string(raw["adapter_sha256"], _DIGEST_RE, "adapter digest") + entrypoint_sha = _string(raw["entrypoint_sha256"], _DIGEST_RE, "entrypoint digest") + lifecycle_config_sha = _string( + raw["lifecycle_config_sha256"], + _DIGEST_RE, + "lifecycle config digest", + ) + if _digest_file(values["adapter_path"]) != "sha256:" + adapter_sha.removeprefix("sha256:"): + raise HostJobError("adapter digest changed") + if _digest_file(values["entrypoint_path"]) != "sha256:" + entrypoint_sha.removeprefix("sha256:"): + raise HostJobError("entrypoint digest changed") + if _digest_file(values["lifecycle_config_path"]) != "sha256:" + lifecycle_config_sha.removeprefix("sha256:"): + raise HostJobError("lifecycle config digest changed") + + return HostJobConfig( + run_id=run_id, + lifecycle_run_id=lifecycle_run_id, + platform=platform, + attempt_ordinal=attempt, + source_commit=source_commit, + job_name=job_name, + host_user=host_user, + adapter_path=values["adapter_path"], + adapter_sha256="sha256:" + adapter_sha.removeprefix("sha256:"), + config_path=values["config_path"], + entrypoint_path=values["entrypoint_path"], + entrypoint_sha256="sha256:" + entrypoint_sha.removeprefix("sha256:"), + lifecycle_config_path=values["lifecycle_config_path"], + lifecycle_config_sha256="sha256:" + lifecycle_config_sha.removeprefix("sha256:"), + evidence_path=values["evidence_path"], + stderr_path=values["stderr_path"], + status_path=values["status_path"], + terminal_path=values["terminal_path"], + working_directory=values["working_directory"], + python_executable=values["python_executable"], + max_run_seconds=_integer( + raw["max_run_seconds"], + "maximum run seconds", + MIN_RUN_SECONDS, + MAX_RUN_SECONDS, + ), + ) + + +def _atomic_json(path: Path, value: Mapping[str, Any], *, exclusive: bool = False) -> None: + payload = (json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8") + if not 1 <= len(payload) <= MAX_STATE_BYTES: + raise HostJobError("state is too large") + if exclusive: + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError as exc: + raise HostJobError("state already exists") from exc + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + return + + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + try: + os.chmod(temporary, 0o600) + except OSError: + pass + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _load_status(config: HostJobConfig) -> Mapping[str, Any] | None: + if not config.status_path.exists(): + return None + raw = _exact_mapping( + _strict_json( + _regular_bytes(config.status_path, MAX_STATE_BYTES), + MAX_STATE_BYTES, + ), + _STATUS_FIELDS, + "status", + ) + if ( + raw["schema_version"] != SCHEMA_VERSION + or raw["run_id"] != config.run_id + or raw["platform"] != config.platform + or raw["attempt_ordinal"] != config.attempt_ordinal + or raw["state"] != "running" + or type(raw["started_at_unix"]) is not int + ): + raise HostJobError("status binding changed") + return raw + + +def _load_terminal(config: HostJobConfig) -> Mapping[str, Any] | None: + if not config.terminal_path.exists(): + return None + raw = _exact_mapping( + _strict_json( + _regular_bytes(config.terminal_path, MAX_STATE_BYTES), + MAX_STATE_BYTES, + ), + _TERMINAL_FIELDS, + "terminal", + ) + if ( + raw["schema_version"] != SCHEMA_VERSION + or raw["run_id"] != config.run_id + or raw["platform"] != config.platform + or raw["attempt_ordinal"] != config.attempt_ordinal + or raw["result"] not in {"passed", "failed"} + or (raw["failure_code"] is not None and not re.fullmatch(r"[a-z0-9_]{1,64}", str(raw["failure_code"]))) + or type(raw["exit_code"]) is not int + or type(raw["finished_at_unix"]) is not int + ): + raise HostJobError("terminal binding changed") + digest = raw["evidence_digest"] + if raw["result"] == "passed": + _string(digest, _DIGEST_RE, "terminal evidence digest") + if raw["failure_code"] is not None or raw["exit_code"] != 0: + raise HostJobError("terminal success is inconsistent") + elif digest is not None or raw["failure_code"] is None: + raise HostJobError("terminal failure is inconsistent") + return raw + + +def _terminal( + config: HostJobConfig, + *, + result: str, + failure_code: str | None, + evidence_digest: str | None, + exit_code: int, + finished_at_unix: int, +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": config.attempt_ordinal, + "result": result, + "failure_code": failure_code, + "evidence_digest": evidence_digest, + "exit_code": exit_code, + "finished_at_unix": finished_at_unix, + } + + +def _entrypoint_argv(config: HostJobConfig) -> list[str]: + if config.platform == "windows" and config.entrypoint_path.suffix.casefold() == ".ps1": + return [ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + os.fspath(config.entrypoint_path), + ] + return [ + os.fspath(config.python_executable), + os.fspath(config.entrypoint_path), + "--config", + os.fspath(config.lifecycle_config_path), + ] + + +def _bounded_environment(config: HostJobConfig) -> dict[str, str]: + allowed = WINDOWS_RUNTIME_ENVIRONMENT if config.platform == "windows" else LINUX_RUNTIME_ENVIRONMENT + return {key: os.environ[key] for key in allowed if key in os.environ} + + +def _bounded_copy( + stream: Any, + destination: Path, + maximum: int, + overflow: threading.Event, + errors: list[BaseException], +) -> None: + total = 0 + try: + with destination.open("xb") as output: + while True: + chunk = stream.read(READ_CHUNK_BYTES) + if not chunk: + break + if not isinstance(chunk, bytes): + raise HostJobError("child output type is invalid") + remaining = max(0, maximum - total) + if remaining: + output.write(chunk[:remaining]) + total += len(chunk) + if total > maximum: + overflow.set() + output.flush() + os.fsync(output.fileno()) + except BaseException as exc: + errors.append(exc) + overflow.set() + finally: + try: + stream.close() + except BaseException: + pass + + +def _wait_for_exit(process: Any, timeout: float) -> bool: + try: + process.wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + + +def _stop_process_tree(config: HostJobConfig, process: Any) -> None: + if config.platform == "windows": + try: + process.send_signal(signal.CTRL_BREAK_EVENT) + except (OSError, ValueError, AttributeError): + pass + if _wait_for_exit(process, SUPERVISOR_GRACE_SECONDS): + return + try: + subprocess.run( + [ + r"C:\Windows\System32\taskkill.exe", + "/PID", + str(process.pid), + "/T", + "/F", + ], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + shell=False, + ) + except (OSError, subprocess.TimeoutExpired): + pass + else: + try: + os.killpg(process.pid, POSIX_SIGTERM) + except (OSError, AttributeError): + try: + process.terminate() + except OSError: + pass + if _wait_for_exit(process, SUPERVISOR_GRACE_SECONDS): + return + try: + os.killpg(process.pid, POSIX_SIGKILL) + except (OSError, AttributeError): + try: + process.kill() + except OSError: + pass + if not _wait_for_exit(process, 30): + raise HostJobError("entrypoint process tree did not stop") + + +def _run_entrypoint(config: HostJobConfig) -> int: + if config.evidence_path.exists() or config.stderr_path.exists(): + raise HostJobError("attempt output already exists") + creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if config.platform == "windows" else 0 + process = subprocess.Popen( + _entrypoint_argv(config), + cwd=config.working_directory, + env=_bounded_environment(config), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + bufsize=0, + start_new_session=config.platform == "linux", + creationflags=creationflags, + ) + if process.stdout is None or process.stderr is None: + _stop_process_tree(config, process) + raise HostJobError("entrypoint pipes are unavailable") + + overflow = threading.Event() + copy_errors: list[BaseException] = [] + threads = [ + threading.Thread( + target=_bounded_copy, + args=(process.stdout, config.evidence_path, MAX_EVIDENCE_BYTES, overflow, copy_errors), + daemon=True, + ), + threading.Thread( + target=_bounded_copy, + args=(process.stderr, config.stderr_path, MAX_STDERR_BYTES, overflow, copy_errors), + daemon=True, + ), + ] + for thread in threads: + thread.start() + + deadline = time.monotonic() + config.max_run_seconds + stop_code: int | None = None + while process.poll() is None: + if overflow.is_set(): + stop_code = 126 + break + if time.monotonic() >= deadline: + stop_code = 124 + break + time.sleep(0.05) + if stop_code is not None: + _stop_process_tree(config, process) + + for thread in threads: + thread.join(SUPERVISOR_GRACE_SECONDS) + if any(thread.is_alive() for thread in threads): + _stop_process_tree(config, process) + raise HostJobError("entrypoint output streams did not close") + if copy_errors: + raise HostJobError("entrypoint output could not be bounded") + if stop_code is not None: + return stop_code + if overflow.is_set(): + return 126 + return int(process.returncode) + + +def _validate_gate13_evidence(payload: bytes) -> Mapping[str, Any]: + document = lifecycle.load_lifecycle_json(payload.decode("utf-8")) + return lifecycle.validate_lifecycle_document(document) + + +EVIDENCE_VALIDATOR: Callable[[bytes], Mapping[str, Any]] = _validate_gate13_evidence + + +def _validate_evidence(config: HostJobConfig) -> tuple[bytes, str]: + payload = _regular_bytes(config.evidence_path, MAX_EVIDENCE_BYTES) + try: + summary = EVIDENCE_VALIDATOR(payload) + except Exception as exc: + raise HostJobError("lifecycle evidence is invalid") from exc + if ( + summary.get("run_id") != config.lifecycle_run_id + or summary.get("platform") != config.platform + or summary.get("source_commit") != config.source_commit + ): + raise HostJobError("lifecycle evidence binding changed") + return payload, _digest_bytes(payload) + + +def execute( + config_path: Path, + *, + clock: Callable[[], float] = time.time, + entrypoint_runner: Callable[[HostJobConfig], int] = _run_entrypoint, +) -> Mapping[str, Any]: + config = load_config(config_path) + existing_terminal = _load_terminal(config) + if existing_terminal is not None: + return existing_terminal + if _load_status(config) is not None: + raise HostJobError("attempt was already started") + + _atomic_json( + config.status_path, + { + "schema_version": SCHEMA_VERSION, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": config.attempt_ordinal, + "state": "running", + "started_at_unix": int(clock()), + }, + exclusive=True, + ) + + exit_code = 125 + failure_code: str | None = "host_job_failed" + evidence_digest: str | None = None + result = "failed" + try: + exit_code = int(entrypoint_runner(config)) + if exit_code == 0: + _payload, evidence_digest = _validate_evidence(config) + result = "passed" + failure_code = None + elif exit_code == 124: + failure_code = "host_job_timed_out" + elif exit_code == 126: + failure_code = "host_job_output_exceeded" + else: + failure_code = "lifecycle_failed" + except Exception: + failure_code = "invalid_lifecycle_evidence" if exit_code == 0 else "host_job_failed" + result = "failed" + evidence_digest = None + + terminal = _terminal( + config, + result=result, + failure_code=failure_code, + evidence_digest=evidence_digest, + exit_code=exit_code, + finished_at_unix=int(clock()), + ) + _atomic_json(config.terminal_path, terminal, exclusive=True) + return terminal + + +def _native_snapshot(value: Mapping[str, Any]) -> Mapping[str, Any]: + raw = _exact_mapping(value, _NATIVE_FIELDS, "native snapshot") + if raw["native_state"] not in {"absent", "starting", "running", "inactive"}: + raise HostJobError("native state is invalid") + if type(raw["binding_ok"]) is not bool: + raise HostJobError("native binding is invalid") + if raw["native_state"] == "absent" and raw["binding_ok"]: + raise HostJobError("absent native job has a binding") + return raw + + +def observe_job(config: HostJobConfig, native: Mapping[str, Any]) -> dict[str, Any]: + snapshot = _native_snapshot(native) + terminal = _load_terminal(config) + status = _load_status(config) + if not snapshot["binding_ok"] and snapshot["native_state"] != "absent": + return {"job_state": "ambiguous", "attempt_ordinal": 1, "evidence_digest": None} + if terminal is not None: + return { + "job_state": "passed" if terminal["result"] == "passed" else "failed", + "attempt_ordinal": config.attempt_ordinal, + "evidence_digest": terminal["evidence_digest"], + } + if status is not None: + state = snapshot["native_state"] + return { + "job_state": "running" if state in {"starting", "running"} else "ambiguous", + "attempt_ordinal": config.attempt_ordinal, + "evidence_digest": None, + } + if snapshot["native_state"] == "absent": + return {"job_state": "absent", "attempt_ordinal": 0, "evidence_digest": None} + if snapshot["binding_ok"] and snapshot["native_state"] in {"starting", "running"}: + return {"job_state": "starting", "attempt_ordinal": 1, "evidence_digest": None} + return {"job_state": "ambiguous", "attempt_ordinal": 1, "evidence_digest": None} + + +def collect(config_path: Path) -> bytes: + config = load_config(config_path) + terminal = _load_terminal(config) + if terminal is None or terminal["result"] != "passed": + raise HostJobError("successful terminal record is absent") + payload, digest = _validate_evidence(config) + if digest != terminal["evidence_digest"]: + raise HostJobError("evidence digest changed") + return payload + + +def _default_runner( + argv: Sequence[str], + *, + timeout: int = 60, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(argv), + check=False, + capture_output=True, + text=True, + timeout=timeout, + shell=False, + ) + + +def _powershell_argv(script: str) -> list[str]: + import base64 + + encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii") + return [ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-EncodedCommand", + encoded, + ] + + +def _ps_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _windows_action_arguments(config: HostJobConfig) -> str: + return f'"{config.adapter_path}" execute --config ' f'"{config.config_path}"' + + +def _windows_register_script(config: HostJobConfig) -> str: + task_path = "\\" + return "\n".join( + [ + "$ErrorActionPreference = 'Stop'", + f"$taskPath = {_ps_quote(task_path)}", + f"$taskName = {_ps_quote(config.job_name)}", + "$identity = [Security.Principal.WindowsIdentity]::GetCurrent()", + "$operator = [Security.Principal.WindowsPrincipal]::new($identity)", + "if (-not $operator.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { throw 'privileged task registration required' }", + f"$targetUser = Get-LocalUser -Name {_ps_quote(config.host_user)} -ErrorAction Stop", + "$targetAccount = [Security.Principal.NTAccount]::new($env:COMPUTERNAME, [string]$targetUser.Name)", + "$targetSid = $targetAccount.Translate([Security.Principal.SecurityIdentifier]).Value", + "$existing = Get-ScheduledTask -TaskPath $taskPath -TaskName $taskName -ErrorAction SilentlyContinue", + "if ($null -ne $existing) { throw 'exact task already exists' }", + ( + "$action = New-ScheduledTaskAction " + f"-Execute {_ps_quote(os.fspath(config.python_executable))} " + f"-Argument {_ps_quote(_windows_action_arguments(config))}" + ), + ( + "$principal = New-ScheduledTaskPrincipal -UserId $targetAccount.Value " + "-LogonType Interactive -RunLevel Limited" + ), + ( + "$settings = New-ScheduledTaskSettingsSet -MultipleInstances IgnoreNew " + f"-ExecutionTimeLimit (New-TimeSpan -Seconds {config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS})" + ), + ( + "Register-ScheduledTask -TaskPath $taskPath -TaskName $taskName " + "-Action $action -Principal $principal -Settings $settings | Out-Null" + ), + "Start-ScheduledTask -TaskPath $taskPath -TaskName $taskName", + ] + ) + + +def _windows_snapshot_script(config: HostJobConfig) -> str: + task_path = "\\" + arguments = _windows_action_arguments(config) + return "\n".join( + [ + "$ErrorActionPreference = 'Stop'", + f"$taskPath = {_ps_quote(task_path)}", + f"$taskName = {_ps_quote(config.job_name)}", + "$task = Get-ScheduledTask -TaskPath $taskPath -TaskName $taskName -ErrorAction SilentlyContinue", + "if ($null -eq $task) {", + " [pscustomobject]@{ native_state = 'absent'; binding_ok = $false } | ConvertTo-Json -Compress", + " exit 0", + "}", + "$identity = [Security.Principal.WindowsIdentity]::GetCurrent()", + "$operator = [Security.Principal.WindowsPrincipal]::new($identity)", + f"$targetUser = Get-LocalUser -Name {_ps_quote(config.host_user)} -ErrorAction Stop", + "$targetAccount = [Security.Principal.NTAccount]::new($env:COMPUTERNAME, [string]$targetUser.Name)", + "$targetSid = $targetAccount.Translate([Security.Principal.SecurityIdentifier]).Value", + "$taskSid = ''", + "try {", + " $taskAccount = [Security.Principal.NTAccount]::new([string]$task.Principal.UserId)", + " $taskSid = $taskAccount.Translate([Security.Principal.SecurityIdentifier]).Value", + "} catch {", + " $taskSid = ''", + "}", + "$action = @($task.Actions)[0]", + f"$expectedLimit = [Xml.XmlConvert]::ToString([TimeSpan]::FromSeconds({config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS}))", + ( + "$binding = (@($task.Actions).Count -eq 1) -and " + f"($action.Execute -eq {_ps_quote(os.fspath(config.python_executable))}) -and " + f"($action.Arguments -eq {_ps_quote(arguments)}) -and " + "($operator.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) -and " + "($taskSid -eq $targetSid) -and " + "($task.Principal.LogonType -eq 'Interactive') -and " + "($task.Principal.RunLevel -eq 'Limited') -and " + "($task.Settings.MultipleInstances -eq 'IgnoreNew') -and " + "($task.Settings.ExecutionTimeLimit -eq $expectedLimit)" + ), + "$state = if ($task.State -eq 'Running') { 'running' } elseif ($task.State -eq 'Queued') { 'starting' } else { 'inactive' }", + "[pscustomobject]@{ native_state = $state; binding_ok = [bool]$binding } | ConvertTo-Json -Compress", + ] + ) + + +def _parse_json_stdout(result: subprocess.CompletedProcess[str]) -> Mapping[str, Any]: + if result.returncode != 0 or len(result.stdout.encode("utf-8")) > 32_768: + raise HostJobError("native supervisor inventory failed") + return _strict_json(result.stdout.encode("utf-8"), 32_768) + + +def _windows_snapshot(config: HostJobConfig, runner: Runner) -> Mapping[str, Any]: + result = runner(_powershell_argv(_windows_snapshot_script(config)), timeout=60) + return _native_snapshot(_parse_json_stdout(result)) + + +def _linux_service(config: HostJobConfig) -> str: + return config.job_name + ".service" + + +def _linux_start_argv(config: HostJobConfig) -> list[str]: + return [ + "sudo", + "-n", + "/usr/bin/systemd-run", + "--quiet", + "--collect", + "--service-type=exec", + "--unit", + config.job_name, + f"--property=User={config.host_user}", + f"--property=Group={config.host_user}", + f"--property=WorkingDirectory={config.working_directory}", + "--property=Restart=no", + "--property=KillMode=control-group", + "--property=UMask=0077", + "--property=NoNewPrivileges=no", + "--property=PrivateTmp=no", + "--property=TimeoutStartSec=120", + f"--property=RuntimeMaxSec={config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS}", + "--setenv=DISPLAY=:99", + f"--setenv=HOME={LINUX_HOME}", + f"--setenv=XDG_RUNTIME_DIR={LINUX_RUNTIME_DIR}", + "/usr/bin/dbus-run-session", + os.fspath(config.python_executable), + os.fspath(config.adapter_path), + "execute-linux-desktop-session", + "--config", + os.fspath(config.config_path), + ] + + +def _parse_systemd_seconds(value: str) -> float: + if value == "0": + return 0.0 + factors = { + "us": 0.000001, + "ms": 0.001, + "s": 1.0, + "min": 60.0, + "h": 3600.0, + "d": 86_400.0, + } + parts = re.findall(r"(\d+(?:\.\d+)?)(us|ms|s|min|h|d)", value) + compact = re.sub(r"\s+", "", value) + if not parts or "".join(number + unit for number, unit in parts) != compact: + raise HostJobError("native supervisor duration is invalid") + return sum(float(number) * factors[unit] for number, unit in parts) + + +def _systemd_exec_start_matches(config: HostJobConfig, value: str) -> bool: + if not value or any(character in value for character in ("\r", "\n", "\x00")): + return False + normalized = " ".join(value.split()) + matched = re.fullmatch( + ( + r"\{ path=(?P\S+) ; argv\[\]=(?P[^;]+) ; " + r"ignore_errors=(?Pyes|no) ; " + r"start_time=\[[^\]]+\] ; stop_time=\[[^\]]+\] ; " + r"pid=(?P\d+) ; code=(?P\(null\)|[a-z-]+) ; " + r"status=(?P[A-Za-z0-9()/+.-]+) \}" + ), + normalized, + ) + if matched is None: + return False + expected_argv = ( + f"/usr/bin/dbus-run-session {config.python_executable} {config.adapter_path} " + f"execute-linux-desktop-session --config {config.config_path}" + ) + return ( + matched["path"] == "/usr/bin/dbus-run-session" + and matched["argv"] == expected_argv + and matched["ignore"] == "no" + ) + + +def _systemd_environment_matches(value: str) -> bool: + assignments = re.findall(r'(?:^|\s)(?:"([^"\\]*(?:\\.[^"\\]*)*)"|(\S+))', value) + rendered = {quoted or plain for quoted, plain in assignments} + return rendered == { + "DISPLAY=:99", + f"HOME={LINUX_HOME}", + f"XDG_RUNTIME_DIR={LINUX_RUNTIME_DIR}", + } + + +def _linux_snapshot(config: HostJobConfig, runner: Runner) -> Mapping[str, Any]: + argv = [ + "sudo", + "-n", + "/usr/bin/systemctl", + "show", + _linux_service(config), + "--no-pager", + "--property=LoadState", + "--property=ActiveState", + "--property=SubState", + "--property=User", + "--property=Group", + "--property=ExecStart", + "--property=WorkingDirectory", + "--property=Restart", + "--property=KillMode", + "--property=UMask", + "--property=NoNewPrivileges", + "--property=PrivateTmp", + "--property=Environment", + "--property=TimeoutStartUSec", + "--property=RuntimeMaxUSec", + ] + result = runner(argv, timeout=60) + if result.returncode != 0: + raise HostJobError("native supervisor inventory failed") + if len(result.stdout.encode("utf-8")) > 32_768: + raise HostJobError("native supervisor inventory is too large") + fields: dict[str, str] = {} + for line in result.stdout.splitlines(): + key, separator, value = line.partition("=") + if not separator or key in fields: + raise HostJobError("native supervisor inventory is invalid") + fields[key] = value + expected_fields = { + "LoadState", + "ActiveState", + "SubState", + "User", + "Group", + "ExecStart", + "WorkingDirectory", + "Restart", + "KillMode", + "UMask", + "NoNewPrivileges", + "PrivateTmp", + "Environment", + "TimeoutStartUSec", + "RuntimeMaxUSec", + } + if fields.get("LoadState") == "not-found": + absent_field_sets = (expected_fields, expected_fields - {"ExecStart"}) + if set(fields) not in absent_field_sets: + raise HostJobError("native supervisor inventory is incomplete") + return {"native_state": "absent", "binding_ok": False} + if set(fields) != expected_fields: + raise HostJobError("native supervisor inventory is incomplete") + binding = ( + fields["LoadState"] == "loaded" + and fields["User"] == config.host_user + and fields["Group"] == config.host_user + and _systemd_exec_start_matches(config, fields["ExecStart"]) + and fields["WorkingDirectory"] == os.fspath(config.working_directory) + and fields["Restart"] == "no" + and fields["KillMode"] == "control-group" + and fields["UMask"] == "0077" + and fields["NoNewPrivileges"] == "no" + and fields["PrivateTmp"] == "no" + and _systemd_environment_matches(fields["Environment"]) + and _parse_systemd_seconds(fields["TimeoutStartUSec"]) == 120.0 + and _parse_systemd_seconds(fields["RuntimeMaxUSec"]) == config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS + ) + if fields["ActiveState"] in {"activating", "reloading"}: + native_state = "starting" + elif fields["ActiveState"] == "active": + native_state = "running" + else: + native_state = "inactive" + return {"native_state": native_state, "binding_ok": binding} + + +def native_snapshot(config: HostJobConfig, runner: Runner = _default_runner) -> Mapping[str, Any]: + return _windows_snapshot(config, runner) if config.platform == "windows" else _linux_snapshot(config, runner) + + +def start(config_path: Path, runner: Runner = _default_runner) -> Mapping[str, Any]: + config = load_config(config_path) + current = observe_job(config, native_snapshot(config, runner)) + if current["job_state"] != "absent" or current["attempt_ordinal"] != 0: + return current + + if config.platform == "windows": + result = runner(_powershell_argv(_windows_register_script(config)), timeout=60) + else: + result = runner(_linux_start_argv(config), timeout=60) + if result.returncode != 0: + raise HostJobError("native supervisor start failed") + observed = observe_job(config, native_snapshot(config, runner)) + if observed["job_state"] == "absent": + raise HostJobError("native supervisor start was not durable") + return observed + + +def cleanup(config_path: Path, runner: Runner = _default_runner) -> Mapping[str, Any]: + config = load_config(config_path) + snapshot = native_snapshot(config, runner) + if snapshot["native_state"] == "absent": + return snapshot + if not snapshot["binding_ok"]: + raise HostJobError("refusing to remove foreign exact-name job") + if config.platform == "windows": + task_path = "\\" + script = "\n".join( + [ + "$ErrorActionPreference = 'Stop'", + f"$taskPath = {_ps_quote(task_path)}", + f"$taskName = {_ps_quote(config.job_name)}", + "Stop-ScheduledTask -TaskPath $taskPath -TaskName $taskName -ErrorAction SilentlyContinue", + "Unregister-ScheduledTask -TaskPath $taskPath -TaskName $taskName -Confirm:$false", + ] + ) + result = runner(_powershell_argv(script), timeout=60) + else: + result = runner( + [ + "sudo", + "-n", + "/usr/bin/systemctl", + "stop", + _linux_service(config), + ], + timeout=60, + ) + if result.returncode != 0: + raise HostJobError("native supervisor cleanup failed") + final = native_snapshot(config, runner) + if final["native_state"] != "absent": + raise HostJobError("native supervisor cleanup is incomplete") + return final + + +def _render(value: Mapping[str, Any]) -> str: + return json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + + +def _execute_linux_desktop_session(config_path: Path) -> Mapping[str, Any]: + if not sys.platform.startswith("linux"): + raise HostJobError("Linux desktop session used on another platform") + expected = { + "DISPLAY": ":99", + "HOME": LINUX_HOME, + "XDG_RUNTIME_DIR": LINUX_RUNTIME_DIR, + } + if any(os.environ.get(key) != value for key, value in expected.items()): + raise HostJobError("Linux desktop environment is not bound") + if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): + raise HostJobError("Linux D-Bus session is absent") + try: + keyring = subprocess.run( + ["/usr/bin/gnome-keyring-daemon", "--unlock", "--components=secrets"], + input="\n", + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise HostJobError("Linux Secret Service could not start") from exc + if keyring.returncode != 0 or len(keyring.stdout.encode("utf-8")) > 32_768: + raise HostJobError("Linux Secret Service could not start") + for line in keyring.stdout.splitlines(): + name, separator, value = line.partition("=") + if not separator or name not in {"GNOME_KEYRING_CONTROL", "SSH_AUTH_SOCK"} or not value: + raise HostJobError("Linux Secret Service environment is invalid") + os.environ[name] = value + return execute(config_path) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument( + "action", + choices=("start", "status", "execute", "execute-linux-desktop-session", "collect", "cleanup"), + ) + parser.add_argument("--config", required=True) + try: + arguments = parser.parse_args(sys.argv[1:] if argv is None else argv) + config_path = Path(arguments.config) + if arguments.action == "start": + print(_render(start(config_path))) + elif arguments.action == "status": + config = load_config(config_path) + print(_render(observe_job(config, native_snapshot(config)))) + elif arguments.action == "execute": + terminal = execute(config_path) + print(_render(terminal)) + return 0 if terminal["result"] == "passed" else 2 + elif arguments.action == "execute-linux-desktop-session": + terminal = _execute_linux_desktop_session(config_path) + print(_render(terminal)) + return 0 if terminal["result"] == "passed" else 2 + elif arguments.action == "collect": + sys.stdout.buffer.write(collect(config_path)) + else: + print(_render(cleanup(config_path))) + return 0 + except (Exception, SystemExit): + print( + _render( + { + "failure_code": "host_job_rejected", + "result": "failed", + "schema_version": SCHEMA_VERSION, + } + ) + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate13_linux_client_startup.sh b/scripts/gate13_linux_client_startup.sh new file mode 100644 index 000000000..336c646a4 --- /dev/null +++ b/scripts/gate13_linux_client_startup.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +metadata_root=http://metadata.google.internal/computeMetadata/v1/instance/attributes +metadata() { + curl -fsS -H 'Metadata-Flavor: Google' "$metadata_root/$1" +} + +export DEBIAN_FRONTEND=noninteractive +bootstrap_status=/var/lib/gate13-bootstrap-status +printf '%s\n' starting >"$bootstrap_status" +trap 'rc=$?; if (( rc != 0 )); then printf "%s\\n" failed >"$bootstrap_status"; fi' EXIT + +# The image's HTTP security mirror is unreachable from this GCP network. +sed -i 's|http://security\.ubuntu\.com/ubuntu|https://security.ubuntu.com/ubuntu|g' \ + /etc/apt/sources.list.d/ubuntu.sources + +apt_deadline=$(( $(date +%s) + 300 )) +apt_options=( + -o APT::Update::Error-Mode=any + -o Acquire::ForceIPv4=true + -o Acquire::Retries=1 + -o Acquire::http::Timeout=20 + -o Acquire::https::Timeout=20 + -o DPkg::Lock::Timeout=30 +) + +apt_updated=false +for attempt in 1 2 3; do + remaining=$(( apt_deadline - $(date +%s) )) + if (( remaining <= 0 )); then + break + fi + attempt_timeout=$(( remaining / (4 - attempt) )) + if (( attempt_timeout > 90 )); then + attempt_timeout=90 + fi + echo "Gate 13 APT update attempt ${attempt}/3 (${attempt_timeout}s maximum)" + if timeout --signal=TERM --kill-after=10s "${attempt_timeout}s" \ + apt-get "${apt_options[@]}" update; then + apt_updated=true + break + fi + echo "Gate 13 APT update attempt ${attempt}/3 failed" >&2 + rm -rf -- /var/lib/apt/lists/partial + install -d -m 0755 /var/lib/apt/lists/partial +done +if [[ "$apt_updated" != true ]]; then + echo "Gate 13 APT update failed after three attempts" >&2 + exit 1 +fi + +remaining=$(( apt_deadline - $(date +%s) )) +if (( remaining <= 0 )); then + echo "Gate 13 APT install had no time remaining" >&2 + exit 1 +fi +echo "Gate 13 APT install (${remaining}s maximum)" +timeout --signal=TERM --kill-after=10s "${remaining}s" \ + apt-get "${apt_options[@]}" install -y \ + python3 xvfb xauth x11-utils xdotool imagemagick dbus-x11 \ + gnome-keyring libsecret-tools libsecret-1-0 libdbus-1-3 \ + libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0 \ + libxkbcommon0 libxkbcommon-x11-0 libegl1 libgl1 libpulse0 libfontconfig1 \ + unzip curl + +if ! id gate13 >/dev/null 2>&1; then + useradd --create-home --home-dir /home/gate13 --shell /bin/bash gate13 +fi + +run_root=/qualification +download_root=/var/tmp/gate13-download +install -d -m 0700 "$download_root" +install -d -o gate13 -g gate13 -m 0700 \ + "$run_root" "$run_root/package" "$run_root/install" "$run_root/runtime" + +package_url="$(metadata package-url)" +package_sha256="$(metadata package-sha256)" +package_bytes="$(metadata package-bytes)" +wrapper="$download_root/artifact.zip" +curl -fL --retry 4 --retry-delay 5 "$package_url" -o "$wrapper" +unzip -q "$wrapper" -d "$download_root/artifact" +archive="$download_root/artifact/communityai-desktop-linux.tar.gz" +test "$(stat -c %s "$archive")" = "$package_bytes" +test "$(sha256sum "$archive" | cut -d' ' -f1)" = "$package_sha256" +mv "$archive" "$run_root/package/communityai-desktop-linux.tar.gz" +tar -xzf "$run_root/package/communityai-desktop-linux.tar.gz" -C "$run_root/install" +rm -rf "$wrapper" "$download_root/artifact" +chown -R gate13:gate13 "$run_root" + +systemd-run --quiet --collect --service-type=exec \ + --unit=communityai-gate13-display \ + --property=User=gate13 \ + --property=Group=gate13 \ + --property=Restart=no \ + --property=KillMode=control-group \ + --property=UMask=0077 \ + --property=RuntimeMaxSec=21600 \ + /usr/bin/Xvfb :99 -screen 0 1280x900x24 -nolisten tcp +for _ in $(seq 1 30); do + if sudo -u gate13 env DISPLAY=:99 xdpyinfo >/dev/null 2>&1; then + printf '%s\n' ready >"$bootstrap_status" + touch /var/lib/gate13-bootstrap-ready + trap - EXIT + exit 0 + fi + sleep 1 +done +echo "Gate 13 X display did not become ready" >&2 +exit 1 diff --git a/scripts/gate13_linux_packaged_lifecycle.py b/scripts/gate13_linux_packaged_lifecycle.py index 1a6e7a18b..b43a1cd1e 100644 --- a/scripts/gate13_linux_packaged_lifecycle.py +++ b/scripts/gate13_linux_packaged_lifecycle.py @@ -1386,6 +1386,67 @@ def _validate_desktop_metrics( return package_version +def _audit_tar_payload(source, artifacts): + """Verify ordinary payloads and backward hardlinks before any extraction.""" + artifact_map = {item["path"]: item for item in artifacts} + members = {} + folded = set() + for member in source.getmembers(): + path = _safe_member_path(member.name, allow_root=True) + if path.casefold() in folded: + raise LifecycleRunError("install archive has duplicate members") + folded.add(path.casefold()) + if not (member.isdir() or member.isfile() or member.issym() or member.islnk()) or member.issparse(): + raise LifecycleRunError("install archive member type is unsafe") + if stat.S_IMODE(member.mode) & 0o7000 or (not member.isfile() and member.size != 0): + raise LifecycleRunError("install archive member mode or size is unsafe") + if member.islnk(): + target = _safe_member_path(member.linkname) + prior = members.get(target) + if member.linkname != target or prior is None or not prior.isfile(): + raise LifecycleRunError("install archive hardlink target is not a prior regular member") + members[path] = member + artifact_members = {path: member for path, member in members.items() if not member.isdir()} + if set(artifact_members) != set(artifact_map): + raise LifecycleRunError("install archive artifacts do not match provenance") + verified_regular = set() + for path, member in artifact_members.items(): + artifact = artifact_map[path] + if artifact["kind"] == "file": + effective_size = member.size + if member.islnk(): + prior = artifact_map.get(member.linkname, {}) + if member.linkname not in verified_regular or any( + artifact[key] != prior.get(key) for key in ("sha256", "size_bytes", "mode") + ): + raise LifecycleRunError("install archive hardlink target is not a verified identical file") + effective_size = prior["size_bytes"] + if ( + not (member.isfile() or member.islnk()) + or effective_size != artifact["size_bytes"] + or stat.S_IMODE(member.mode) != artifact["mode"] + ): + raise LifecycleRunError("install archive file identity is invalid") + stream = source.extractfile(member) + if stream is None: + raise LifecycleRunError("install archive file is unreadable") + digest = hashlib.sha256() + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != artifact["sha256"]: + raise LifecycleRunError("install archive file digest is invalid") + if member.isfile(): + verified_regular.add(path) + elif ( + not member.issym() + or _canonical_link_target(path, member.linkname) != artifact["link_target"] + or artifact["link_target"] not in artifact_map + or artifact_map[artifact["link_target"]]["kind"] != "file" + ): + raise LifecycleRunError("install archive symlink identity is invalid") + return members + + def _audit_package(root: Path, expected_digest: str, expected_bytes: int) -> PackageAudit: archive = root / ARCHIVE_NAME metadata_path = root / "release-metadata.json" @@ -1489,45 +1550,11 @@ def _audit_package(root: Path, expected_digest: str, expected_bytes: int) -> Pac if checksums_path.read_bytes() != expected_checksums: raise LifecycleRunError("package checksum inventory is invalid") - members: dict[str, tarfile.TarInfo] = {} try: with tarfile.open(archive, "r:gz") as source: - for member in source.getmembers(): - path = _safe_member_path(member.name, allow_root=True) - if path.casefold() in {candidate.casefold() for candidate in members}: - raise LifecycleRunError("install archive has duplicate members") - if not (member.isdir() or member.isfile() or member.issym()) or member.islnk() or member.issparse(): - raise LifecycleRunError("install archive member type is unsafe") - members[path] = member + members = _audit_tar_payload(source, artifacts) if len(members) != archive_record["entry_count"]: raise LifecycleRunError("install archive member count is invalid") - artifact_members = {path: member for path, member in members.items() if not member.isdir()} - if set(artifact_members) != set(artifact_map): - raise LifecycleRunError("install archive artifacts do not match provenance") - for path, artifact in artifact_map.items(): - member = artifact_members[path] - if artifact["kind"] == "file": - if ( - not member.isfile() - or member.size != artifact["size_bytes"] - or stat.S_IMODE(member.mode) != artifact["mode"] - ): - raise LifecycleRunError("install archive file identity is invalid") - stream = source.extractfile(member) - if stream is None: - raise LifecycleRunError("install archive file is unreadable") - digest_stream = hashlib.sha256() - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest_stream.update(chunk) - if digest_stream.hexdigest() != artifact["sha256"]: - raise LifecycleRunError("install archive file digest is invalid") - elif ( - not member.issym() - or _canonical_link_target(path, member.linkname) != artifact["link_target"] - or artifact["link_target"] not in artifact_map - or artifact_map[artifact["link_target"]]["kind"] != "file" - ): - raise LifecycleRunError("install archive symlink identity is invalid") except (OSError, tarfile.TarError) as exc: raise LifecycleRunError("install archive is unreadable") from exc @@ -1578,7 +1605,7 @@ def _extract_package(audit: PackageAudit, install_root: Path) -> Path: product_root = install_root / "CommunityAI" try: with tarfile.open(audit.archive, "r:gz") as source: - members = source.getmembers() + members = list(_audit_tar_payload(source, audit.artifacts).values()) for member in members: path = _safe_member_path(member.name, allow_root=True) target = install_root.joinpath(*PurePosixPath(path).parts) @@ -1598,6 +1625,15 @@ def _extract_package(audit: PackageAudit, install_root: Path) -> Path: destination.flush() os.fsync(destination.fileno()) os.chmod(target, stat.S_IMODE(member.mode)) + for member in members: + if member.islnk(): + path = _safe_member_path(member.name) + target = install_root.joinpath(*PurePosixPath(path).parts) + prior = install_root.joinpath(*PurePosixPath(member.linkname).parts) + if not stat.S_ISREG(prior.lstat().st_mode): + raise LifecycleRunError("install archive hardlink target is not a regular file") + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.link(prior, target, follow_symlinks=False) for member in members: if member.issym(): path = _safe_member_path(member.name) @@ -2712,27 +2748,37 @@ def run_from_config(path: str) -> Mapping[str, Any]: raise LifecycleRunError("lifecycle cleanup was not proved") +def _termination_requested(_signum: int, _frame: Any) -> None: + raise LifecycleRunError("lifecycle termination was requested") + + def main(argv: Sequence[str] | None = None) -> int: arguments = list(sys.argv[1:] if argv is None else argv) os.umask(0o077) + previous_sigterm = signal.signal(signal.SIGTERM, _termination_requested) + previous_sigint = signal.signal(signal.SIGINT, _termination_requested) try: - _disable_core_dumps() - if len(arguments) != 2 or arguments[0] != "--config": - raise LifecycleRunError("exactly one config path is required") - document = run_from_config(arguments[1]) - except BaseException: - print( - _canonical_json( - { - "failure_code": "linux_lifecycle_failed", - "result": "failed", - "schema_version": SCHEMA_VERSION, - } + try: + _disable_core_dumps() + if len(arguments) != 2 or arguments[0] != "--config": + raise LifecycleRunError("exactly one config path is required") + document = run_from_config(arguments[1]) + except BaseException: + print( + _canonical_json( + { + "failure_code": "linux_lifecycle_failed", + "result": "failed", + "schema_version": SCHEMA_VERSION, + } + ) ) - ) - return 2 - print(_canonical_json(document)) - return 0 + return 2 + print(_canonical_json(document)) + return 0 + finally: + signal.signal(signal.SIGTERM, previous_sigterm) + signal.signal(signal.SIGINT, previous_sigint) if __name__ == "__main__": diff --git a/scripts/gate13_packaged_lifecycle.py b/scripts/gate13_packaged_lifecycle.py index 2bf2c09b7..1e27d0716 100644 --- a/scripts/gate13_packaged_lifecycle.py +++ b/scripts/gate13_packaged_lifecycle.py @@ -18,6 +18,13 @@ SCHEMA_VERSION = 1 SCOPE = "gate13-packaged-lifecycle" +AUTOMATED_REPLAY_SCOPE = "gate13-automated-desktop-replay" +AUTOMATED_REPLAY_SCHEMA_VERSION = 2 +AUTOMATED_REPLAY_POLICY_PROFILE = "gate13-manual-cpu-v1" +AUTOMATED_REPLAY_SEQUENCE_PROFILES = { + "windows": "gate13-manual-windows-v1", + "linux": "gate13-manual-linux-v1", +} MAX_INPUT_BYTES = 1_048_576 MAX_COUNT = 1_000_000 MAX_BYTES = 1 << 50 @@ -74,6 +81,31 @@ _DIGEST_RE = re.compile(r"(?:sha256:)?[0-9a-f]{64}") _LABEL_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,63}") _DISPLAY_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._+()-]{0,127}") +_AUTOMATED_REPLAY_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "result", + "source_commit", + "package", + "model_id", + "manifest_digest", + "real_window_sessions", + "localhost_inference_count", + "policy_dialog_saved", + "start_clicked", + "pause_control_observed", + "restart_resume_observed", + "pause_clicked", + "sharing_intent_paused", + "policy_profile", + "sequence_profile", + "start_observation_seconds", + "session_duration_seconds", + "privacy_safe", + "qualification_temporaries_removed", +} class LifecycleEvidenceError(ValueError): @@ -655,7 +687,107 @@ def finalize(self) -> dict[str, Any]: } +def _validate_automated_replay(raw_document: Mapping[str, Any]) -> dict[str, Any]: + document = dict(_mapping(raw_document)) + _exact_fields(document, _AUTOMATED_REPLAY_FIELDS) + if ( + document["schema_version"] != AUTOMATED_REPLAY_SCHEMA_VERSION + or document["scope"] != AUTOMATED_REPLAY_SCOPE + or document["result"] != "passed" + or not isinstance(document["run_id"], str) + or _LABEL_RE.fullmatch(document["run_id"]) is None + or document["platform"] not in ("windows", "linux") + or not isinstance(document["source_commit"], str) + or _HEX40_RE.fullmatch(document["source_commit"]) is None + or document["model_id"] not in MODEL_PROFILES + or document["policy_profile"] != AUTOMATED_REPLAY_POLICY_PROFILE + or document["sequence_profile"] != AUTOMATED_REPLAY_SEQUENCE_PROFILES.get(document["platform"]) + ): + _fail() + profile = MODEL_PROFILES[document["model_id"]] + expected_manifest = "sha256:" + profile["manifest_digest"] + if document["manifest_digest"] != expected_manifest: + _fail() + package = _mapping(document["package"]) + _exact_fields(package, {"sha256", "bytes", "verified_before_run", "self_test_count"}) + if ( + not isinstance(package["sha256"], str) + or _DIGEST_RE.fullmatch(package["sha256"]) is None + or type(package["bytes"]) is not int + or not 1 <= package["bytes"] <= 8 * 1024**3 + or package["verified_before_run"] is not True + or package["self_test_count"] != 4 + ): + _fail() + expected_inferences = 1 if document["platform"] == "windows" else 2 + expected_resume = document["platform"] == "linux" + expected_start_observation = 25.0 if document["platform"] == "windows" else 20.0 + if ( + document["real_window_sessions"] != 2 + or document["localhost_inference_count"] != expected_inferences + or type(document["restart_resume_observed"]) is not bool + or document["restart_resume_observed"] is not expected_resume + or type(document["start_observation_seconds"]) not in (int, float) + or float(document["start_observation_seconds"]) != expected_start_observation + ): + _fail() + for field in ( + "policy_dialog_saved", + "start_clicked", + "pause_control_observed", + "pause_clicked", + "sharing_intent_paused", + "privacy_safe", + "qualification_temporaries_removed", + ): + if document[field] is not True: + _fail() + durations = _mapping(document["session_duration_seconds"]) + _exact_fields(durations, {"initial", "restart"}) + for value in durations.values(): + if type(value) not in (int, float) or not math.isfinite(float(value)) or not 0 <= float(value) <= 3_630: + _fail() + return { + "schema_version": AUTOMATED_REPLAY_SCHEMA_VERSION, + "scope": AUTOMATED_REPLAY_SCOPE, + "result": "passed", + "run_id": document["run_id"], + "platform": document["platform"], + "source_commit": document["source_commit"], + "package_sha256": package["sha256"].removeprefix("sha256:"), + "package_bytes": package["bytes"], + "model_id": document["model_id"], + "manifest_digest": expected_manifest.removeprefix("sha256:"), + "package": dict(package), + "model": { + "id": document["model_id"], + "manifest_digest": expected_manifest, + }, + "lifecycle": { + "real_window_sessions": 2, + "localhost_inference_count": expected_inferences, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": expected_resume, + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": AUTOMATED_REPLAY_POLICY_PROFILE, + "sequence_profile": AUTOMATED_REPLAY_SEQUENCE_PROFILES[document["platform"]], + "start_observation_seconds": expected_start_observation, + "response_content_retained": False, + "token_identifier_count": 0, + }, + "cleanup": { + "qualification_temporaries_removed": True, + "complete": True, + }, + } + + def validate_lifecycle_document(raw_document: Mapping[str, Any]) -> dict[str, Any]: + if isinstance(raw_document, dict) and raw_document.get("scope") == AUTOMATED_REPLAY_SCOPE: + return _validate_automated_replay(raw_document) document = dict(_mapping(raw_document)) _exact_fields(document, _DOCUMENT_FIELDS) phases = document["phases"] diff --git a/scripts/gate13_route_fence.py b/scripts/gate13_route_fence.py new file mode 100644 index 000000000..d9c7ec517 --- /dev/null +++ b/scripts/gate13_route_fence.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Fence the Gate 13 product route to one exact client model. + +Run this as root on the already-qualified route VM immediately before each client. +It stops the other product service, restarts the requested service so its DHT +advertisement is fresh, and verifies the exact local product view twice. Only +bounded route facts are emitted; credentials, endpoints, paths, and model outputs +never leave the process. +""" + +from __future__ import annotations + +import argparse +import json +import os +import stat +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +SCHEMA_VERSION = 1 +SCOPE = "gate13-route-client-fence" +MAX_RESPONSE_BYTES = 1_048_576 +MAX_SECRET_BYTES = 512 +SERVICE_ACTION_TIMEOUT_SECONDS = 300 + + +@dataclass(frozen=True) +class Profile: + target: str + service: str + other_service: str + origin: str + local_key: Path + control_key: Path + model_id: str + manifest_digest: str + total_blocks: int + + +PROFILES = { + "windows": Profile( + target="windows", + service="communityai-qwen.service", + other_service="communityai-gemma.service", + origin="http://127.0.0.1:8081", + local_key=Path("/srv/communityai/qwen/local-api.key"), + control_key=Path("/srv/communityai/qwen/control-api.key"), + model_id="Qwen3.5 2B", + manifest_digest="sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + total_blocks=24, + ), + "linux": Profile( + target="linux", + service="communityai-gemma.service", + other_service="communityai-qwen.service", + origin="http://127.0.0.1:8082", + local_key=Path("/srv/communityai/gemma/local-api.key"), + control_key=Path("/srv/communityai/gemma/control-api.key"), + model_id="Gemma 4 E2B IT", + manifest_digest="sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + total_blocks=35, + ), +} + + +class FenceError(RuntimeError): + """The exact route service could not be made stable for one client.""" + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ARG002 + return None + + +def _secret(path: Path) -> str: + try: + metadata = path.lstat() + except OSError as exc: + raise FenceError("route credential is unavailable") from exc + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= MAX_SECRET_BYTES: + raise FenceError("route credential is unsafe") + try: + value = path.read_text(encoding="ascii").strip() + except (OSError, UnicodeError) as exc: + raise FenceError("route credential is unreadable") from exc + if not value or any(character.isspace() for character in value): + raise FenceError("route credential is invalid") + return value + + +def _request_json(opener: Any, url: str, secret: str) -> Mapping[str, Any]: + request = Request(url, headers={"Authorization": f"Bearer {secret}", "Accept": "application/json"}) + try: + with opener.open(request, timeout=10) as response: + if response.status != 200 or response.headers.get_content_type() != "application/json": + raise FenceError("route API rejected the readiness probe") + payload = response.read(MAX_RESPONSE_BYTES + 1) + except (HTTPError, URLError, OSError, TimeoutError) as exc: + raise FenceError("route API is unavailable") from exc + if not 1 <= len(payload) <= MAX_RESPONSE_BYTES: + raise FenceError("route API response is invalid") + try: + document = json.loads(payload.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as exc: + raise FenceError("route API response is invalid") from exc + if not isinstance(document, dict): + raise FenceError("route API response is invalid") + return document + + +def _systemctl( + arguments: Sequence[str], + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, + *, + timeout_seconds: float = 60, +) -> None: + try: + result = runner( + ["/usr/bin/systemctl", *arguments], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=timeout_seconds, + close_fds=True, + ) + except subprocess.TimeoutExpired as exc: + raise FenceError(f"systemctl_{arguments[0]}_timeout") from exc + except (OSError, subprocess.SubprocessError) as exc: + raise FenceError(f"systemctl_{arguments[0]}_unavailable") from exc + if result.returncode != 0: + raise FenceError(f"systemctl_{arguments[0]}_failed") + + +def _snapshot(profile: Profile, opener: Any) -> bool: + local_secret = _secret(profile.local_key) + control_secret = _secret(profile.control_key) + models = _request_json(opener, f"{profile.origin}/v1/models", local_secret) + status = _request_json(opener, f"{profile.origin}/control/v1/status", control_secret) + local_secret = control_secret = "" + data = models.get("data") + if not isinstance(data, list): + return False + model = next((item for item in data if isinstance(item, dict) and item.get("id") == profile.model_id), None) + selection = status.get("auto_selection") + if not isinstance(model, dict) or not isinstance(selection, dict): + return False + return bool( + model.get("availability") == "complete" + and model.get("manifest_digest") == profile.manifest_digest + and selection.get("status") == "selected" + and selection.get("model") == profile.model_id + and selection.get("manifest_digest") == profile.manifest_digest + and selection.get("covered_blocks") == profile.total_blocks + and selection.get("total_blocks") == profile.total_blocks + and isinstance(selection.get("peer_count"), int) + and selection["peer_count"] > 0 + ) + + +def fence_route( + profile: Profile, + *, + timeout_seconds: float, + settle_seconds: float, + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, + opener: Any = None, + clock: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, +) -> Mapping[str, Any]: + """Run the fence and require systemd to report the standby as inactive.""" + + if opener is None: + opener = build_opener(ProxyHandler({}), _RejectRedirects()) + _systemctl( + ("stop", profile.other_service), + runner, + timeout_seconds=SERVICE_ACTION_TIMEOUT_SECONDS, + ) + _systemctl( + ("restart", profile.service), + runner, + timeout_seconds=SERVICE_ACTION_TIMEOUT_SECONDS, + ) + deadline = clock() + timeout_seconds + while clock() < deadline: + candidate_ready = False + try: + _systemctl(("is-active", "--quiet", profile.service), runner) + candidate_ready = _snapshot(profile, opener) + except FenceError: + pass + if candidate_ready: + sleeper(settle_seconds) + try: + _systemctl(("is-active", "--quiet", profile.service), runner) + result = runner( + ["/usr/bin/systemctl", "is-active", "--quiet", profile.other_service], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=60, + close_fds=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise FenceError("standby route state is unavailable") from exc + if result.returncode == 0: + raise FenceError("route fence did not remain stable") + try: + if _snapshot(profile, opener): + break + except FenceError: + pass + sleeper(5.0) + else: + raise FenceError("route did not become ready before the deadline") + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "passed", + "target": profile.target, + "model_id": profile.model_id, + "manifest_digest": profile.manifest_digest, + "covered_blocks": profile.total_blocks, + "total_blocks": profile.total_blocks, + "peer_count_minimum": 1, + "target_service_restarted": True, + "standby_service_stopped": True, + "stable_rechecks": 2, + "settle_seconds": settle_seconds, + "privacy_safe": True, + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Fence the Gate 13 route for one exact client") + parser.add_argument("--target", choices=tuple(PROFILES), required=True) + parser.add_argument("--timeout-seconds", type=float, default=1_200.0) + parser.add_argument("--settle-seconds", type=float, default=30.0) + args = parser.parse_args(argv) + try: + if hasattr(os, "geteuid") and os.geteuid() != 0: + raise FenceError("route fence requires root") + if not 30 <= args.timeout_seconds <= 1_800 or not 5 <= args.settle_seconds <= 120: + raise FenceError("route fence bounds are invalid") + result = fence_route( + PROFILES[args.target], + timeout_seconds=args.timeout_seconds, + settle_seconds=args.settle_seconds, + ) + except FenceError as exc: + result = { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "failed", + "failure_code": str(exc), + } + except BaseException: + result = { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "failed", + "failure_code": "route_fence_failed", + } + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 if result.get("result") == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gate13_route_setup.sh b/scripts/gate13_route_setup.sh new file mode 100644 index 000000000..a7b43ba0d --- /dev/null +++ b/scripts/gate13_route_setup.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root=/tmp/gate13-route +wheel="$root/drift-2.3.0.dev2-py3-none-any.whl" +test "$(stat -c %s "$wheel")" = "389449" +test "$(sha256sum "$wheel" | cut -d' ' -f1)" = "edfd4598c293719d4d7701c9613b64f47f9fd20c3a2dc2e4c0fcacacad3c493a" +test "$(sha256sum "$root/configure_product_route_node.py" | cut -d' ' -f1)" = "fc385f74e02ca955203b1fc5e8ae493c7f4ccd31bd7383c2ae0a1c461c91363e" +test "$(sha256sum "$root/gate11_product_node_acceptance.py" | cut -d' ' -f1)" = "bdcc9f499a7cd6b727c0e33a0c4c2b0e71e76e28f3f21cb99804a8f39edfa0d2" +apt-get update -qq +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-venv python3-pip curl +if ! id communityai >/dev/null 2>&1; then + useradd --system --create-home --home-dir /srv/communityai --shell /usr/sbin/nologin communityai +fi +install -d -m 0755 /opt/communityai +if [ ! -x /opt/communityai/venv/bin/drift ]; then + python3 -m venv /opt/communityai/venv + /opt/communityai/venv/bin/pip install --no-cache-dir "$wheel[api]" +fi +chmod -R a+rX /opt/communityai/venv +install -d -o communityai -g communityai -m 0700 /srv/communityai/qwen /srv/communityai/gemma /srv/communityai/cache +install -d -o root -g root -m 0755 /opt/communityai/bootstrap +cp -a "$root/catalog-v1/." /opt/communityai/bootstrap/ +public_ip="$(curl -fsS -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)" +test -n "$public_ip" +for role in qwen gemma; do + data="/srv/communityai/$role" + sudo -u communityai /opt/communityai/venv/bin/drift bootstrap /opt/communityai/bootstrap/catalog-bootstrap.json --data_dir "$data" --node_config "$data/node-config.json" >/dev/null +done +/opt/communityai/venv/bin/python "$root/configure_product_route_node.py" --config /srv/communityai/qwen/node-config.json --role primary --public-ip "$public_ip" --cache-root /srv/communityai/cache >/dev/null +/opt/communityai/venv/bin/python "$root/configure_product_route_node.py" --config /srv/communityai/gemma/node-config.json --role standby --public-ip "$public_ip" --cache-root /srv/communityai/cache >/dev/null +chown -R communityai:communityai /srv/communityai +cat >/etc/systemd/system/communityai-qwen.service <<'UNIT' +[Unit] +Description=CommunityAI Qwen public route +After=network-online.target +Wants=network-online.target +[Service] +Type=simple +User=communityai +Group=communityai +WorkingDirectory=/srv/communityai/qwen +ExecStart=/opt/communityai/venv/bin/drift node --config /srv/communityai/qwen/node-config.json --data_dir /srv/communityai/qwen --host 127.0.0.1 --port 8081 +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +LimitCORE=0 +[Install] +WantedBy=multi-user.target +UNIT +cat >/etc/systemd/system/communityai-gemma.service <<'UNIT' +[Unit] +Description=CommunityAI Gemma public route +After=network-online.target +Wants=network-online.target +[Service] +Type=simple +User=communityai +Group=communityai +WorkingDirectory=/srv/communityai/gemma +ExecStart=/opt/communityai/venv/bin/drift node --config /srv/communityai/gemma/node-config.json --data_dir /srv/communityai/gemma --host 127.0.0.1 --port 8082 +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +LimitCORE=0 +[Install] +WantedBy=multi-user.target +UNIT +systemctl daemon-reload +systemctl enable --now communityai-qwen.service communityai-gemma.service +rm -rf "$root" +printf 'route-setup=started\n' diff --git a/scripts/gate13_run_controller.py b/scripts/gate13_run_controller.py new file mode 100644 index 000000000..87573273c --- /dev/null +++ b/scripts/gate13_run_controller.py @@ -0,0 +1,807 @@ +"""Durable state contract for one bounded Gate 13 GCP lifecycle. + +This module is deliberately provider-command agnostic. The paid-run adapter supplies a +fresh, exact provider/host observation before every transition and executes only the +returned allowlisted action. Persisting the transition before returning makes a local +operator crash recoverable: the next invocation inventories first and either reattaches +to the same durable host job or proceeds to cleanup. + +A lifecycle is never resumed after a product attempt fails. Such a client is consumed +for acceptance even when its product-owned files were removed successfully. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import gate13_packaged_lifecycle as lifecycle +import qualification_cost_guard as cost_guard + +SCHEMA_VERSION = 1 +STATE_SCHEMA_VERSION = 1 +MAX_JSON_BYTES = 1_048_576 +MAX_STATE_BYTES = 262_144 +MIN_ROUTE_RUNWAY_SECONDS = 3_600 +ALLOWED_COMBINED_CLOUD_CEILINGS = frozenset({100.0, 500.0}) +PROTECTED_INSTANCE = "communityai-bootstrap-1" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_NAME_RE = re.compile(r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?") +_DIGEST_RE = re.compile(r"(?:sha256:)?[0-9a-f]{64}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") + +PHASES = { + "ABSENT", + "ROUTE_STARTING", + "ROUTE_ACCEPTING", + "ROUTE_ACCEPTED", + "WINDOWS_RUNNING", + "WINDOWS_COLLECTING", + "WINDOWS_COLLECTED", + "WINDOWS_DELETING", + "LINUX_RUNNING", + "LINUX_COLLECTING", + "LINUX_COLLECTED", + "LINUX_DELETING", + "ROUTE_DELETING", + "CLEANING_FAILED", + "CLEANED_PASS", + "CLEANED_FAILURE", +} +TERMINAL_PHASES = {"CLEANED_PASS", "CLEANED_FAILURE"} +JOB_STATES = {"absent", "starting", "running", "passed", "failed", "ambiguous"} +ACTION_STATES = { + "start_route", + "accept_route", + "start_windows", + "collect_windows", + "delete_windows", + "start_linux", + "collect_linux", + "delete_linux", + "delete_route", + "cleanup_failure", + "none", +} +CLEANUP_ACTIONS = frozenset({"delete_windows", "delete_linux", "delete_route", "cleanup_failure"}) + +_STATE_FIELDS = { + "schema_version", + "run_id", + "authorization_sha256", + "provider_plan_digest", + "revision", + "phase", + "failure_code", + "route_acceptance_digest", + "windows_evidence_digest", + "linux_evidence_digest", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + "next_action", +} +_OBSERVATION_FIELDS = { + "schema_version", + "run_id", + "observed_at_unix", + "instances", + "disks", + "firewalls", + "protected_bootstrap_running", + "route_acceptance", + "clients", +} +_INSTANCE_FIELDS = { + "present", + "run_id", + "source_commit", + "termination_unix", +} +_CLIENT_FIELDS = {"job_state", "attempt_ordinal", "evidence_digest"} +_ROUTE_ACCEPTANCE_FIELDS = {"job_state", "evidence_digest"} + + +class RunControllerError(ValueError): + """The run state, authorization, or provider observation failed closed.""" + + +@dataclass(frozen=True) +class RunPlan: + run_id: str + authorization_sha256: str + provider_plan_digest: str + ledger_state: str + project: str + zone: str + route_instance: str + route_disk: str + route_firewalls: tuple[str, str] + route_source_commit: str + windows_instance: str + windows_disk: str + windows_source_commit: str + linux_instance: str + linux_disk: str + linux_source_commit: str + windows_package_sha256: str + windows_package_bytes: int + linux_package_sha256: str + linux_package_bytes: int + qwen_manifest: str + gemma_manifest: str + clients_may_run_concurrently: bool + + @property + def instance_names(self) -> tuple[str, str, str]: + return (self.route_instance, self.windows_instance, self.linux_instance) + + @property + def disk_names(self) -> tuple[str, str, str]: + return (self.route_disk, self.windows_disk, self.linux_disk) + + +def _reject_constant(_value: str) -> None: + raise RunControllerError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise RunControllerError("duplicate JSON field") + value[key] = item + return value + + +def _strict_json_bytes(payload: bytes, maximum: int = MAX_JSON_BYTES) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= maximum: + raise RunControllerError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RunControllerError("invalid JSON") from exc + if not isinstance(value, dict): + raise RunControllerError("JSON root is invalid") + return value + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise RunControllerError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise RunControllerError("required file is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise RunControllerError("required file is unreadable") from exc + + +def _mapping(value: Any, fields: set[str], label: str) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise RunControllerError(f"{label} schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str], label: str) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise RunControllerError(f"{label} is invalid") + return value + + +def _boolean(value: Any, label: str) -> bool: + if type(value) is not bool: + raise RunControllerError(f"{label} is invalid") + return value + + +def _integer(value: Any, label: str, *, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise RunControllerError(f"{label} is invalid") + return value + + +def _provider_digest(provider_plan: Mapping[str, Any]) -> str: + return cost_guard._provider_plan_digest(provider_plan) + + +def load_plan(authorization_path: Path, ledger_path: Path) -> RunPlan: + authorization_payload = _regular_bytes(authorization_path, MAX_JSON_BYTES) + authorization = _strict_json_bytes(authorization_payload) + if authorization.get("schema_version") != 1 or authorization.get("gate") != 13: + raise RunControllerError("authorization scope is invalid") + if authorization.get("result") != "authorized": + raise RunControllerError("authorization is not active") + + run_id = _string(authorization.get("run_id"), _RUN_RE, "run id") + provider_plan = authorization.get("provider_plan") + if not isinstance(provider_plan, dict): + raise RunControllerError("provider plan is invalid") + provider_plan_digest = _provider_digest(provider_plan) + if authorization.get("provider_plan_digest") != provider_plan_digest: + raise RunControllerError("provider plan digest changed") + + authorization_section = authorization.get("authorization") + if not isinstance(authorization_section, dict): + raise RunControllerError("cost authorization is invalid") + try: + ceiling = float(authorization_section["combined_cloud_ceiling_usd"]) + before = float(authorization_section["ledger_committed_before_run_usd"]) + maximum = float(authorization_section["maximum_estimate_usd"]) + remaining = float(authorization_section["remaining_after_run_maximum_usd"]) + except (KeyError, TypeError, ValueError) as exc: + raise RunControllerError("cost authorization is invalid") from exc + if ( + not all(math.isfinite(value) for value in (ceiling, before, maximum, remaining)) + or ceiling not in ALLOWED_COMBINED_CLOUD_CEILINGS + or before < 0 + or maximum <= 0 + or before + maximum > ceiling + or abs((ceiling - before - maximum) - remaining) > 0.001 + or authorization_section.get("reservation_recorded") is not True + or authorization_section.get("provisioning_authorized_after_fail_closed_preflight") is not True + ): + raise RunControllerError("cost authorization is inconsistent") + + prohibited = authorization.get("prohibited") + if not isinstance(prohibited, dict) or any(value != 0 or type(value) is not int for value in prohibited.values()): + raise RunControllerError("prohibited work is present") + + source = authorization.get("source") + immutable = authorization.get("immutable_inputs") + route = provider_plan.get("route") + clients = provider_plan.get("clients") + sequencing = provider_plan.get("sequencing") + if not all(isinstance(value, dict) for value in (source, immutable, route, sequencing)): + raise RunControllerError("authorization bindings are invalid") + legacy_lifecycle = sequencing.get("all_16_phases_required_per_platform") is True + automated_replay = sequencing.get("automated_gate13_replay_required") is True + if ( + not isinstance(clients, list) + or len(clients) != 2 + or sequencing.get("route_live_for_both_lifecycles") is not True + or legacy_lifecycle == automated_replay + or sequencing.get("exact_cleanup_before_pass") is not True + ): + raise RunControllerError("execution sequencing is invalid") + + by_platform = { + client.get("platform"): client + for client in clients + if isinstance(client, dict) and isinstance(client.get("platform"), str) + } + if set(by_platform) != {"windows", "linux"}: + raise RunControllerError("client plan is invalid") + windows = by_platform["windows"] + linux = by_platform["linux"] + + project = _string(provider_plan.get("project"), _NAME_RE, "project") + route_instance = _string(route.get("instance"), _NAME_RE, "route instance") + zone = route.get("zone") + if not isinstance(zone, str) or not zone or route_instance == PROTECTED_INSTANCE: + raise RunControllerError("route target is invalid") + if windows.get("zone") != zone or linux.get("zone") != zone: + raise RunControllerError("client zones are inconsistent") + firewalls = route.get("firewalls") + if not isinstance(firewalls, list) or len(firewalls) != 2: + raise RunControllerError("firewall plan is invalid") + firewall_names = tuple(_string(value, _NAME_RE, "firewall") for value in firewalls) + instance_names = ( + route_instance, + _string(windows.get("instance"), _NAME_RE, "Windows instance"), + _string(linux.get("instance"), _NAME_RE, "Linux instance"), + ) + if len(set(instance_names)) != 3 or PROTECTED_INSTANCE in instance_names: + raise RunControllerError("instance targets are unsafe") + + ledger = _regular_bytes(ledger_path, MAX_JSON_BYTES * 4).decode("utf-8") + ledger_rows = [line for line in ledger.splitlines() if line.startswith(f"| {run_id} |")] + if len(ledger_rows) != 1 or provider_plan_digest not in ledger_rows[0]: + raise RunControllerError("ledger reservation is absent") + ledger_cells = [cell.strip() for cell in ledger_rows[0].strip().strip("|").split("|")] + if len(ledger_cells) != 7 or ledger_cells[0] != run_id: + raise RunControllerError("ledger reservation is invalid") + ledger_state = ledger_cells[-1] + if ledger_state not in {"RESERVED", "CLEANED-COMMITTED", "CLEANED-RELEASED"}: + raise RunControllerError("ledger state is invalid") + + windows_package = immutable.get("windows_package") + linux_package = immutable.get("linux_package") + if not isinstance(windows_package, dict) or not isinstance(linux_package, dict): + raise RunControllerError("package bindings are invalid") + + route_source = _string(source.get("route_runtime_commit"), _COMMIT_RE, "route source") + package_source = _string(source.get("package_commit"), _COMMIT_RE, "package source") + qwen_manifest = _string(immutable.get("qwen_manifest"), _DIGEST_RE, "Qwen manifest") + gemma_manifest = _string(immutable.get("gemma_manifest"), _DIGEST_RE, "Gemma manifest") + windows_sha = _string(windows_package.get("sha256"), _DIGEST_RE, "Windows package digest") + linux_sha = _string(linux_package.get("sha256"), _DIGEST_RE, "Linux package digest") + + authorization_sha256 = "sha256:" + hashlib.sha256(authorization_payload).hexdigest() + return RunPlan( + run_id=run_id, + authorization_sha256=authorization_sha256, + provider_plan_digest=provider_plan_digest, + ledger_state=ledger_state, + project=project, + zone=zone, + route_instance=route_instance, + route_disk=route_instance, + route_firewalls=(firewall_names[0], firewall_names[1]), + route_source_commit=route_source, + windows_instance=instance_names[1], + windows_disk=instance_names[1], + windows_source_commit=package_source, + linux_instance=instance_names[2], + linux_disk=instance_names[2], + linux_source_commit=package_source, + windows_package_sha256=windows_sha, + windows_package_bytes=_integer(windows_package.get("bytes"), "Windows package bytes", minimum=1), + linux_package_sha256=linux_sha, + linux_package_bytes=_integer(linux_package.get("bytes"), "Linux package bytes", minimum=1), + qwen_manifest=qwen_manifest, + gemma_manifest=gemma_manifest, + clients_may_run_concurrently=_boolean( + sequencing.get("clients_may_run_concurrently"), + "client concurrency policy", + ), + ) + + +def initial_state(plan: RunPlan) -> dict[str, Any]: + if plan.ledger_state != "RESERVED": + raise RunControllerError("authorization is not reserved for provisioning") + if plan.clients_may_run_concurrently: + raise RunControllerError("concurrent clients are forbidden") + return { + "schema_version": STATE_SCHEMA_VERSION, + "run_id": plan.run_id, + "authorization_sha256": plan.authorization_sha256, + "provider_plan_digest": plan.provider_plan_digest, + "revision": 0, + "phase": "ABSENT", + "failure_code": None, + "route_acceptance_digest": None, + "windows_evidence_digest": None, + "linux_evidence_digest": None, + "windows_consumed": False, + "linux_consumed": False, + "cleanup_verified": False, + "next_action": "start_route", + } + + +def validate_state(raw: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + state = dict(_mapping(raw, _STATE_FIELDS, "state")) + if ( + state["schema_version"] != STATE_SCHEMA_VERSION + or state["run_id"] != plan.run_id + or state["authorization_sha256"] != plan.authorization_sha256 + or state["provider_plan_digest"] != plan.provider_plan_digest + ): + raise RunControllerError("state authorization binding changed") + _integer(state["revision"], "state revision") + if state["phase"] not in PHASES or state["next_action"] not in ACTION_STATES: + raise RunControllerError("state transition is invalid") + for field in ("windows_consumed", "linux_consumed", "cleanup_verified"): + _boolean(state[field], field) + for field in ("route_acceptance_digest", "windows_evidence_digest", "linux_evidence_digest"): + if state[field] is not None: + _string(state[field], _DIGEST_RE, field) + failure = state["failure_code"] + if failure is not None and (not isinstance(failure, str) or not re.fullmatch(r"[a-z0-9_]{1,64}", failure)): + raise RunControllerError("failure code is invalid") + if state["phase"] in TERMINAL_PHASES and state["cleanup_verified"] is not True: + raise RunControllerError("terminal state lacks cleanup proof") + return state + + +def load_state(path: Path, plan: RunPlan) -> dict[str, Any]: + path = Path(path) + if not path.exists(): + return initial_state(plan) + return validate_state(_strict_json_bytes(_regular_bytes(path, MAX_STATE_BYTES), MAX_STATE_BYTES), plan) + + +def _atomic_write(path: Path, state: Mapping[str, Any]) -> None: + payload = (json.dumps(state, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8") + if len(payload) > MAX_STATE_BYTES: + raise RunControllerError("state exceeds its bound") + path = Path(os.path.abspath(os.fspath(path))) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and (path.is_symlink() or not path.is_file()): + raise RunControllerError("state path is unsafe") + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + try: + os.chmod(temporary, 0o600) + except OSError: + pass + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _present(mapping: Mapping[str, Any]) -> bool: + return any(value is True for value in mapping.values()) + + +def _instance_present(observation: Mapping[str, Any], name: str) -> bool: + return bool(observation["instances"][name]["present"]) + + +def validate_observation(raw: Mapping[str, Any], plan: RunPlan, now_unix: int) -> dict[str, Any]: + observation = dict(_mapping(raw, _OBSERVATION_FIELDS, "observation")) + if observation["schema_version"] != SCHEMA_VERSION or observation["run_id"] != plan.run_id: + raise RunControllerError("observation binding is invalid") + observed_at = _integer(observation["observed_at_unix"], "observation time", minimum=1) + if abs(observed_at - now_unix) > 300: + raise RunControllerError("observation is stale") + _boolean(observation["protected_bootstrap_running"], "protected bootstrap state") + if observation["protected_bootstrap_running"] is not True: + raise RunControllerError("protected bootstrap is unavailable") + + instances = observation["instances"] + disks = observation["disks"] + firewalls = observation["firewalls"] + if ( + not isinstance(instances, dict) + or set(instances) != set(plan.instance_names) + or not isinstance(disks, dict) + or set(disks) != set(plan.disk_names) + or not isinstance(firewalls, dict) + or set(firewalls) != set(plan.route_firewalls) + ): + raise RunControllerError("resource inventory is not exact") + expected_sources = { + plan.route_instance: plan.route_source_commit, + plan.windows_instance: plan.windows_source_commit, + plan.linux_instance: plan.linux_source_commit, + } + for name, value in instances.items(): + item = _mapping(value, _INSTANCE_FIELDS, "instance") + present = _boolean(item["present"], "instance presence") + if present: + if item["run_id"] != plan.run_id or item["source_commit"] != expected_sources[name]: + raise RunControllerError("foreign exact-name instance is present") + termination = _integer(item["termination_unix"], "termination deadline", minimum=1) + if termination <= observed_at: + raise RunControllerError("instance deadline expired") + elif any(item[field] is not None for field in ("run_id", "source_commit", "termination_unix")): + raise RunControllerError("absent instance carries identity") + for value in (*disks.values(), *firewalls.values()): + _boolean(value, "resource presence") + + route_acceptance = _mapping(observation["route_acceptance"], _ROUTE_ACCEPTANCE_FIELDS, "route acceptance") + if route_acceptance["job_state"] not in JOB_STATES: + raise RunControllerError("route acceptance state is invalid") + if route_acceptance["evidence_digest"] is not None: + _string(route_acceptance["evidence_digest"], _DIGEST_RE, "route acceptance digest") + clients = observation["clients"] + if not isinstance(clients, dict) or set(clients) != {"windows", "linux"}: + raise RunControllerError("client job inventory is invalid") + for value in clients.values(): + client = _mapping(value, _CLIENT_FIELDS, "client job") + if client["job_state"] not in JOB_STATES: + raise RunControllerError("client job state is invalid") + _integer(client["attempt_ordinal"], "attempt ordinal", maximum=1) + if client["evidence_digest"] is not None: + _string(client["evidence_digest"], _DIGEST_RE, "client evidence digest") + return observation + + +def _all_resources_absent(observation: Mapping[str, Any]) -> bool: + return ( + not any(value["present"] for value in observation["instances"].values()) + and not _present(observation["disks"]) + and not _present(observation["firewalls"]) + ) + + +def _fail(state: dict[str, Any], code: str, observation: Mapping[str, Any]) -> dict[str, Any]: + state["failure_code"] = code + state["phase"] = "CLEANING_FAILED" + state["next_action"] = "cleanup_failure" + for platform in ("windows", "linux"): + if observation["clients"][platform]["job_state"] != "absent": + state[f"{platform}_consumed"] = True + return state + + +def begin_action(state: Mapping[str, Any], plan: RunPlan, *, action: str) -> dict[str, Any]: + """Persist an action intent before its first provider or host mutation. + + A missing resource after one of these transitions is a consumed failed attempt, not + permission to recreate it. Repeating ``start`` must inventory and reconcile the + durable job instead of calling this function again. + """ + + current = validate_state(state, plan) + if plan.ledger_state != "RESERVED" and action not in CLEANUP_ACTIONS: + raise RunControllerError("authorization is not reserved for forward action") + if action != current["next_action"] or action not in ACTION_STATES - {"none"}: + raise RunControllerError("action intent is out of order") + phases = { + "start_route": "ROUTE_STARTING", + "accept_route": "ROUTE_ACCEPTING", + "start_windows": "WINDOWS_RUNNING", + "collect_windows": "WINDOWS_COLLECTING", + "delete_windows": "WINDOWS_DELETING", + "start_linux": "LINUX_RUNNING", + "collect_linux": "LINUX_COLLECTING", + "delete_linux": "LINUX_DELETING", + "delete_route": "ROUTE_DELETING", + "cleanup_failure": "CLEANING_FAILED", + } + result = dict(current) + result["phase"] = phases[action] + result["next_action"] = "none" + result["revision"] += 1 + return validate_state(result, plan) + + +def reconcile( + state: Mapping[str, Any], + observation: Mapping[str, Any], + plan: RunPlan, + *, + now_unix: int, +) -> dict[str, Any]: + current = validate_state(state, plan) + observed = validate_observation(observation, plan, now_unix) + result = dict(current) + + if current["phase"] in TERMINAL_PHASES: + if not _all_resources_absent(observed): + raise RunControllerError("resource reappeared after terminal cleanup") + return result + + if _all_resources_absent(observed): + if current["phase"] == "ABSENT": + result["next_action"] = "start_route" + elif current["phase"] in {"CLEANING_FAILED", "ROUTE_DELETING", "LINUX_COLLECTED", "LINUX_DELETING"}: + passed = ( + current["failure_code"] is None + and current["route_acceptance_digest"] is not None + and current["windows_evidence_digest"] is not None + and current["linux_evidence_digest"] is not None + ) + result["phase"] = "CLEANED_PASS" if passed else "CLEANED_FAILURE" + result["cleanup_verified"] = True + result["next_action"] = "none" + else: + result["phase"] = "CLEANED_FAILURE" + result["failure_code"] = "resources_disappeared_before_completion" + result["cleanup_verified"] = True + result["next_action"] = "none" + result["revision"] += 1 + return validate_state(result, plan) + + route_present = _instance_present(observed, plan.route_instance) + windows_present = _instance_present(observed, plan.windows_instance) + linux_present = _instance_present(observed, plan.linux_instance) + route_job = observed["route_acceptance"]["job_state"] + windows_job = observed["clients"]["windows"]["job_state"] + linux_job = observed["clients"]["linux"]["job_state"] + windows_attempt = observed["clients"]["windows"]["attempt_ordinal"] + linux_attempt = observed["clients"]["linux"]["attempt_ordinal"] + + if not route_present or route_job in {"failed", "ambiguous"}: + return _fail(result, "route_failed_or_ambiguous", observed) + route_deadline = observed["instances"][plan.route_instance]["termination_unix"] + if route_deadline - now_unix < MIN_ROUTE_RUNWAY_SECONDS: + return _fail(result, "route_runway_exhausted", observed) + + if route_job != "passed": + if windows_present or linux_present: + return _fail(result, "client_started_before_route_acceptance", observed) + if current["phase"] == "ROUTE_ACCEPTING" and current["next_action"] == "none": + if route_job == "absent": + return _fail(result, "route_acceptance_disappeared", observed) + result["phase"] = "ROUTE_ACCEPTING" + result["next_action"] = "none" + else: + result["phase"] = "ROUTE_ACCEPTING" + result["next_action"] = "accept_route" + else: + route_digest = observed["route_acceptance"]["evidence_digest"] + if route_digest is None: + return _fail(result, "route_acceptance_digest_absent", observed) + result["route_acceptance_digest"] = route_digest + if current["windows_evidence_digest"] is None and windows_attempt == 1 and windows_job == "absent": + return _fail(result, "windows_attempt_disappeared", observed) + if current["linux_evidence_digest"] is None and linux_attempt == 1 and linux_job == "absent": + return _fail(result, "linux_attempt_disappeared", observed) + if linux_present and current["windows_evidence_digest"] is None: + return _fail(result, "linux_started_before_windows_evidence", observed) + if current["phase"] == "WINDOWS_DELETING": + if windows_present or observed["disks"][plan.windows_disk]: + result["next_action"] = "delete_windows" + else: + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + elif current["phase"] == "LINUX_DELETING": + if linux_present or observed["disks"][plan.linux_disk]: + result["next_action"] = "delete_linux" + else: + result["phase"] = "LINUX_COLLECTED" + result["next_action"] = "delete_route" + elif windows_present: + result["windows_consumed"] = windows_job != "absent" + if windows_job in {"failed", "ambiguous"}: + return _fail(result, "windows_failed_or_ambiguous", observed) + if windows_job == "passed": + result["phase"] = "WINDOWS_COLLECTING" + result["next_action"] = "collect_windows" + elif windows_job == "absent": + if current["phase"] == "WINDOWS_RUNNING" and current["next_action"] == "none": + result["phase"] = "WINDOWS_RUNNING" + result["next_action"] = "none" + else: + result["phase"] = "ROUTE_ACCEPTED" + result["next_action"] = "start_windows" + else: + result["phase"] = "WINDOWS_RUNNING" + result["next_action"] = "none" + elif current["windows_evidence_digest"] is None: + if current["windows_consumed"]: + return _fail(result, "windows_consumed_without_evidence", observed) + if current["phase"] == "WINDOWS_RUNNING" and current["next_action"] == "none": + return _fail(result, "windows_disappeared_after_start_intent", observed) + result["phase"] = "ROUTE_ACCEPTED" + result["next_action"] = "start_windows" + elif linux_present: + result["linux_consumed"] = linux_job != "absent" + if linux_job in {"failed", "ambiguous"}: + return _fail(result, "linux_failed_or_ambiguous", observed) + if linux_job == "passed": + result["phase"] = "LINUX_COLLECTING" + result["next_action"] = "collect_linux" + elif linux_job == "absent": + if current["phase"] == "LINUX_RUNNING" and current["next_action"] == "none": + result["phase"] = "LINUX_RUNNING" + result["next_action"] = "none" + else: + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + else: + result["phase"] = "LINUX_RUNNING" + result["next_action"] = "none" + elif current["linux_evidence_digest"] is None: + if current["linux_consumed"]: + return _fail(result, "linux_consumed_without_evidence", observed) + if current["phase"] == "LINUX_RUNNING" and current["next_action"] == "none": + return _fail(result, "linux_disappeared_after_start_intent", observed) + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + else: + result["phase"] = "LINUX_COLLECTED" + result["next_action"] = "delete_route" + + result["revision"] += 1 + return validate_state(result, plan) + + +def collect_platform( + state: Mapping[str, Any], + plan: RunPlan, + *, + platform: str, + evidence_payload: bytes, + observed_digest: str, +) -> dict[str, Any]: + current = validate_state(state, plan) + if platform not in {"windows", "linux"}: + raise RunControllerError("platform is invalid") + expected_phase = "WINDOWS_COLLECTING" if platform == "windows" else "LINUX_COLLECTING" + if current["phase"] != expected_phase: + raise RunControllerError("evidence collection is out of order") + digest = "sha256:" + hashlib.sha256(evidence_payload).hexdigest() + if digest != observed_digest: + raise RunControllerError("host evidence digest changed") + try: + raw = lifecycle.load_lifecycle_json(evidence_payload.decode("utf-8")) + validated = lifecycle.validate_lifecycle_document(raw) + except Exception as exc: + raise RunControllerError("lifecycle evidence is invalid") from exc + expected = { + "windows": { + "source_commit": plan.windows_source_commit, + "package_sha256": plan.windows_package_sha256, + "package_bytes": plan.windows_package_bytes, + "model_id": "Qwen3.5 2B", + "manifest_digest": plan.qwen_manifest.removeprefix("sha256:"), + }, + "linux": { + "source_commit": plan.linux_source_commit, + "package_sha256": plan.linux_package_sha256, + "package_bytes": plan.linux_package_bytes, + "model_id": "Gemma 4 E2B IT", + "manifest_digest": plan.gemma_manifest.removeprefix("sha256:"), + }, + }[platform] + for field, value in expected.items(): + if validated.get(field) != value: + raise RunControllerError("lifecycle evidence binding changed") + + result = dict(current) + result[f"{platform}_evidence_digest"] = digest + result[f"{platform}_consumed"] = True + if platform == "windows": + result["phase"] = "WINDOWS_DELETING" + result["next_action"] = "delete_windows" + else: + result["phase"] = "LINUX_DELETING" + result["next_action"] = "delete_linux" + result["revision"] += 1 + return validate_state(result, plan) + + +def mark_client_absent( + state: Mapping[str, Any], + plan: RunPlan, + *, + platform: str, + observation: Mapping[str, Any], + now_unix: int, +) -> dict[str, Any]: + current = validate_state(state, plan) + expected = "WINDOWS_DELETING" if platform == "windows" else "LINUX_DELETING" + if current["phase"] != expected or current[f"{platform}_evidence_digest"] is None: + raise RunControllerError("client deletion is out of order") + observed = validate_observation(observation, plan, now_unix) + instance = plan.windows_instance if platform == "windows" else plan.linux_instance + disk = plan.windows_disk if platform == "windows" else plan.linux_disk + if _instance_present(observed, instance) or observed["disks"][disk]: + raise RunControllerError("client absence is not proved") + if not _instance_present(observed, plan.route_instance): + raise RunControllerError("route disappeared during client deletion") + result = dict(current) + if platform == "windows": + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + else: + result["phase"] = "LINUX_COLLECTED" + result["next_action"] = "delete_route" + result["revision"] += 1 + return validate_state(result, plan) + + +def persist(path: Path, state: Mapping[str, Any], plan: RunPlan) -> None: + _atomic_write(path, validate_state(state, plan)) + + +def public_status(state: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + current = validate_state(state, plan) + return { + "schema_version": 1, + "run_id": current["run_id"], + "phase": current["phase"], + "next_action": current["next_action"], + "failure_code": current["failure_code"], + "windows_consumed": current["windows_consumed"], + "linux_consumed": current["linux_consumed"], + "cleanup_verified": current["cleanup_verified"], + } diff --git a/scripts/gate13_windows_client_startup.ps1 b/scripts/gate13_windows_client_startup.ps1 new file mode 100644 index 000000000..a3be52f1f --- /dev/null +++ b/scripts/gate13_windows_client_startup.ps1 @@ -0,0 +1,142 @@ +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$metadataHeaders = @{ "Metadata-Flavor" = "Google" } +$metadataRoot = "http://metadata.google.internal/computeMetadata/v1/instance/attributes" +$bootstrapRoot = "C:\Gate13Bootstrap" +$runRoot = "C:\Gate13Run" +$downloadRoot = "C:\Gate13Download" +New-Item -ItemType Directory -Force -Path $bootstrapRoot, $runRoot, $downloadRoot | Out-Null +$readyMarker = Join-Path $bootstrapRoot "ready.txt" +if (Test-Path -LiteralPath $readyMarker -PathType Leaf) { + Set-Service -Name sshd -StartupType Automatic + if ((Get-Service -Name sshd).Status -ne "Running") { Start-Service -Name sshd } + $sshFirewall = Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue + if ($null -eq $sshFirewall) { + New-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -DisplayName "OpenSSH Server (sshd)" -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 | Out-Null + } else { + Set-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -Enabled True -Profile Any + } + return +} + +$capability = Get-WindowsCapability -Online -Name "OpenSSH.Server~~~~0.0.1.0" +if ($capability.State -ne "Installed") { + Add-WindowsCapability -Online -Name "OpenSSH.Server~~~~0.0.1.0" | Out-Null +} +Set-Service -Name sshd -StartupType Automatic +Start-Service -Name sshd +if (-not (Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue)) { + New-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -DisplayName "OpenSSH Server (sshd)" -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 | Out-Null +} +Set-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -Enabled True -Profile Any + +$randomBytes = New-Object byte[] 32 +$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create() +try { + $randomGenerator.GetBytes($randomBytes) +} finally { + $randomGenerator.Dispose() +} +$plainPassword = [Convert]::ToBase64String($randomBytes) + "aA1!" +$securePassword = ConvertTo-SecureString $plainPassword -AsPlainText -Force +if (-not (Get-LocalUser -Name "M" -ErrorAction SilentlyContinue)) { + New-LocalUser -Name "M" -Password $securePassword -PasswordNeverExpires -UserMayNotChangePassword | Out-Null +} else { + Set-LocalUser -Name "M" -Password $securePassword +} +if (-not (Get-LocalUser -Name "Gate13Admin" -ErrorAction SilentlyContinue)) { + New-LocalUser -Name "Gate13Admin" -NoPassword -AccountNeverExpires -UserMayNotChangePassword | Out-Null +} +$administratorMemberNames = @(Get-LocalGroupMember -Group "Administrators" | ForEach-Object Name) +if ($administratorMemberNames -notcontains "$env:COMPUTERNAME\Gate13Admin") { + Add-LocalGroupMember -Group "Administrators" -Member "Gate13Admin" +} +$openSshGroup = Get-LocalGroup -Name "OpenSSH Users" +$openSshMemberNames = @(Get-LocalGroupMember -Group $openSshGroup | ForEach-Object Name) +if ($openSshMemberNames -notcontains "$env:COMPUTERNAME\M") { + Add-LocalGroupMember -Group $openSshGroup -Member "M" +} +Remove-LocalGroupMember -Group "Administrators" -Member "M" -ErrorAction SilentlyContinue + +$publicKey = (Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/gate13-ssh-public-key").Trim() +$profileRoot = "C:\Users\M" +$sshRoot = Join-Path $profileRoot ".ssh" +New-Item -ItemType Directory -Force -Path $profileRoot, $sshRoot | Out-Null +$authorizedKeys = Join-Path $sshRoot "authorized_keys" +[IO.File]::WriteAllText($authorizedKeys, $publicKey + "`n", [Text.UTF8Encoding]::new($false)) +& icacls.exe $sshRoot /inheritance:r /grant:r "M:(OI)(CI)F" "SYSTEM:(OI)(CI)F" | Out-Null +& icacls.exe $authorizedKeys /inheritance:r /grant:r "M:F" "SYSTEM:F" | Out-Null + +$programDataSsh = Join-Path $env:ProgramData "ssh" +New-Item -ItemType Directory -Force -Path $programDataSsh | Out-Null +$administratorKeys = Join-Path $programDataSsh "administrators_authorized_keys" +$ordinaryKeys = Join-Path $programDataSsh "communityai_gate13_m_authorized_keys" +[IO.File]::WriteAllText($administratorKeys, $publicKey + "`n", [Text.UTF8Encoding]::new($false)) +[IO.File]::WriteAllText($ordinaryKeys, $publicKey + "`n", [Text.UTF8Encoding]::new($false)) +& icacls.exe $administratorKeys /inheritance:r /grant:r "Administrators:F" "SYSTEM:F" | Out-Null +& icacls.exe $ordinaryKeys /inheritance:r /grant:r "Administrators:F" "SYSTEM:F" | Out-Null +$sshdConfig = Join-Path $programDataSsh "sshd_config" +if (-not (Test-Path -LiteralPath $sshdConfig -PathType Leaf)) { + Copy-Item -LiteralPath "$env:WINDIR\System32\OpenSSH\sshd_config_default" -Destination $sshdConfig +} +$marker = "# CommunityAI Gate13 ordinary user" +if (-not (Select-String -LiteralPath $sshdConfig -SimpleMatch $marker -Quiet)) { + [IO.File]::AppendAllText( + $sshdConfig, + "`n$marker`nMatch User M`n AuthorizedKeysFile __PROGRAMDATA__/ssh/communityai_gate13_m_authorized_keys`n", + [Text.UTF8Encoding]::new($false) + ) +} +& "$env:WINDIR\System32\OpenSSH\sshd.exe" -t +if ($LASTEXITCODE -ne 0) { throw "OpenSSH configuration invalid" } +Restart-Service -Name sshd + +$pythonRoot = "C:\Gate13Python" +if (-not (Test-Path -LiteralPath "$pythonRoot\python.exe" -PathType Leaf)) { + $installer = Join-Path $downloadRoot "python-3.12.9-amd64.exe" + & curl.exe -fL --retry 4 --retry-delay 5 "https://www.python.org/ftp/python/3.12.9/python-3.12.9-amd64.exe" -o $installer + if ($LASTEXITCODE -ne 0) { throw "Python download failed" } + if ((Get-Item -LiteralPath $installer).Length -ne 26923696) { throw "Python installer size changed" } + if ((Get-FileHash -LiteralPath $installer -Algorithm SHA256).Hash.ToLowerInvariant() -cne "2a52993092a19cfdffe126e2eeac46a4265e25705614546604ad44988e040c0f") { throw "Python installer digest changed" } + $process = Start-Process -FilePath $installer -ArgumentList "/quiet InstallAllUsers=1 TargetDir=$pythonRoot Include_pip=0 Include_test=0 PrependPath=0" -Wait -PassThru + if ($process.ExitCode -ne 0) { throw "Python installation failed" } + Remove-Item -LiteralPath $installer -Force +} + +$packageUrl = (Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/package-url").Trim() +$packageSha256 = (Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/package-sha256").Trim().ToLowerInvariant() +$packageBytes = [int64](Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/package-bytes") +$wrapper = Join-Path $downloadRoot "artifact.zip" +& curl.exe -fL --retry 4 --retry-delay 5 $packageUrl -o $wrapper +if ($LASTEXITCODE -ne 0) { throw "Package wrapper download failed" } +$staging = Join-Path $downloadRoot "artifact" +New-Item -ItemType Directory -Force -Path $staging | Out-Null +& tar.exe -xf $wrapper -C $staging +if ($LASTEXITCODE -ne 0) { throw "Package wrapper extraction failed" } +$archive = Join-Path $staging "communityai-desktop-windows.zip" +if ((Get-Item -LiteralPath $archive).Length -ne $packageBytes) { throw "Package byte size changed" } +if ((Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $packageSha256) { throw "Package digest changed" } +$packageRoot = Join-Path $runRoot "package" +$installRoot = Join-Path $runRoot "install" +New-Item -ItemType Directory -Force -Path $packageRoot, $installRoot | Out-Null +Move-Item -LiteralPath $archive -Destination (Join-Path $packageRoot "communityai-desktop-windows.zip") +& tar.exe -xf (Join-Path $packageRoot "communityai-desktop-windows.zip") -C $installRoot +if ($LASTEXITCODE -ne 0) { throw "Product extraction failed" } +Remove-Item -LiteralPath $wrapper, $staging -Recurse -Force +& icacls.exe $runRoot /inheritance:r /grant:r "M:(OI)(CI)F" "Administrators:(OI)(CI)F" "SYSTEM:(OI)(CI)F" | Out-Null + +$winlogon = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" +Set-ItemProperty -Path $winlogon -Name AutoAdminLogon -Value "1" -Type String +Set-ItemProperty -Path $winlogon -Name DefaultUserName -Value "M" -Type String +Set-ItemProperty -Path $winlogon -Name DefaultDomainName -Value $env:COMPUTERNAME -Type String +Set-ItemProperty -Path $winlogon -Name DefaultPassword -Value $plainPassword -Type String +Set-ItemProperty -Path $winlogon -Name AutoLogonCount -Value 1 -Type DWord +$clearArgument = '-NoProfile -WindowStyle Hidden -Command "Start-Sleep -Seconds 60; $p = ''HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon''; Remove-ItemProperty -Path $p -Name DefaultPassword -ErrorAction SilentlyContinue; Set-ItemProperty -Path $p -Name AutoAdminLogon -Value ''0'' -Type String"' +$clearAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $clearArgument +$clearTrigger = New-ScheduledTaskTrigger -AtLogOn -User "M" +Register-ScheduledTask -TaskName "Gate13ClearAutoLogon" -Action $clearAction -Trigger $clearTrigger -User "SYSTEM" -RunLevel Highest -Force | Out-Null +$plainPassword = $null +$securePassword = $null +[IO.File]::WriteAllText($readyMarker, "ready`n", [Text.UTF8Encoding]::new($false)) +Restart-Computer -Force diff --git a/scripts/gate13_windows_packaged_lifecycle.ps1 b/scripts/gate13_windows_packaged_lifecycle.ps1 index 59646ff1b..a3865c000 100644 --- a/scripts/gate13_windows_packaged_lifecycle.ps1 +++ b/scripts/gate13_windows_packaged_lifecycle.ps1 @@ -36,6 +36,8 @@ $script:LifecycleProcess = $null $script:LifecycleAcquisitionInvoked = $false $script:LifecycleOwnWorkRoot = $false $script:LifecycleOwnPersistentRoot = $false +$script:LifecycleFailurePhase = "initialization" +$script:LifecycleFailureOperation = "initialization" function Initialize-Gate13NativeHost { if ($null -ne ("Gate13.NativeHost" -as [type])) { @@ -48,6 +50,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text; using System.Threading; using Microsoft.Win32.SafeHandles; @@ -101,6 +104,32 @@ namespace Gate13 } } + public bool ContainsProcessId(int candidateProcessId) + { + if (closed || job == IntPtr.Zero || candidateProcessId < 1) + { + return false; + } + return NativeHost.IsJobMember(job, checked((UInt32)candidateProcessId)); + } + + public void KillMemberProcess(int candidateProcessId, int timeoutMilliseconds) + { + if (closed || job == IntPtr.Zero || candidateProcessId < 1) + { + throw new InvalidOperationException("contained process identity unavailable"); + } + if (timeoutMilliseconds < 1 || timeoutMilliseconds > 300000) + { + throw new ArgumentOutOfRangeException("timeoutMilliseconds"); + } + NativeHost.TerminateJobMember( + job, + checked((UInt32)candidateProcessId), + timeoutMilliseconds + ); + } + private void ReleaseHandles() { if (process != IntPtr.Zero) @@ -169,35 +198,77 @@ namespace Gate13 throw new ArgumentOutOfRangeException("timeoutMilliseconds"); } - Exception failure = null; - try + Exception terminationFailure = null; + if (job != IntPtr.Zero && !NativeHost.TerminateJobObject(job, 209)) { - if (job != IntPtr.Zero && !NativeHost.TerminateJobObject(job, 209)) + int error = Marshal.GetLastWin32Error(); + if (error != NativeHost.ErrorAccessDenied) { - int error = Marshal.GetLastWin32Error(); - if (error != NativeHost.ErrorAccessDenied) - { - failure = new Win32Exception(error); - } + terminationFailure = new Win32Exception(error); } - if (!WaitForEmpty(timeoutMilliseconds)) + } + if (!WaitForEmpty(timeoutMilliseconds)) + { + if (terminationFailure != null) { - failure = new InvalidOperationException("contained process tree survived"); + throw terminationFailure; } + throw new InvalidOperationException("contained process tree survived"); } - finally + ReleaseHandles(); + } + + public void Dispose() + { + ForceAndVerify(30000); + } + } + + public sealed class LockedPath : IDisposable + { + private IntPtr handle; + private readonly bool directory; + + public bool IsDirectory { get { return directory; } } + public long Length { get; private set; } + public string FileIdentity { get; private set; } + + internal LockedPath( + IntPtr nativeHandle, + bool isDirectory, + long length, + string fileIdentity + ) + { + handle = nativeHandle; + directory = isDirectory; + Length = length; + FileIdentity = fileIdentity; + } + + public string Sha256() + { + if (handle == IntPtr.Zero || directory) { - ReleaseHandles(); + throw new InvalidOperationException("locked file is unavailable"); } - if (failure != null) + using (SafeFileHandle safe = new SafeFileHandle(handle, false)) + using (FileStream stream = new FileStream(safe, FileAccess.Read, 4096, false)) + using (SHA256 hasher = SHA256.Create()) { - throw failure; + stream.Position = 0; + byte[] digest = hasher.ComputeHash(stream); + return BitConverter.ToString(digest).Replace("-", "").ToLowerInvariant(); } } public void Dispose() { - ForceAndVerify(30000); + if (handle != IntPtr.Zero) + { + NativeHost.CloseHandle(handle); + handle = IntPtr.Zero; + } } } @@ -215,11 +286,19 @@ namespace Gate13 private const UInt32 JobObjectBasicAccountingInformation = 1; private const UInt32 JobObjectLimitKillOnJobClose = 0x00002000; private const UInt32 StdInputHandle = 0xfffffff6; + private const UInt32 GenericRead = 0x80000000; private const UInt32 GenericWrite = 0x40000000; private const UInt32 FileShareRead = 0x00000001; private const UInt32 FileShareWrite = 0x00000002; private const UInt32 OpenExisting = 3; + private const UInt32 FileAttributeDirectory = 0x00000010; + private const UInt32 FileAttributeReparsePoint = 0x00000400; private const UInt32 FileAttributeNormal = 0x00000080; + private const UInt32 FileFlagBackupSemantics = 0x02000000; + private const UInt32 FileFlagOpenReparsePoint = 0x00200000; + private const UInt32 ProcessTerminate = 0x00000001; + private const UInt32 ProcessQueryLimitedInformation = 0x00001000; + private const UInt32 Synchronize = 0x00100000; [StructLayout(LayoutKind.Sequential)] private struct SecurityAttributes @@ -230,6 +309,28 @@ namespace Gate13 public bool InheritHandle; } + [StructLayout(LayoutKind.Sequential)] + private struct FileTime + { + public UInt32 LowDateTime; + public UInt32 HighDateTime; + } + + [StructLayout(LayoutKind.Sequential)] + private struct ByHandleFileInformation + { + public UInt32 FileAttributes; + public FileTime CreationTime; + public FileTime LastAccessTime; + public FileTime LastWriteTime; + public UInt32 VolumeSerialNumber; + public UInt32 FileSizeHigh; + public UInt32 FileSizeLow; + public UInt32 NumberOfLinks; + public UInt32 FileIndexHigh; + public UInt32 FileIndexLow; + } + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] private struct StartupInfo { @@ -368,6 +469,25 @@ namespace Gate13 [DllImport("kernel32.dll", SetLastError = true)] private static extern UInt32 GetProcessId(IntPtr process); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr OpenProcess( + UInt32 desiredAccess, + [MarshalAs(UnmanagedType.Bool)] bool inheritHandle, + UInt32 processId + ); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool IsProcessInJob( + IntPtr processHandle, + IntPtr jobHandle, + [MarshalAs(UnmanagedType.Bool)] out bool result + ); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool TerminateProcess(IntPtr processHandle, UInt32 exitCode); + [DllImport("kernel32.dll", SetLastError = true)] private static extern UInt32 ResumeThread(IntPtr thread); @@ -397,6 +517,12 @@ namespace Gate13 IntPtr templateFile ); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetFileInformationByHandle( + IntPtr file, + out ByHandleFileInformation information + ); + [DllImport("kernel32.dll")] private static extern IntPtr GetStdHandle(UInt32 standardHandle); @@ -414,6 +540,75 @@ namespace Gate13 out ProcessInformation processInformation ); + public static LockedPath OpenReadOnlyNoFollow(string path, bool directory) + { + if (String.IsNullOrWhiteSpace(path) || !Path.IsPathRooted(path)) + { + throw new ArgumentException("locked path is invalid", "path"); + } + SecurityAttributes attributes = new SecurityAttributes(); + attributes.Length = checked((UInt32)Marshal.SizeOf(typeof(SecurityAttributes))); + UInt32 flags = FileFlagOpenReparsePoint; + if (directory) + { + flags |= FileFlagBackupSemantics; + } + IntPtr handle = CreateFile( + path, + GenericRead, + FileShareRead, + ref attributes, + OpenExisting, + flags, + IntPtr.Zero + ); + if (handle == new IntPtr(-1)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + try + { + ByHandleFileInformation information; + if (!GetFileInformationByHandle(handle, out information)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + if ((information.FileAttributes & FileAttributeReparsePoint) != 0) + { + throw new InvalidOperationException("locked path is a reparse point"); + } + bool actualDirectory = + (information.FileAttributes & FileAttributeDirectory) != 0; + if (actualDirectory != directory) + { + throw new InvalidOperationException("locked path type changed"); + } + long length = + ((long)information.FileSizeHigh << 32) | information.FileSizeLow; + string identity = String.Format( + "{0:x8}:{1:x8}{2:x8}", + information.VolumeSerialNumber, + information.FileIndexHigh, + information.FileIndexLow + ); + LockedPath result = new LockedPath( + handle, + actualDirectory, + length, + identity + ); + handle = IntPtr.Zero; + return result; + } + finally + { + if (handle != IntPtr.Zero) + { + CloseHandle(handle); + } + } + } + private static string Quote(string value) { if (value == null) @@ -564,6 +759,69 @@ namespace Gate13 return QueryActiveProcesses(job) == 0; } + internal static bool IsJobMember(IntPtr job, UInt32 processId) + { + IntPtr member = OpenProcess(ProcessQueryLimitedInformation, false, processId); + if (member == IntPtr.Zero) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + try + { + bool result; + if (!IsProcessInJob(member, job, out result)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + return result; + } + finally + { + CloseHandle(member); + } + } + + internal static void TerminateJobMember( + IntPtr job, + UInt32 processId, + int timeoutMilliseconds + ) + { + IntPtr member = OpenProcess( + ProcessTerminate | ProcessQueryLimitedInformation | Synchronize, + false, + processId + ); + if (member == IntPtr.Zero) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + try + { + bool inJob; + if (!IsProcessInJob(member, job, out inJob)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + if (!inJob) + { + throw new InvalidOperationException("process is outside the contained job"); + } + if (!TerminateProcess(member, 215)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + if (WaitForSingleObject(member, checked((UInt32)timeoutMilliseconds)) != WaitObject0) + { + throw new InvalidOperationException("contained member survived termination"); + } + } + finally + { + CloseHandle(member); + } + } + internal static int QueryActiveProcesses(IntPtr job) { IntPtr buffer = IntPtr.Zero; @@ -1099,11 +1357,11 @@ function Stop-Gate13Product { return } $owned = $script:LifecycleProcess - $script:LifecycleProcess = $null $graceful = $owned.StopGracefully(60000) if (-not $graceful -or $owned.ActiveProcessCount -ne 0) { throw "graceful product cleanup failed" } + $script:LifecycleProcess = $null } function Stop-Gate13ProductForFault { @@ -1111,11 +1369,11 @@ function Stop-Gate13ProductForFault { throw "fault target unavailable" } $owned = $script:LifecycleProcess - $script:LifecycleProcess = $null $owned.ForceAndVerify(30000) if ($owned.ActiveProcessCount -ne 0) { throw "fault cleanup failed" } + $script:LifecycleProcess = $null } function Force-Gate13ProductCleanup { @@ -1123,11 +1381,11 @@ function Force-Gate13ProductCleanup { return } $owned = $script:LifecycleProcess - $script:LifecycleProcess = $null $owned.ForceAndVerify(30000) if ($owned.ActiveProcessCount -ne 0) { throw "product cleanup failed" } + $script:LifecycleProcess = $null } function Get-Gate13CredentialCount { @@ -1147,6 +1405,8 @@ function Measure-Gate13Phase { [Parameter(Mandatory = $true)] [string] $Name, [Parameter(Mandatory = $true)] [scriptblock] $Action ) + $script:LifecycleFailurePhase = $Name + $script:LifecycleFailureOperation = $Name $timer = [System.Diagnostics.Stopwatch]::StartNew() $facts = & $Action $timer.Stop() @@ -1199,7 +1459,33 @@ function Get-Gate13Sha256 { if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "required file missing" } - return (Get-FileHash -LiteralPath $Path -Algorithm SHA256 -ErrorAction Stop).Hash.ToLowerInvariant() + $stream = $null + $hasher = $null + $digest = $null + try { + $stream = [System.IO.FileStream]::new( + $Path, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::Read, + [System.IO.FileShare]::Read, + 1048576, + [System.IO.FileOptions]::SequentialScan + ) + $hasher = [System.Security.Cryptography.SHA256]::Create() + $digest = $hasher.ComputeHash($stream) + return [System.BitConverter]::ToString($digest).Replace("-", "").ToLowerInvariant() + } + finally { + if ($null -ne $digest) { + [Array]::Clear($digest, 0, $digest.Length) + } + if ($null -ne $hasher) { + $hasher.Dispose() + } + if ($null -ne $stream) { + $stream.Dispose() + } + } } function Read-Gate13JsonFile { @@ -2814,7 +3100,9 @@ function Invoke-Gate13WindowsPackagedLifecycle { })) [void]$phases.Add((Measure-Gate13Phase -Name "signed_bootstrap" -Action { + $script:LifecycleFailureOperation = "bootstrap_command" $state.Bootstrap = Invoke-Gate13Bootstrap + $script:LifecycleFailureOperation = "bootstrap_binding" if ( $state.Bootstrap.CatalogId -cne $state.Audit.PublicationCatalogId -or [int64]$state.Bootstrap.CatalogSequence -ne @@ -2826,8 +3114,11 @@ function Invoke-Gate13WindowsPackagedLifecycle { ) { throw "installed bootstrap did not match release provenance" } + $script:LifecycleFailureOperation = "product_start" Start-Gate13Product + $script:LifecycleFailureOperation = "product_readiness" $state.ProductStatus = Wait-Gate13ProductStatus -TimeoutSeconds 300 + $script:LifecycleFailureOperation = "profile_binding" $state.Profile = $state.ProductStatus.Profile if ( $state.Profile.ModelId -cne $state.Audit.ExpectedModelId -or @@ -2835,6 +3126,7 @@ function Invoke-Gate13WindowsPackagedLifecycle { ) { throw "operator-bound selected model identity rejected" } + $script:LifecycleFailureOperation = "selected_manifest_context" $state.Context = Get-Gate13SelectedManifestContext -Profile $state.Profile return [ordered]@{ catalog_id = $state.Bootstrap.CatalogId @@ -3186,6 +3478,8 @@ function Invoke-Gate13WindowsPackagedLifecycle { } })) + $script:LifecycleFailurePhase = "evidence_validation" + $script:LifecycleFailureOperation = "evidence_validation" $document = [ordered]@{ schema_version = 1 run_id = $state.Audit.RunId @@ -3252,10 +3546,24 @@ function Start-Gate13WindowsPackagedLifecycle { return 0 } catch { + $failurePhase = [string]$script:LifecycleFailurePhase + if ($failurePhase -notmatch '^[a-z_]{1,64}$') { + $failurePhase = "initialization" + } + $failureOperation = [string]$script:LifecycleFailureOperation + if ($failureOperation -notmatch '^[a-z_]{1,64}$') { + $failureOperation = $failurePhase + } Invoke-Gate13FailureCleanup - [Console]::Out.WriteLine( - '{"failure_code":"windows_packaged_lifecycle_failed","result":"failed","schema_version":1}' - ) + [Console]::Out.WriteLine(( + [ordered]@{ + failure_code = "windows_packaged_lifecycle_failed" + failure_operation = $failureOperation + failure_phase = $failurePhase + result = "failed" + schema_version = 1 + } | ConvertTo-Json -Compress + )) return 2 } finally { diff --git a/scripts/gate14_cache_materializer.py b/scripts/gate14_cache_materializer.py new file mode 100644 index 000000000..c704f2b93 --- /dev/null +++ b/scripts/gate14_cache_materializer.py @@ -0,0 +1,1627 @@ +"""Materialize, verify, and promote one exact Gate 14 warm cache.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, BinaryIO, Callable, Mapping, Sequence +from urllib.request import getproxies + +import gate14_packaged_lifecycle as lifecycle + +SCHEMA_VERSION = 1 +PLAN_SCOPE = "gate14-cache-materialization-plan" +HANDOFF_SCOPE = "gate14-cache-materialization-handoff" +RESULT_SCOPE = "gate14-cache-materialization" +PLAN_NAME = "gate14-cache-plan.json" +TEMPLATE_NAME = "gate14-lifecycle-template.json" +CONFIG_NAME = "gate14-lifecycle.json" +CACHE_NAME = "gate14-warm-cache" +RECORD_NAME = lifecycle._MATERIALIZATION_RECORD_NAME +BINDING_NAME = "gate14-warm-cache-binding.json" +HANDOFF_NAME = "gate14-cache-handoff.json" +MAX_JSON_BYTES = lifecycle.MAX_MATERIALIZATION_RECORD_BYTES +MAX_SOURCE_BYTES = 8 * 1024 * 1024 +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_MANIFEST_NAMES = { + "windows": "qwen3.5-2b-bfloat16-eager.json", + "linux": "gemma-4-e2b-it-bfloat16-eager.json", +} +_ACQUIRER_NAMES = { + "windows": "CommunityAI-Node.exe", + "linux": "CommunityAI-Node", +} +_ACQUIRER_RUNTIME_PARTS = ("acquirer-runtime", "CommunityAI", "node") +MAX_ACQUIRER_BYTES = 2 * 1024**3 +MAX_ACQUIRER_OUTPUT_BYTES = MAX_JSON_BYTES +ACQUIRER_TIMEOUT_SECONDS = 14_400 +ACQUIRER_RESULT_NAME = ".gate14-acquirer-result.json" +_OVERRIDE_NAMES = ( + "HF_ENDPOINT", + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + "SSL_CERT_FILE", + "SSL_CERT_DIR", +) +_SOURCE_NAMES = ( + "gate14_cache_materializer.py", + "gate14_packaged_lifecycle.py", +) +_PLAN_FIELDS = { + "schema_version", + "scope", + "platform", + "source_commit", + "manifest_path", + "manifest_sha256", + "acquirer_path", + "acquirer_sha256", + "acquirer_bytes", + "work_root", + "staging_root", + "lifecycle_template_sha256", + "sources", +} +_HANDOFF_FIELDS = { + "schema_version", + "scope", + "plan_sha256", + "platform", + "source_commit", + "model_id", + "manifest_digest", + "materializer_sources_sha256", + "materialization_record_sha256", + "materialization_record_bytes", + "warm_cache_binding_sha256", + "warm_cache_binding_file_sha256", + "warm_cache_binding_bytes", + "artifact_count", + "artifact_bytes", +} + + +class Gate14CacheMaterializationError(RuntimeError): + """Fresh-cache materialization, verification, or promotion failed closed.""" + + +Acquirer = Callable[..., Mapping[str, Any]] +OwnershipVerifier = Callable[..., None] + + +@dataclass(frozen=True) +class ManifestProfile: + name: str + + +@dataclass(frozen=True) +class CachePlan: + platform: str + source_commit: str + manifest_path: Path + manifest_sha256: str + acquirer_path: Path + acquirer_sha256: str + acquirer_bytes: int + work_root: Path + staging_root: Path + template_path: Path + template_sha256: str + sources: Mapping[str, str] + sources_sha256: str + plan_sha256: str + + +@dataclass +class CacheLease: + handles: list[Any] + identities: list[tuple[Path, os.stat_result, bool]] + directory_entries: list[tuple[Path, frozenset[str]]] + + def assert_stable(self) -> None: + for path, expected, directory in self.identities: + try: + observed = path.lstat() + except OSError as exc: + raise Gate14CacheMaterializationError("warm cache changed after verification") from exc + if ( + _reparse(observed) + or path.is_symlink() + or bool(stat.S_ISDIR(observed.st_mode)) is not directory + or _file_identity(observed) != _file_identity(expected) + ): + raise Gate14CacheMaterializationError("warm cache changed after verification") + for path, expected_names in self.directory_entries: + try: + observed_names = frozenset(entry.name for entry in os.scandir(path)) + except OSError as exc: + raise Gate14CacheMaterializationError("warm cache changed after verification") from exc + if observed_names != expected_names: + raise Gate14CacheMaterializationError("warm cache changed after verification") + + def close(self) -> None: + while self.handles: + handle = self.handles.pop() + try: + handle.close() + except OSError: + pass + + +class _RawHandle: + def __init__(self, value: int, closer: Callable[[int], Any]): + self.value = value + self._closer = closer + + def close(self) -> None: + if self.value: + value, self.value = self.value, 0 + self._closer(value) + + +@dataclass +class _LockedRegular: + path: Path + handle: BinaryIO + opened: os.stat_result + + def assert_stable(self, label: str) -> None: + try: + after_handle = os.fstat(self.handle.fileno()) + after_path = self.path.lstat() + except (OSError, ValueError) as exc: + raise Gate14CacheMaterializationError(f"{label} identity changed") from exc + if ( + _file_identity(self.opened) != _file_identity(after_handle) + or _file_identity(self.opened) != _file_identity(after_path) + or _reparse(after_path) + or self.path.is_symlink() + ): + raise Gate14CacheMaterializationError(f"{label} identity changed") + + def close(self) -> None: + self.handle.close() + + +def _native_platform() -> str: + if sys.platform == "win32": + return "windows" + if sys.platform.startswith("linux"): + return "linux" + raise Gate14CacheMaterializationError("unsupported materialization platform") + + +def _reparse(metadata: os.stat_result) -> bool: + return bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + + +def _file_identity(metadata: os.stat_result) -> tuple[int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + ) + + +def _canonical(value: Mapping[str, Any]) -> bytes: + try: + payload = ( + json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Gate14CacheMaterializationError("JSON value is invalid") from exc + if not 1 <= len(payload) <= MAX_JSON_BYTES: + raise Gate14CacheMaterializationError("JSON value exceeded its bound") + return payload + + +def _strict_canonical(payload: bytes, label: str) -> Mapping[str, Any]: + try: + value = lifecycle._strict_json(payload, MAX_JSON_BYTES) + except lifecycle.Gate14LifecycleError as exc: + raise Gate14CacheMaterializationError(f"{label} is invalid") from exc + if _canonical(value) != payload: + raise Gate14CacheMaterializationError(f"{label} is not canonical") + return value + + +def _safe_directory(path: Path, label: str) -> Path: + candidate = Path(path) + if not candidate.is_absolute(): + raise Gate14CacheMaterializationError(f"{label} must be absolute") + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14CacheMaterializationError(f"{label} is unavailable") from exc + if _reparse(metadata) or candidate.is_symlink() or not stat.S_ISDIR(metadata.st_mode): + raise Gate14CacheMaterializationError(f"{label} is unsafe") + return candidate.resolve(strict=True) + + +def _windows_open(path: Path, *, directory: bool) -> tuple[int, Callable[[int], Any]]: + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + create_file = kernel32.CreateFileW + create_file.argtypes = ( + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ) + create_file.restype = wintypes.HANDLE + close_handle = kernel32.CloseHandle + close_handle.argtypes = (wintypes.HANDLE,) + close_handle.restype = wintypes.BOOL + + generic_read = 0x80000000 + share_read_only = 0x00000001 + open_existing = 3 + open_reparse_point = 0x00200000 + backup_semantics = 0x02000000 + sequential_scan = 0x08000000 + flags = open_reparse_point | (backup_semantics if directory else sequential_scan) + raw_handle = create_file( + str(path), + generic_read, + share_read_only, + None, + open_existing, + flags, + None, + ) + invalid_handle = ctypes.c_void_p(-1).value + if raw_handle == invalid_handle: + error = ctypes.get_last_error() + raise OSError(error, "could not open locked cache entry", str(path)) + return int(raw_handle), close_handle + + +def _open_locked_regular( + path: Path, + maximum: int, + *, + minimum: int = 1, +) -> tuple[BinaryIO, os.stat_result]: + candidate = Path(path) + try: + before = candidate.lstat() + except OSError as exc: + raise Gate14CacheMaterializationError("required file is unavailable") from exc + if ( + _reparse(before) + or candidate.is_symlink() + or not stat.S_ISREG(before.st_mode) + or not minimum <= before.st_size <= maximum + ): + raise Gate14CacheMaterializationError("required file is unsafe") + + handle: BinaryIO | None = None + try: + if os.name == "nt": + import msvcrt + + raw_handle, close_handle = _windows_open(candidate, directory=False) + try: + descriptor = msvcrt.open_osfhandle(raw_handle, os.O_RDONLY) + except BaseException: + close_handle(raw_handle) + raise + else: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(candidate, flags) + handle = os.fdopen(descriptor, "rb", closefd=True) + opened = os.fstat(handle.fileno()) + if _reparse(opened) or not stat.S_ISREG(opened.st_mode) or _file_identity(before) != _file_identity(opened): + raise Gate14CacheMaterializationError("required file changed while opening") + return handle, opened + except Gate14CacheMaterializationError: + if handle is not None: + handle.close() + raise + except OSError as exc: + if handle is not None: + handle.close() + raise Gate14CacheMaterializationError("required file is unreadable") from exc + + +def _read_locked_regular( + path: Path, + maximum: int, + *, + minimum: int = 1, +) -> bytes: + handle, opened = _open_locked_regular(path, maximum, minimum=minimum) + try: + payload = handle.read(maximum + 1) + after_handle = os.fstat(handle.fileno()) + after_path = Path(path).lstat() + except OSError as exc: + raise Gate14CacheMaterializationError("required file is unreadable") from exc + finally: + handle.close() + if ( + len(payload) != opened.st_size + or _file_identity(opened) != _file_identity(after_handle) + or _file_identity(opened) != _file_identity(after_path) + or _reparse(after_path) + or Path(path).is_symlink() + ): + raise Gate14CacheMaterializationError("required file changed while reading") + return payload + + +def _hash_locked_regular(path: Path, maximum: int) -> tuple[int, str]: + handle, opened = _open_locked_regular(path, maximum) + digest = hashlib.sha256() + total = 0 + try: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + total += len(chunk) + digest.update(chunk) + after_handle = os.fstat(handle.fileno()) + after_path = Path(path).lstat() + except OSError as exc: + raise Gate14CacheMaterializationError("required file is unreadable") from exc + finally: + handle.close() + if ( + total != opened.st_size + or _file_identity(opened) != _file_identity(after_handle) + or _file_identity(opened) != _file_identity(after_path) + or _reparse(after_path) + or Path(path).is_symlink() + ): + raise Gate14CacheMaterializationError("required file changed while hashing") + return total, "sha256:" + digest.hexdigest() + + +def _lock_verified_payload( + path: Path, + maximum: int, + expected_sha256: str, + label: str, +) -> tuple[_LockedRegular, bytes]: + try: + handle, opened = _open_locked_regular(path, maximum) + except Gate14CacheMaterializationError as exc: + raise Gate14CacheMaterializationError(f"{label} identity changed") from exc + lease = _LockedRegular(Path(path), handle, opened) + try: + payload = handle.read(maximum + 1) + if len(payload) != opened.st_size or lifecycle._digest(payload) != expected_sha256: + raise Gate14CacheMaterializationError(f"{label} identity changed") + lease.assert_stable(label) + return lease, payload + except BaseException: + lease.close() + raise + + +def _lock_verified_acquirer( + path: Path, + maximum: int, + expected_bytes: int, + expected_sha256: str, +) -> _LockedRegular: + try: + handle, opened = _open_locked_regular(path, maximum) + except Gate14CacheMaterializationError as exc: + raise Gate14CacheMaterializationError("packaged acquirer identity changed") from exc + lease = _LockedRegular(Path(path), handle, opened) + digest = hashlib.sha256() + total = 0 + try: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + total += len(chunk) + digest.update(chunk) + if total != expected_bytes or "sha256:" + digest.hexdigest() != expected_sha256: + raise Gate14CacheMaterializationError("packaged acquirer identity changed") + lease.assert_stable("packaged acquirer") + return lease + except BaseException: + lease.close() + raise + + +def _bound_acquirer_execution( + acquirer_path: Path, + lease: _LockedRegular, +) -> tuple[str, Mapping[str, Any]]: + original_path = os.fspath(acquirer_path) + if os.name == "nt": + return original_path, {} + if not sys.platform.startswith("linux"): + raise Gate14CacheMaterializationError("unsupported packaged acquirer platform") + descriptor = lease.handle.fileno() + descriptor_path = f"/proc/self/fd/{descriptor}" + try: + descriptor_identity = os.stat(descriptor_path) + handle_identity = os.fstat(descriptor) + except OSError as exc: + raise Gate14CacheMaterializationError("verified acquirer descriptor is unavailable") from exc + if _file_identity(descriptor_identity) != _file_identity(handle_identity): + raise Gate14CacheMaterializationError("verified acquirer descriptor identity changed") + return original_path, { + "executable": descriptor_path, + "pass_fds": (descriptor,), + "close_fds": True, + } + + +def _open_locked_directory(path: Path) -> tuple[Any, os.stat_result]: + candidate = Path(path) + try: + before = candidate.lstat() + except OSError as exc: + raise Gate14CacheMaterializationError("warm cache directory is unavailable") from exc + if _reparse(before) or candidate.is_symlink() or not stat.S_ISDIR(before.st_mode): + raise Gate14CacheMaterializationError("warm cache contains an unsafe directory") + try: + if os.name == "nt": + raw_handle, closer = _windows_open(candidate, directory=True) + handle: Any = _RawHandle(raw_handle, closer) + else: + descriptor = os.open( + candidate, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + handle = os.fdopen(descriptor, "rb", closefd=True) + opened = os.fstat(handle.fileno()) + if _file_identity(before) != _file_identity(opened): + raise Gate14CacheMaterializationError("warm cache directory changed while opening") + after = candidate.lstat() + if ( + _reparse(after) + or candidate.is_symlink() + or not stat.S_ISDIR(after.st_mode) + or _file_identity(before) != _file_identity(after) + ): + raise Gate14CacheMaterializationError("warm cache directory changed while opening") + return handle, before + except Gate14CacheMaterializationError: + try: + handle.close() + except (OSError, UnboundLocalError): + pass + raise + except OSError as exc: + raise Gate14CacheMaterializationError("warm cache directory is unreadable") from exc + + +def _source_paths() -> Mapping[str, Path]: + return { + "gate14_cache_materializer.py": Path(__file__), + "gate14_packaged_lifecycle.py": Path(lifecycle.__file__), + } + + +def current_source_bindings() -> Mapping[str, str]: + result: dict[str, str] = {} + for name, path in _source_paths().items(): + payload = _read_locked_regular(path, MAX_SOURCE_BYTES) + normalized = payload.replace(b"\r\n", b"\n") + if b"\r" in normalized: + raise Gate14CacheMaterializationError("materializer source line endings are invalid") + result[name] = "sha256:" + hashlib.sha256(normalized).hexdigest() + return result + + +def _load_plan( + plan_path: Path, + *, + ownership_verifier: OwnershipVerifier = lifecycle._assert_controller_owned, +) -> CachePlan: + payload = _read_locked_regular(plan_path, lifecycle.MAX_CONFIG_BYTES) + raw = _strict_canonical(payload, "cache materialization plan") + if ( + set(raw) != _PLAN_FIELDS + or type(raw.get("schema_version")) is not int + or raw.get("schema_version") != SCHEMA_VERSION + or raw.get("scope") != PLAN_SCOPE + or raw.get("platform") not in _MANIFEST_NAMES + or not isinstance(raw.get("source_commit"), str) + or _COMMIT_RE.fullmatch(raw["source_commit"]) is None + or not isinstance(raw.get("manifest_sha256"), str) + or _DIGEST_RE.fullmatch(raw["manifest_sha256"]) is None + or not isinstance(raw.get("acquirer_sha256"), str) + or _DIGEST_RE.fullmatch(raw["acquirer_sha256"]) is None + or type(raw.get("acquirer_bytes")) is not int + or not 1 <= raw["acquirer_bytes"] <= MAX_ACQUIRER_BYTES + or not isinstance(raw.get("lifecycle_template_sha256"), str) + or _DIGEST_RE.fullmatch(raw["lifecycle_template_sha256"]) is None + or not isinstance(raw.get("sources"), dict) + or set(raw["sources"]) != set(_SOURCE_NAMES) + or any(not isinstance(value, str) or _DIGEST_RE.fullmatch(value) is None for value in raw["sources"].values()) + ): + raise Gate14CacheMaterializationError("cache materialization plan binding is invalid") + try: + manifest_path = Path(raw["manifest_path"]) + acquirer_path = Path(raw["acquirer_path"]) + work_root = Path(raw["work_root"]) + staging_root = Path(raw["staging_root"]) + except TypeError as exc: + raise Gate14CacheMaterializationError("cache materialization plan path is invalid") from exc + work = _safe_directory(work_root, "work root") + staging = _safe_directory(staging_root, "staging root") + if work == staging or work in staging.parents or staging in work.parents: + raise Gate14CacheMaterializationError("materialization roots overlap") + exact_plan = staging / PLAN_NAME + exact_template = staging / TEMPLATE_NAME + exact_manifest = staging / _MANIFEST_NAMES[raw["platform"]] + acquirer_root = staging.joinpath(*_ACQUIRER_RUNTIME_PARTS) + exact_acquirer = acquirer_root / _ACQUIRER_NAMES[raw["platform"]] + if ( + Path(os.path.abspath(os.fspath(plan_path))) != exact_plan + or Path(os.path.abspath(os.fspath(manifest_path))) != exact_manifest + or Path(os.path.abspath(os.fspath(acquirer_path))) != exact_acquirer + ): + raise Gate14CacheMaterializationError("cache materialization plan path binding is invalid") + acquirer_parents = [ + staging.joinpath(*_ACQUIRER_RUNTIME_PARTS[:index]) for index in range(1, len(_ACQUIRER_RUNTIME_PARTS) + 1) + ] + try: + ownership_verifier(staging.parent, directory=True) + ownership_verifier(staging, directory=True) + for directory in acquirer_parents: + _safe_directory(directory, "packaged acquirer directory") + ownership_verifier(directory, directory=True) + for staged_file in (exact_plan, exact_template, exact_manifest, exact_acquirer): + ownership_verifier(staged_file, directory=False) + except lifecycle.Gate14LifecycleError as exc: + raise Gate14CacheMaterializationError("cache materialization plan is not controller-owned") from exc + if _hash_locked_regular(exact_manifest, lifecycle.MAX_CONFIG_BYTES)[1] != raw["manifest_sha256"]: + raise Gate14CacheMaterializationError("manifest file identity changed") + if _hash_locked_regular(exact_acquirer, MAX_ACQUIRER_BYTES) != ( + raw["acquirer_bytes"], + raw["acquirer_sha256"], + ): + raise Gate14CacheMaterializationError("packaged acquirer identity changed") + sources = dict(raw["sources"]) + return CachePlan( + platform=raw["platform"], + source_commit=raw["source_commit"], + manifest_path=exact_manifest, + manifest_sha256=raw["manifest_sha256"], + acquirer_path=exact_acquirer, + acquirer_sha256=raw["acquirer_sha256"], + acquirer_bytes=raw["acquirer_bytes"], + work_root=work, + staging_root=staging, + template_path=exact_template, + template_sha256=raw["lifecycle_template_sha256"], + sources=sources, + sources_sha256=lifecycle._digest(lifecycle._canonical(sources)), + plan_sha256=lifecycle._digest(payload), + ) + + +def _verify_sources(plan: CachePlan) -> None: + if current_source_bindings() != plan.sources: + raise Gate14CacheMaterializationError("cache materializer source binding changed") + + +def _safe_manifest(plan: CachePlan) -> Mapping[str, Any]: + payload = _read_locked_regular(plan.manifest_path, lifecycle.MAX_CONFIG_BYTES) + if lifecycle._digest(payload) != plan.manifest_sha256: + raise Gate14CacheMaterializationError("manifest file identity changed") + try: + return lifecycle._strict_json(payload, lifecycle.MAX_CONFIG_BYTES) + except lifecycle.Gate14LifecycleError as exc: + raise Gate14CacheMaterializationError("manifest is invalid") from exc + + +def _validate_manifest( + manifest: Mapping[str, Any], + *, + platform_name: str, +) -> tuple[str, str]: + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[platform_name] + profile = lifecycle.acceptance.MODEL_PROFILES[model_id] + repository, dtype = lifecycle._MODEL_SOURCE[model_id] + expected = lifecycle._GATE9_WARM_CACHE[platform_name]["artifacts"] + try: + source = manifest["source"] + runtime = manifest["runtime"] + artifacts = manifest["artifacts"] + if ( + set(manifest) != {"schema_version", "name", "aliases", "source", "model", "runtime", "artifacts"} + or manifest["schema_version"] != 1 + or not isinstance(source, dict) + or set(source) != {"repository", "revision"} + or not isinstance(runtime, dict) + or not isinstance(artifacts, list) + or any( + not isinstance(item, dict) + or set(item) != {"path", "role", "sha256", "size"} + or not isinstance(item["path"], str) + or not isinstance(item["role"], str) + or not isinstance(item["sha256"], str) + or type(item["size"]) is not int + for item in artifacts + ) + ): + raise Gate14CacheMaterializationError("manifest profile binding changed") + canonical = dict(manifest) + aliases = canonical["aliases"] + if not isinstance(aliases, list) or any(not isinstance(item, str) for item in aliases): + raise Gate14CacheMaterializationError("manifest profile binding changed") + canonical["aliases"] = sorted(aliases) + canonical["artifacts"] = sorted(artifacts, key=lambda item: (item["path"], item["role"])) + manifest_digest = lifecycle._digest(lifecycle._canonical(canonical)) + observed = tuple( + (item["path"], item["role"], "sha256:" + item["sha256"], item["size"]) + for item in sorted(artifacts, key=lambda item: item["path"]) + ) + except (KeyError, TypeError) as exc: + raise Gate14CacheMaterializationError("manifest profile binding changed") from exc + if ( + manifest["name"] != model_id + or manifest_digest != profile["manifest_digest"] + or source["repository"] != repository + or source["revision"] != profile["revision_commit"] + or runtime.get("dtype") != dtype + or runtime.get("adapter_profile") != "none" + or observed != expected + ): + raise Gate14CacheMaterializationError("manifest profile binding changed") + return model_id, profile["manifest_digest"] + + +def _packaged_acquirer( + manifest: ManifestProfile, + *, + cache_dir: Path, + token: bool, + max_resumptions: int, + require_direct_upstream: bool, + manifest_path: Path, + manifest_sha256: str, + acquirer_path: Path, + acquirer_sha256: str, + acquirer_bytes: int, +) -> Mapping[str, Any]: + if token is not False or max_resumptions != 3 or require_direct_upstream is not True: + raise Gate14CacheMaterializationError("packaged acquirer invocation is invalid") + + result_path = cache_dir / ACQUIRER_RESULT_NAME + if _entry_exists(result_path): + raise Gate14CacheMaterializationError("packaged acquirer output is not fresh") + allowed_environment = { + name: os.environ[name] + for name in ( + "COMSPEC", + "PATH", + "SYSTEMROOT", + "TEMP", + "TMP", + "WINDIR", + ) + if name in os.environ + } + allowed_environment["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1" + manifest_lease: _LockedRegular | None = None + acquirer_lease: _LockedRegular | None = None + try: + manifest_lease, manifest_payload = _lock_verified_payload( + manifest_path, + lifecycle.MAX_CONFIG_BYTES, + manifest_sha256, + "manifest file", + ) + acquirer_lease = _lock_verified_acquirer( + acquirer_path, + MAX_ACQUIRER_BYTES, + acquirer_bytes, + acquirer_sha256, + ) + executable, platform_options = _bound_acquirer_execution(acquirer_path, acquirer_lease) + argv = [ + executable, + "edge-acquire", + "--manifest_stdin_sha256", + manifest_sha256, + "--cache_dir", + os.fspath(cache_dir), + "--max_resumptions", + "3", + "--require_direct_upstream", + "--no_token", + "--output", + os.fspath(result_path), + ] + try: + result = subprocess.run( + argv, + check=False, + input=manifest_payload, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=ACQUIRER_TIMEOUT_SECONDS, + shell=False, + cwd=os.fspath(acquirer_path.parent), + env=allowed_environment, + **platform_options, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Gate14CacheMaterializationError("packaged acquirer failed") from exc + if result.returncode != 0: + raise Gate14CacheMaterializationError("packaged acquirer failed") + manifest_lease.assert_stable("manifest file") + acquirer_lease.assert_stable("packaged acquirer") + try: + payload = _read_locked_regular(result_path, MAX_ACQUIRER_OUTPUT_BYTES) + return lifecycle._strict_json(payload, MAX_ACQUIRER_OUTPUT_BYTES) + except (Gate14CacheMaterializationError, lifecycle.Gate14LifecycleError) as exc: + raise Gate14CacheMaterializationError("packaged acquirer result is invalid") from exc + finally: + if _entry_exists(result_path): + _unlink_retry(result_path) + if acquirer_lease is not None: + acquirer_lease.close() + if manifest_lease is not None: + manifest_lease.close() + + +def _transport_overridden() -> bool: + if any(os.environ.get(name) for name in _OVERRIDE_NAMES): + return True + try: + proxies = getproxies() + except OSError: + return True + return any(value for key, value in proxies.items() if key.casefold() not in {"no", "no_proxy"}) + + +def _clear_acquisition_metadata( + cache_root: Path, + manifest_digest: str, + artifact_paths: Sequence[str], +) -> None: + manifest_root = cache_root / "manifest-artifacts" / manifest_digest.removeprefix("sha256:") + partial = manifest_root / "partial" + locks = manifest_root / "locks" + if _entry_exists(partial): + metadata = partial.lstat() + if _reparse(metadata) or partial.is_symlink() or not stat.S_ISDIR(metadata.st_mode) or any(partial.iterdir()): + raise Gate14CacheMaterializationError("acquisition retained partial material") + partial.rmdir() + if _entry_exists(locks): + metadata = locks.lstat() + if _reparse(metadata) or locks.is_symlink() or not stat.S_ISDIR(metadata.st_mode): + raise Gate14CacheMaterializationError("acquisition lock directory is unsafe") + expected = {hashlib.sha256(path.encode("utf-8")).hexdigest() + ".lock" for path in artifact_paths} + entries = list(locks.iterdir()) + if {entry.name for entry in entries} != expected: + raise Gate14CacheMaterializationError("acquisition lock inventory changed") + for entry in entries: + metadata = entry.lstat() + if _reparse(metadata) or entry.is_symlink() or not stat.S_ISREG(metadata.st_mode) or metadata.st_size != 0: + raise Gate14CacheMaterializationError("acquisition lock entry is unsafe") + entry.unlink() + locks.rmdir() + + +def _expected_cache( + binding: Mapping[str, Any], +) -> tuple[dict[str, tuple[int, str]], set[str]]: + artifacts = binding.get("artifacts") + record_digest = binding.get("materialization_record_sha256") + if ( + not isinstance(artifacts, list) + or not isinstance(record_digest, str) + or _DIGEST_RE.fullmatch(record_digest) is None + ): + raise Gate14CacheMaterializationError("warm-cache binding is invalid") + manifest_digest = None + for platform_name, model_id in lifecycle.acceptance.EXPECTED_PLATFORM_MODELS.items(): + profile = lifecycle.acceptance.MODEL_PROFILES[model_id] + expected_artifacts = [ + { + "path": path, + "role": role, + "sha256": digest, + "size_bytes": size, + } + for path, role, digest, size in lifecycle._GATE9_WARM_CACHE[platform_name]["artifacts"] + ] + if expected_artifacts == artifacts: + manifest_digest = profile["manifest_digest"].removeprefix("sha256:") + break + if manifest_digest is None: + raise Gate14CacheMaterializationError("warm-cache artifact profile is unknown") + expected_files: dict[str, tuple[int, str]] = {} + expected_directories: set[str] = set() + prefix = f"manifest-artifacts/{manifest_digest}/snapshot" + for item in artifacts: + relative = f"{prefix}/{item['path']}" + expected_files[relative] = ( + item["size_bytes"], + item["sha256"].removeprefix("sha256:"), + ) + parts = relative.split("/") + expected_directories.update("/".join(parts[:index]) for index in range(1, len(parts))) + return expected_files, expected_directories + + +def _hash_open_file( + handle: BinaryIO, + opened: os.stat_result, +) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + size += len(chunk) + digest.update(chunk) + after = os.fstat(handle.fileno()) + if _file_identity(opened) != _file_identity(after): + raise Gate14CacheMaterializationError("warm cache artifact changed while hashing") + return size, digest.hexdigest() + + +def verify_exact_cache( + cache_root: Path, + binding: Mapping[str, Any], +) -> CacheLease: + cache_root = Path(cache_root) + expected_files, expected_directories = _expected_cache(binding) + lease = CacheLease([], [], []) + seen_files: set[str] = set() + seen_directories: set[str] = set() + seen_casefold: set[str] = set() + pending = [cache_root] + try: + while pending: + root_path = pending.pop() + directory_handle, directory_metadata = _open_locked_directory(root_path) + lease.handles.append(directory_handle) + lease.identities.append((root_path, directory_metadata, True)) + try: + entries = list(os.scandir(root_path)) + except OSError as exc: + raise Gate14CacheMaterializationError("warm cache directory is unreadable") from exc + lease.directory_entries.append((root_path, frozenset(entry.name for entry in entries))) + for entry in entries: + path = root_path / entry.name + relative = path.relative_to(cache_root).as_posix() + folded = relative.casefold() + if folded in seen_casefold: + raise Gate14CacheMaterializationError("warm cache path collides") + seen_casefold.add(folded) + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14CacheMaterializationError("warm cache entry is unavailable") from exc + if ( + _reparse(metadata) + or path.is_symlink() + or not (stat.S_ISDIR(metadata.st_mode) or stat.S_ISREG(metadata.st_mode)) + ): + raise Gate14CacheMaterializationError("warm cache contains an unsafe entry") + if stat.S_ISDIR(metadata.st_mode): + if relative not in expected_directories: + raise Gate14CacheMaterializationError("warm cache contains an unexpected directory") + seen_directories.add(relative) + pending.append(path) + continue + if relative not in expected_files: + raise Gate14CacheMaterializationError("warm cache contains an unexpected file") + handle, opened = _open_locked_regular( + path, + expected_files[relative][0], + ) + lease.handles.append(handle) + lease.identities.append((path, opened, False)) + size, digest = _hash_open_file(handle, opened) + if (size, digest) != expected_files[relative]: + raise Gate14CacheMaterializationError("warm cache artifact verification failed") + seen_files.add(relative) + if seen_files != set(expected_files) or seen_directories != expected_directories: + raise Gate14CacheMaterializationError("warm cache inventory is incomplete") + lease.assert_stable() + return lease + except BaseException: + lease.close() + raise + + +def _entry_exists(path: Path) -> bool: + try: + path.lstat() + except FileNotFoundError: + return False + except OSError as exc: + raise Gate14CacheMaterializationError("materialization output state is unavailable") from exc + return True + + +def _unlink_retry(path: Path) -> None: + last_error: OSError | None = None + for _attempt in range(2): + try: + path.unlink() + break + except FileNotFoundError: + break + except OSError as exc: + last_error = exc + if _entry_exists(path): + raise Gate14CacheMaterializationError("materialization output cleanup did not complete") from last_error + + +def _remove_tree_retry(path: Path) -> None: + last_error: OSError | None = None + for _attempt in range(2): + if not _entry_exists(path): + return + try: + metadata = path.lstat() + if _reparse(metadata) or path.is_symlink(): + if stat.S_ISDIR(metadata.st_mode): + path.rmdir() + else: + path.unlink() + elif stat.S_ISDIR(metadata.st_mode): + shutil.rmtree(path) + else: + path.unlink() + except OSError as exc: + last_error = exc + if _entry_exists(path): + raise Gate14CacheMaterializationError("materialization cache cleanup did not complete") from last_error + + +def _write_new(path: Path, payload: bytes) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + except BaseException as exc: + try: + _unlink_retry(path) + except Gate14CacheMaterializationError as cleanup_exc: + raise Gate14CacheMaterializationError("incomplete materialization output could not be removed") from exc + raise + + +_WINDOWS_CONTROLLER_SDDL = "O:BAG:BAD:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;GR;;;AU)" +_POSIX_CONTROLLER_FILE_MODE = 0o644 + + +def _windows_protect_controller_output(path: Path) -> None: + import ctypes + from ctypes import wintypes + + advapi32 = ctypes.WinDLL("advapi32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + local_free = kernel32.LocalFree + local_free.argtypes = (ctypes.c_void_p,) + local_free.restype = ctypes.c_void_p + + descriptor = ctypes.c_void_p() + descriptor_size = wintypes.DWORD() + convert = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW + convert.argtypes = ( + wintypes.LPCWSTR, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(wintypes.DWORD), + ) + convert.restype = wintypes.BOOL + if not convert( + _WINDOWS_CONTROLLER_SDDL, + 1, + ctypes.byref(descriptor), + ctypes.byref(descriptor_size), + ): + error = ctypes.get_last_error() + raise Gate14CacheMaterializationError( + "controller output security descriptor could not be created" + ) from OSError(error, "security descriptor conversion failed") + + try: + owner = ctypes.c_void_p() + owner_defaulted = wintypes.BOOL() + get_owner = advapi32.GetSecurityDescriptorOwner + get_owner.argtypes = ( + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(wintypes.BOOL), + ) + get_owner.restype = wintypes.BOOL + present = wintypes.BOOL() + dacl = ctypes.c_void_p() + dacl_defaulted = wintypes.BOOL() + get_dacl = advapi32.GetSecurityDescriptorDacl + get_dacl.argtypes = ( + ctypes.c_void_p, + ctypes.POINTER(wintypes.BOOL), + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(wintypes.BOOL), + ) + get_dacl.restype = wintypes.BOOL + if ( + not get_owner( + descriptor, + ctypes.byref(owner), + ctypes.byref(owner_defaulted), + ) + or not get_dacl( + descriptor, + ctypes.byref(present), + ctypes.byref(dacl), + ctypes.byref(dacl_defaulted), + ) + or not owner.value + or not present.value + or not dacl.value + ): + error = ctypes.get_last_error() + raise Gate14CacheMaterializationError("controller output security descriptor is invalid") from OSError( + error, "security descriptor parsing failed" + ) + + set_security = advapi32.SetNamedSecurityInfoW + set_security.argtypes = ( + wintypes.LPWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ) + set_security.restype = wintypes.DWORD + result = set_security( + os.fspath(path), + 1, + 0x1 | 0x4 | 0x80000000, + owner, + None, + dacl, + None, + ) + if result != 0: + raise Gate14CacheMaterializationError( + "controller output security descriptor could not be installed" + ) from OSError(result, "security descriptor installation failed") + finally: + local_free(descriptor) + + +def _protect_promoted_output( + path: Path, + *, + os_name: str | None = None, +) -> None: + observed_os = os.name if os_name is None else os_name + if observed_os == "nt": + _windows_protect_controller_output(path) + else: + try: + # The ordinary qualification identity must read promoted inputs, + # while only the controller may replace or modify them. + path.chmod(_POSIX_CONTROLLER_FILE_MODE) + except OSError as exc: + raise Gate14CacheMaterializationError("controller output permissions could not be installed") from exc + try: + lifecycle._assert_controller_managed(path, directory=False) + except lifecycle.Gate14LifecycleError as exc: + raise Gate14CacheMaterializationError("controller output protection is invalid") from exc + + +def _cleanup_materialization( + *, + outputs: Sequence[tuple[Path, bool]], + cache: Path, + cache_created: bool, +) -> None: + failures = [] + for path, created in outputs: + if not created: + continue + try: + _unlink_retry(path) + except Gate14CacheMaterializationError as exc: + failures.append(exc) + if cache_created: + try: + _remove_tree_retry(cache) + except Gate14CacheMaterializationError as exc: + failures.append(exc) + if failures: + raise Gate14CacheMaterializationError("materialization failed and cleanup did not complete") from failures[0] + + +def materialize( + *, + plan_path: Path, + acquirer: Acquirer | None = None, + ownership_verifier: OwnershipVerifier = lifecycle._assert_controller_owned, + cache_verifier: Callable[[Path, Mapping[str, Any]], CacheLease] = verify_exact_cache, + native_platform: str | None = None, +) -> Mapping[str, Any]: + plan = _load_plan(plan_path, ownership_verifier=ownership_verifier) + observed_platform = _native_platform() if native_platform is None else native_platform + if observed_platform != plan.platform: + raise Gate14CacheMaterializationError("native materialization platform changed") + _validate_template( + plan=plan, + template_payload=_read_locked_regular( + plan.template_path, + lifecycle.MAX_CONFIG_BYTES, + ), + ) + if _transport_overridden(): + raise Gate14CacheMaterializationError("official upstream transport is overridden") + _verify_sources(plan) + manifest = _safe_manifest(plan) + model_id, manifest_digest = _validate_manifest( + manifest, + platform_name=plan.platform, + ) + + cache = plan.work_root / CACHE_NAME + record_path = plan.work_root / RECORD_NAME + binding_path = plan.work_root / BINDING_NAME + handoff_path = plan.work_root / HANDOFF_NAME + if any(_entry_exists(path) for path in (cache, record_path, binding_path, handoff_path)): + raise Gate14CacheMaterializationError("materialization outputs are not fresh") + + cache_created = False + record_created = False + binding_created = False + handoff_created = False + lease: CacheLease | None = None + try: + cache.mkdir(mode=0o700) + cache_created = True + acquisition = _packaged_acquirer if acquirer is None else acquirer + record = acquisition( + ManifestProfile(name=model_id), + cache_dir=cache, + token=False, + max_resumptions=3, + require_direct_upstream=True, + manifest_path=plan.manifest_path, + manifest_sha256=plan.manifest_sha256, + acquirer_path=plan.acquirer_path, + acquirer_sha256=plan.acquirer_sha256, + acquirer_bytes=plan.acquirer_bytes, + ) + record_payload = _canonical(record) + binding = lifecycle.build_warm_cache_binding( + record_payload, + platform=plan.platform, + source_commit=plan.source_commit, + materialization_plan_sha256=plan.plan_sha256, + materializer_sources_sha256=plan.sources_sha256, + model_id=model_id, + manifest_digest=manifest_digest, + ) + _clear_acquisition_metadata( + cache, + manifest_digest, + [item["path"] for item in binding["artifacts"]], + ) + lease = cache_verifier(cache, binding) + binding_payload = _canonical(binding) + binding_sha256 = lifecycle._digest(lifecycle._canonical(binding)) + handoff = { + "schema_version": SCHEMA_VERSION, + "scope": HANDOFF_SCOPE, + "plan_sha256": plan.plan_sha256, + "platform": plan.platform, + "source_commit": plan.source_commit, + "model_id": model_id, + "manifest_digest": manifest_digest, + "materializer_sources_sha256": plan.sources_sha256, + "materialization_record_sha256": binding["materialization_record_sha256"], + "materialization_record_bytes": len(record_payload), + "warm_cache_binding_sha256": binding_sha256, + "warm_cache_binding_file_sha256": lifecycle._digest(binding_payload), + "warm_cache_binding_bytes": len(binding_payload), + "artifact_count": binding["artifact_count"], + "artifact_bytes": binding["artifact_bytes"], + } + _write_new(record_path, record_payload) + record_created = True + _write_new(binding_path, binding_payload) + binding_created = True + _write_new(handoff_path, _canonical(handoff)) + handoff_created = True + lease.assert_stable() + except BaseException as exc: + if lease is not None: + lease.close() + try: + _cleanup_materialization( + outputs=( + (handoff_path, handoff_created), + (binding_path, binding_created), + (record_path, record_created), + ), + cache=cache, + cache_created=cache_created, + ) + except Gate14CacheMaterializationError as cleanup_exc: + raise cleanup_exc from exc + raise + finally: + if lease is not None: + lease.close() + + return { + "schema_version": SCHEMA_VERSION, + "scope": RESULT_SCOPE, + "result": "passed", + "phase": "materialized", + "platform": plan.platform, + "source_commit": plan.source_commit, + "plan_sha256": plan.plan_sha256, + "materializer_sources_sha256": plan.sources_sha256, + "model_id": model_id, + "manifest_digest": manifest_digest, + "materialization_record_sha256": binding["materialization_record_sha256"], + "artifact_count": binding["artifact_count"], + "artifact_bytes": binding["artifact_bytes"], + "warm_cache_binding_sha256": binding_sha256, + } + + +def _validate_handoff( + payload: bytes, + *, + plan: CachePlan, + record_payload: bytes, + binding_payload: bytes, +) -> tuple[Mapping[str, Any], Mapping[str, Any]]: + handoff = _strict_canonical(payload, "cache materialization handoff") + binding = _strict_canonical(binding_payload, "warm-cache binding") + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[plan.platform] + manifest_digest = lifecycle.acceptance.MODEL_PROFILES[model_id]["manifest_digest"] + expected_binding = lifecycle.build_warm_cache_binding( + record_payload, + platform=plan.platform, + source_commit=plan.source_commit, + materialization_plan_sha256=plan.plan_sha256, + materializer_sources_sha256=plan.sources_sha256, + model_id=model_id, + manifest_digest=manifest_digest, + ) + expected_handoff = { + "schema_version": SCHEMA_VERSION, + "scope": HANDOFF_SCOPE, + "plan_sha256": plan.plan_sha256, + "platform": plan.platform, + "source_commit": plan.source_commit, + "model_id": model_id, + "manifest_digest": manifest_digest, + "materializer_sources_sha256": plan.sources_sha256, + "materialization_record_sha256": expected_binding["materialization_record_sha256"], + "materialization_record_bytes": len(record_payload), + "warm_cache_binding_sha256": lifecycle._digest(lifecycle._canonical(expected_binding)), + "warm_cache_binding_file_sha256": lifecycle._digest(binding_payload), + "warm_cache_binding_bytes": len(binding_payload), + "artifact_count": expected_binding["artifact_count"], + "artifact_bytes": expected_binding["artifact_bytes"], + } + if set(handoff) != _HANDOFF_FIELDS or handoff != expected_handoff or binding != expected_binding: + raise Gate14CacheMaterializationError("cache materialization handoff binding changed") + return handoff, binding + + +def _validate_template( + *, + plan: CachePlan, + template_payload: bytes, +) -> Mapping[str, Any]: + if lifecycle._digest(template_payload) != plan.template_sha256: + raise Gate14CacheMaterializationError("lifecycle template identity changed") + template = _strict_canonical(template_payload, "lifecycle template") + expected_model = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[plan.platform] + expected_manifest = lifecycle.acceptance.MODEL_PROFILES[expected_model]["manifest_digest"] + if ( + set(template) != lifecycle._CONFIG_FIELDS + or template.get("warm_cache") is not None + or template.get("platform") != plan.platform + or template.get("source_commit") != plan.source_commit + or template.get("model_id") != expected_model + or template.get("manifest_digest") != expected_manifest + or template.get("work_root") != str(plan.work_root) + or template.get("staging_root") != str(plan.staging_root) + ): + raise Gate14CacheMaterializationError("lifecycle template binding changed") + return template + + +def _final_config( + *, + plan: CachePlan, + template_payload: bytes, + binding: Mapping[str, Any], +) -> tuple[Mapping[str, Any], bytes]: + value = dict( + _validate_template( + plan=plan, + template_payload=template_payload, + ) + ) + value["warm_cache"] = dict(binding) + return value, _canonical(value) + + +def _load_promoted_config(path: Path) -> Any: + return lifecycle.load_config( + path, + ownership_verifier=lifecycle._assert_controller_managed, + ) + + +def _validate_promoted_config( + *, + plan: CachePlan, + config_path: Path, + config_payload: bytes, + record_payload: bytes, + lifecycle_loader: Callable[[Path], Any], +) -> Any: + loaded = lifecycle_loader(config_path) + if ( + loaded.platform != plan.platform + or loaded.source_commit != plan.source_commit + or loaded.config_sha256 != lifecycle._digest(config_payload) + or loaded.warm_cache.materialization_plan_sha256 != plan.plan_sha256 + or loaded.warm_cache.materializer_sources_sha256 != plan.sources_sha256 + or loaded.warm_cache.materialization_record_sha256 != lifecycle._digest(record_payload) + or loaded.warm_cache.materialization_record_bytes != len(record_payload) + ): + raise Gate14CacheMaterializationError("promoted lifecycle configuration changed") + return loaded + + +def _cleanup_promoted_handoff(paths: Sequence[Path]) -> None: + failures = [] + for path in paths: + try: + _unlink_retry(path) + except Gate14CacheMaterializationError as exc: + failures.append(exc) + if failures or any(_entry_exists(path) for path in paths): + raise Gate14CacheMaterializationError( + "promoted lifecycle inputs were committed but handoff cleanup did not complete" + ) from (failures[0] if failures else None) + + +def _promotion_result( + *, + plan: CachePlan, + config_payload: bytes, + binding: Mapping[str, Any], +) -> Mapping[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "scope": RESULT_SCOPE, + "result": "passed", + "phase": "promoted", + "platform": plan.platform, + "source_commit": plan.source_commit, + "plan_sha256": plan.plan_sha256, + "materializer_sources_sha256": plan.sources_sha256, + "model_id": lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[plan.platform], + "manifest_digest": lifecycle.acceptance.MODEL_PROFILES[ + lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[plan.platform] + ]["manifest_digest"], + "materialization_record_sha256": binding["materialization_record_sha256"], + "artifact_count": binding["artifact_count"], + "artifact_bytes": binding["artifact_bytes"], + "warm_cache_binding_sha256": lifecycle._digest(lifecycle._canonical(binding)), + "lifecycle_config_sha256": lifecycle._digest(config_payload), + } + + +def promote( + *, + plan_path: Path, + ownership_verifier: OwnershipVerifier = lifecycle._assert_controller_managed, + cache_verifier: Callable[[Path, Mapping[str, Any]], CacheLease] = verify_exact_cache, + lifecycle_loader: Callable[[Path], Any] = _load_promoted_config, + output_protector: Callable[[Path], None] = _protect_promoted_output, +) -> Mapping[str, Any]: + plan = _load_plan(plan_path, ownership_verifier=ownership_verifier) + _verify_sources(plan) + cache = plan.work_root / CACHE_NAME + work_record = plan.work_root / RECORD_NAME + work_binding = plan.work_root / BINDING_NAME + work_handoff = plan.work_root / HANDOFF_NAME + handoff_paths = (work_handoff, work_binding, work_record) + staged_record = plan.staging_root / RECORD_NAME + config_path = plan.staging_root / CONFIG_NAME + template_payload = _read_locked_regular( + plan.template_path, + lifecycle.MAX_CONFIG_BYTES, + ) + _validate_template(plan=plan, template_payload=template_payload) + + staged_state = ( + _entry_exists(staged_record), + _entry_exists(config_path), + ) + if any(staged_state) and not all(staged_state): + raise Gate14CacheMaterializationError("promoted lifecycle outputs are incomplete") + if all(staged_state): + record_payload = _read_locked_regular(staged_record, MAX_JSON_BYTES) + config_payload = _read_locked_regular( + config_path, + lifecycle.MAX_CONFIG_BYTES, + ) + config = _strict_canonical(config_payload, "promoted lifecycle configuration") + binding = config.get("warm_cache") + if not isinstance(binding, dict): + raise Gate14CacheMaterializationError("promoted lifecycle configuration changed") + _validate_promoted_config( + plan=plan, + config_path=config_path, + config_payload=config_payload, + record_payload=record_payload, + lifecycle_loader=lifecycle_loader, + ) + lease = cache_verifier(cache, binding) + try: + lease.assert_stable() + _cleanup_promoted_handoff(handoff_paths) + lease.assert_stable() + finally: + lease.close() + return _promotion_result( + plan=plan, + config_payload=config_payload, + binding=binding, + ) + + record_payload = _read_locked_regular(work_record, MAX_JSON_BYTES) + binding_payload = _read_locked_regular(work_binding, MAX_JSON_BYTES) + handoff_payload = _read_locked_regular(work_handoff, MAX_JSON_BYTES) + handoff, binding = _validate_handoff( + handoff_payload, + plan=plan, + record_payload=record_payload, + binding_payload=binding_payload, + ) + _config, config_payload = _final_config( + plan=plan, + template_payload=template_payload, + binding=binding, + ) + + record_created = False + config_created = False + committed = False + lease: CacheLease | None = None + try: + lease = cache_verifier(cache, binding) + _write_new(staged_record, record_payload) + record_created = True + output_protector(staged_record) + _write_new(config_path, config_payload) + config_created = True + output_protector(config_path) + loaded = _validate_promoted_config( + plan=plan, + config_path=config_path, + config_payload=config_payload, + record_payload=record_payload, + lifecycle_loader=lifecycle_loader, + ) + if loaded.warm_cache.binding_sha256 != handoff["warm_cache_binding_sha256"]: + raise Gate14CacheMaterializationError("promoted lifecycle configuration changed") + lease.assert_stable() + committed = True + _cleanup_promoted_handoff(handoff_paths) + lease.assert_stable() + except BaseException as exc: + if lease is not None: + lease.close() + if committed: + raise Gate14CacheMaterializationError( + "promoted lifecycle outputs were committed; retry promotion cleanup" + ) from exc + failures = [] + for path, created in ( + (config_path, config_created), + (staged_record, record_created), + ): + if not created: + continue + try: + _unlink_retry(path) + except Gate14CacheMaterializationError as cleanup_exc: + failures.append(cleanup_exc) + if failures: + raise Gate14CacheMaterializationError( + "promotion failed and protected-output cleanup did not complete" + ) from exc + raise + finally: + if lease is not None: + lease.close() + + return _promotion_result( + plan=plan, + config_payload=config_payload, + binding=binding, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + for name in ("materialize", "promote"): + command = commands.add_parser(name) + command.add_argument("--plan", type=Path, required=True) + commands.add_parser("source-bindings") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "source-bindings": + result: Mapping[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": "gate14-cache-materializer-sources", + "sources": current_source_bindings(), + } + elif args.command == "materialize": + result = materialize(plan_path=args.plan) + else: + result = promote(plan_path=args.plan) + except ( + Gate14CacheMaterializationError, + lifecycle.Gate14LifecycleError, + OSError, + ValueError, + ) as exc: + raise SystemExit(str(exc)) from exc + print( + json.dumps( + result, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_calibration_challenge.py b/scripts/gate14_calibration_challenge.py new file mode 100644 index 000000000..6963234db --- /dev/null +++ b/scripts/gate14_calibration_challenge.py @@ -0,0 +1,241 @@ +"""Short-lived, source-bound calibration challenges for Gate 14 host probes.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import secrets +import stat +import tempfile +import time +from pathlib import Path +from typing import Any, Mapping, Sequence + +SCHEMA_VERSION = 1 +SCOPE = "gate14-calibration-challenge" +MAX_JSON_BYTES = 16_384 +MAX_LIFETIME_SECONDS = 900 +MIN_LIFETIME_SECONDS = 60 +MAX_SAMPLE_WINDOW_SECONDS = 120 + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_NONCE_RE = re.compile(r"[0-9a-f]{64}") +_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "source_commit", + "package_sha256", + "checkpoint_sha256", + "controller_state_revision", + "issued_at_unix", + "expires_at_unix", + "nonce", +} + + +class Gate14ChallengeError(ValueError): + """A calibration challenge is malformed, stale, or incorrectly bound.""" + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14ChallengeError("duplicate challenge field") + result[key] = value + return result + + +def _reject_constant(_value: str) -> None: + raise Gate14ChallengeError("non-finite challenge value") + + +def canonical_payload(value: Mapping[str, Any]) -> bytes: + return json.dumps(value, allow_nan=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def digest(value: Mapping[str, Any]) -> str: + return "sha256:" + hashlib.sha256(canonical_payload(value)).hexdigest() + + +def parse_payload(payload: bytes) -> Mapping[str, Any]: + if not 1 <= len(payload) <= MAX_JSON_BYTES: + raise Gate14ChallengeError("challenge size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14ChallengeError("challenge is invalid JSON") from exc + if not isinstance(value, dict): + raise Gate14ChallengeError("challenge must be an object") + return value + + +def regular_payload(path: Path) -> bytes: + path = Path(path) + try: + before = path.lstat() + except OSError as exc: + raise Gate14ChallengeError("challenge is unavailable") from exc + reparse = bool(getattr(before, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(before.st_mode) or not 1 <= before.st_size <= MAX_JSON_BYTES: + raise Gate14ChallengeError("challenge path is unsafe") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as handle: + opened = os.fstat(handle.fileno()) + if ( + not stat.S_ISREG(opened.st_mode) + or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino) + or not 1 <= opened.st_size <= MAX_JSON_BYTES + ): + raise Gate14ChallengeError("challenge changed while opening") + payload = handle.read(MAX_JSON_BYTES + 1) + after = os.fstat(handle.fileno()) + except Gate14ChallengeError: + raise + except OSError as exc: + raise Gate14ChallengeError("challenge is unreadable") from exc + if len(payload) != opened.st_size or (after.st_dev, after.st_ino, after.st_size) != ( + opened.st_dev, + opened.st_ino, + opened.st_size, + ): + raise Gate14ChallengeError("challenge changed while reading") + return payload + + +def load(path: Path) -> Mapping[str, Any]: + return parse_payload(regular_payload(path)) + + +def validate( + value: Mapping[str, Any], + *, + run_id: str, + platform: str, + source_commit: str, + package_sha256: str, + checkpoint_sha256: str | None = None, + now_unix: float | None = None, +) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != _FIELDS: + raise Gate14ChallengeError("challenge schema is invalid") + if ( + value["schema_version"] != SCHEMA_VERSION + or value["scope"] != SCOPE + or not isinstance(value["run_id"], str) + or _RUN_RE.fullmatch(value["run_id"]) is None + or value["run_id"] != run_id + or value["platform"] not in {"windows", "linux"} + or value["platform"] != platform + or not isinstance(value["source_commit"], str) + or _COMMIT_RE.fullmatch(value["source_commit"]) is None + or value["source_commit"] != source_commit + or not isinstance(value["package_sha256"], str) + or _DIGEST_RE.fullmatch(value["package_sha256"]) is None + or value["package_sha256"] != package_sha256 + or not isinstance(value["checkpoint_sha256"], str) + or _DIGEST_RE.fullmatch(value["checkpoint_sha256"]) is None + or (checkpoint_sha256 is not None and value["checkpoint_sha256"] != checkpoint_sha256) + or not isinstance(value["nonce"], str) + or _NONCE_RE.fullmatch(value["nonce"]) is None + ): + raise Gate14ChallengeError("challenge binding is invalid") + revision = value["controller_state_revision"] + issued = value["issued_at_unix"] + expires = value["expires_at_unix"] + if ( + type(revision) is not int + or revision < 0 + or type(issued) is not int + or type(expires) is not int + or not MIN_LIFETIME_SECONDS <= expires - issued <= MAX_LIFETIME_SECONDS + ): + raise Gate14ChallengeError("challenge lifetime is invalid") + if now_unix is not None: + if type(now_unix) not in (int, float): + raise Gate14ChallengeError("challenge clock is invalid") + now = float(now_unix) + if not issued <= now <= expires: + raise Gate14ChallengeError("challenge is not currently valid") + return dict(value) + + +def create( + *, + run_id: str, + platform: str, + source_commit: str, + package_sha256: str, + checkpoint_sha256: str, + controller_state_revision: int, + issued_at_unix: int | None = None, + lifetime_seconds: int = MAX_LIFETIME_SECONDS, + nonce: str | None = None, +) -> Mapping[str, Any]: + issued = int(time.time()) if issued_at_unix is None else issued_at_unix + value = { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "run_id": run_id, + "platform": platform, + "source_commit": source_commit, + "package_sha256": package_sha256, + "checkpoint_sha256": checkpoint_sha256, + "controller_state_revision": controller_state_revision, + "issued_at_unix": issued, + "expires_at_unix": issued + lifetime_seconds, + "nonce": secrets.token_hex(32) if nonce is None else nonce, + } + return validate( + value, + run_id=run_id, + platform=platform, + source_commit=source_commit, + package_sha256=package_sha256, + checkpoint_sha256=checkpoint_sha256, + now_unix=issued, + ) + + +def write_new(path: Path, value: Mapping[str, Any]) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + raise Gate14ChallengeError("challenge output already exists") + payload = canonical_payload(value) + os.linesep.encode("ascii") + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, path) + except FileExistsError as exc: + raise Gate14ChallengeError("challenge output already exists") from exc + temporary.unlink() + except BaseException: + try: + temporary.unlink() + except OSError: + pass + raise diff --git a/scripts/gate14_gcp_executor.py b/scripts/gate14_gcp_executor.py new file mode 100644 index 000000000..22bae7a38 --- /dev/null +++ b/scripts/gate14_gcp_executor.py @@ -0,0 +1,664 @@ +"""Execute exact GCP actions selected by the durable Gate 14 controller. + +Every operation revalidates native gcloud authentication, inventories the two planned +instances/disks plus all project L4 usage, and verifies the protected bootstrap. Only +the controller's allowlisted next action may mutate provider state. Exact-name foreign +or ambiguous resources fail closed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate14_hardware_acceptance as acceptance +import gate14_run_controller as controller + +MAX_OUTPUT_BYTES = 1_048_576 +MAX_JSON_BYTES = 262_144 +SCOPE_LABEL = "gate14-hardware" +PROTECTED_INSTANCE = controller.PROTECTED_INSTANCE + + +class Gate14GcpError(RuntimeError): + """A provider observation or exact action failed closed.""" + + +@dataclass(frozen=True) +class CommandResult: + returncode: int + stdout: bytes + stderr: bytes + + +Runner = Callable[[Sequence[str], int], CommandResult] + + +def _default_runner(argv: Sequence[str], timeout: int) -> CommandResult: + if not argv or any(not isinstance(item, str) or not item for item in argv): + raise Gate14GcpError("provider command is invalid") + try: + result = subprocess.run( + list(argv), + check=False, + capture_output=True, + timeout=timeout, + shell=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise Gate14GcpError("provider command failed") from exc + if len(result.stdout) > MAX_OUTPUT_BYTES or len(result.stderr) > MAX_OUTPUT_BYTES: + raise Gate14GcpError("provider command output exceeded its bound") + return CommandResult(result.returncode, result.stdout, result.stderr) + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14GcpError("duplicate provider JSON field") + result[key] = value + return result + + +def _reject_constant(_value: str) -> None: + raise Gate14GcpError("non-finite provider JSON value") + + +def _json_bytes(payload: bytes, label: str) -> Any: + if not 1 <= len(payload) <= MAX_OUTPUT_BYTES: + raise Gate14GcpError(f"{label} output is invalid") + try: + return json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14GcpError(f"{label} returned invalid JSON") from exc + + +def _basename(value: Any) -> str: + return value.rsplit("/", 1)[-1] if isinstance(value, str) else "" + + +def _metadata(value: Mapping[str, Any]) -> Mapping[str, str]: + raw = value.get("metadata") + items = raw.get("items") if isinstance(raw, dict) else None + if items is None: + return {} + if not isinstance(items, list): + raise Gate14GcpError("instance metadata is invalid") + result: dict[str, str] = {} + for item in items: + if not isinstance(item, dict) or set(item) != {"key", "value"}: + raise Gate14GcpError("instance metadata item is invalid") + key, field = item["key"], item["value"] + if not isinstance(key, str) or not isinstance(field, str) or key in result: + raise Gate14GcpError("instance metadata item is ambiguous") + result[key] = field + return result + + +def _labels(plan: controller.RunPlan, client: controller.ClientPlan) -> dict[str, str]: + return { + "communityai-run": plan.run_id, + "communityai-scope": SCOPE_LABEL, + "communityai-source": client.source_commit, + } + + +def _label_argument(value: Mapping[str, str]) -> str: + return ",".join(f"{key}={value[key]}" for key in sorted(value)) + + +def _jobs_document(value: Any) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != {"schema_version", "run_id", "clients"}: + raise Gate14GcpError("host job observation schema is invalid") + clients = value["clients"] + if ( + value["schema_version"] != 1 + or not isinstance(value["run_id"], str) + or not isinstance(clients, dict) + or set(clients) != {"windows", "linux"} + ): + raise Gate14GcpError("host job observation binding is invalid") + for item in clients.values(): + if not isinstance(item, dict) or set(item) != controller._CLIENT_FIELDS: + raise Gate14GcpError("host job observation client is invalid") + return dict(value) + + +def load_jobs(path: Path, plan: controller.RunPlan) -> Mapping[str, Any]: + path = Path(path) + if not path.exists(): + return { + "schema_version": 1, + "run_id": plan.run_id, + "clients": { + platform: { + "job_state": "absent", + "attempt_ordinal": 0, + "evidence_digest": None, + } + for platform in ("windows", "linux") + }, + } + value = _jobs_document(controller._strict_json(controller._regular_bytes(path))) + if value["run_id"] != plan.run_id: + raise Gate14GcpError("host job observation run changed") + return value + + +def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise Gate14GcpError("output target is unsafe") + payload = (json.dumps(value, allow_nan=False, sort_keys=True, separators=(",", ":")) + os.linesep).encode("utf-8") + if len(payload) > MAX_JSON_BYTES: + raise Gate14GcpError("output exceeded its size bound") + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + temporary.unlink() + except OSError: + pass + raise + + +class GcpExecutor: + def __init__( + self, + plan: controller.RunPlan, + *, + runner: Runner = _default_runner, + clock: Callable[[], float] = time.time, + ) -> None: + self.plan = plan + self.runner = runner + self.clock = clock + + def _gcloud( + self, + *arguments: str, + timeout: int = 300, + check: bool = True, + ) -> CommandResult: + result = self.runner(("gcloud", *arguments, "--quiet"), timeout) + if check and result.returncode != 0: + raise Gate14GcpError("gcloud action failed") + return result + + def _gcloud_json(self, *arguments: str, timeout: int = 300) -> Any: + result = self._gcloud(*arguments, "--format=json", timeout=timeout) + return _json_bytes(result.stdout, "gcloud") + + def _check_auth(self) -> None: + result = self._gcloud( + "auth", + "list", + "--filter=status:ACTIVE", + "--format=value(account)", + timeout=60, + ) + accounts = [line for line in result.stdout.decode("utf-8", "strict").splitlines() if line.strip()] + if len(accounts) != 1: + raise Gate14GcpError("exactly one active native gcloud account is required") + project = self._gcloud_json("projects", "describe", self.plan.project, timeout=60) + if not isinstance(project, dict) or project.get("lifecycleState") != "ACTIVE": + raise Gate14GcpError("authorized GCP project is unavailable") + + def _describe(self, kind: str, name: str) -> Mapping[str, Any] | None: + result = self._gcloud( + "compute", + kind, + "describe", + name, + f"--project={self.plan.project}", + f"--zone={self.plan.zone}", + "--format=json", + timeout=60, + check=False, + ) + if result.returncode != 0: + text = result.stderr.decode("utf-8", "replace").casefold() + if "not found" in text or "was not found" in text: + return None + raise Gate14GcpError("provider inventory failed") + value = _json_bytes(result.stdout, f"{kind} inventory") + if not isinstance(value, dict): + raise Gate14GcpError("provider inventory item is invalid") + return value + + def _validate_disk(self, client: controller.ClientPlan, value: Mapping[str, Any]) -> None: + if value.get("name") != client.disk or value.get("status") not in {"READY", "CREATING"}: + raise Gate14GcpError("planned disk identity is invalid") + labels = value.get("labels") + if not isinstance(labels, dict) or any( + labels.get(key) != field for key, field in _labels(self.plan, client).items() + ): + raise Gate14GcpError("planned disk ownership is invalid") + source_image = value.get("sourceImage") + api_prefix = "https://www.googleapis.com/compute/v1/" + if isinstance(source_image, str) and source_image.startswith(api_prefix): + source_image = source_image.removeprefix(api_prefix) + expected_image = f"projects/{client.image_project}/global/images/{client.image}" + if ( + _basename(value.get("type")) != client.boot_disk_type + or source_image != expected_image + or str(value.get("sizeGb")) != str(client.boot_disk_gib) + ): + raise Gate14GcpError("planned disk shape is invalid") + + def _validate_instance(self, client: controller.ClientPlan, value: Mapping[str, Any]) -> None: + if value.get("name") != client.instance: + raise Gate14GcpError("planned instance identity is invalid") + labels = value.get("labels") + if not isinstance(labels, dict) or any( + labels.get(key) != field for key, field in _labels(self.plan, client).items() + ): + raise Gate14GcpError("planned instance ownership is invalid") + disks = value.get("disks") + accelerators = value.get("guestAccelerators") + metadata = _metadata(value) + no_service_account = "serviceAccounts" not in value or value["serviceAccounts"] == [] + if ( + not no_service_account + or _basename(value.get("machineType")) != client.machine_type + or not isinstance(disks, list) + or len(disks) != 1 + or _basename(disks[0].get("source") if isinstance(disks[0], dict) else None) != client.disk + or not isinstance(accelerators, list) + or len(accelerators) != 1 + or _basename(accelerators[0].get("acceleratorType") if isinstance(accelerators[0], dict) else None) + != "nvidia-l4" + or accelerators[0].get("acceleratorCount") != 1 + or metadata.get("communityai-run-id") != self.plan.run_id + or metadata.get("communityai-source-commit") != client.source_commit + or metadata.get("communityai-termination-unix") != str(client.termination_unix) + ): + raise Gate14GcpError("planned instance shape is invalid") + + def inventory(self, jobs: Mapping[str, Any]) -> Mapping[str, Any]: + self._check_auth() + jobs = _jobs_document(jobs) + if jobs["run_id"] != self.plan.run_id: + raise Gate14GcpError("host job observation run changed") + instances: dict[str, Any] = {} + disks: dict[str, bool] = {} + for client in (self.plan.windows, self.plan.linux): + instance = self._describe("instances", client.instance) + disk = self._describe("disks", client.disk) + if instance is not None: + self._validate_instance(client, instance) + if disk is None: + raise Gate14GcpError("planned instance lost its disk") + if disk is not None: + self._validate_disk(client, disk) + instances[client.instance] = { + "present": instance is not None, + "run_id": self.plan.run_id if instance is not None else None, + "source_commit": client.source_commit if instance is not None else None, + "termination_unix": client.termination_unix if instance is not None else None, + } + disks[client.disk] = disk is not None + + bootstrap = self._describe("instances", PROTECTED_INSTANCE) + firewalls = self._gcloud_json( + "compute", + "firewall-rules", + "list", + f"--project={self.plan.project}", + f"--filter=labels.communityai-run={self.plan.run_id}", + timeout=60, + ) + if not isinstance(firewalls, list) or firewalls: + raise Gate14GcpError("unexpected run-scoped firewall inventory") + l4 = self._gcloud_json( + "compute", + "instances", + "list", + f"--project={self.plan.project}", + "--filter=status=RUNNING AND guestAccelerators.acceleratorType:nvidia-l4", + timeout=60, + ) + if not isinstance(l4, list): + raise Gate14GcpError("accelerator inventory is invalid") + observation = { + "schema_version": 1, + "run_id": self.plan.run_id, + "observed_at_unix": int(self.clock()), + "instances": instances, + "disks": disks, + "clients": jobs["clients"], + "l4_usage": len(l4), + "protected_bootstrap_running": bootstrap is not None and bootstrap.get("status") == "RUNNING", + } + return controller.validate_observation(observation, self.plan) + + def preflight(self, jobs: Mapping[str, Any]) -> Mapping[str, Any]: + observation = self.inventory(jobs) + if any(item["present"] for item in observation["instances"].values()) or any(observation["disks"].values()): + raise Gate14GcpError("planned resources already exist") + if observation["l4_usage"] != 0 or any( + item["job_state"] != "absent" for item in observation["clients"].values() + ): + raise Gate14GcpError("preflight inventory is not clean") + accelerator = self._gcloud_json( + "compute", + "accelerator-types", + "describe", + "nvidia-l4", + f"--project={self.plan.project}", + f"--zone={self.plan.zone}", + timeout=60, + ) + if not isinstance(accelerator, dict) or accelerator.get("name") != "nvidia-l4": + raise Gate14GcpError("L4 capacity is unavailable in the authorized zone") + project_info = self._gcloud_json( + "compute", + "project-info", + "describe", + f"--project={self.plan.project}", + timeout=60, + ) + quotas = project_info.get("quotas") if isinstance(project_info, dict) else None + gpu_quota = ( + next( + (item for item in quotas if isinstance(item, dict) and item.get("metric") == "GPUS_ALL_REGIONS"), + None, + ) + if isinstance(quotas, list) + else None + ) + try: + quota_available = float(gpu_quota["limit"]) - float(gpu_quota.get("usage", 0)) + except (KeyError, TypeError, ValueError) as exc: + raise Gate14GcpError("global GPU quota is unavailable") from exc + if quota_available < 1: + raise Gate14GcpError("global GPU quota has no headroom") + for client in (self.plan.windows, self.plan.linux): + image = self._gcloud_json( + "compute", + "images", + "describe", + client.image, + f"--project={client.image_project}", + timeout=60, + ) + if not isinstance(image, dict) or image.get("status") != "READY": + raise Gate14GcpError("authorized image is unavailable") + return { + "schema_version": 1, + "run_id": self.plan.run_id, + "result": "passed", + "native_auth_revalidated": True, + "planned_resources_absent": True, + "l4_usage": 0, + "protected_bootstrap_running": True, + "maximum_estimate_usd": "44.00", + } + + def _create_client(self, client: controller.ClientPlan) -> None: + if self._describe("instances", client.instance) is not None or self._describe("disks", client.disk) is not None: + raise Gate14GcpError("fresh client resources are required") + labels = _label_argument(_labels(self.plan, client)) + self._gcloud( + "compute", + "disks", + "create", + client.disk, + f"--project={self.plan.project}", + f"--zone={self.plan.zone}", + f"--type={client.boot_disk_type}", + f"--size={client.boot_disk_gib}GB", + f"--image={client.image}", + f"--image-project={client.image_project}", + f"--labels={labels}", + timeout=900, + ) + try: + disk = self._describe("disks", client.disk) + if disk is None: + raise Gate14GcpError("planned disk creation was not observable") + self._validate_disk(client, disk) + self._gcloud( + "compute", + "instances", + "create", + client.instance, + f"--project={self.plan.project}", + f"--zone={self.plan.zone}", + f"--machine-type={client.machine_type}", + f"--disk=name={client.disk},boot=yes,auto-delete=yes", + f"--labels={labels}", + "--no-address", + "--no-service-account", + "--maintenance-policy=TERMINATE", + "--restart-on-failure", + f"--max-run-duration={client.max_run_seconds}s", + "--instance-termination-action=DELETE", + ( + "--metadata=" + f"communityai-run-id={self.plan.run_id}," + f"communityai-source-commit={client.source_commit}," + f"communityai-termination-unix={client.termination_unix}" + ), + timeout=900, + ) + instance = self._describe("instances", client.instance) + if instance is None: + raise Gate14GcpError("planned instance creation was not observable") + self._validate_instance(client, instance) + except BaseException: + self._delete_client(client) + raise + + def _delete_client(self, client: controller.ClientPlan) -> None: + instance = self._describe("instances", client.instance) + disk = self._describe("disks", client.disk) + if disk is not None: + self._validate_disk(client, disk) + if instance is not None: + self._validate_instance(client, instance) + self._gcloud( + "compute", + "instances", + "delete", + client.instance, + f"--project={self.plan.project}", + f"--zone={self.plan.zone}", + "--delete-disks=all", + timeout=900, + ) + disk = self._describe("disks", client.disk) + if disk is not None: + self._validate_disk(client, disk) + self._gcloud( + "compute", + "disks", + "delete", + client.disk, + f"--project={self.plan.project}", + f"--zone={self.plan.zone}", + timeout=900, + ) + if self._describe("instances", client.instance) is not None or self._describe("disks", client.disk) is not None: + raise Gate14GcpError("provider deletion was not verified") + + def execute( + self, + action: str, + *, + state: Mapping[str, Any], + jobs: Mapping[str, Any], + ) -> None: + try: + bound_state = controller.validate_state(state, self.plan) + observation = self.inventory(jobs) + reconciled = controller.reconcile(bound_state, observation, self.plan) + except controller.Gate14ControllerError as exc: + raise Gate14GcpError("controller execution precondition failed") from exc + if bound_state["next_action"] != action or reconciled["next_action"] != action: + raise Gate14GcpError("controller action is stale or unbound") + + if action == "start_windows": + self._create_client(self.plan.windows) + elif action == "start_linux": + self._create_client(self.plan.linux) + elif action == "delete_windows": + self._delete_client(self.plan.windows) + elif action == "delete_linux": + self._delete_client(self.plan.linux) + elif action == "cleanup_failure": + errors = 0 + for client in (self.plan.windows, self.plan.linux): + try: + self._delete_client(client) + except Gate14GcpError: + errors += 1 + if errors: + raise Gate14GcpError("provider cleanup is incomplete") + elif action != "none": + raise Gate14GcpError("controller action is not executable by GCP") + + +def _evidence_for(platform_name: str, args: argparse.Namespace) -> Path: + value = args.windows_evidence if platform_name == "windows" else args.linux_evidence + if value is None: + raise Gate14GcpError(f"{platform_name} evidence is required") + return value + + +def _challenge_for(platform_name: str, args: argparse.Namespace) -> Path: + value = args.windows_challenge if platform_name == "windows" else args.linux_challenge + if value is None: + raise Gate14GcpError(f"{platform_name} calibration challenge is required") + return value + + +def _cleanup_document( + plan: controller.RunPlan, + state_path: Path, + state: Mapping[str, Any], +) -> Mapping[str, Any]: + if state["phase"] != "CLEANED_PASS": + raise Gate14GcpError("passing cleanup evidence requires a cleaned pass") + terminal_payload = controller._regular_bytes(state_path) + return { + "schema_version": 1, + "scope": acceptance.CLEANUP_SCOPE, + "run_id": plan.run_id, + "result": "passed", + "provider": "GCP", + "controller_source_commit": plan.source_commit, + "provider_plan_digest": plan.provider_plan_digest, + "project": plan.project, + "zone": plan.zone, + "deleted_instances": list(plan.instances), + "deleted_disks": list(plan.disks), + "controller_terminal_state_sha256": "sha256:" + hashlib.sha256(terminal_payload).hexdigest(), + "native_auth_revalidated": True, + "expected_instances": 2, + "remaining_instances": 0, + "expected_disks": 2, + "remaining_disks": 0, + "remaining_firewalls": 0, + "l4_usage": 0, + "protected_bootstrap_running": True, + "product_processes_remaining": 0, + "temporary_credentials_remaining": 0, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("preflight", "observe", "step")) + parser.add_argument("--authorization", type=Path, required=True) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--state", type=Path) + parser.add_argument("--jobs", type=Path, required=True) + parser.add_argument("--observation-output", type=Path) + parser.add_argument("--windows-evidence", type=Path) + parser.add_argument("--linux-evidence", type=Path) + parser.add_argument("--windows-challenge", type=Path) + parser.add_argument("--linux-challenge", type=Path) + parser.add_argument("--cleanup-output", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + plan = controller.load_plan(args.authorization, args.ledger) + jobs = load_jobs(args.jobs, plan) + executor = GcpExecutor(plan) + if args.operation == "preflight": + result = executor.preflight(jobs) + else: + observation = executor.inventory(jobs) + if args.observation_output is not None: + _atomic_json(args.observation_output, observation) + if args.operation == "observe": + result = observation + else: + if args.state is None: + raise Gate14GcpError("state is required for a controller step") + state = ( + controller.load_state(args.state, plan) if args.state.exists() else controller.initial_state(plan) + ) + state = controller.reconcile(state, observation, plan) + action = state["next_action"] + if action in {"collect_windows", "collect_linux"}: + platform_name = action.removeprefix("collect_") + state = controller.collect_platform( + state, + plan, + platform_name, + _evidence_for(platform_name, args), + _challenge_for(platform_name, args), + ) + action = state["next_action"] + controller.save_state(args.state, state, plan) + executor.execute(action, state=state, jobs=jobs) + result = state + if state["phase"] == "CLEANED_PASS" and args.cleanup_output is not None: + _atomic_json(args.cleanup_output, _cleanup_document(plan, args.state, state)) + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + except ( + Gate14GcpError, + controller.Gate14ControllerError, + acceptance.Gate14EvidenceError, + ) as exc: + raise SystemExit(str(exc)) from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_hardware_acceptance.py b/scripts/gate14_hardware_acceptance.py new file mode 100644 index 000000000..c9c57e6da --- /dev/null +++ b/scripts/gate14_hardware_acceptance.py @@ -0,0 +1,969 @@ +"""Validate privacy-safe Gate 14 packaged hardware evidence. + +The real host probes are deliberately separate from this verifier. They may use private +paths and provider details while running, but only the strict bounded documents accepted +here can enter the evidence archive. The aggregate binds the exact controller source, +production packages, manifests, Gate 9 envelopes, device profiles, and final cleanup. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import stat +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping, Sequence + +SCHEMA_VERSION = 1 +PLATFORM_SCOPE = "gate14-packaged-hardware" +CLEANUP_SCOPE = "gate14-provider-cleanup" +AGGREGATE_SCOPE = "gate14-hardware-acceptance" +MAX_INPUT_BYTES = 262_144 +MAX_DURATION_SECONDS = 300.0 +MAX_CALIBRATION_CHALLENGE_SECONDS = 900 +MAX_CALIBRATION_SAMPLE_SECONDS = 120.0 +MAX_BYTES = 1 << 50 +MAX_BLOCKS = 512 +PROTECTED_INSTANCE = "communityai-bootstrap-1" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_NAME_RE = re.compile(r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?") +_PROJECT_RE = re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]") +_ZONE_RE = re.compile(r"[a-z]+(?:-[a-z0-9]+)+-[a-z]") +_OS_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._+()/-]{0,127}") + +MODEL_PROFILES = { + "Qwen3.5 2B": { + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "revision_commit": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "selected_artifact_count": 8, + "selected_artifact_bytes": 4_571_197_320, + "total_blocks": 24, + }, + "Gemma 4 E2B IT": { + "manifest_digest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "revision_commit": "3e22461f65e89153144f8adb70e3b8c2cc9845a7", + "selected_artifact_count": 5, + "selected_artifact_bytes": 10_278_818_149, + "total_blocks": 35, + }, +} +EXPECTED_PLATFORM_MODELS = {"windows": "Qwen3.5 2B", "linux": "Gemma 4 E2B IT"} +EXPECTED_PLATFORM_OS = {"windows": "Windows Server 2022", "linux": "Ubuntu 24.04"} +EXPECTED_GATE9_ENVELOPES = { + "windows": "sha256:cd68afb67d9b0f3cb8c82db0d3314ad89b558c20880998ea4d8c4493e9f4bc9f", + "linux": "sha256:2eb0bcf6419ba085665fad34310453a1b9dc2e89d90e9177f41566df012996c8", +} +EXPECTED_GATE13_EVIDENCE_SHA256 = "sha256:ad4f892f4af9a9aee0dd428d74695981d0cca6241f79c0270c9fcea3a229b72e" + +_DOCUMENT_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "result", + "source_commit", + "gate13_evidence_sha256", + "package", + "model", + "hardware", + "cache", + "placement", + "limits", + "calibration_challenge", + "suspensions", + "recovery", + "pause", + "restart", + "unsupported_telemetry", + "privacy", + "qualification_temporaries_removed", +} +_PACKAGE_FIELDS = { + "source_commit", + "archive_sha256", + "archive_bytes", + "release_metadata_sha256", +} +_MODEL_FIELDS = { + "id", + "manifest_digest", + "revision_commit", + "gate9_envelope_sha256", + "selected_artifact_count", + "selected_artifact_bytes", + "total_blocks", +} +_HARDWARE_FIELDS = { + "os_name", + "accelerator", + "accelerator_count", + "accelerator_memory_bytes", +} +_CACHE_FIELDS = { + "verified_bytes_before", + "verified_bytes_after", + "transfer_bytes_during_gate", + "digest_mismatch_count", + "forbidden_model_acquired", +} +_PLACEMENT_FIELDS = { + "automatic", + "worker_count", + "block_start", + "block_end", + "intent_published", + "remote_acknowledged", +} +_LIMIT_FIELDS = { + "disk_bytes", + "vram_bytes", + "bandwidth_mbps", + "power_watts", + "schedule_timezone", + "resource_limit_count", + "configured_and_resolved_match", + "low_vram_rejected", +} +_SUSPENSION_FIELDS = { + "kind", + "suspended", + "resumed", + "desired_intent_preserved", + "worker_count_during", + "duration_seconds", + "calibration", +} +_CALIBRATION_CHALLENGE_FIELDS = { + "challenge_sha256", + "controller_state_revision", + "issued_at_unix", + "expires_at_unix", +} +_CALIBRATION_FIELDS = { + "measurement_source", + "measurement_scope", + "sample_count", + "sample_interval_seconds", + "baseline_value", + "configured_limit", + "trigger_value", + "resume_value", + "challenge_sha256", + "sample_started_at_unix", + "sample_ended_at_unix", +} +_RECOVERY_FIELDS = { + "worker_crash_observed", + "worker_restarted", + "restart_seconds", + "previous_worker_absent", + "manifest_unchanged", + "automatic_block_range_valid", + "desired_intent_preserved", +} +_PAUSE_FIELDS = { + "requested", + "completed", + "duration_seconds", + "worker_count_after", + "descendant_count_after", +} +_RESTART_FIELDS = { + "node_restarted", + "policy_persisted", + "desired_intent_persisted", + "worker_resumed", + "duration_seconds", + "cache_reused", +} +_UNSUPPORTED_FIELDS = { + "device", + "configured_limit", + "start_rejected", + "reason_code", + "private_detail_retained", +} +_PRIVACY_FIELDS = { + "prompt_retained", + "response_retained", + "token_identifiers_retained", + "credentials_retained", + "paths_retained", + "endpoints_retained", + "provider_output_retained", +} +_CLEANUP_FIELDS = { + "schema_version", + "scope", + "run_id", + "result", + "provider", + "controller_source_commit", + "provider_plan_digest", + "project", + "zone", + "deleted_instances", + "deleted_disks", + "controller_terminal_state_sha256", + "native_auth_revalidated", + "expected_instances", + "remaining_instances", + "expected_disks", + "remaining_disks", + "remaining_firewalls", + "l4_usage", + "protected_bootstrap_running", + "product_processes_remaining", + "temporary_credentials_remaining", +} +_TERMINAL_STATE_FIELDS = { + "schema_version", + "run_id", + "authorization_sha256", + "provider_plan_digest", + "revision", + "phase", + "failure_code", + "windows_evidence_digest", + "linux_evidence_digest", + "windows_challenge_sha256", + "linux_challenge_sha256", + "windows_challenge_consumed", + "linux_challenge_consumed", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + "next_action", +} +_AUTH_FIELDS = { + "schema_version", + "gate", + "result", + "run_id", + "source_commit", + "provider_plan_digest", + "provider_plan", + "authorization", + "prohibited", +} +_AUTHORIZATION_FIELDS = { + "combined_cloud_ceiling_usd", + "ledger_committed_before_run_usd", + "maximum_estimate_usd", + "remaining_after_run_maximum_usd", + "reservation_recorded", + "native_auth_revalidated", + "provisioning_authorized_after_fail_closed_preflight", +} +_PLAN_FIELDS = {"project", "zone", "clients", "sequencing"} +_CLIENT_PLAN_FIELDS = { + "platform", + "instance", + "disk", + "source_commit", + "termination_unix", + "package_sha256", + "model_id", + "manifest_digest", + "machine_type", + "image_project", + "image", + "boot_disk_gib", + "boot_disk_type", + "service_account_disabled", + "max_run_seconds", + "termination_action", +} +_SEQUENCING_FIELDS = { + "clients_may_run_concurrently", + "windows_first", + "fresh_host_per_platform", +} + + +class Gate14EvidenceError(ValueError): + """A Gate 14 input was malformed, unsafe, incomplete, or inconsistent.""" + + +def _reject_constant(_value: str) -> None: + raise Gate14EvidenceError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14EvidenceError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14EvidenceError("required evidence is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or path.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or not 1 <= metadata.st_size <= MAX_INPUT_BYTES + ): + raise Gate14EvidenceError("required evidence is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise Gate14EvidenceError("required evidence is unreadable") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_INPUT_BYTES: + raise Gate14EvidenceError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14EvidenceError("invalid JSON") from exc + if not isinstance(value, dict): + raise Gate14EvidenceError("JSON root is invalid") + return value + + +def _mapping(value: Any, fields: set[str]) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise Gate14EvidenceError("evidence schema is invalid") + return value + + +def _true(value: Any) -> None: + if value is not True: + raise Gate14EvidenceError("required proof is absent") + + +def _false(value: Any) -> None: + if value is not False: + raise Gate14EvidenceError("forbidden retention or result is present") + + +def _integer(value: Any, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise Gate14EvidenceError("integer evidence is invalid") + return value + + +def _number(value: Any, minimum: float, maximum: float) -> float: + if type(value) not in (int, float): + raise Gate14EvidenceError("numeric evidence is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise Gate14EvidenceError("numeric evidence is invalid") + return rendered + + +def _string(value: Any, pattern: re.Pattern[str]) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise Gate14EvidenceError("string evidence is invalid") + return value + + +def _validate_package(value: Any, source_commit: str) -> Mapping[str, Any]: + package = _mapping(value, _PACKAGE_FIELDS) + if package["source_commit"] != source_commit: + raise Gate14EvidenceError("package source is inconsistent") + _string(package["source_commit"], _COMMIT_RE) + _string(package["archive_sha256"], _DIGEST_RE) + _integer(package["archive_bytes"], 1, 8 * 1024**3) + _string(package["release_metadata_sha256"], _DIGEST_RE) + return package + + +def _validate_model(value: Any, platform: str) -> Mapping[str, Any]: + model = _mapping(value, _MODEL_FIELDS) + model_id = model["id"] + if model_id != EXPECTED_PLATFORM_MODELS[platform]: + raise Gate14EvidenceError("platform model is invalid") + profile = MODEL_PROFILES[model_id] + for field in ( + "manifest_digest", + "revision_commit", + "selected_artifact_count", + "selected_artifact_bytes", + "total_blocks", + ): + if model[field] != profile[field]: + raise Gate14EvidenceError("model identity is inconsistent") + if model["gate9_envelope_sha256"] != EXPECTED_GATE9_ENVELOPES[platform]: + raise Gate14EvidenceError("Gate 9 envelope is inconsistent") + return model + + +def _validate_hardware(value: Any, platform: str) -> Mapping[str, Any]: + hardware = _mapping(value, _HARDWARE_FIELDS) + _string(hardware["os_name"], _OS_RE) + if hardware["os_name"] != EXPECTED_PLATFORM_OS[platform]: + raise Gate14EvidenceError("platform operating system is inconsistent") + if hardware["accelerator"] != "NVIDIA L4": + raise Gate14EvidenceError("real L4 hardware is required") + if hardware["accelerator_count"] != 1: + raise Gate14EvidenceError("exactly one accelerator is required") + _integer(hardware["accelerator_memory_bytes"], 20 * 1024**3, 32 * 1024**3) + return hardware + + +def _validate_cache(value: Any, selected_bytes: int) -> None: + cache = _mapping(value, _CACHE_FIELDS) + if ( + cache["verified_bytes_before"] != selected_bytes + or cache["verified_bytes_after"] != selected_bytes + or cache["transfer_bytes_during_gate"] != 0 + or cache["digest_mismatch_count"] != 0 + ): + raise Gate14EvidenceError("verified cache reuse is inconsistent") + _false(cache["forbidden_model_acquired"]) + + +def _validate_placement(value: Any, total_blocks: int) -> tuple[int, int]: + placement = _mapping(value, _PLACEMENT_FIELDS) + _true(placement["automatic"]) + if placement["worker_count"] != 1: + raise Gate14EvidenceError("exactly one automatic worker is required") + start = _integer(placement["block_start"], 0, total_blocks - 1) + end = _integer(placement["block_end"], 1, total_blocks) + if end <= start: + raise Gate14EvidenceError("automatic block range is empty") + _true(placement["intent_published"]) + _true(placement["remote_acknowledged"]) + return start, end + + +def _validate_limits(value: Any, selected_bytes: int, accelerator_memory: int) -> None: + limits = _mapping(value, _LIMIT_FIELDS) + disk = _integer(limits["disk_bytes"], selected_bytes, MAX_BYTES) + vram = _integer(limits["vram_bytes"], 1, accelerator_memory) + if disk < selected_bytes or vram >= accelerator_memory: + raise Gate14EvidenceError("resource ceilings are not bounded") + _number(limits["bandwidth_mbps"], 0.001, 1_000_000.0) + _number(limits["power_watts"], 0.001, 1_000.0) + if limits["schedule_timezone"] != "UTC" or limits["resource_limit_count"] != 5: + raise Gate14EvidenceError("all five resource classes are required") + _true(limits["configured_and_resolved_match"]) + _true(limits["low_vram_rejected"]) + + +def _validate_calibration_challenge(value: Any) -> Mapping[str, Any]: + challenge = _mapping(value, _CALIBRATION_CHALLENGE_FIELDS) + _string(challenge["challenge_sha256"], _DIGEST_RE) + _integer(challenge["controller_state_revision"]) + issued = _integer(challenge["issued_at_unix"]) + expires = _integer(challenge["expires_at_unix"]) + if not 60 <= expires - issued <= MAX_CALIBRATION_CHALLENGE_SECONDS: + raise Gate14EvidenceError("calibration challenge lifetime is invalid") + return challenge + + +def _validate_suspensions( + value: Any, + limits: Mapping[str, Any], + challenge: Mapping[str, Any], +) -> None: + if not isinstance(value, list) or len(value) != 3: + raise Gate14EvidenceError("three suspension classes are required") + expected_calibration = { + "bandwidth": ("host-network-counters", "aggregate-host-network", limits["bandwidth_mbps"]), + "power": ("nvidia-nvml-device-power", "selected-nvidia-l4-device", limits["power_watts"]), + "schedule": ("utc-policy-clock", "utc-schedule-policy", 0.5), + } + seen: set[str] = set() + for raw in value: + item = _mapping(raw, _SUSPENSION_FIELDS) + kind = item["kind"] + if kind not in expected_calibration or kind in seen: + raise Gate14EvidenceError("suspension class is invalid") + seen.add(kind) + _true(item["suspended"]) + _true(item["resumed"]) + _true(item["desired_intent_preserved"]) + if item["worker_count_during"] != 0: + raise Gate14EvidenceError("worker remained active while suspended") + _number(item["duration_seconds"], 0.0, MAX_DURATION_SECONDS) + calibration = _mapping(item["calibration"], _CALIBRATION_FIELDS) + source, scope, expected_limit = expected_calibration[kind] + if calibration["measurement_source"] != source or calibration["measurement_scope"] != scope: + raise Gate14EvidenceError("suspension measurement source is invalid") + sample_count = _integer(calibration["sample_count"], 3, 10_000) + sample_interval = _number(calibration["sample_interval_seconds"], 0.05, 30.0) + baseline = _number(calibration["baseline_value"], 0.0, 1_000_000.0) + configured = _number(calibration["configured_limit"], 0.001, 1_000_000.0) + trigger = _number(calibration["trigger_value"], 0.0, 1_000_000.0) + resume = _number(calibration["resume_value"], 0.0, 1_000_000.0) + if calibration["challenge_sha256"] != challenge["challenge_sha256"]: + raise Gate14EvidenceError("calibration challenge binding is invalid") + started = _number(calibration["sample_started_at_unix"], 0.0, 4_102_444_800.0) + ended = _number(calibration["sample_ended_at_unix"], 0.0, 4_102_444_800.0) + sample_span = ended - started + if ( + started < challenge["issued_at_unix"] + or ended > challenge["expires_at_unix"] + or sample_span < (sample_count - 1) * sample_interval + or sample_span > MAX_CALIBRATION_SAMPLE_SECONDS + ): + raise Gate14EvidenceError("calibration measurement window is stale or invalid") + if configured != expected_limit: + raise Gate14EvidenceError("suspension limit does not match resolved policy") + if kind == "schedule": + if (baseline, trigger, resume) != (1.0, 0.0, 1.0): + raise Gate14EvidenceError("schedule trigger calibration is invalid") + elif not (baseline < configured < trigger and resume < configured): + raise Gate14EvidenceError("physical trigger calibration did not cross its limit") + + +def _validate_recovery(value: Any) -> None: + recovery = _mapping(value, _RECOVERY_FIELDS) + for field in _RECOVERY_FIELDS - {"restart_seconds"}: + _true(recovery[field]) + _number(recovery["restart_seconds"], 0.0, MAX_DURATION_SECONDS) + + +def _validate_pause(value: Any) -> None: + pause = _mapping(value, _PAUSE_FIELDS) + _true(pause["requested"]) + _true(pause["completed"]) + _number(pause["duration_seconds"], 0.0, MAX_DURATION_SECONDS) + if pause["worker_count_after"] != 0 or pause["descendant_count_after"] != 0: + raise Gate14EvidenceError("pause cleanup is incomplete") + + +def _validate_restart(value: Any) -> None: + restart = _mapping(value, _RESTART_FIELDS) + for field in _RESTART_FIELDS - {"duration_seconds"}: + _true(restart[field]) + _number(restart["duration_seconds"], 0.0, MAX_DURATION_SECONDS) + + +def _validate_unsupported(value: Any) -> None: + unsupported = _mapping(value, _UNSUPPORTED_FIELDS) + if ( + unsupported["device"] != "cpu" + or unsupported["configured_limit"] != "power_watts" + or unsupported["reason_code"] != "power-telemetry-unavailable" + ): + raise Gate14EvidenceError("unsupported telemetry classification is invalid") + _true(unsupported["start_rejected"]) + _false(unsupported["private_detail_retained"]) + + +def _validate_privacy(value: Any) -> None: + privacy = _mapping(value, _PRIVACY_FIELDS) + for field in _PRIVACY_FIELDS: + _false(privacy[field]) + + +def validate_platform_document(value: Mapping[str, Any]) -> Mapping[str, Any]: + document = _mapping(value, _DOCUMENT_FIELDS) + if ( + document["schema_version"] != SCHEMA_VERSION + or document["scope"] != PLATFORM_SCOPE + or document["result"] != "passed" + ): + raise Gate14EvidenceError("platform evidence header is invalid") + run_id = _string(document["run_id"], _RUN_RE) + platform = document["platform"] + if platform not in EXPECTED_PLATFORM_MODELS: + raise Gate14EvidenceError("platform is invalid") + source_commit = _string(document["source_commit"], _COMMIT_RE) + gate13_evidence_sha256 = _string(document["gate13_evidence_sha256"], _DIGEST_RE) + if gate13_evidence_sha256 != EXPECTED_GATE13_EVIDENCE_SHA256: + raise Gate14EvidenceError("Gate 13 lifecycle evidence is inconsistent") + package = _validate_package(document["package"], source_commit) + model = _validate_model(document["model"], platform) + hardware = _validate_hardware(document["hardware"], platform) + _validate_cache(document["cache"], model["selected_artifact_bytes"]) + block_start, block_end = _validate_placement(document["placement"], model["total_blocks"]) + _validate_limits( + document["limits"], + model["selected_artifact_bytes"], + hardware["accelerator_memory_bytes"], + ) + challenge = _validate_calibration_challenge(document["calibration_challenge"]) + _validate_suspensions(document["suspensions"], document["limits"], challenge) + _validate_recovery(document["recovery"]) + _validate_pause(document["pause"]) + _validate_restart(document["restart"]) + _validate_unsupported(document["unsupported_telemetry"]) + _validate_privacy(document["privacy"]) + _true(document["qualification_temporaries_removed"]) + return { + "run_id": run_id, + "platform": platform, + "source_commit": source_commit, + "gate13_evidence_sha256": gate13_evidence_sha256, + "package_sha256": package["archive_sha256"], + "model_id": model["id"], + "manifest_digest": model["manifest_digest"], + "gate9_envelope_sha256": model["gate9_envelope_sha256"], + "accelerator": hardware["accelerator"], + "calibration_challenge_sha256": challenge["challenge_sha256"], + "block_start": block_start, + "block_end": block_end, + } + + +def _digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _resource_names(value: Sequence[str], field: str) -> tuple[str, str]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence) or len(value) != 2: + raise Gate14EvidenceError(f"{field} inventory is invalid") + names = tuple(_string(item, _NAME_RE) for item in value) + if len(set(names)) != 2: + raise Gate14EvidenceError(f"{field} inventory is not unique") + if PROTECTED_INSTANCE in names: + raise Gate14EvidenceError("protected resource is targeted") + return names + + +def validate_authorization_document( + value: Mapping[str, Any], + *, + run_id: str, + source_commit: str, + provider_plan_digest: str, + project: str, + zone: str, + expected_instances: Sequence[str], + expected_disks: Sequence[str], + package_sha256: Mapping[str, str], +) -> Mapping[str, Any]: + authorization = _mapping(value, _AUTH_FIELDS) + if ( + authorization["schema_version"] != SCHEMA_VERSION + or authorization["gate"] != 14 + or authorization["result"] != "authorized" + or authorization["run_id"] != run_id + or authorization["source_commit"] != source_commit + or authorization["provider_plan_digest"] != provider_plan_digest + ): + raise Gate14EvidenceError("authorization binding is invalid") + + provider_plan = _mapping(authorization["provider_plan"], _PLAN_FIELDS) + canonical_plan = json.dumps(provider_plan, sort_keys=True, separators=(",", ":")).encode("utf-8") + if ( + _digest(canonical_plan) != provider_plan_digest + or provider_plan["project"] != project + or provider_plan["zone"] != zone + ): + raise Gate14EvidenceError("authorized provider plan is inconsistent") + sequencing = _mapping(provider_plan["sequencing"], _SEQUENCING_FIELDS) + if ( + sequencing["clients_may_run_concurrently"] is not False + or sequencing["windows_first"] is not True + or sequencing["fresh_host_per_platform"] is not True + ): + raise Gate14EvidenceError("authorized sequencing is inconsistent") + clients = provider_plan["clients"] + if not isinstance(clients, list) or len(clients) != 2: + raise Gate14EvidenceError("authorized client plan is invalid") + by_platform = { + item.get("platform"): item + for item in clients + if isinstance(item, dict) and isinstance(item.get("platform"), str) + } + if set(by_platform) != {"windows", "linux"}: + raise Gate14EvidenceError("authorized platform plan is invalid") + for index, platform in enumerate(("windows", "linux")): + client = _mapping(by_platform[platform], _CLIENT_PLAN_FIELDS) + model_id = EXPECTED_PLATFORM_MODELS[platform] + expected_image_project = "windows-cloud" if platform == "windows" else "ubuntu-os-cloud" + expected_image_pattern = ( + re.compile(r"windows-server-2022-dc-v[0-9]{8}") + if platform == "windows" + else re.compile(r"ubuntu-2404-noble-amd64-v[0-9]{8}") + ) + if ( + client["platform"] != platform + or client["instance"] != expected_instances[index] + or client["disk"] != expected_disks[index] + or client["source_commit"] != source_commit + or client["package_sha256"] != package_sha256[platform] + or client["model_id"] != model_id + or client["manifest_digest"] != MODEL_PROFILES[model_id]["manifest_digest"] + or client["machine_type"] != "g2-standard-8" + or client["image_project"] != expected_image_project + or client["boot_disk_type"] != "pd-balanced" + or client["service_account_disabled"] is not True + or client["termination_action"] != "DELETE" + ): + raise Gate14EvidenceError("authorized client binding is inconsistent") + _string(client["image"], expected_image_pattern) + _integer(client["boot_disk_gib"], 100, 200) + _integer(client["max_run_seconds"], 1_800, 14_400) + _integer(client["termination_unix"], 1) + + cost = _mapping(authorization["authorization"], _AUTHORIZATION_FIELDS) + try: + ceiling = Decimal(str(cost["combined_cloud_ceiling_usd"])) + before = Decimal(str(cost["ledger_committed_before_run_usd"])) + maximum = Decimal(str(cost["maximum_estimate_usd"])) + remaining = Decimal(str(cost["remaining_after_run_maximum_usd"])) + except (InvalidOperation, TypeError, ValueError) as exc: + raise Gate14EvidenceError("cost authorization is invalid") from exc + if ( + (ceiling, before, maximum, remaining) + != (Decimal("100.00"), Decimal("56.00"), Decimal("44.00"), Decimal("0.00")) + or not all(item.is_finite() for item in (ceiling, before, maximum, remaining)) + or cost["reservation_recorded"] is not True + or cost["native_auth_revalidated"] is not True + or cost["provisioning_authorized_after_fail_closed_preflight"] is not True + ): + raise Gate14EvidenceError("cost authorization is inconsistent") + prohibited = authorization["prohibited"] + if ( + not isinstance(prohibited, dict) + or set(prohibited) != {"credits", "macos", "fly_gpu"} + or any(type(item) is not int or item != 0 for item in prohibited.values()) + ): + raise Gate14EvidenceError("prohibited work is present") + return authorization + + +def validate_terminal_state( + value: Mapping[str, Any], + *, + run_id: str, + authorization_sha256: str, + provider_plan_digest: str, + windows_evidence_sha256: str, + linux_evidence_sha256: str, + windows_challenge_sha256: str, + linux_challenge_sha256: str, +) -> Mapping[str, Any]: + state = _mapping(value, _TERMINAL_STATE_FIELDS) + if ( + state["schema_version"] != SCHEMA_VERSION + or state["run_id"] != run_id + or state["authorization_sha256"] != authorization_sha256 + or state["provider_plan_digest"] != provider_plan_digest + or state["phase"] != "CLEANED_PASS" + or state["failure_code"] is not None + or state["windows_evidence_digest"] != windows_evidence_sha256 + or state["linux_evidence_digest"] != linux_evidence_sha256 + or state["windows_challenge_sha256"] != windows_challenge_sha256 + or state["linux_challenge_sha256"] != linux_challenge_sha256 + or state["windows_challenge_consumed"] is not True + or state["linux_challenge_consumed"] is not True + or state["windows_consumed"] is not True + or state["linux_consumed"] is not True + or state["cleanup_verified"] is not True + or state["next_action"] != "none" + ): + raise Gate14EvidenceError("controller terminal state is inconsistent") + _integer(state["revision"], 1) + return state + + +def validate_cleanup_document( + value: Mapping[str, Any], + *, + run_id: str, + controller_source_commit: str, + provider_plan_digest: str, + project: str, + zone: str, + expected_instances: Sequence[str], + expected_disks: Sequence[str], + terminal_state_sha256: str, +) -> Mapping[str, Any]: + cleanup = _mapping(value, _CLEANUP_FIELDS) + if ( + cleanup["schema_version"] != SCHEMA_VERSION + or cleanup["scope"] != CLEANUP_SCOPE + or cleanup["run_id"] != run_id + or cleanup["result"] != "passed" + or cleanup["provider"] != "GCP" + or cleanup["controller_source_commit"] != controller_source_commit + or cleanup["provider_plan_digest"] != provider_plan_digest + or cleanup["project"] != project + or cleanup["zone"] != zone + or cleanup["deleted_instances"] != list(expected_instances) + or cleanup["deleted_disks"] != list(expected_disks) + or cleanup["controller_terminal_state_sha256"] != terminal_state_sha256 + ): + raise Gate14EvidenceError("cleanup evidence binding is invalid") + _true(cleanup["native_auth_revalidated"]) + if cleanup["expected_instances"] != 2 or cleanup["expected_disks"] != 2: + raise Gate14EvidenceError("cleanup target count is invalid") + for field in ( + "remaining_instances", + "remaining_disks", + "remaining_firewalls", + "l4_usage", + "product_processes_remaining", + "temporary_credentials_remaining", + ): + if cleanup[field] != 0 or type(cleanup[field]) is not int: + raise Gate14EvidenceError("cleanup is incomplete") + _true(cleanup["protected_bootstrap_running"]) + return cleanup + + +def validate_files( + windows_path: Path, + linux_path: Path, + cleanup_path: Path, + controller_source_commit: str, + *, + provider_plan_digest: str, + project: str, + zone: str, + expected_instances: Sequence[str], + expected_disks: Sequence[str], + terminal_state_path: Path, + authorization_path: Path, +) -> Mapping[str, Any]: + controller_source_commit = _string(controller_source_commit, _COMMIT_RE) + provider_plan_digest = _string(provider_plan_digest, _DIGEST_RE) + project = _string(project, _PROJECT_RE) + zone = _string(zone, _ZONE_RE) + expected_instances = _resource_names(expected_instances, "instance") + expected_disks = _resource_names(expected_disks, "disk") + payloads = { + "windows": _regular_bytes(windows_path), + "linux": _regular_bytes(linux_path), + "cleanup": _regular_bytes(cleanup_path), + "terminal_state": _regular_bytes(terminal_state_path), + "authorization": _regular_bytes(authorization_path), + } + windows = validate_platform_document(_strict_json(payloads["windows"])) + linux = validate_platform_document(_strict_json(payloads["linux"])) + if windows["platform"] != "windows" or linux["platform"] != "linux": + raise Gate14EvidenceError("platform evidence ordering is invalid") + if windows["run_id"] != linux["run_id"]: + raise Gate14EvidenceError("run identity is inconsistent") + if windows["source_commit"] != linux["source_commit"] or windows["source_commit"] != controller_source_commit: + raise Gate14EvidenceError("package source identity is inconsistent") + authorization_sha256 = _digest(payloads["authorization"]) + validate_authorization_document( + _strict_json(payloads["authorization"]), + run_id=windows["run_id"], + source_commit=controller_source_commit, + provider_plan_digest=provider_plan_digest, + project=project, + zone=zone, + expected_instances=expected_instances, + expected_disks=expected_disks, + package_sha256={ + "windows": windows["package_sha256"], + "linux": linux["package_sha256"], + }, + ) + terminal_state_sha256 = _digest(payloads["terminal_state"]) + validate_terminal_state( + _strict_json(payloads["terminal_state"]), + run_id=windows["run_id"], + authorization_sha256=authorization_sha256, + provider_plan_digest=provider_plan_digest, + windows_evidence_sha256=_digest(payloads["windows"]), + linux_evidence_sha256=_digest(payloads["linux"]), + windows_challenge_sha256=windows["calibration_challenge_sha256"], + linux_challenge_sha256=linux["calibration_challenge_sha256"], + ) + cleanup = validate_cleanup_document( + _strict_json(payloads["cleanup"]), + run_id=windows["run_id"], + controller_source_commit=controller_source_commit, + provider_plan_digest=provider_plan_digest, + project=project, + zone=zone, + expected_instances=expected_instances, + expected_disks=expected_disks, + terminal_state_sha256=terminal_state_sha256, + ) + return { + "schema_version": SCHEMA_VERSION, + "scope": AGGREGATE_SCOPE, + "run_id": windows["run_id"], + "result": "passed", + "controller_source_commit": controller_source_commit, + "package_source_commit": windows["source_commit"], + "provider_plan_digest": provider_plan_digest, + "authorization_sha256": authorization_sha256, + "platforms": [ + { + **windows, + "evidence_sha256": _digest(payloads["windows"]), + }, + { + **linux, + "evidence_sha256": _digest(payloads["linux"]), + }, + ], + "cleanup": { + "evidence_sha256": _digest(payloads["cleanup"]), + "provider": cleanup["provider"], + "project": project, + "zone": zone, + "deleted_instances": list(expected_instances), + "deleted_disks": list(expected_disks), + "terminal_state_sha256": terminal_state_sha256, + "resource_absence_proved": True, + "protected_bootstrap_running": True, + }, + "credits_in_scope": False, + "macos_in_scope": False, + "privacy_safe": True, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--windows", type=Path, required=True) + parser.add_argument("--linux", type=Path, required=True) + parser.add_argument("--cleanup", type=Path, required=True) + parser.add_argument("--controller-state", type=Path, required=True) + parser.add_argument("--authorization", type=Path, required=True) + parser.add_argument("--controller-source-commit", required=True) + parser.add_argument("--provider-plan-digest", required=True) + parser.add_argument("--project", required=True) + parser.add_argument("--zone", required=True) + parser.add_argument("--instances", nargs=2, required=True) + parser.add_argument("--disks", nargs=2, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + result = validate_files( + args.windows, + args.linux, + args.cleanup, + args.controller_source_commit, + provider_plan_digest=args.provider_plan_digest, + project=args.project, + zone=args.zone, + expected_instances=args.instances, + expected_disks=args.disks, + terminal_state_path=args.controller_state, + authorization_path=args.authorization, + ) + except Gate14EvidenceError as exc: + raise SystemExit(str(exc)) from exc + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_host_job.py b/scripts/gate14_host_job.py new file mode 100644 index 000000000..cf74a40c0 --- /dev/null +++ b/scripts/gate14_host_job.py @@ -0,0 +1,154 @@ +"""Gate 14 specialization of the durable native qualification host job. + +The shared Gate 13 adapter owns the process, Scheduled Task, and systemd safety +mechanics. It is loaded into a private module namespace here so Gate 14 can +supply a separate identity, roots, ordinary host user, lifecycle binding, and +strict platform-evidence validator without mutating Gate 13 runtime state. +""" + +from __future__ import annotations + +import hashlib +import re +import stat +import sys +import time +import types +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate14_hardware_acceptance as acceptance + +_EXPECTED_SHARED_CORE_SHA256 = "c4a94fda88f25ad0bbab6e500fada7bd78f63a6cad34063fe71a363cf5638bd4" +_MAX_SHARED_CORE_BYTES = 8 * 1024 * 1024 + + +def _verified_shared_core() -> tuple[Path, bytes]: + candidate = Path(__file__).with_name("gate13_host_job.py") + try: + metadata = candidate.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or candidate.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise ImportError("Gate 14 host-job core is unsafe") + if not 1 <= metadata.st_size <= _MAX_SHARED_CORE_BYTES: + raise ImportError("Gate 14 host-job core size is invalid") + payload = candidate.read_bytes() + except OSError as exc: + raise ImportError("Gate 14 host-job core is unavailable") from exc + canonical = payload.replace(b"\r\n", b"\n") + if b"\r" in canonical or hashlib.sha256(canonical).hexdigest() != _EXPECTED_SHARED_CORE_SHA256: + raise ImportError("Gate 14 host-job core digest changed") + return candidate.resolve(), canonical + + +_SHARED_CORE_PATH, _SHARED_CORE_SOURCE = _verified_shared_core() +_CORE_MODULE_NAME = "_communityai_gate14_host_job_core" +core = types.ModuleType(_CORE_MODULE_NAME) +core.__file__ = str(_SHARED_CORE_PATH) +core.__package__ = "" +sys.modules[_CORE_MODULE_NAME] = core +exec(compile(_SHARED_CORE_SOURCE, str(_SHARED_CORE_PATH), "exec"), core.__dict__) +del _SHARED_CORE_SOURCE + +GATE_NAME = "gate14" +HOST_ROOTS = { + "windows": Path(r"C:\Gate14Run"), + "linux": Path("/qualification/gate14"), +} +HOST_PYTHON = { + "windows": Path(r"C:\Gate14Python\python.exe"), + "linux": Path("/usr/bin/python3"), +} +ADAPTER_PATH = Path(__file__).resolve() +LINUX_HOST_USER = "gate14" +LINUX_HOME = "/home/gate14" +LINUX_RUNTIME_DIR = "/qualification/gate14/runtime" +LIFECYCLE_CONFIG_NAMES = { + "windows": "gate14-lifecycle.json", + "linux": "gate14-lifecycle.json", +} +_JOB_RE = re.compile(r"communityai-gate14-[a-z0-9-]{1,63}-(?:windows|linux)") + +HostJobConfig = core.HostJobConfig +HostJobError = core.HostJobError +Runner = core.Runner + + +def _lifecycle_run_id(run_id: str, _platform: str) -> str: + return run_id + + +def _validate_platform_evidence(payload: bytes) -> Mapping[str, Any]: + document = acceptance._strict_json(payload) + acceptance.validate_platform_document(document) + return { + "run_id": document["run_id"], + "platform": document["platform"], + "source_commit": document["source_commit"], + } + + +def _configure_core() -> None: + core.HOST_ROOTS = HOST_ROOTS + core.HOST_PYTHON = HOST_PYTHON + core.ADAPTER_PATH = ADAPTER_PATH + core.GATE_NAME = GATE_NAME + core.LINUX_HOST_USER = LINUX_HOST_USER + core.LINUX_HOME = LINUX_HOME + core.LINUX_RUNTIME_DIR = LINUX_RUNTIME_DIR + core.LIFECYCLE_CONFIG_NAMES = LIFECYCLE_CONFIG_NAMES + core.LIFECYCLE_RUN_ID_BUILDER = _lifecycle_run_id + core._JOB_RE = _JOB_RE + core.MAX_EVIDENCE_BYTES = acceptance.MAX_INPUT_BYTES + core.EVIDENCE_VALIDATOR = _validate_platform_evidence + + +def load_config(path: Path) -> HostJobConfig: + _configure_core() + return core.load_config(path) + + +def execute( + config_path: Path, + *, + clock: Callable[[], float] = time.time, + entrypoint_runner: Callable[[HostJobConfig], int] | None = None, +) -> Mapping[str, Any]: + _configure_core() + if entrypoint_runner is None: + return core.execute(config_path, clock=clock) + return core.execute(config_path, clock=clock, entrypoint_runner=entrypoint_runner) + + +def native_snapshot(config: HostJobConfig, runner: Runner = core._default_runner) -> Mapping[str, Any]: + _configure_core() + return core.native_snapshot(config, runner) + + +def observe_job(config: HostJobConfig, native: Mapping[str, Any]) -> dict[str, Any]: + _configure_core() + return core.observe_job(config, native) + + +def start(config_path: Path, runner: Runner = core._default_runner) -> Mapping[str, Any]: + _configure_core() + return core.start(config_path, runner) + + +def collect(config_path: Path) -> bytes: + _configure_core() + return core.collect(config_path) + + +def cleanup(config_path: Path, runner: Runner = core._default_runner) -> Mapping[str, Any]: + _configure_core() + return core.cleanup(config_path, runner) + + +def main(argv: Sequence[str] | None = None) -> int: + _configure_core() + return core.main(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_host_probe.py b/scripts/gate14_host_probe.py new file mode 100644 index 000000000..f87ff859c --- /dev/null +++ b/scripts/gate14_host_probe.py @@ -0,0 +1,433 @@ +"""Finalize one source-bound, privacy-safe Gate 14 host probe. + +The platform wrappers invoke this module on the qualification host. Private control +credentials, process identifiers, paths, endpoints, and raw provider output remain in +the host action workspace. This module independently measures OS/GPU identity, hashes +the exact package inputs, validates calibrated action facts, and emits only the strict +Gate 14 platform document. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate14_calibration_challenge as challenge_contract +import gate14_hardware_acceptance as acceptance + +SCHEMA_VERSION = 1 +FACT_SCOPE = "gate14-host-action-facts" +MAX_JSON_BYTES = 262_144 +MAX_PACKAGE_BYTES = 16 * 1024**3 +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") + +_FACT_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "source_commit", + "gate13_evidence_sha256", + "expected_package_sha256", + "model", + "cache", + "placement", + "limits", + "suspensions", + "recovery", + "pause", + "restart", + "unsupported_telemetry", + "qualification_temporaries_removed", +} + + +class Gate14ProbeError(ValueError): + """Host facts or independently measured identity failed closed.""" + + +Runner = Callable[[Sequence[str], int], subprocess.CompletedProcess[str]] +HardwareProbe = Callable[[str], Mapping[str, Any]] + + +def _reject_constant(_value: str) -> None: + raise Gate14ProbeError("non-finite JSON value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14ProbeError("duplicate JSON field") + result[key] = value + return result + + +def _open_regular(path: Path, maximum: int): + path = Path(path) + try: + before = path.lstat() + except OSError as exc: + raise Gate14ProbeError("required input is unavailable") from exc + reparse = bool(getattr(before, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(before.st_mode) or not 1 <= before.st_size <= maximum: + raise Gate14ProbeError("required input is unsafe") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + handle = os.fdopen(descriptor, "rb") + except OSError as exc: + raise Gate14ProbeError("required input is unreadable") from exc + try: + opened = os.fstat(handle.fileno()) + if ( + not stat.S_ISREG(opened.st_mode) + or not 1 <= opened.st_size <= maximum + or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino) + ): + raise Gate14ProbeError("required input changed while opening") + return handle, opened + except BaseException: + handle.close() + raise + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + handle, metadata = _open_regular(path, maximum) + try: + payload = handle.read(maximum + 1) + after = os.fstat(handle.fileno()) + except OSError as exc: + raise Gate14ProbeError("required input is unreadable") from exc + finally: + handle.close() + if len(payload) != metadata.st_size or (after.st_dev, after.st_ino, after.st_size) != ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + ): + raise Gate14ProbeError("required input changed while reading") + return payload + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not 1 <= len(payload) <= MAX_JSON_BYTES: + raise Gate14ProbeError("host facts exceeded the size bound") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14ProbeError("host facts are invalid JSON") from exc + if not isinstance(value, dict) or set(value) != _FACT_FIELDS: + raise Gate14ProbeError("host facts schema is invalid") + return value + + +def _sha256(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _hash_regular_file(path: Path, maximum: int) -> tuple[int, str]: + stream, metadata = _open_regular(path, maximum) + digest = hashlib.sha256() + try: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + after = os.fstat(stream.fileno()) + except OSError as exc: + raise Gate14ProbeError("required input is unreadable") from exc + finally: + stream.close() + if (after.st_dev, after.st_ino, after.st_size) != ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + ): + raise Gate14ProbeError("required input changed while reading") + return metadata.st_size, "sha256:" + digest.hexdigest() + + +def _default_runner(argv: Sequence[str], timeout: int) -> subprocess.CompletedProcess[str]: + if not argv or any(not isinstance(item, str) or not item for item in argv): + raise Gate14ProbeError("hardware command is invalid") + try: + return subprocess.run( + list(argv), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + timeout=timeout, + shell=False, + ) + except (OSError, subprocess.SubprocessError, UnicodeError) as exc: + raise Gate14ProbeError("hardware command failed") from exc + + +def _operating_system(platform_name: str) -> str: + if platform_name == "windows": + if os.name != "nt": + raise Gate14ProbeError("Windows probe requires native Windows") + try: + import winreg + + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows NT\CurrentVersion", + ) as key: + product_name, _ = winreg.QueryValueEx(key, "ProductName") + except (ImportError, OSError) as exc: + raise Gate14ProbeError("Windows product identity is unavailable") from exc + if not isinstance(product_name, str) or "Windows Server 2022" not in product_name: + raise Gate14ProbeError("Windows Server 2022 is required") + return "Windows Server 2022" + + if platform_name != "linux" or os.name == "nt": + raise Gate14ProbeError("Linux probe requires native Linux") + values: dict[str, str] = {} + try: + payload = Path("/etc/os-release").read_text(encoding="utf-8") + except OSError as exc: + raise Gate14ProbeError("Linux release identity is unavailable") from exc + for line in payload.splitlines(): + if "=" not in line: + continue + key, raw = line.split("=", 1) + values[key] = raw.strip().strip('"') + if values.get("ID") != "ubuntu" or values.get("VERSION_ID") != "24.04": + raise Gate14ProbeError("Ubuntu 24.04 is required") + return "Ubuntu 24.04" + + +def probe_hardware( + platform_name: str, + *, + runner: Runner = _default_runner, +) -> Mapping[str, Any]: + os_name = _operating_system(platform_name) + result = runner( + ( + "nvidia-smi", + "--query-gpu=name,memory.total", + "--format=csv,noheader,nounits", + ), + 30, + ) + if result.returncode != 0 or len(result.stdout) > 4096 or result.stderr and len(result.stderr) > 4096: + raise Gate14ProbeError("accelerator identity is unavailable") + rows = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if len(rows) != 1: + raise Gate14ProbeError("exactly one accelerator is required") + fields = [item.strip() for item in rows[0].split(",")] + if len(fields) != 2 or fields[0] != "NVIDIA L4": + raise Gate14ProbeError("NVIDIA L4 is required") + try: + memory_bytes = int(fields[1]) * 1024**2 + except ValueError as exc: + raise Gate14ProbeError("accelerator memory is invalid") from exc + if not 20 * 1024**3 <= memory_bytes <= 32 * 1024**3: + raise Gate14ProbeError("accelerator memory is outside the L4 profile") + return { + "os_name": os_name, + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": memory_bytes, + } + + +def build_document( + facts: Mapping[str, Any], + *, + platform_name: str, + package_sha256: str, + package_bytes: int, + release_metadata_payload: bytes, + hardware: Mapping[str, Any], + challenge_value: Mapping[str, Any], + now_unix: float, +) -> Mapping[str, Any]: + if set(facts) != _FACT_FIELDS: + raise Gate14ProbeError("host facts schema is invalid") + if facts["schema_version"] != SCHEMA_VERSION or facts["scope"] != FACT_SCOPE or facts["platform"] != platform_name: + raise Gate14ProbeError("host facts scope is invalid") + source_commit = facts["source_commit"] + expected_package = facts["expected_package_sha256"] + if ( + not isinstance(source_commit, str) + or _COMMIT_RE.fullmatch(source_commit) is None + or not isinstance(expected_package, str) + or _DIGEST_RE.fullmatch(expected_package) is None + or package_sha256 != expected_package + or type(package_bytes) is not int + or not 1 <= package_bytes <= MAX_PACKAGE_BYTES + ): + raise Gate14ProbeError("package source binding is invalid") + if facts["gate13_evidence_sha256"] != acceptance.EXPECTED_GATE13_EVIDENCE_SHA256: + raise Gate14ProbeError("Gate 13 lifecycle binding is invalid") + challenge = challenge_contract.validate( + challenge_value, + run_id=facts["run_id"], + platform=platform_name, + source_commit=source_commit, + package_sha256=package_sha256, + now_unix=now_unix, + ) + challenge_sha256 = challenge_contract.digest(challenge) + + document = { + "schema_version": SCHEMA_VERSION, + "scope": acceptance.PLATFORM_SCOPE, + "run_id": facts["run_id"], + "platform": platform_name, + "result": "passed", + "source_commit": source_commit, + "gate13_evidence_sha256": facts["gate13_evidence_sha256"], + "package": { + "source_commit": source_commit, + "archive_sha256": expected_package, + "archive_bytes": package_bytes, + "release_metadata_sha256": _sha256(release_metadata_payload), + }, + "model": facts["model"], + "hardware": dict(hardware), + "cache": facts["cache"], + "placement": facts["placement"], + "limits": facts["limits"], + "calibration_challenge": { + "challenge_sha256": challenge_sha256, + "controller_state_revision": challenge["controller_state_revision"], + "issued_at_unix": challenge["issued_at_unix"], + "expires_at_unix": challenge["expires_at_unix"], + }, + "suspensions": facts["suspensions"], + "recovery": facts["recovery"], + "pause": facts["pause"], + "restart": facts["restart"], + "unsupported_telemetry": facts["unsupported_telemetry"], + "privacy": { + "prompt_retained": False, + "response_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + "provider_output_retained": False, + }, + "qualification_temporaries_removed": facts["qualification_temporaries_removed"], + } + acceptance.validate_platform_document(document) + for suspension in document["suspensions"]: + ended_at = float(suspension["calibration"]["sample_ended_at_unix"]) + if ended_at > now_unix: + raise Gate14ProbeError("calibration measurement is future-dated") + return document + + +def _atomic_output(path: Path, value: Mapping[str, Any]) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and (path.is_symlink() or not path.is_file()): + raise Gate14ProbeError("output target is unsafe") + payload = (json.dumps(value, allow_nan=False, sort_keys=True, separators=(",", ":")) + os.linesep).encode("utf-8") + if len(payload) > MAX_JSON_BYTES: + raise Gate14ProbeError("platform evidence exceeded the size bound") + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + temporary.unlink() + except OSError: + pass + raise + + +def run_probe( + *, + platform_name: str, + facts_path: Path, + challenge_path: Path, + package_path: Path, + release_metadata_path: Path, + output_path: Path, + hardware_probe: HardwareProbe = probe_hardware, + now_unix: float | None = None, +) -> Mapping[str, Any]: + facts = _strict_json(_regular_bytes(facts_path, MAX_JSON_BYTES)) + challenge_value = challenge_contract.load(challenge_path) + package_bytes, package_sha256 = _hash_regular_file(package_path, MAX_PACKAGE_BYTES) + metadata_payload = _regular_bytes(release_metadata_path, MAX_JSON_BYTES) + hardware = hardware_probe(platform_name) + document = build_document( + facts, + platform_name=platform_name, + package_sha256=package_sha256, + package_bytes=package_bytes, + release_metadata_payload=metadata_payload, + hardware=hardware, + challenge_value=challenge_value, + now_unix=time.time() if now_unix is None else now_unix, + ) + _atomic_output(output_path, document) + return document + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--platform", choices=("windows", "linux"), required=True) + parser.add_argument("--facts", type=Path, required=True) + parser.add_argument("--challenge", type=Path, required=True) + parser.add_argument("--package", type=Path, required=True) + parser.add_argument("--release-metadata", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + document = run_probe( + platform_name=args.platform, + facts_path=args.facts, + challenge_path=args.challenge, + package_path=args.package, + release_metadata_path=args.release_metadata, + output_path=args.output, + ) + except ( + Gate14ProbeError, + challenge_contract.Gate14ChallengeError, + acceptance.Gate14EvidenceError, + ) as exc: + raise SystemExit(str(exc)) from exc + print(_sha256(_regular_bytes(args.output, MAX_JSON_BYTES))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_linux_action_transport.py b/scripts/gate14_linux_action_transport.py new file mode 100644 index 000000000..bc068fde5 --- /dev/null +++ b/scripts/gate14_linux_action_transport.py @@ -0,0 +1,531 @@ +"""Persistent, bounded RPC transport for Gate 14 Linux lifecycle actions.""" + +from __future__ import annotations + +import hashlib +import json +import os +import queue +import re +import secrets +import signal +import stat +import subprocess +import sys +import threading +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate14_calibration_challenge as challenge_contract + +SCHEMA_VERSION = 1 +SCOPE = "gate14-linux-lifecycle-actions" +MAX_FRAME_BYTES = 262_144 +MAX_SOURCE_BYTES = 8 * 1024 * 1024 +DEFAULT_OPERATION_TIMEOUT_SECONDS = { + "prepare": 3_600.0, + "calibrate": 1_800.0, + "cleanup": 300.0, +} +DEFAULT_CLOSE_TIMEOUT_SECONDS = 30.0 + +_GATE13_LIFECYCLE_SHA256 = "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535" +_GATE13_INFERENCE_SHA256 = "ccf10f9b19f505afb4efde4a86a49e73e3e7c88e9d51ead4991dec68f0c15209" +_PRODUCT_ACTIONS_SHA256 = "7a904b1c4653eb2a392bb64b1f97404beaee3fff9a2102f3c9c7949f0a2aa973" +_ACTION_HOST_SHA256 = "495028ae72a6a1c37a8c718356ed7e7bbae437156b0cc8b7328ddb4fce8e1a36" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_FAILURE_RE = re.compile(r"[a-z][a-z0-9-]{0,63}") +_FORBIDDEN_KEYS = { + "api_key", + "api_token", + "authorization", + "argv", + "command", + "control_key", + "control_token", + "credential", + "endpoint", + "environment", + "gpu_uuid", + "hostname", + "output", + "password", + "path", + "prompt", + "secret", + "token", + "url", + "username", +} +_RESPONSE_FIELDS = { + "failure_code", + "operation", + "payload", + "request_id", + "result", + "schema_version", + "scope", + "session_id", +} + + +class Gate14ActionTransportError(ValueError): + """The Linux action host source, RPC stream, or process failed closed.""" + + +ProcessFactory = Callable[..., subprocess.Popen[bytes]] + + +def _reject_constant(_value: str) -> None: + raise Gate14ActionTransportError("non-finite RPC value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14ActionTransportError("duplicate RPC field") + result[key] = value + return result + + +def _canonical(value: Mapping[str, Any]) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Gate14ActionTransportError("RPC value is not canonical JSON") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_FRAME_BYTES: + raise Gate14ActionTransportError("RPC frame size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14ActionTransportError("RPC frame is invalid") from exc + if not isinstance(value, dict): + raise Gate14ActionTransportError("RPC root is invalid") + if _canonical(value) != payload: + raise Gate14ActionTransportError("RPC frame is not canonical") + return value + + +def _normalized_source(path: Path, expected_sha256: str) -> Path: + candidate = Path(path) + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14ActionTransportError("action source is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or not 1 <= metadata.st_size <= MAX_SOURCE_BYTES + ): + raise Gate14ActionTransportError("action source is unsafe") + try: + payload = candidate.read_bytes() + except OSError as exc: + raise Gate14ActionTransportError("action source is unreadable") from exc + normalized = payload.replace(b"\r\n", b"\n") + if b"\r" in normalized or hashlib.sha256(normalized).hexdigest() != expected_sha256: + raise Gate14ActionTransportError("action source digest changed") + return candidate.resolve() + + +def _assert_safe_payload(value: Any) -> None: + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str) or key.casefold() in _FORBIDDEN_KEYS: + raise Gate14ActionTransportError("action response contains private material") + _assert_safe_payload(item) + return + if isinstance(value, (list, tuple)): + for item in value: + _assert_safe_payload(item) + return + if isinstance(value, str) and ( + value.startswith("drift_control_") or "\r" in value or "\n" in value or "\x00" in value + ): + raise Gate14ActionTransportError("action response contains private material") + + +def _binding(config: Any) -> dict[str, Any]: + value = { + "attempt_ordinal": config.attempt_ordinal, + "lifecycle_config_sha256": config.config_sha256, + "package_sha256": config.package_sha256, + "platform": config.platform, + "run_id": config.run_id, + "source_commit": config.source_commit, + } + if ( + type(value["attempt_ordinal"]) is not int + or not 1 <= value["attempt_ordinal"] <= 100 + or value["platform"] != "linux" + or not isinstance(value["run_id"], str) + or _RUN_RE.fullmatch(value["run_id"]) is None + or not isinstance(value["source_commit"], str) + or _COMMIT_RE.fullmatch(value["source_commit"]) is None + or not isinstance(value["package_sha256"], str) + or _DIGEST_RE.fullmatch(value["package_sha256"]) is None + or not isinstance(value["lifecycle_config_sha256"], str) + or _DIGEST_RE.fullmatch(value["lifecycle_config_sha256"]) is None + ): + raise Gate14ActionTransportError("lifecycle action binding is invalid") + return value + + +def _file_digest(path: Path, maximum: int) -> str: + candidate = Path(path) + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14ActionTransportError("lifecycle configuration is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or candidate.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise Gate14ActionTransportError("lifecycle configuration is unsafe") + try: + return "sha256:" + hashlib.sha256(candidate.read_bytes()).hexdigest() + except OSError as exc: + raise Gate14ActionTransportError("lifecycle configuration is unreadable") from exc + + +def _timeouts(value: Mapping[str, float] | None) -> dict[str, float]: + result = dict(DEFAULT_OPERATION_TIMEOUT_SECONDS if value is None else value) + if set(result) != set(DEFAULT_OPERATION_TIMEOUT_SECONDS): + raise Gate14ActionTransportError("action transport timeout schema is invalid") + caps = {"prepare": 7_200.0, "calibrate": 3_600.0, "cleanup": 600.0} + for operation, maximum in caps.items(): + timeout = result[operation] + if type(timeout) not in (int, float) or not 0.1 <= float(timeout) <= maximum: + raise Gate14ActionTransportError("action transport timeout is invalid") + result[operation] = float(timeout) + return result + + +def _challenge_payload(challenge: Mapping[str, Any]) -> dict[str, Any]: + return { + "challenge_sha256": challenge_contract.digest(challenge), + "controller_state_revision": challenge["controller_state_revision"], + "issued_at_unix": challenge["issued_at_unix"], + "expires_at_unix": challenge["expires_at_unix"], + } + + +class LinuxActionTransport: + """Own exactly one source-bound Linux action host.""" + + def __init__( + self, + config: Any, + *, + python: str | None = None, + process_factory: ProcessFactory = subprocess.Popen, + operation_timeouts: Mapping[str, float] | None = None, + close_timeout_seconds: float = DEFAULT_CLOSE_TIMEOUT_SECONDS, + transport_self_test: bool = False, + self_test_cleanup_marker: Path | None = None, + ) -> None: + if type(close_timeout_seconds) not in (int, float) or not 0.1 <= float(close_timeout_seconds) <= 60.0: + raise Gate14ActionTransportError("action transport timeout is invalid") + self._timeouts = _timeouts(operation_timeouts) + self._binding = _binding(config) + config_path = Path(config.config_path) + if config_path.name != "gate14-lifecycle.json" or _file_digest(config_path, 65_536) != config.config_sha256: + raise Gate14ActionTransportError("lifecycle configuration binding changed") + + directory = Path(__file__).resolve().parent + self._host_path = _normalized_source( + directory / "gate14_linux_lifecycle_actions.py", + _ACTION_HOST_SHA256, + ) + self._gate13_lifecycle_path = _normalized_source( + directory / "gate13_linux_packaged_lifecycle.py", + _GATE13_LIFECYCLE_SHA256, + ) + self._gate13_inference_path = _normalized_source( + directory / "gate13_linux_localhost_inference.py", + _GATE13_INFERENCE_SHA256, + ) + self._product_actions_path = _normalized_source( + directory / "gate14_linux_product_actions.py", + _PRODUCT_ACTIONS_SHA256, + ) + if not ( + self._host_path.parent == self._gate13_lifecycle_path.parent + and self._gate13_lifecycle_path.parent == self._gate13_inference_path.parent + and self._gate13_inference_path.parent == self._product_actions_path.parent + ): + raise Gate14ActionTransportError("action sources are not colocated") + + executable = python or sys.executable + self._session_id = secrets.token_hex(32) + arguments = [ + executable, + "-u", + os.fspath(self._host_path), + "--session-id", + self._session_id, + "--run-id", + self._binding["run_id"], + "--attempt-ordinal", + str(self._binding["attempt_ordinal"]), + "--source-commit", + self._binding["source_commit"], + "--package-sha256", + self._binding["package_sha256"], + "--gate13-lifecycle", + os.fspath(self._gate13_lifecycle_path), + "--gate13-inference", + os.fspath(self._gate13_inference_path), + "--gate13-lifecycle-sha256", + _GATE13_LIFECYCLE_SHA256, + "--gate13-inference-sha256", + _GATE13_INFERENCE_SHA256, + "--product-actions", + os.fspath(self._product_actions_path), + "--product-actions-sha256", + _PRODUCT_ACTIONS_SHA256, + "--lifecycle-config", + os.fspath(config_path), + "--lifecycle-config-sha256", + config.config_sha256, + ] + if transport_self_test: + arguments.append("--transport-self-test") + if self_test_cleanup_marker is not None: + arguments.extend(("--self-test-cleanup-marker", os.fspath(Path(self_test_cleanup_marker)))) + + try: + self._process = process_factory( + arguments, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + start_new_session=True, + ) + except (OSError, ValueError) as exc: + raise Gate14ActionTransportError("action host could not start") from exc + if self._process.stdin is None or self._process.stdout is None or self._process.stderr is None: + self._terminate() + raise Gate14ActionTransportError("action host pipes are unavailable") + + self._close_timeout = float(close_timeout_seconds) + self._responses: queue.Queue[bytes | BaseException | None] = queue.Queue(maxsize=2) + self._stderr_bytes = 0 + self._stderr_overflow = False + self._next_request_id = 1 + self._phase = "new" + self._closed = False + self._reader = threading.Thread( + target=self._read_stdout, + name="gate14-linux-action-stdout", + daemon=True, + ) + self._stderr_reader = threading.Thread( + target=self._drain_stderr, + name="gate14-linux-action-stderr", + daemon=True, + ) + self._reader.start() + self._stderr_reader.start() + + def _read_stdout(self) -> None: + try: + while True: + line = self._process.stdout.readline(MAX_FRAME_BYTES + 2) + if not line: + self._responses.put(None) + return + if len(line) > MAX_FRAME_BYTES + 1 or not line.endswith(b"\n"): + self._responses.put(Gate14ActionTransportError("action response framing is invalid")) + return + frame = line[:-1] + if frame.endswith(b"\r"): + frame = frame[:-1] + self._responses.put(frame) + except BaseException as exc: + self._responses.put(exc) + + def _drain_stderr(self) -> None: + try: + while chunk := self._process.stderr.read(65_536): + self._stderr_bytes += len(chunk) + if self._stderr_bytes > MAX_FRAME_BYTES: + self._stderr_overflow = True + except BaseException: + self._stderr_overflow = True + + def _terminate(self) -> None: + process = getattr(self, "_process", None) + if process is None: + return + try: + if process.stdin is not None: + process.stdin.close() + except OSError: + pass + try: + process.wait(timeout=getattr(self, "_close_timeout", DEFAULT_CLOSE_TIMEOUT_SECONDS)) + except (subprocess.TimeoutExpired, OSError): + try: + if hasattr(os, "killpg"): + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except OSError: + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=DEFAULT_CLOSE_TIMEOUT_SECONDS) + except (subprocess.TimeoutExpired, OSError): + pass + + def _fail(self, message: str) -> None: + self._closed = True + self._terminate() + raise Gate14ActionTransportError(message) + + def request(self, operation: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: + if self._closed: + raise Gate14ActionTransportError("action transport is closed") + if operation not in {"prepare", "calibrate", "cleanup"}: + self._fail("action operation is invalid") + if not isinstance(payload, dict): + self._fail("action payload is invalid") + request_id = self._next_request_id + frame = { + "binding": self._binding, + "operation": operation, + "payload": payload, + "request_id": request_id, + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "session_id": self._session_id, + } + rendered = _canonical(frame) + b"\n" + if len(rendered) > MAX_FRAME_BYTES: + self._fail("action request is too large") + try: + self._process.stdin.write(rendered) + self._process.stdin.flush() + except (BrokenPipeError, OSError): + self._fail("action host ended before request") + + try: + response_item = self._responses.get(timeout=self._timeouts[operation]) + except queue.Empty: + self._fail("action response timed out") + if response_item is None: + self._fail("action host ended before response") + if isinstance(response_item, BaseException): + self._fail("action response could not be read") + try: + response = _strict_json(response_item) + except Gate14ActionTransportError: + self._fail("action response is invalid") + if ( + set(response) != _RESPONSE_FIELDS + or type(response.get("schema_version")) is not int + or response.get("schema_version") != SCHEMA_VERSION + or response.get("scope") != SCOPE + or response.get("session_id") != self._session_id + or type(response.get("request_id")) is not int + or response.get("request_id") != request_id + or response.get("operation") != operation + or response.get("result") not in {"passed", "failed"} + ): + self._fail("action response binding is invalid") + self._next_request_id += 1 + if response["result"] == "failed": + failure_code = response["failure_code"] + if ( + response["payload"] is not None + or not isinstance(failure_code, str) + or _FAILURE_RE.fullmatch(failure_code) is None + ): + self._fail("action failure response is invalid") + raise Gate14ActionTransportError(f"action host rejected {operation}: {failure_code}") + if response["failure_code"] is not None or not isinstance(response["payload"], dict): + self._fail("action success response is invalid") + _assert_safe_payload(response["payload"]) + if self._stderr_overflow: + self._fail("action host diagnostics exceeded the bound") + return dict(response["payload"]) + + def prepare(self, _config: Any) -> Mapping[str, Any]: + if self._phase != "new": + self._fail("action operation order is invalid") + try: + result = self.request("prepare", {}) + except BaseException: + self._phase = "failed" + raise + self._phase = "prepared" + return result + + def calibrate( + self, + _config: Any, + challenge: Mapping[str, Any], + ) -> Sequence[Mapping[str, Any]]: + if self._phase != "prepared": + self._fail("action operation order is invalid") + try: + result = self.request("calibrate", _challenge_payload(challenge)) + except BaseException: + self._phase = "failed" + raise + self._phase = "calibrated" + observations = result.get("suspensions") + if not isinstance(observations, list): + raise Gate14ActionTransportError("action calibration response is not an observation list") + return observations + + def cleanup(self, config: Any) -> Mapping[str, Any]: + if self._closed: + raise Gate14ActionTransportError("action transport is closed") + result = self.request("cleanup", {}) + expected = { + "action_temporaries_removed": True, + "attempt_ordinal": config.attempt_ordinal, + "credentials_removed": True, + "platform": "linux", + "processes_absent": True, + "run_id": config.run_id, + "schema_version": SCHEMA_VERSION, + "scope": "gate14-host-lifecycle-cleanup", + } + if result != expected: + self._fail("action cleanup response is invalid") + self._phase = "cleaned" + return result + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._terminate() + + def __enter__(self) -> "LinuxActionTransport": + return self + + def __exit__(self, _type, _value, _traceback) -> None: + self.close() diff --git a/scripts/gate14_linux_lifecycle_actions.py b/scripts/gate14_linux_lifecycle_actions.py new file mode 100644 index 000000000..deaa977f9 --- /dev/null +++ b/scripts/gate14_linux_lifecycle_actions.py @@ -0,0 +1,473 @@ +"""Persistent Gate 14 Linux lifecycle action host. + +This process is intentionally long-lived so the production handler retains its +systemd-owned product tree, Secret Service credential, and verified cache across +the controller-owned calibration challenge. The bounded self-test path exercises +only transport lifetime and cleanup. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import sys +import types +from pathlib import Path +from typing import Any, Mapping, Sequence + +SCHEMA_VERSION = 1 +SCOPE = "gate14-linux-lifecycle-actions" +MAX_FRAME_BYTES = 262_144 +MAX_SOURCE_BYTES = 8 * 1024 * 1024 + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_SESSION_RE = re.compile(r"[0-9a-f]{64}") +_FAILURE_RE = re.compile(r"[a-z][a-z0-9-]{0,63}") +_FRAME_FIELDS = { + "binding", + "operation", + "payload", + "request_id", + "schema_version", + "scope", + "session_id", +} +_BINDING_FIELDS = { + "attempt_ordinal", + "lifecycle_config_sha256", + "package_sha256", + "platform", + "run_id", + "source_commit", +} + + +class Gate14LinuxActionError(ValueError): + """A Linux action-host input or operation failed closed.""" + + +def _reject_constant(_value: str) -> None: + raise Gate14LinuxActionError("non-finite RPC value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14LinuxActionError("duplicate RPC field") + result[key] = value + return result + + +def _canonical(value: Mapping[str, Any]) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Gate14LinuxActionError("RPC value is not canonical JSON") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_FRAME_BYTES: + raise Gate14LinuxActionError("RPC frame size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14LinuxActionError("RPC frame is invalid") from exc + if not isinstance(value, dict) or _canonical(value) != payload: + raise Gate14LinuxActionError("RPC frame is not canonical") + return value + + +def _verified_source(path: Path, expected_sha256: str) -> tuple[Path, bytes]: + candidate = Path(path) + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14LinuxActionError("action helper is unavailable") from exc + if ( + candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or (os.name != "nt" and metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) + or not 1 <= metadata.st_size <= MAX_SOURCE_BYTES + ): + raise Gate14LinuxActionError("action helper is unsafe") + try: + payload = candidate.read_bytes() + except OSError as exc: + raise Gate14LinuxActionError("action helper is unreadable") from exc + normalized = payload.replace(b"\r\n", b"\n") + if b"\r" in normalized or hashlib.sha256(normalized).hexdigest() != expected_sha256: + raise Gate14LinuxActionError("action helper binding changed") + return candidate.resolve(), normalized + + +def _normalized_source(path: Path, expected_sha256: str) -> Path: + return _verified_source(path, expected_sha256)[0] + + +def _file_digest(path: Path, maximum: int) -> str: + candidate = Path(path) + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14LinuxActionError("lifecycle configuration is unavailable") from exc + if ( + candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or (os.name != "nt" and metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) + or not 1 <= metadata.st_size <= maximum + ): + raise Gate14LinuxActionError("lifecycle configuration is unsafe") + try: + return "sha256:" + hashlib.sha256(candidate.read_bytes()).hexdigest() + except OSError as exc: + raise Gate14LinuxActionError("lifecycle configuration is unreadable") from exc + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--session-id", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--attempt-ordinal", required=True, type=int) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--package-sha256", required=True) + parser.add_argument("--gate13-lifecycle", required=True) + parser.add_argument("--gate13-inference", required=True) + parser.add_argument("--gate13-lifecycle-sha256", required=True) + parser.add_argument("--gate13-inference-sha256", required=True) + parser.add_argument("--product-actions", required=True) + parser.add_argument("--product-actions-sha256", required=True) + parser.add_argument("--lifecycle-config", required=True) + parser.add_argument("--lifecycle-config-sha256", required=True) + parser.add_argument("--transport-self-test", action="store_true") + parser.add_argument("--self-test-cleanup-marker") + return parser + + +def _arguments(argv: Sequence[str] | None) -> argparse.Namespace: + value = _parser().parse_args(sys.argv[1:] if argv is None else argv) + if ( + _SESSION_RE.fullmatch(value.session_id) is None + or _RUN_RE.fullmatch(value.run_id) is None + or type(value.attempt_ordinal) is not int + or not 1 <= value.attempt_ordinal <= 100 + or _COMMIT_RE.fullmatch(value.source_commit) is None + or _DIGEST_RE.fullmatch(value.package_sha256) is None + or _DIGEST_RE.fullmatch(value.lifecycle_config_sha256) is None + or not re.fullmatch(r"[0-9a-f]{64}", value.gate13_lifecycle_sha256) + or not re.fullmatch(r"[0-9a-f]{64}", value.gate13_inference_sha256) + or not re.fullmatch(r"[0-9a-f]{64}", value.product_actions_sha256) + ): + raise Gate14LinuxActionError("action-host binding is invalid") + return value + + +def _assert_binding(value: Any, arguments: argparse.Namespace) -> None: + expected = { + "attempt_ordinal": arguments.attempt_ordinal, + "lifecycle_config_sha256": arguments.lifecycle_config_sha256, + "package_sha256": arguments.package_sha256, + "platform": "linux", + "run_id": arguments.run_id, + "source_commit": arguments.source_commit, + } + if ( + not isinstance(value, dict) + or set(value) != _BINDING_FIELDS + or any(type(value[key]) is not type(expected[key]) for key in expected) + or value != expected + ): + raise Gate14LinuxActionError("RPC binding is invalid") + + +def _response( + arguments: argparse.Namespace, + request_id: int, + operation: str, + *, + result: str, + payload: Mapping[str, Any] | None, + failure_code: str | None, +) -> None: + if result not in {"passed", "failed"}: + raise Gate14LinuxActionError("RPC result is invalid") + if failure_code is not None and _FAILURE_RE.fullmatch(failure_code) is None: + raise Gate14LinuxActionError("RPC failure code is invalid") + value = { + "failure_code": failure_code, + "operation": operation, + "payload": payload, + "request_id": request_id, + "result": result, + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "session_id": arguments.session_id, + } + rendered = _canonical(value) + if len(rendered) > MAX_FRAME_BYTES: + raise Gate14LinuxActionError("RPC response is too large") + sys.stdout.buffer.write(rendered + b"\n") + sys.stdout.buffer.flush() + + +def _cleanup( + arguments: argparse.Namespace, + cleaned: list[bool], + product: Any | None = None, +) -> Mapping[str, Any] | None: + if cleaned[0]: + return None + cleaned[0] = True + if product is not None: + result = product.cleanup() + if not isinstance(result, dict): + raise Gate14LinuxActionError("product cleanup result is invalid") + return result + if arguments.transport_self_test and arguments.self_test_cleanup_marker: + marker = Path(arguments.self_test_cleanup_marker) + if marker.exists() or marker.is_symlink() or not marker.parent.is_dir(): + raise Gate14LinuxActionError("self-test cleanup marker is unsafe") + descriptor = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: + handle.write("cleaned") + return None + + +def _load_verified_module(name: str, path: Path, source: bytes) -> types.ModuleType: + module = types.ModuleType(name) + module.__file__ = os.fspath(path) + module.__package__ = "" + sys.modules[name] = module + try: + exec(compile(source, os.fspath(path), "exec"), module.__dict__) + except BaseException: + sys.modules.pop(name, None) + raise + return module + + +def serve(argv: Sequence[str] | None = None) -> int: + os.umask(0o077) + arguments = _arguments(argv) + lifecycle_path, lifecycle_source = _verified_source( + Path(arguments.gate13_lifecycle), + arguments.gate13_lifecycle_sha256, + ) + inference_path, inference_source = _verified_source( + Path(arguments.gate13_inference), + arguments.gate13_inference_sha256, + ) + product_path, product_source = _verified_source( + Path(arguments.product_actions), + arguments.product_actions_sha256, + ) + if ( + lifecycle_path.name != "gate13_linux_packaged_lifecycle.py" + or inference_path.name != "gate13_linux_localhost_inference.py" + or product_path.name != "gate14_linux_product_actions.py" + or lifecycle_path.parent != inference_path.parent + or lifecycle_path.parent != product_path.parent + ): + raise Gate14LinuxActionError("action helper identity is invalid") + config_path = Path(arguments.lifecycle_config) + if ( + config_path.name != "gate14-lifecycle.json" + or _file_digest(config_path, 65_536) != arguments.lifecycle_config_sha256 + ): + raise Gate14LinuxActionError("lifecycle configuration binding changed") + + product = None + actions = None + if not arguments.transport_self_test: + sys.modules["gate13_packaged_lifecycle"] = types.ModuleType("gate13_packaged_lifecycle") + _load_verified_module("gate13_linux_localhost_inference", inference_path, inference_source) + gate13 = _load_verified_module("gate13_linux_packaged_lifecycle", lifecycle_path, lifecycle_source) + actions = _load_verified_module("gate14_linux_product_actions", product_path, product_source) + actions.bind_gate13(gate13) + + phase = "new" + expected_request_id = 1 + state_nonce = os.urandom(32).hex() + cleaned = [False] + try: + while True: + line = sys.stdin.buffer.readline(MAX_FRAME_BYTES + 2) + if not line: + break + if len(line) > MAX_FRAME_BYTES + 1 or not line.endswith(b"\n"): + raise Gate14LinuxActionError("RPC frame is invalid") + frame = _strict_json(line[:-1]) + if ( + set(frame) != _FRAME_FIELDS + or type(frame.get("schema_version")) is not int + or frame.get("schema_version") != SCHEMA_VERSION + or frame.get("scope") != SCOPE + or frame.get("session_id") != arguments.session_id + or type(frame.get("request_id")) is not int + or frame.get("request_id") != expected_request_id + or frame.get("operation") not in {"prepare", "calibrate", "cleanup"} + or not isinstance(frame.get("payload"), dict) + ): + raise Gate14LinuxActionError("RPC frame binding is invalid") + _assert_binding(frame["binding"], arguments) + request_id = frame["request_id"] + operation = frame["operation"] + expected_request_id += 1 + + if operation == "prepare": + if phase != "new" or frame["payload"]: + raise Gate14LinuxActionError("RPC operation order is invalid") + if arguments.transport_self_test: + payload = { + "helpers_verified": True, + "host_process_id": os.getpid(), + "state_nonce": state_nonce, + } + else: + try: + product = actions.LinuxProductActions( + config_path=config_path, + run_id=arguments.run_id, + attempt_ordinal=arguments.attempt_ordinal, + source_commit=arguments.source_commit, + package_sha256=arguments.package_sha256, + ) + payload = product.prepare() + except Exception: + phase = "failed" + _response( + arguments, + request_id, + operation, + result="failed", + payload=None, + failure_code="product-prepare-failed", + ) + continue + phase = "prepared" + _response( + arguments, + request_id, + operation, + result="passed", + failure_code=None, + payload=payload, + ) + continue + + if operation == "calibrate": + if phase != "prepared" or set(frame["payload"]) != { + "challenge_sha256", + "controller_state_revision", + "issued_at_unix", + "expires_at_unix", + }: + raise Gate14LinuxActionError("RPC operation order is invalid") + challenge_sha256 = frame["payload"]["challenge_sha256"] + revision = frame["payload"]["controller_state_revision"] + issued = frame["payload"]["issued_at_unix"] + expires = frame["payload"]["expires_at_unix"] + if ( + not isinstance(challenge_sha256, str) + or _DIGEST_RE.fullmatch(challenge_sha256) is None + or type(revision) is not int + or revision < 0 + or type(issued) is not int + or type(expires) is not int + or not 60 <= expires - issued <= 900 + ): + raise Gate14LinuxActionError("RPC calibration binding is invalid") + if arguments.transport_self_test: + payload = { + "challenge_sha256": challenge_sha256, + "host_process_id": os.getpid(), + "state_nonce": state_nonce, + } + else: + try: + payload = {"suspensions": list(product.calibrate(frame["payload"]))} + except Exception: + phase = "failed" + _response( + arguments, + request_id, + operation, + result="failed", + payload=None, + failure_code="product-calibration-failed", + ) + continue + phase = "calibrated" + _response( + arguments, + request_id, + operation, + result="passed", + failure_code=None, + payload=payload, + ) + continue + + if frame["payload"]: + raise Gate14LinuxActionError("RPC cleanup payload is invalid") + cleanup_payload = _cleanup(arguments, cleaned, product) + if cleanup_payload is None: + cleanup_payload = { + "action_temporaries_removed": True, + "attempt_ordinal": arguments.attempt_ordinal, + "credentials_removed": True, + "platform": "linux", + "processes_absent": True, + "run_id": arguments.run_id, + "schema_version": SCHEMA_VERSION, + "scope": "gate14-host-lifecycle-cleanup", + } + phase = "cleaned" + _response( + arguments, + request_id, + operation, + result="passed", + failure_code=None, + payload=cleanup_payload, + ) + except Gate14LinuxActionError: + _cleanup(arguments, cleaned, product) + _response( + arguments, + expected_request_id, + "invalid", + result="failed", + payload=None, + failure_code="invalid-action-frame", + ) + return 2 + finally: + _cleanup(arguments, cleaned, product) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + return serve(argv) + except (Exception, SystemExit): + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_linux_probe.sh b/scripts/gate14_linux_probe.sh new file mode 100644 index 000000000..f00504a9f --- /dev/null +++ b/scripts/gate14_linux_probe.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: gate14_linux_probe.sh PYTHON FACTS CHALLENGE PACKAGE RELEASE_METADATA OUTPUT" >&2 + exit 2 +fi + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +exec "$1" "$script_dir/gate14_host_probe.py" \ + --platform linux \ + --facts "$2" \ + --challenge "$3" \ + --package "$4" \ + --release-metadata "$5" \ + --output "$6" diff --git a/scripts/gate14_linux_product_actions.py b/scripts/gate14_linux_product_actions.py new file mode 100644 index 000000000..7dd7d34e4 --- /dev/null +++ b/scripts/gate14_linux_product_actions.py @@ -0,0 +1,1043 @@ +"""Concrete Gate 14 Linux packaged-product actions. + +The action host keeps this object alive across prepare, controller challenge, +calibrate, and cleanup. It reuses the source-bound Gate 13 package/process/API +primitives, but owns a separate work namespace and never accepts observed pass +claims from the lifecycle configuration. +""" + +from __future__ import annotations + +import json +import os +import secrets +import shutil +import signal +import socket +import subprocess +import threading +import time +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Mapping, Sequence + +# The source-bound action host injects the verified Gate 13 helper module before +# constructing LinuxProductActions. Keeping this explicit prevents a pathname +# reopen between digest verification and execution. +gate13: Any = None + + +def bind_gate13(module: Any) -> None: + global gate13 + required = ( + "ARCHIVE_NAME", + "ProxyHandler", + "SystemdUnitOwner", + "_RejectRedirects", + "_assert_cache_unchanged", + "_audit_package", + "_bootstrap", + "_clear_control_token", + "_control_request", + "_credential_count", + "_extract_package", + "_run_self_tests", + "_start_products", + "_status_identity", + "_stop_products", + "_store_control_token", + "_strict_json", + "_verify_cache", + "build_opener", + ) + if gate13 is not None or module is None or any(not hasattr(module, name) for name in required): + raise Gate14LinuxProductError("Gate 13 helper binding is invalid") + gate13 = module + + +CONTROL_ORIGIN = "http://127.0.0.1:8080" +WARM_CACHE_NAME = "gate14-warm-cache" +ACTION_ROOT_NAME = "gate14-product-action" +MAX_TRANSITION_SECONDS = 300.0 +MODEL_PROFILES = { + "Qwen3.5 2B": { + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "revision_commit": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "selected_artifact_count": 8, + "selected_artifact_bytes": 4_571_197_320, + "total_blocks": 24, + "gate9_envelope_sha256": "sha256:cd68afb67d9b0f3cb8c82db0d3314ad89b558c20880998ea4d8c4493e9f4bc9f", + }, + "Gemma 4 E2B IT": { + "manifest_digest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "revision_commit": "3e22461f65e89153144f8adb70e3b8c2cc9845a7", + "selected_artifact_count": 5, + "selected_artifact_bytes": 10_278_818_149, + "total_blocks": 35, + "gate9_envelope_sha256": "sha256:2eb0bcf6419ba085665fad34310453a1b9dc2e89d90e9177f41566df012996c8", + }, +} + + +class Gate14LinuxProductError(RuntimeError): + """A concrete packaged action or its physical observation failed closed.""" + + +def _canonical(value: Mapping[str, Any]) -> bytes: + return json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def _load_config(path: Path) -> Mapping[str, Any]: + return gate13._strict_json(path.read_bytes(), maximum=65_536) + + +def _safe_artifacts(config: Mapping[str, Any]) -> tuple[dict[str, Any], ...]: + raw = config.get("warm_cache") + artifacts = None if not isinstance(raw, dict) else raw.get("artifacts") + if not isinstance(artifacts, list) or not artifacts: + raise Gate14LinuxProductError("warm cache artifact inventory is absent") + result = [] + seen = set() + for item in artifacts: + if not isinstance(item, dict) or set(item) != {"path", "role", "sha256", "size_bytes"}: + raise Gate14LinuxProductError("warm cache artifact inventory is invalid") + path = item["path"] + pure = PurePosixPath(path) if isinstance(path, str) else None + digest = item.get("sha256") + size = item.get("size_bytes") + if ( + pure is None + or pure.is_absolute() + or pure.as_posix() != path + or any(part in ("", ".", "..") for part in pure.parts) + or path.casefold() in seen + or not isinstance(digest, str) + or not digest.startswith("sha256:") + or len(digest) != 71 + or type(size) is not int + or size < 1 + ): + raise Gate14LinuxProductError("warm cache artifact inventory is invalid") + seen.add(path.casefold()) + result.append( + { + "path": path, + "role": item["role"], + "sha256": digest.removeprefix("sha256:"), + "size_bytes": size, + } + ) + return tuple(result) + + +def _schedule(allowed: bool, now: float | None = None) -> Mapping[str, Any]: + if allowed: + days = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] + return {"timezone": "UTC", "windows": [{"days": days, "start": "00:00", "end": "23:59"}]} + weekday = time.gmtime(time.time() if now is None else now).tm_wday + names = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] + return { + "timezone": "UTC", + "windows": [{"days": [names[(weekday + 1) % 7]], "start": "00:00", "end": "00:01"}], + } + + +def _active(snapshot: Mapping[str, Any]) -> bool: + return snapshot.get("state") in {"starting", "running", "stopping"} + + +def _pid_exists(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +class _LoopbackLoad: + def __init__(self) -> None: + self._stop = threading.Event() + self._threads: list[threading.Thread] = [] + self._listener: socket.socket | None = None + + def start(self) -> None: + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(1.0) + self._listener = listener + address = listener.getsockname() + + def receive() -> None: + connection = None + try: + while not self._stop.is_set(): + try: + connection, _peer = listener.accept() + break + except socket.timeout: + continue + if connection is None: + return + with connection: + connection.settimeout(1.0) + while not self._stop.is_set(): + try: + if not connection.recv(1 << 20): + return + except socket.timeout: + continue + except OSError: + if not self._stop.is_set(): + self._stop.set() + + def send() -> None: + try: + with socket.create_connection(address, timeout=5.0) as connection: + payload = b"\0" * (1 << 20) + while not self._stop.is_set(): + connection.sendall(payload) + except OSError: + if not self._stop.is_set(): + self._stop.set() + + self._threads = [ + threading.Thread(target=receive, name="gate14-loopback-receive", daemon=True), + threading.Thread(target=send, name="gate14-loopback-send", daemon=True), + ] + for thread in self._threads: + thread.start() + + def close(self) -> None: + self._stop.set() + if self._listener is not None: + try: + self._listener.close() + except OSError: + pass + for thread in self._threads: + thread.join(timeout=5.0) + + +class LinuxProductActions: + """Own one real packaged desktop/node/worker lifecycle on Linux.""" + + def __init__( + self, + *, + config_path: Path, + run_id: str, + attempt_ordinal: int, + source_commit: str, + package_sha256: str, + clock: Callable[[], float] = time.time, + monotonic: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + self.config_path = Path(config_path) + self.config = _load_config(self.config_path) + expected = { + "run_id": run_id, + "attempt_ordinal": attempt_ordinal, + "source_commit": source_commit, + "package_sha256": package_sha256, + "platform": "linux", + } + if any(self.config.get(field) != value for field, value in expected.items()): + raise Gate14LinuxProductError("product action binding changed") + self.run_id = run_id + self.attempt_ordinal = attempt_ordinal + self.source_commit = source_commit + self.package_sha256 = package_sha256 + self.clock = clock + self.monotonic = monotonic + self.sleeper = sleeper + self.model_id = self.config["model_id"] + self.profile = MODEL_PROFILES.get(self.model_id) + if self.profile is None or self.config.get("manifest_digest") != self.profile["manifest_digest"]: + raise Gate14LinuxProductError("product model binding changed") + self.work_root = Path(self.config["work_root"]).resolve() + self.action_root = self.work_root / ACTION_ROOT_NAME + self.warm_cache = self.work_root / WARM_CACHE_NAME + self.release_root = self.action_root / "release" + self.install_root = self.action_root / "install" + self.persistent_root = self.action_root / "persistent" + self.cache_root = self.persistent_root / "model-cache" / self.profile["manifest_digest"].removeprefix("sha256:") + self.owner: gate13.SystemdUnitOwner | None = None + self.product: gate13.OwnedUnit | None = None + self.token = "" + self.credential_created = False + self.prepared = False + self.cleaned = False + self.context = None + self.cache_identities: Mapping[str, tuple[int, int, int, int, int]] = {} + self.baseline_processes = 0 + self.worker_pid = 0 + self.expected_policy: Mapping[str, Any] | None = None + self._burns: list[subprocess.Popen[bytes]] = [] + self._opener = gate13.build_opener(gate13.ProxyHandler({}), gate13._RejectRedirects()) + + def _poll(self, action: Callable[[], Any], timeout: float, label: str) -> Any: + deadline = self.monotonic() + timeout + last_error: Exception | None = None + while self.monotonic() < deadline: + try: + return action() + except Exception as exc: + last_error = exc + self.sleeper(0.25) + raise Gate14LinuxProductError(f"{label} did not reach the required state") from last_error + + def _request( + self, + method: str, + path: str, + payload: Mapping[str, Any] | None = None, + ) -> Mapping[str, Any]: + if not self.token: + raise Gate14LinuxProductError("product control credential is unavailable") + return gate13._control_request(self._opener, method, path, self.token, payload) + + def _policy(self, *, vram_bytes: int | None = None, schedule: Mapping[str, Any] | None = None) -> Mapping[str, Any]: + snapshot = self._request("GET", "/control/v1/contribution-policy") + revision = snapshot.get("config_revision") + if ( + set(snapshot) != {"schema_version", "config_revision", "policy"} + or snapshot.get("schema_version") != 1 + or not isinstance(revision, str) + ): + raise Gate14LinuxProductError("contribution policy snapshot is invalid") + vram = self.config["vram_bytes"] if vram_bytes is None else vram_bytes + policy = { + "sharing_enabled": True, + "allowed_models": [self.model_id], + "preferred_models": [self.model_id], + "denied_models": [], + "max_disk_space": f"{self.config['disk_bytes']}B", + "max_vram": f"{vram}B", + "max_bandwidth_mbps": float(self.config["bandwidth_mbps"]), + "max_power_watts": float(self.config["power_watts"]), + "pause_timeout": float(self.config["pause_timeout_seconds"]), + "schedule": _schedule(True) if schedule is None else schedule, + } + response = self._request( + "PUT", + "/control/v1/contribution-policy", + {"schema_version": 1, "expected_config_revision": revision, "policy": policy}, + ) + if ( + set(response) != {"schema_version", "config_revision", "policy"} + or response.get("schema_version") != 1 + or response.get("policy") != policy + ): + raise Gate14LinuxProductError("contribution policy update was not preserved") + self.expected_policy = policy + return response + + def _worker(self, *, running: bool = False) -> Mapping[str, Any]: + response = self._request("GET", "/control/v1/workers") + workers = response.get("workers") + if set(response) != {"workers"} or not isinstance(workers, list): + raise Gate14LinuxProductError("exact worker snapshot is invalid") + automatic = [item for item in workers if isinstance(item, dict) and item.get("automatic") is True] + active = [item for item in workers if isinstance(item, dict) and _active(item)] + if len(automatic) != 1 or (running and (len(active) != 1 or active[0] is not automatic[0])): + raise Gate14LinuxProductError("automatic worker identity is invalid") + worker = automatic[0] + if running and ( + worker.get("state") != "running" + or worker.get("desired_running") is not True + or worker.get("model") != self.model_id + or worker.get("intent_published") is not True + or worker.get("remote_acknowledged") is not True + or type(worker.get("pid")) is not int + or worker["pid"] < 1 + ): + raise Gate14LinuxProductError("automatic worker is not running with acknowledged intent") + return worker + + def _running(self) -> Mapping[str, Any]: + worker = self._worker(running=True) + if self.owner is None or self.product is None: + raise Gate14LinuxProductError("product owner is unavailable") + if worker["pid"] not in self.owner.process_ids(self.product): + raise Gate14LinuxProductError("automatic worker escaped the owned product unit") + return worker + + def _wait_running(self, timeout: float = 300.0) -> Mapping[str, Any]: + return self._poll(self._running, timeout, "automatic worker") + + def _status_worker(self) -> Mapping[str, Any]: + status = self._request("GET", "/control/v1/status") + gate13._status_identity( + status, + self.model_id, + self.profile["manifest_digest"].removeprefix("sha256:"), + ) + contribution = status.get("contribution") + workers = None if not isinstance(contribution, dict) else contribution.get("workers") + if not isinstance(workers, list): + raise Gate14LinuxProductError("public contribution status is invalid") + automatic = [item for item in workers if isinstance(item, dict) and item.get("id") == "automatic"] + if len(automatic) != 1: + raise Gate14LinuxProductError("public automatic worker status is invalid") + return automatic[0] + + def _wait_inactive( + self, + *, + resource: bool = False, + schedule: bool = False, + prior_pid: int | None = None, + ) -> tuple[Mapping[str, Any], Mapping[str, Any]]: + def observe() -> tuple[Mapping[str, Any], Mapping[str, Any]]: + private = self._worker() + public = self._status_worker() + if private.get("desired_running") is not True or private.get("pid") is not None or _active(private): + raise Gate14LinuxProductError("desired worker remains active") + if resource and private.get("resource_suspended") is not True: + raise Gate14LinuxProductError("resource suspension is absent") + if schedule and private.get("schedule_suspended") is not True: + raise Gate14LinuxProductError("schedule suspension is absent") + if prior_pid is not None and _pid_exists(prior_pid): + raise Gate14LinuxProductError("previous worker process remains") + if public.get("desired_running") is not True or _active(public): + raise Gate14LinuxProductError("public status still contains an active worker") + return private, public + + return self._poll(observe, MAX_TRANSITION_SECONDS, "worker suspension") + + def _write_node_config(self, value: Mapping[str, Any]) -> None: + path = self.persistent_root / "node-config.json" + original_mode = path.stat().st_mode & 0o777 + temporary = path.with_name(".gate14-node-config.tmp") + if temporary.exists() or temporary.is_symlink(): + raise Gate14LinuxProductError("temporary node configuration already exists") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, original_mode) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(_canonical(value)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + def _cpu_power_probe(self) -> Mapping[str, Any]: + path = self.persistent_root / "node-config.json" + original = path.read_bytes() + parsed = gate13._strict_json(original) + workers = parsed.get("workers") + automatic = ( + [] + if not isinstance(workers, list) + else [item for item in workers if isinstance(item, dict) and item.get("id") == "automatic"] + ) + if len(automatic) != 1: + raise Gate14LinuxProductError("automatic worker configuration is invalid") + changed = json.loads(json.dumps(parsed)) + changed_worker = next(item for item in changed["workers"] if item.get("id") == "automatic") + changed_worker["device"] = "cpu" + try: + self._write_node_config(changed) + + def rejected() -> Mapping[str, Any]: + worker = self._worker() + reason = worker.get("resource_reason") + if ( + worker.get("pid") is not None + or worker.get("resource_admitted") is not False + or not isinstance(reason, str) + or "power telemetry is unavailable" not in reason + ): + raise Gate14LinuxProductError("CPU power telemetry was not rejected") + return worker + + self._poll(rejected, MAX_TRANSITION_SECONDS, "CPU power telemetry rejection") + finally: + restored = gate13._strict_json(original) + self._write_node_config(restored) + self.worker_pid = self._wait_running()["pid"] + return { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + } + + def _low_vram_probe(self) -> None: + prior = self._running()["pid"] + self._policy(vram_bytes=1) + + def rejected() -> None: + worker = self._worker() + if worker.get("pid") is not None or worker.get("resource_admitted") is not False: + raise Gate14LinuxProductError("low VRAM policy was not rejected") + if _pid_exists(prior): + raise Gate14LinuxProductError("low VRAM worker process remains") + + self._poll(rejected, MAX_TRANSITION_SECONDS, "low VRAM rejection") + self._policy() + self.worker_pid = self._wait_running()["pid"] + + def _crash_recovery(self) -> Mapping[str, Any]: + before = self._running() + old_pid = before["pid"] + started = self.monotonic() + os.kill(old_pid, signal.SIGKILL) + + def recovered() -> Mapping[str, Any]: + worker = self._running() + if worker["pid"] == old_pid: + raise Gate14LinuxProductError("worker crash was not observed") + try: + os.kill(old_pid, 0) + except ProcessLookupError: + return worker + raise Gate14LinuxProductError("previous worker process remains") + + after = self._poll(recovered, MAX_TRANSITION_SECONDS, "worker crash recovery") + duration = round(self.monotonic() - started, 6) + self.worker_pid = after["pid"] + return { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": duration, + "previous_worker_absent": True, + "manifest_unchanged": after.get("model") == self.model_id, + "automatic_block_range_valid": isinstance(after.get("block_indices"), str), + "desired_intent_preserved": after.get("desired_running") is True, + } + + def _pause(self) -> Mapping[str, Any]: + before = self._running() + started = self.monotonic() + self._request("POST", "/control/v1/workers/automatic/pause") + if self.owner is None or self.product is None: + raise Gate14LinuxProductError("product owner is unavailable") + + def paused() -> None: + worker = self._worker() + if ( + worker.get("state") != "paused" + or worker.get("desired_running") is not False + or worker.get("operator_paused") is not True + or worker.get("pid") is not None + or before["pid"] in self.owner.process_ids(self.product) + ): + raise Gate14LinuxProductError("automatic worker did not pause") + + self._poll(paused, MAX_TRANSITION_SECONDS, "operator pause") + duration = round(self.monotonic() - started, 6) + result = { + "requested": True, + "completed": True, + "duration_seconds": duration, + "worker_count_after": 0, + "descendant_count_after": 0, + } + self._request("POST", "/control/v1/workers/automatic/start") + self.worker_pid = self._wait_running()["pid"] + return result + + def _restart(self) -> Mapping[str, Any]: + if self.owner is None or self.product is None or self.context is None: + raise Gate14LinuxProductError("product restart state is unavailable") + started = self.monotonic() + before = gate13._assert_cache_unchanged( + self.context.cache, + self.profile["manifest_digest"].removeprefix("sha256:"), + self.cache_identities, + ) + gate13._stop_products(self.owner, self.product, self.product) + self.product = None + product, _node = gate13._start_products( + self.owner, + self.product_root, + self.persistent_root, + self.token, + self._opener, + self.model_id, + self.profile["manifest_digest"].removeprefix("sha256:"), + ) + self.product = product + worker = self._wait_running() + after = gate13._assert_cache_unchanged( + self.context.cache, + self.profile["manifest_digest"].removeprefix("sha256:"), + self.cache_identities, + ) + policy = self._request("GET", "/control/v1/contribution-policy").get("policy") + if policy != self.expected_policy or before != after: + raise Gate14LinuxProductError("restart did not preserve policy and cache") + self.worker_pid = worker["pid"] + return { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": worker.get("desired_running") is True, + "worker_resumed": True, + "duration_seconds": round(self.monotonic() - started, 6), + "cache_reused": True, + } + + @property + def product_root(self) -> Path: + candidate = self.install_root / "CommunityAI" + if not candidate.is_dir(): + raise Gate14LinuxProductError("installed product root is unavailable") + return candidate + + def prepare(self) -> Mapping[str, Any]: + if self.prepared or self.cleaned: + raise Gate14LinuxProductError("product prepare order is invalid") + if self.action_root.exists() or self.action_root.is_symlink(): + raise Gate14LinuxProductError("product action root is not fresh") + if not self.warm_cache.is_dir() or self.warm_cache.is_symlink(): + raise Gate14LinuxProductError("fresh materialized cache is unavailable") + artifacts = _safe_artifacts(self.config) + try: + self.action_root.mkdir(mode=0o700) + self.release_root.mkdir(mode=0o700) + package = Path(self.config["package_path"]) + os.link(package, self.release_root / gate13.ARCHIVE_NAME) + for name in ("SHA256SUMS", "desktop-metrics.json", "provenance.json", "release-metadata.json"): + source = Path(self.config["staging_root"]) / "release-audit" / name + shutil.copyfile(source, self.release_root / name) + audit = gate13._audit_package( + self.release_root, + self.package_sha256.removeprefix("sha256:"), + self.config["package_bytes"], + ) + if audit.source_commit != self.source_commit: + raise Gate14LinuxProductError("package source binding changed") + self.owner = gate13.SystemdUnitOwner(f"{self.run_id}-a{self.attempt_ordinal}") + gate13._extract_package(audit, self.install_root) + gate13._run_self_tests(self.owner, self.product_root, audit.package_version) + if gate13._credential_count() != 0: + raise Gate14LinuxProductError("clean host credential baseline is not empty") + self.persistent_root.mkdir(mode=0o700) + _bootstrap, manifest = gate13._bootstrap( + self.owner, + self.product_root, + self.persistent_root, + self.model_id, + self.profile["manifest_digest"].removeprefix("sha256:"), + audit, + ) + context_cache = self.cache_root + context_cache.parent.mkdir(mode=0o700, parents=True) + os.replace(self.warm_cache, context_cache) + verified, identities = gate13._verify_cache( + context_cache, + self.profile["manifest_digest"].removeprefix("sha256:"), + artifacts, + ) + if verified != self.profile["selected_artifact_bytes"]: + raise Gate14LinuxProductError("materialized cache byte total changed") + self.context = type( + "Gate14Context", + (), + {"cache": context_cache, "manifest": manifest}, + )() + self.cache_identities = identities + self.token = "drift_control_" + secrets.token_urlsafe(32) + gate13._store_control_token(self.token) + self.credential_created = True + product, _node = gate13._start_products( + self.owner, + self.product_root, + self.persistent_root, + self.token, + self._opener, + self.model_id, + self.profile["manifest_digest"].removeprefix("sha256:"), + ) + self.product = product + self._policy() + try: + self._request("POST", "/control/v1/workers/automatic/start") + except BaseException: + pass + worker = self._wait_running(1_800.0) + status_worker = self._status_worker() + placement = status_worker.get("placement") + resources = status_worker.get("resources") + limits = None if not isinstance(resources, dict) else resources.get("limits") + if ( + not isinstance(placement, dict) + or placement.get("automatic") is not True + or not isinstance(limits, dict) + or worker.get("intent_published") is not True + or worker.get("remote_acknowledged") is not True + ): + raise Gate14LinuxProductError("automatic placement evidence is incomplete") + block_indices = placement.get("block_indices") + if not isinstance(block_indices, str) or ":" not in block_indices: + raise Gate14LinuxProductError("automatic block placement is invalid") + block_start, block_end = (int(part) for part in block_indices.split(":", 1)) + resolved = { + "disk_bytes": self.config["disk_bytes"], + "vram_bytes": self.config["vram_bytes"], + "bandwidth_mbps": float(self.config["bandwidth_mbps"]), + "power_watts": float(self.config["power_watts"]), + } + for field, expected in resolved.items(): + if float(limits.get(field, -1)) != float(expected): + raise Gate14LinuxProductError("resolved contribution limits changed") + self.worker_pid = worker["pid"] + self.baseline_processes = len(self.owner.process_ids(self.product)) - 1 + self._low_vram_probe() + unsupported = self._cpu_power_probe() + recovery = self._crash_recovery() + pause = self._pause() + restart = self._restart() + after = gate13._assert_cache_unchanged( + context_cache, + self.profile["manifest_digest"].removeprefix("sha256:"), + self.cache_identities, + ) + self.prepared = True + return { + "schema_version": 1, + "scope": "gate14-prepared-host-observations", + "run_id": self.run_id, + "platform": "linux", + "attempt_ordinal": self.attempt_ordinal, + "source_commit": self.source_commit, + "package_sha256": self.package_sha256, + "model": { + "id": self.model_id, + "manifest_digest": self.profile["manifest_digest"], + "revision_commit": self.profile["revision_commit"], + "gate9_envelope_sha256": self.profile["gate9_envelope_sha256"], + "selected_artifact_count": self.profile["selected_artifact_count"], + "selected_artifact_bytes": self.profile["selected_artifact_bytes"], + "total_blocks": self.profile["total_blocks"], + }, + "cache": { + "verified_bytes_before": verified, + "verified_bytes_after": after, + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": block_start, + "block_end": block_end, + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + **resolved, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "recovery": recovery, + "pause": pause, + "restart": restart, + "unsupported_telemetry": unsupported, + } + except BaseException as exc: + try: + self.cleanup() + except BaseException as cleanup_exc: + raise Gate14LinuxProductError("prepare failed and cleanup was incomplete") from cleanup_exc + if isinstance(exc, Gate14LinuxProductError): + raise + raise Gate14LinuxProductError("packaged product prepare failed") from exc + + def _measurement(self, worker: Mapping[str, Any], field: str) -> float: + value = worker.get(field) + if type(value) not in (int, float): + raise Gate14LinuxProductError("physical resource measurement is unavailable") + return float(value) + + def _calibration_record( + self, + *, + kind: str, + challenge: Mapping[str, Any], + started_at: float, + ended_at: float, + baseline: float, + trigger: float, + resume: float, + source: str, + scope: str, + configured: float, + duration: float, + ) -> Mapping[str, Any]: + interval = float(self.config["sample_interval_seconds"]) + if ended_at - started_at < 2 * interval or ended_at - started_at > 120: + raise Gate14LinuxProductError("physical calibration sample window is invalid") + return { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": round(duration, 6), + "calibration": { + "measurement_source": source, + "measurement_scope": scope, + "sample_count": 3, + "sample_interval_seconds": interval, + "baseline_value": baseline, + "configured_limit": configured, + "trigger_value": trigger, + "resume_value": resume, + "challenge_sha256": challenge["challenge_sha256"], + "sample_started_at_unix": started_at, + "sample_ended_at_unix": ended_at, + }, + } + + def _calibrate_bandwidth(self, challenge: Mapping[str, Any]) -> Mapping[str, Any]: + limit = float(self.config["bandwidth_mbps"]) + started_wall = self.clock() + started = self.monotonic() + interval = float(self.config["sample_interval_seconds"]) + baseline_samples = [] + for _index in range(3): + baseline_samples.append(self._measurement(self._worker(running=True), "current_bandwidth_mbps")) + self.sleeper(interval) + baseline = min(baseline_samples) + if baseline >= limit: + raise Gate14LinuxProductError("bandwidth baseline already exceeds its limit") + load = _LoopbackLoad() + load.start() + try: + private, _public = self._wait_inactive(resource=True, prior_pid=self.worker_pid) + trigger = self._measurement(private, "current_bandwidth_mbps") + finally: + load.close() + resumed = self._wait_running() + resume = self._measurement(resumed, "current_bandwidth_mbps") + ended_wall = self.clock() + if not (trigger > limit and resume < limit): + raise Gate14LinuxProductError("bandwidth calibration did not cross its limit") + self.worker_pid = resumed["pid"] + return self._calibration_record( + kind="bandwidth", + challenge=challenge, + started_at=started_wall, + ended_at=ended_wall, + baseline=baseline, + trigger=trigger, + resume=resume, + source="host-network-counters", + scope="aggregate-host-network", + configured=limit, + duration=self.monotonic() - started, + ) + + def _start_power_burn(self) -> subprocess.Popen[bytes]: + if self.context is None: + raise Gate14LinuxProductError("power calibration context is unavailable") + node = self.product_root / "node" / "CommunityAI-Node" + process = subprocess.Popen( + [ + str(node), + "edge-benchmark", + str(self.context.manifest), + "--cache_dir", + str(self.context.cache), + "--allow_warm_cache", + "--prompt", + "CommunityAI Gate 14 calibration", + "--max_new_tokens", + "128", + "--supervisor_timeout", + "120", + ], + cwd=self.product_root, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + self._burns.append(process) + return process + + def _stop_burn(self, process: subprocess.Popen[bytes]) -> None: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=10) + except BaseException: + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + pass + try: + process.wait(timeout=10) + except BaseException: + pass + if process in self._burns: + self._burns.remove(process) + + def _calibrate_power(self, challenge: Mapping[str, Any]) -> Mapping[str, Any]: + limit = float(self.config["power_watts"]) + interval = float(self.config["sample_interval_seconds"]) + started_wall = self.clock() + started = self.monotonic() + baseline_samples = [] + for _index in range(3): + baseline_samples.append(self._measurement(self._worker(running=True), "current_power_watts")) + self.sleeper(interval) + baseline = min(baseline_samples) + if baseline >= limit: + raise Gate14LinuxProductError("power baseline already exceeds its limit") + burn = self._start_power_burn() + try: + private, _public = self._wait_inactive(resource=True, prior_pid=self.worker_pid) + trigger = self._measurement(private, "current_power_watts") + finally: + self._stop_burn(burn) + resumed = self._wait_running() + resume = self._measurement(resumed, "current_power_watts") + ended_wall = self.clock() + if not (trigger > limit and resume < limit): + raise Gate14LinuxProductError("power calibration did not cross its limit") + self.worker_pid = resumed["pid"] + return self._calibration_record( + kind="power", + challenge=challenge, + started_at=started_wall, + ended_at=ended_wall, + baseline=baseline, + trigger=trigger, + resume=resume, + source="nvidia-nvml-device-power", + scope="selected-nvidia-l4-device", + configured=limit, + duration=self.monotonic() - started, + ) + + def _calibrate_schedule(self, challenge: Mapping[str, Any]) -> Mapping[str, Any]: + interval = float(self.config["sample_interval_seconds"]) + started_wall = self.clock() + started = self.monotonic() + self.sleeper(2 * interval) + self._policy(schedule=_schedule(False, self.clock())) + try: + self._wait_inactive(schedule=True, prior_pid=self.worker_pid) + finally: + self._policy(schedule=_schedule(True)) + resumed = self._wait_running() + ended_wall = self.clock() + self.worker_pid = resumed["pid"] + return self._calibration_record( + kind="schedule", + challenge=challenge, + started_at=started_wall, + ended_at=ended_wall, + baseline=1.0, + trigger=0.0, + resume=1.0, + source="utc-policy-clock", + scope="utc-schedule-policy", + configured=0.5, + duration=self.monotonic() - started, + ) + + def calibrate(self, challenge: Mapping[str, Any]) -> Sequence[Mapping[str, Any]]: + if not self.prepared or self.cleaned: + raise Gate14LinuxProductError("product calibration order is invalid") + now = self.clock() + if ( + set(challenge) + != { + "challenge_sha256", + "controller_state_revision", + "issued_at_unix", + "expires_at_unix", + } + or type(challenge.get("issued_at_unix")) is not int + or type(challenge.get("expires_at_unix")) is not int + or not challenge["issued_at_unix"] <= now <= challenge["expires_at_unix"] + ): + raise Gate14LinuxProductError("controller challenge is invalid or stale") + records = [ + self._calibrate_bandwidth(challenge), + self._calibrate_power(challenge), + self._calibrate_schedule(challenge), + ] + if self.clock() > challenge["expires_at_unix"]: + raise Gate14LinuxProductError("controller challenge expired during calibration") + return records + + def cleanup(self) -> Mapping[str, Any]: + if self.cleaned: + return { + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + "run_id": self.run_id, + "platform": "linux", + "attempt_ordinal": self.attempt_ordinal, + "processes_absent": True, + "credentials_removed": True, + "action_temporaries_removed": True, + } + failed = False + for process in tuple(self._burns): + try: + self._stop_burn(process) + except BaseException: + failed = True + if self.owner is not None: + try: + self.owner.stop_all() + except BaseException: + failed = True + self.product = None + if self.credential_created: + try: + gate13._clear_control_token() + except BaseException: + failed = True + self.credential_created = False + self.token = "" + for path in (self.warm_cache, self.action_root): + try: + if path.exists() or path.is_symlink(): + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + except BaseException: + failed = True + try: + if gate13._credential_count() != 0: + failed = True + except BaseException: + failed = True + if self.owner is not None: + try: + if self.owner.process_count() != 0: + failed = True + except BaseException: + failed = True + if self.action_root.exists() or self.warm_cache.exists() or failed: + raise Gate14LinuxProductError("packaged product cleanup was not proved") + self.cleaned = True + return { + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + "run_id": self.run_id, + "platform": "linux", + "attempt_ordinal": self.attempt_ordinal, + "processes_absent": True, + "credentials_removed": True, + "action_temporaries_removed": True, + } diff --git a/scripts/gate14_packaged_lifecycle.py b/scripts/gate14_packaged_lifecycle.py new file mode 100644 index 000000000..8b6f5a3ae --- /dev/null +++ b/scripts/gate14_packaged_lifecycle.py @@ -0,0 +1,2281 @@ +"""Shared, fail-closed Gate 14 packaged lifecycle boundary. + +Platform adapters perform the real desktop, control-API, process, cache, and +measurement operations. This module owns the source/package configuration, +immutable challenge-ready checkpoint, challenge ordering, strict observation +validation, privacy cleanup boundary, and final host-probe invocation. The +configuration deliberately has no fields for claimed passes, suspension +results, calibration samples, or cleanup success. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +import tempfile +import time +import zipfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Mapping, Protocol, Sequence + +import gate14_calibration_challenge as challenge_contract +import gate14_hardware_acceptance as acceptance +import gate14_host_probe as host_probe + +SCHEMA_VERSION = 1 +SCOPE = "gate14-packaged-lifecycle" +PREPARED_SCOPE = "gate14-prepared-host-observations" +CHECKPOINT_SCOPE = "gate14-host-lifecycle-checkpoint" +CLEANUP_SCOPE = "gate14-host-lifecycle-cleanup" +MAX_CONFIG_BYTES = 65_536 +MAX_PRIVATE_JSON_BYTES = 262_144 +MAX_RELEASE_METADATA_BYTES = host_probe.MAX_JSON_BYTES +MAX_RELEASE_PROVENANCE_BYTES = 8 * 1024**2 +MAX_DESKTOP_METRICS_BYTES = 1024**2 +MAX_RELEASE_CHECKSUMS_BYTES = 4 * 1024**2 +MAX_RELEASE_AUDIT_BYTES = 32 * 1024**2 +MAX_MATERIALIZATION_RECORD_BYTES = 2 * 1024**2 +MAX_PACKAGE_BYTES = 8 * 1024**3 +MAX_CHALLENGE_WAIT_SECONDS = 1_200.0 + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") + +_CONFIG_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "attempt_ordinal", + "source_commit", + "package_path", + "package_sha256", + "package_bytes", + "release_metadata_path", + "release_metadata_sha256", + "release_audit", + "warm_cache", + "model_id", + "manifest_digest", + "gate13_evidence_sha256", + "staging_root", + "work_root", + "challenge_path", + "checkpoint_path", + "facts_path", + "evidence_path", + "disk_bytes", + "vram_bytes", + "bandwidth_mbps", + "power_watts", + "pause_timeout_seconds", + "sample_interval_seconds", + "max_challenge_wait_seconds", +} +_PREPARED_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "attempt_ordinal", + "source_commit", + "package_sha256", + "model", + "cache", + "placement", + "limits", + "recovery", + "pause", + "restart", + "unsupported_telemetry", +} +_CHECKPOINT_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "attempt_ordinal", + "source_commit", + "lifecycle_config_sha256", + "package_sha256", + "release_metadata_sha256", + "release_audit_sha256", + "warm_cache_sha256", + "materialization_record_sha256", + "prepared_facts_sha256", + "phase", + "created_at_unix", +} +_CLEANUP_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "attempt_ordinal", + "processes_absent", + "credentials_removed", + "action_temporaries_removed", +} +_PACKAGE_NAMES = { + "windows": "communityai-desktop-windows.zip", + "linux": "communityai-desktop-linux.tar.gz", +} +_OUTPUT_NAMES = { + "checkpoint_path": "gate14-checkpoint.json", + "facts_path": "gate14-facts.json", + "evidence_path": "gate14-platform-evidence.json", +} +_CHALLENGE_NAME = "gate14-challenge.json" +_RELEASE_AUDIT_ARCHIVE_NAME = "release-audit.zip" +_RELEASE_AUDIT_DIRECTORY_NAME = "release-audit" +_MATERIALIZATION_RECORD_NAME = "gate14-cache-materialization.json" +_RELEASE_AUDIT_MEMBERS = ( + "SHA256SUMS", + "desktop-metrics.json", + "provenance.json", + "release-metadata.json", +) +_MODEL_SOURCE = { + "Qwen3.5 2B": ("Qwen/Qwen3.5-2B", "bfloat16"), + "Gemma 4 E2B IT": ("google/gemma-4-E2B-it", "bfloat16"), +} +_GATE9_WARM_CACHE = { + "windows": { + "gate9_acquisition_record_sha256": "sha256:557c9a5a5441d095f780bfe20620450502a1b941fd7e33fe72b83bec5e147c52", + "gate9_resource_envelope_sha256": "sha256:cd68afb67d9b0f3cb8c82db0d3314ad89b558c20880998ea4d8c4493e9f4bc9f", + "artifacts": ( + ( + "chat_template.jinja", + "chat_template", + "sha256:273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80", + 7755, + ), + ("config.json", "config", "sha256:ed1c1723241f23f7f4e23430759cbd7dcfb4103cbdfe052bfe7626b57c2615b4", 2908), + ( + "merges.txt", + "tokenizer", + "sha256:a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + 3353259, + ), + ( + "model.safetensors-00001-of-00001.safetensors", + "weight", + "sha256:aa33250c4fc64891ddfaba3a314fd9542ea371843c387178b425fbcc5ed680b1", + 4548221488, + ), + ( + "model.safetensors.index.json", + "weight_index", + "sha256:aca8afed9da75b0f050b408d270766fd77627f1af401e240f61c3b47d0db02f9", + 64460, + ), + ( + "tokenizer.json", + "tokenizer", + "sha256:5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + 12807982, + ), + ( + "tokenizer_config.json", + "tokenizer", + "sha256:49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c", + 16709, + ), + ( + "vocab.json", + "tokenizer", + "sha256:ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + 6722759, + ), + ), + }, + "linux": { + "gate9_acquisition_record_sha256": "sha256:1628f87f1baaa2f562ca6c7340d2863034cdb0d17dbdf8995e38d9ae792fe0b5", + "gate9_resource_envelope_sha256": "sha256:2eb0bcf6419ba085665fad34310453a1b9dc2e89d90e9177f41566df012996c8", + "artifacts": ( + ( + "chat_template.jinja", + "chat_template", + "sha256:0a2c8073c878ab1da004bee933a998606537bbb62016310352c7285c3f01c5b5", + 18569, + ), + ("config.json", "config", "sha256:1b28f3d2c3100f6c594754b81107428bd7b822a7f48272ca681dae9d2ec38330", 4954), + ( + "model.safetensors", + "weight", + "sha256:2db5482b20d746879bb3ef79b5203e9075a2e2b98f54ec7c2f281c1477ddc550", + 10246621918, + ), + ( + "tokenizer.json", + "tokenizer", + "sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f", + 32169626, + ), + ( + "tokenizer_config.json", + "tokenizer", + "sha256:9f4fec4b1dc6ecddf8f4a92e9caea5971c0e67d81309f3f9066a2bee8c362633", + 3082, + ), + ), + }, +} +_PENDING_EVIDENCE_NAME = ".gate14-platform-evidence.pending.json" +_RELEASE_METADATA = { + "schema_version": 1, + "product": "CommunityAI", + "package": "communityai-desktop", + "release_channel": "public-alpha", + "warning": ( + "Unsigned public-alpha engineering bundle: verify SHA256SUMS before use. " + "No publisher signature or authenticated automatic update is provided." + ), + "unsigned": True, + "publisher_signature": False, + "automatic_updates": False, + "supported_platforms": ["Windows", "Linux"], + "macos_supported": False, + "credits_enabled": False, + "complete_release_qualification": False, + "artifact_root": "CommunityAI", + "artifact_inventory": "regular-files-and-relative-internal-file-symlinks-with-file-modes", + "checksum_manifest": "SHA256SUMS", + "install_archive_required": True, + "install_archive_provenance": "provenance.json#install_archive", + "desktop_metrics": "desktop-metrics.json", + "provenance": "provenance.json", +} + + +class Gate14LifecycleError(ValueError): + """A packaged lifecycle input or observation failed closed.""" + + +class LifecycleActions(Protocol): + """Source-bound platform actions used by the shared sequencer.""" + + def prepare(self, config: "LifecycleConfig") -> Mapping[str, Any]: + """Perform and observe every non-calibration qualification drill.""" + + def calibrate( + self, + config: "LifecycleConfig", + challenge: Mapping[str, Any], + ) -> Sequence[Mapping[str, Any]]: + """Measure the three physical suspension and recovery windows.""" + + def cleanup(self, config: "LifecycleConfig") -> Mapping[str, Any]: + """Stop product processes and remove credentials/action temporaries.""" + + +HardwareProbe = Callable[[str], Mapping[str, Any]] + + +@dataclass(frozen=True) +class AuditMemberBinding: + name: str + sha256: str + size_bytes: int + path: Path + + +@dataclass(frozen=True) +class ReleaseAuditBinding: + artifact_name: str + artifact_sha256: str + artifact_bytes: int + archive_path: Path + members: tuple[AuditMemberBinding, ...] + binding_sha256: str + + +@dataclass(frozen=True) +class CacheArtifactBinding: + path: str + role: str + sha256: str + size_bytes: int + + +@dataclass(frozen=True) +class WarmCacheBinding: + gate9_acquisition_record_sha256: str + gate9_resource_envelope_sha256: str + source_commit: str + materialization_plan_sha256: str + materializer_sources_sha256: str + materialization_record_sha256: str + materialization_record_bytes: int + materialization_record_path: Path + artifact_count: int + artifact_bytes: int + artifacts: tuple[CacheArtifactBinding, ...] + binding_sha256: str + + +@dataclass(frozen=True) +class LifecycleConfig: + run_id: str + platform: str + attempt_ordinal: int + source_commit: str + config_sha256: str + config_path: Path + package_path: Path + package_sha256: str + package_bytes: int + release_metadata_path: Path + release_metadata_sha256: str + release_audit: ReleaseAuditBinding + warm_cache: WarmCacheBinding + model_id: str + manifest_digest: str + gate13_evidence_sha256: str + staging_root: Path + work_root: Path + challenge_path: Path + checkpoint_path: Path + facts_path: Path + evidence_path: Path + disk_bytes: int + vram_bytes: int + bandwidth_mbps: float + power_watts: float + pause_timeout_seconds: float + sample_interval_seconds: float + max_challenge_wait_seconds: float + + +def _reject_constant(_value: str) -> None: + raise Gate14LifecycleError("non-finite JSON value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14LifecycleError("duplicate JSON field") + result[key] = value + return result + + +def _canonical(value: Mapping[str, Any]) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Gate14LifecycleError("observation is not canonical JSON") from exc + + +def _digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _exact_equal(value: Any, expected: Any) -> bool: + if type(value) is not type(expected): + return False + if isinstance(expected, dict): + return set(value) == set(expected) and all(_exact_equal(value[key], expected[key]) for key in expected) + if isinstance(expected, list): + return len(value) == len(expected) and all( + _exact_equal(observed, required) for observed, required in zip(value, expected) + ) + return value == expected + + +def _open_regular( + path: Path, + maximum: int, + *, + minimum: int = 1, +): + """Open one exact regular file without following a raced path.""" + + path = Path(path) + try: + before = path.lstat() + except OSError as exc: + raise Gate14LifecycleError("required file is unavailable") from exc + reparse = bool(getattr(before, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or path.is_symlink() + or not stat.S_ISREG(before.st_mode) + or (os.name != "nt" and before.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) + or not minimum <= before.st_size <= maximum + ): + raise Gate14LifecycleError("required file is unsafe") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + handle = os.fdopen(descriptor, "rb") + except OSError as exc: + raise Gate14LifecycleError("required file is unreadable") from exc + try: + opened = os.fstat(handle.fileno()) + if ( + not stat.S_ISREG(opened.st_mode) + or not minimum <= opened.st_size <= maximum + or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino) + ): + raise Gate14LifecycleError("required file changed while opening") + return handle, opened + except BaseException: + handle.close() + raise + + +def _regular_metadata( + path: Path, + maximum: int, + *, + minimum: int = 1, +) -> os.stat_result: + handle, metadata = _open_regular(path, maximum, minimum=minimum) + handle.close() + return metadata + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + handle, metadata = _open_regular(path, maximum) + try: + payload = handle.read(maximum + 1) + after = os.fstat(handle.fileno()) + except OSError as exc: + raise Gate14LifecycleError("required file is unreadable") from exc + finally: + handle.close() + if len(payload) != metadata.st_size or (after.st_dev, after.st_ino, after.st_size) != ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + ): + raise Gate14LifecycleError("required file changed while reading") + return payload + + +def _strict_json(payload: bytes, maximum: int) -> Mapping[str, Any]: + if not 1 <= len(payload) <= maximum: + raise Gate14LifecycleError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14LifecycleError("JSON is invalid") from exc + if not isinstance(value, dict): + raise Gate14LifecycleError("JSON root is invalid") + return value + + +def _absolute_path(value: Any, label: str) -> Path: + if not isinstance(value, str) or not value or "\x00" in value: + raise Gate14LifecycleError(f"{label} is invalid") + path = Path(value) + if not path.is_absolute(): + raise Gate14LifecycleError(f"{label} is not absolute") + return Path(os.path.abspath(os.fspath(path))) + + +def _bounded_number( + value: Any, + label: str, + minimum: float, + maximum: float, +) -> float: + if type(value) not in (int, float): + raise Gate14LifecycleError(f"{label} is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise Gate14LifecycleError(f"{label} is invalid") + return rendered + + +def _bounded_integer( + value: Any, + label: str, + minimum: int, + maximum: int, +) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise Gate14LifecycleError(f"{label} is invalid") + return value + + +def _work_root(path: Path, *, controller_owned: bool = False) -> Path: + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14LifecycleError("work root is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or path.is_symlink() + or not stat.S_ISDIR(metadata.st_mode) + or (controller_owned and os.name != "nt" and metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) + ): + raise Gate14LifecycleError("work root is unsafe") + try: + return path.resolve(strict=True) + except OSError as exc: + raise Gate14LifecycleError("work root is unavailable") from exc + + +def _assert_windows_access_denied( + *, + directory: bool, + opener: Callable[[int], Any], + closer: Callable[[Any], Any], + invalid_handle: Any, + get_last_error: Callable[[], int], +) -> None: + masks = [0x00010000, 0x00040000, 0x00080000, 0x40000000] + if directory: + masks.extend((0x00000002, 0x00000004, 0x00000040)) + for access_mask in masks: + handle = opener(access_mask) + if handle != invalid_handle: + closer(handle) + raise Gate14LifecycleError("controller staging is writable by the qualification process") + if get_last_error() != 5: + raise Gate14LifecycleError("controller staging write access could not be disproved") + + +def _windows_controller_owned( + path: Path, + *, + directory: bool, + require_qualification_denied: bool = True, +) -> None: + import ctypes + from ctypes import wintypes + + advapi32 = ctypes.WinDLL("advapi32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + local_free = kernel32.LocalFree + local_free.argtypes = (ctypes.c_void_p,) + local_free.restype = ctypes.c_void_p + close_handle = kernel32.CloseHandle + close_handle.argtypes = (wintypes.HANDLE,) + close_handle.restype = wintypes.BOOL + owner = ctypes.c_void_p() + dacl = ctypes.c_void_p() + descriptor = ctypes.c_void_p() + get_security = advapi32.GetNamedSecurityInfoW + get_security.argtypes = ( + wintypes.LPWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ) + get_security.restype = wintypes.DWORD + result = get_security( + os.fspath(path), + 1, + 0x1 | 0x4, + ctypes.byref(owner), + None, + ctypes.byref(dacl), + None, + ctypes.byref(descriptor), + ) + if result != 0: + raise Gate14LifecycleError("controller staging ACL is unavailable") + try: + if not owner.value or not dacl.value: + raise Gate14LifecycleError("controller staging ACL is unsafe") + owner_text = wintypes.LPWSTR() + convert_sid = advapi32.ConvertSidToStringSidW + convert_sid.argtypes = (ctypes.c_void_p, ctypes.POINTER(wintypes.LPWSTR)) + convert_sid.restype = wintypes.BOOL + if not convert_sid(owner, ctypes.byref(owner_text)): + raise Gate14LifecycleError("controller staging owner is unavailable") + try: + if owner_text.value not in {"S-1-5-18", "S-1-5-32-544"}: + raise Gate14LifecycleError("controller staging owner is unsafe") + finally: + local_free(ctypes.cast(owner_text, ctypes.c_void_p)) + control = wintypes.WORD() + revision = wintypes.DWORD() + get_control = advapi32.GetSecurityDescriptorControl + get_control.argtypes = ( + ctypes.c_void_p, + ctypes.POINTER(wintypes.WORD), + ctypes.POINTER(wintypes.DWORD), + ) + get_control.restype = wintypes.BOOL + if not get_control(descriptor, ctypes.byref(control), ctypes.byref(revision)) or not control.value & 0x1000: + raise Gate14LifecycleError("controller staging DACL is not protected") + finally: + local_free(descriptor) + + if not require_qualification_denied: + return + + create_file = kernel32.CreateFileW + create_file.argtypes = ( + wintypes.LPWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ) + create_file.restype = wintypes.HANDLE + + def open_with_access(access_mask: int): + ctypes.set_last_error(0) + return create_file( + os.fspath(path), + access_mask, + 0x1 | 0x2 | 0x4, + None, + 3, + 0x02000000 if directory else 0, + None, + ) + + _assert_windows_access_denied( + directory=directory, + opener=open_with_access, + closer=close_handle, + invalid_handle=ctypes.c_void_p(-1).value, + get_last_error=ctypes.get_last_error, + ) + + +def _assert_controller_managed(path: Path, *, directory: bool) -> None: + """Prove structural controller ownership without constraining the caller token.""" + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14LifecycleError("controller staging is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + expected_type = stat.S_ISDIR(metadata.st_mode) if directory else stat.S_ISREG(metadata.st_mode) + if reparse or path.is_symlink() or not expected_type: + raise Gate14LifecycleError("controller staging type is unsafe") + if os.name == "nt": + _windows_controller_owned( + path, + directory=directory, + require_qualification_denied=False, + ) + return + if metadata.st_uid != 0 or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise Gate14LifecycleError("controller staging ownership is unsafe") + + +def _assert_controller_owned(path: Path, *, directory: bool) -> None: + _assert_controller_managed(path, directory=directory) + if os.name == "nt": + _windows_controller_owned(path, directory=directory) + return + if os.geteuid() == 0 or os.access(path, os.W_OK, effective_ids=True): + raise Gate14LifecycleError("controller staging ownership is unsafe") + + +def _path_under_root(path: Path, root: Path, expected_name: str, label: str) -> Path: + path = _absolute_path(os.fspath(path), label) + if path.name != expected_name: + raise Gate14LifecycleError("private lifecycle filename is invalid") + try: + parent = path.parent.resolve(strict=True) + except OSError as exc: + raise Gate14LifecycleError("private lifecycle parent is unavailable") from exc + if parent != root: + raise Gate14LifecycleError(f"{label} escaped its root") + if path.exists(): + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise Gate14LifecycleError(f"{label} is unsafe") + return path + + +def _private_path(raw: Mapping[str, Any], field: str, root: Path) -> Path: + return _path_under_root( + _absolute_path(raw[field], field.replace("_", " ")), + root, + _OUTPUT_NAMES[field], + "private lifecycle path", + ) + + +def _exact_staged_path(path: Path, expected: Path, label: str, *, directory: bool) -> Path: + candidate = _absolute_path(os.fspath(path), label) + expected = Path(os.path.abspath(os.fspath(expected))) + if candidate != expected: + raise Gate14LifecycleError(f"{label} escaped its fixed location") + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14LifecycleError(f"{label} is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + expected_type = stat.S_ISDIR(metadata.st_mode) if directory else stat.S_ISREG(metadata.st_mode) + if reparse or candidate.is_symlink() or not expected_type: + raise Gate14LifecycleError(f"{label} is unsafe") + return candidate + + +def _public_path(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or "\\" in value or any(ord(character) < 32 for character in value): + raise Gate14LifecycleError(f"{label} is unsafe") + pure = PurePosixPath(value) + if pure.is_absolute() or pure.as_posix() != value or any(part in ("", ".", "..") for part in pure.parts): + raise Gate14LifecycleError(f"{label} is unsafe") + return value + + +def _digest_field(value: Any, label: str) -> str: + if not isinstance(value, str) or _DIGEST_RE.fullmatch(value) is None: + raise Gate14LifecycleError(f"{label} is invalid") + return value + + +def _audit_member_payloads( + binding: ReleaseAuditBinding, + *, + ownership_verifier: Callable[..., None] | None = None, +) -> dict[str, bytes]: + if ownership_verifier is None: + ownership_verifier = _assert_controller_owned + ownership_verifier(binding.archive_path, directory=False) + archive = _regular_bytes(binding.archive_path, MAX_RELEASE_AUDIT_BYTES) + if len(archive) != binding.artifact_bytes or _digest(archive) != binding.artifact_sha256: + raise Gate14LifecycleError("release audit artifact binding changed") + + expected = {member.name: member for member in binding.members} + payloads: dict[str, bytes] = {} + try: + import io + + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + infos = source.infolist() + names = [info.filename for info in infos] + if names != list(_RELEASE_AUDIT_MEMBERS) or any( + info.is_dir() or info.flag_bits & 0x1 or stat.S_IFMT(info.external_attr >> 16) not in {0, stat.S_IFREG} + for info in infos + ): + raise Gate14LifecycleError("release audit archive members are invalid") + for info in infos: + member = expected[info.filename] + if info.file_size != member.size_bytes: + raise Gate14LifecycleError("release audit archive member size changed") + payload = source.read(info) + if len(payload) != member.size_bytes or _digest(payload) != member.sha256: + raise Gate14LifecycleError("release audit archive member digest changed") + payloads[member.name] = payload + except (OSError, zipfile.BadZipFile, RuntimeError) as exc: + raise Gate14LifecycleError("release audit archive is unreadable") from exc + + for member in binding.members: + ownership_verifier(member.path, directory=False) + staged = _regular_bytes( + member.path, + { + "provenance.json": MAX_RELEASE_PROVENANCE_BYTES, + "desktop-metrics.json": MAX_DESKTOP_METRICS_BYTES, + "SHA256SUMS": MAX_RELEASE_CHECKSUMS_BYTES, + }.get(member.name, MAX_RELEASE_METADATA_BYTES), + ) + if staged != payloads[member.name]: + raise Gate14LifecycleError("release audit extracted member changed") + return payloads + + +def _validate_release_semantics( + payloads: Mapping[str, bytes], + *, + platform: str, + source_commit: str, + package_sha256: str, + package_bytes: int, +) -> None: + provenance = _strict_json( + payloads["provenance.json"], + MAX_RELEASE_PROVENANCE_BYTES, + ) + metrics = _strict_json( + payloads["desktop-metrics.json"], + MAX_DESKTOP_METRICS_BYTES, + ) + release_metadata = _strict_json( + payloads["release-metadata.json"], + MAX_RELEASE_METADATA_BYTES, + ) + if not _exact_equal(release_metadata, _RELEASE_METADATA): + raise Gate14LifecycleError("release audit metadata claims are invalid") + + provenance_fields = { + "schema_version", + "product", + "package", + "release_channel", + "source_commit", + "source_tree", + "build_workflow", + "build_platform", + "build_python", + "build_pyinstaller", + "artifact_root", + "checksum_manifest", + "artifacts", + "install_archive", + "desktop_metrics", + "catalog_publication_bundle", + "unsigned", + "publisher_signature", + "automatic_updates", + "complete_release_qualification", + } + title = platform.title() + archive = provenance.get("install_archive") + metrics_record = provenance.get("desktop_metrics") + artifacts = provenance.get("artifacts") + if ( + set(provenance) != provenance_fields + or type(provenance.get("schema_version")) is not int + or provenance.get("schema_version") != 1 + or provenance.get("product") != "CommunityAI" + or provenance.get("package") != "communityai-desktop" + or provenance.get("release_channel") != "public-alpha" + or provenance.get("source_commit") != source_commit + or not isinstance(provenance.get("source_tree"), str) + or _COMMIT_RE.fullmatch(provenance["source_tree"]) is None + or not isinstance(provenance.get("build_platform"), str) + or not provenance["build_platform"].startswith(title) + or provenance.get("artifact_root") != "CommunityAI" + or provenance.get("checksum_manifest") != "SHA256SUMS" + or provenance.get("unsigned") is not True + or provenance.get("publisher_signature") is not False + or provenance.get("automatic_updates") is not False + or provenance.get("complete_release_qualification") is not False + or not isinstance(artifacts, list) + or not artifacts + or not isinstance(archive, dict) + or not isinstance(metrics_record, dict) + ): + raise Gate14LifecycleError("release provenance binding is invalid") + + expected_archive = { + "schema_version": 1, + "path": _PACKAGE_NAMES[platform], + "format": "zip" if platform == "windows" else "tar.gz", + "platform": title, + "artifact_root": "CommunityAI", + "sha256": package_sha256.removeprefix("sha256:"), + "size_bytes": package_bytes, + "entry_count": archive.get("entry_count"), + "preserves_executable_modes": platform == "linux", + "preserves_internal_file_symlinks": platform == "linux", + } + if ( + set(archive) != set(expected_archive) + or type(archive.get("schema_version")) is not int + or type(archive.get("entry_count")) is not int + or archive["entry_count"] < 1 + or not _exact_equal(archive, expected_archive) + ): + raise Gate14LifecycleError("release archive provenance is invalid") + + metrics_payload = payloads["desktop-metrics.json"] + if not _exact_equal( + metrics_record, + { + "schema_version": 1, + "path": "desktop-metrics.json", + "sha256": hashlib.sha256(metrics_payload).hexdigest(), + "size_bytes": len(metrics_payload), + }, + ): + raise Gate14LifecycleError("desktop metrics provenance is invalid") + + artifact_paths: list[str] = [] + artifact_kinds: dict[str, str] = {} + link_targets: dict[str, str] = {} + checksum_lines: list[str] = [] + for artifact in artifacts: + if not isinstance(artifact, dict): + raise Gate14LifecycleError("release artifact inventory is invalid") + kind = artifact.get("kind") + expected_fields = {"path", "kind", "sha256", "size_bytes"} + expected_fields.add("mode" if kind == "file" else "link_target") + path = _public_path(artifact.get("path"), "release artifact path") + digest = artifact.get("sha256") + size = artifact.get("size_bytes") + if ( + set(artifact) != expected_fields + or not path.startswith("CommunityAI/") + or not isinstance(digest, str) + or re.fullmatch(r"[0-9a-f]{64}", digest) is None + or type(size) is not int + or size < 0 + or kind not in {"file", "symlink"} + ): + raise Gate14LifecycleError("release artifact inventory is invalid") + if kind == "file": + if type(artifact.get("mode")) is not int or not 0 <= artifact["mode"] <= 0o7777: + raise Gate14LifecycleError("release artifact mode is invalid") + else: + target = _public_path(artifact.get("link_target"), "release link target") + if target == path or not target.startswith("CommunityAI/"): + raise Gate14LifecycleError("release artifact link is invalid") + link_targets[path] = target + artifact_paths.append(path) + artifact_kinds[path] = kind + checksum_lines.append(f"{digest} {path}\n") + if any(artifact_kinds.get(target) != "file" for target in link_targets.values()): + raise Gate14LifecycleError("release artifact link target is invalid") + if ( + artifact_paths != sorted(artifact_paths) + or len({item.casefold() for item in artifact_paths}) != len(artifact_paths) + or payloads["SHA256SUMS"] != "".join(checksum_lines).encode("utf-8") + ): + raise Gate14LifecycleError("release checksum inventory is invalid") + + release_artifacts = metrics.get("release_artifacts") + node_sidecar = metrics.get("node_sidecar") + if ( + type(metrics.get("schema_version")) is not int + or metrics.get("schema_version") != 1 + or metrics.get("application") != "CommunityAI" + or metrics.get("package") != "communityai-desktop" + or not isinstance(metrics.get("platform"), str) + or not metrics["platform"].startswith(title) + or metrics.get("signed") is not False + or metrics.get("catalog_bootstrap_bundled") is not True + or not _exact_equal( + metrics.get("catalog_publication_bundle"), + provenance.get("catalog_publication_bundle"), + ) + or not isinstance(release_artifacts, dict) + or set(release_artifacts) + != { + "schema_version", + "artifact_count", + "artifact_bytes", + "checksums_sha256", + "source_commit", + "source_tree", + "unsigned", + "complete_release_qualification", + "install_archive", + } + or type(release_artifacts.get("schema_version")) is not int + or release_artifacts.get("schema_version") != 1 + or type(release_artifacts.get("artifact_count")) is not int + or release_artifacts.get("artifact_count") != len(artifacts) + or type(release_artifacts.get("artifact_bytes")) is not int + or release_artifacts.get("artifact_bytes") != sum(item["size_bytes"] for item in artifacts) + or release_artifacts.get("checksums_sha256") != hashlib.sha256(payloads["SHA256SUMS"]).hexdigest() + or release_artifacts.get("source_commit") != source_commit + or release_artifacts.get("source_tree") != provenance.get("source_tree") + or release_artifacts.get("unsigned") is not True + or not _exact_equal(release_artifacts.get("install_archive"), archive) + or release_artifacts.get("complete_release_qualification") is not False + or not isinstance(node_sidecar, dict) + or node_sidecar.get("self_test_passed") is not True + or node_sidecar.get("node_entrypoint_smoke_passed") is not True + or node_sidecar.get("worker_entrypoint_smoke_passed") is not True + or node_sidecar.get("worker_self_test_passed") is not True + ): + raise Gate14LifecycleError("desktop metrics binding is invalid") + + +def _load_release_audit( + value: Any, + *, + staging_root: Path, + ownership_verifier: Callable[..., None], + release_metadata_path: Path, + release_metadata_sha256: str, + platform: str, + source_commit: str, + package_sha256: str, + package_bytes: int, +) -> ReleaseAuditBinding: + if not isinstance(value, dict) or set(value) != { + "schema_version", + "artifact_name", + "artifact_sha256", + "artifact_bytes", + "members", + }: + raise Gate14LifecycleError("release audit binding schema is invalid") + expected_artifact_name = f"communityai-desktop-audit-{platform}" + raw_members = value.get("members") + if ( + type(value.get("schema_version")) is not int + or value.get("schema_version") != 1 + or value.get("artifact_name") != expected_artifact_name + or not isinstance(raw_members, list) + or len(raw_members) != len(_RELEASE_AUDIT_MEMBERS) + ): + raise Gate14LifecycleError("release audit identity is invalid") + artifact_sha256 = _digest_field( + value.get("artifact_sha256"), + "release audit artifact digest", + ) + artifact_bytes = _bounded_integer( + value.get("artifact_bytes"), + "release audit artifact size", + 1, + MAX_RELEASE_AUDIT_BYTES, + ) + + audit_directory = _exact_staged_path( + staging_root / _RELEASE_AUDIT_DIRECTORY_NAME, + staging_root / _RELEASE_AUDIT_DIRECTORY_NAME, + "release audit directory", + directory=True, + ) + ownership_verifier(audit_directory, directory=True) + members: list[AuditMemberBinding] = [] + for index, raw in enumerate(raw_members): + if not isinstance(raw, dict) or set(raw) != {"name", "sha256", "size_bytes"}: + raise Gate14LifecycleError("release audit member schema is invalid") + name = raw.get("name") + if name != _RELEASE_AUDIT_MEMBERS[index]: + raise Gate14LifecycleError("release audit members are not exact and sorted") + maximum = { + "provenance.json": MAX_RELEASE_PROVENANCE_BYTES, + "desktop-metrics.json": MAX_DESKTOP_METRICS_BYTES, + "SHA256SUMS": MAX_RELEASE_CHECKSUMS_BYTES, + }.get(name, MAX_RELEASE_METADATA_BYTES) + member = AuditMemberBinding( + name=name, + sha256=_digest_field(raw.get("sha256"), "release audit member digest"), + size_bytes=_bounded_integer( + raw.get("size_bytes"), + "release audit member size", + 1, + maximum, + ), + path=_exact_staged_path( + audit_directory / name, + audit_directory / name, + "release audit member", + directory=False, + ), + ) + members.append(member) + metadata_member = next(item for item in members if item.name == "release-metadata.json") + if metadata_member.path != release_metadata_path or metadata_member.sha256 != release_metadata_sha256: + raise Gate14LifecycleError("release metadata is not bound to the full audit") + + binding = ReleaseAuditBinding( + artifact_name=expected_artifact_name, + artifact_sha256=artifact_sha256, + artifact_bytes=artifact_bytes, + archive_path=_exact_staged_path( + staging_root / _RELEASE_AUDIT_ARCHIVE_NAME, + staging_root / _RELEASE_AUDIT_ARCHIVE_NAME, + "release audit archive", + directory=False, + ), + members=tuple(members), + binding_sha256=_digest(_canonical(value)), + ) + payloads = _audit_member_payloads( + binding, + ownership_verifier=ownership_verifier, + ) + _validate_release_semantics( + payloads, + platform=platform, + source_commit=source_commit, + package_sha256=package_sha256, + package_bytes=package_bytes, + ) + return binding + + +def _validate_cache_artifact(value: Any) -> CacheArtifactBinding: + if not isinstance(value, dict) or set(value) != { + "path", + "role", + "sha256", + "size_bytes", + }: + raise Gate14LifecycleError("warm-cache artifact schema is invalid") + path = _public_path(value.get("path"), "warm-cache artifact path") + role = value.get("role") + if role not in {"chat_template", "config", "tokenizer", "weight", "weight_index"}: + raise Gate14LifecycleError("warm-cache artifact role is invalid") + return CacheArtifactBinding( + path=path, + role=role, + sha256=_digest_field(value.get("sha256"), "warm-cache artifact digest"), + size_bytes=_bounded_integer( + value.get("size_bytes"), + "warm-cache artifact size", + 1, + acceptance.MAX_BYTES, + ), + ) + + +def _validate_materialization_record( + payload: bytes, + *, + platform: str, + model_id: str, + manifest_digest: str, + warm_cache: WarmCacheBinding, +) -> None: + raw = _strict_json(payload, MAX_MATERIALIZATION_RECORD_BYTES) + expected_fields = { + "schema_version", + "acquired_at_unix", + "runtime", + "model", + "selection", + "artifacts", + "transfer", + "storage", + "privacy", + } + runtime = raw.get("runtime") + model = raw.get("model") + selection = raw.get("selection") + transfer = raw.get("transfer") + storage = raw.get("storage") + privacy = raw.get("privacy") + artifacts = raw.get("artifacts") + profile = acceptance.MODEL_PROFILES[model_id] + repository, dtype = _MODEL_SOURCE[model_id] + nested = (runtime, model, selection, transfer, storage, privacy) + runtime_values = runtime.values() if isinstance(runtime, dict) else () + if ( + set(raw) != expected_fields + or type(raw.get("schema_version")) is not int + or raw.get("schema_version") != 1 + or type(raw.get("acquired_at_unix")) is not int + or raw["acquired_at_unix"] < 1 + or not all(isinstance(item, dict) for item in nested) + or set(runtime) != {"python", "platform", "drift"} + or any( + not isinstance(item, str) or not item or len(item) > 256 or any(ord(character) < 32 for character in item) + for item in runtime_values + ) + or not ( + runtime.get("platform", "").casefold() == platform + or runtime.get("platform", "").casefold().startswith(platform + "-") + ) + or set(model) != {"id", "manifest_digest", "repository", "revision", "dtype"} + or set(selection) + != { + "startup_artifact_paths", + "weight_artifact_paths", + "artifact_count", + "artifact_bytes", + "weight_artifact_bytes", + } + or set(transfer) + != { + "direct_upstream_transfer", + "mirror_used", + "source_class_verified", + "transport_override_present", + "elapsed_seconds", + "max_resumptions", + "resumptions", + "completed", + } + or not isinstance(artifacts, list) + or model.get("id") != model_id + or model.get("manifest_digest") != manifest_digest + or model.get("repository") != repository + or model.get("revision") != profile["revision_commit"] + or model.get("dtype") != dtype + ): + raise Gate14LifecycleError("cache materialization record binding is invalid") + + expected_artifacts = [ + { + "path": item.path, + "role": item.role, + "sha256": item.sha256.removeprefix("sha256:"), + "size_bytes": item.size_bytes, + } + for item in warm_cache.artifacts + ] + observed_artifacts: list[dict[str, Any]] = [] + total_resumptions = 0 + for artifact in artifacts: + if not isinstance(artifact, dict) or set(artifact) != { + "path", + "role", + "sha256", + "size_bytes", + "materialization_attempts", + "resumptions", + "resumed_from_bytes", + "elapsed_seconds", + }: + raise Gate14LifecycleError("cache materialization artifact schema is invalid") + observed = { + "path": artifact.get("path"), + "role": artifact.get("role"), + "sha256": artifact.get("sha256"), + "size_bytes": artifact.get("size_bytes"), + } + attempts = artifact.get("materialization_attempts") + resumptions = artifact.get("resumptions") + resumed = artifact.get("resumed_from_bytes") + elapsed = artifact.get("elapsed_seconds") + if ( + type(attempts) is not int + or attempts < 1 + or type(resumptions) is not int + or resumptions < 0 + or not isinstance(resumed, list) + or len(resumed) != resumptions + or any(type(item) is not int or item < 1 for item in resumed) + or type(elapsed) not in (int, float) + or not math.isfinite(float(elapsed)) + or elapsed < 0 + ): + raise Gate14LifecycleError("cache materialization artifact proof is invalid") + observed_artifacts.append(observed) + total_resumptions += resumptions + if not _exact_equal(observed_artifacts, expected_artifacts): + raise Gate14LifecycleError("cache materialization artifacts changed") + + startup_paths = [ + item.path + for item in warm_cache.artifacts + if item.role in {"chat_template", "config", "tokenizer", "weight_index"} + ] + weight_paths = [item.path for item in warm_cache.artifacts if item.role == "weight"] + if ( + selection.get("startup_artifact_paths") != sorted(startup_paths) + or selection.get("weight_artifact_paths") != sorted(weight_paths) + or type(selection.get("artifact_count")) is not int + or selection.get("artifact_count") != warm_cache.artifact_count + or type(selection.get("artifact_bytes")) is not int + or selection.get("artifact_bytes") != warm_cache.artifact_bytes + or type(selection.get("weight_artifact_bytes")) is not int + or selection["weight_artifact_bytes"] + != sum(item.size_bytes for item in warm_cache.artifacts if item.role == "weight") + or type(transfer.get("elapsed_seconds")) not in (int, float) + or not math.isfinite(float(transfer["elapsed_seconds"])) + or transfer["elapsed_seconds"] < 0 + or transfer.get("direct_upstream_transfer") is not True + or transfer.get("mirror_used") is not False + or transfer.get("source_class_verified") is not True + or transfer.get("transport_override_present") is not False + or transfer.get("completed") is not True + or type(transfer.get("max_resumptions")) is not int + or transfer.get("max_resumptions") != 3 + or type(transfer.get("resumptions")) is not int + or transfer.get("resumptions") != total_resumptions + or not 0 <= total_resumptions <= 3 + or not _exact_equal( + storage, + { + "cold_start": True, + "cache_bytes_before": 0, + "cache_bytes_after": warm_cache.artifact_bytes, + "cache_growth_bytes": warm_cache.artifact_bytes, + "verified": True, + }, + ) + or not _exact_equal( + privacy, + { + "credentials_retained": False, + "local_paths_retained": False, + "response_bodies_retained": False, + "urls_retained": False, + }, + ) + ): + raise Gate14LifecycleError("cache materialization proof is invalid") + + +def build_warm_cache_binding( + payload: bytes, + *, + platform: str, + source_commit: str, + materialization_plan_sha256: str, + materializer_sources_sha256: str, + model_id: str, + manifest_digest: str, +) -> dict[str, Any]: + """Build the exact lifecycle fragment for one verified fresh materialization.""" + expected_model = acceptance.EXPECTED_PLATFORM_MODELS.get(platform) + profile = acceptance.MODEL_PROFILES.get(model_id) + expected = _GATE9_WARM_CACHE.get(platform) + if ( + expected_model != model_id + or profile is None + or expected is None + or profile["manifest_digest"] != manifest_digest + or not isinstance(source_commit, str) + or _COMMIT_RE.fullmatch(source_commit) is None + or not isinstance(materialization_plan_sha256, str) + or _DIGEST_RE.fullmatch(materialization_plan_sha256) is None + or not isinstance(materializer_sources_sha256, str) + or _DIGEST_RE.fullmatch(materializer_sources_sha256) is None + ): + raise Gate14LifecycleError("cache materialization profile is invalid") + artifacts = tuple(CacheArtifactBinding(*item) for item in expected["artifacts"]) + value = { + "schema_version": 1, + "layout": "manifest-artifacts-v1", + "gate9_acquisition_record_sha256": expected["gate9_acquisition_record_sha256"], + "gate9_resource_envelope_sha256": expected["gate9_resource_envelope_sha256"], + "source_commit": source_commit, + "materialization_plan_sha256": materialization_plan_sha256, + "materializer_sources_sha256": materializer_sources_sha256, + "materialization_record_sha256": _digest(payload), + "materialization_record_bytes": len(payload), + "artifact_count": len(artifacts), + "artifact_bytes": sum(item.size_bytes for item in artifacts), + "artifacts": [ + { + "path": item.path, + "role": item.role, + "sha256": item.sha256, + "size_bytes": item.size_bytes, + } + for item in artifacts + ], + } + binding = WarmCacheBinding( + gate9_acquisition_record_sha256=value["gate9_acquisition_record_sha256"], + gate9_resource_envelope_sha256=value["gate9_resource_envelope_sha256"], + source_commit=value["source_commit"], + materialization_plan_sha256=value["materialization_plan_sha256"], + materializer_sources_sha256=value["materializer_sources_sha256"], + materialization_record_sha256=value["materialization_record_sha256"], + materialization_record_bytes=value["materialization_record_bytes"], + materialization_record_path=Path(_MATERIALIZATION_RECORD_NAME), + artifact_count=value["artifact_count"], + artifact_bytes=value["artifact_bytes"], + artifacts=artifacts, + binding_sha256=_digest(_canonical(value)), + ) + _validate_materialization_record( + payload, + platform=platform, + model_id=model_id, + manifest_digest=manifest_digest, + warm_cache=binding, + ) + return value + + +def _load_warm_cache( + value: Any, + *, + staging_root: Path, + ownership_verifier: Callable[..., None], + platform: str, + source_commit: str, + model_id: str, + manifest_digest: str, +) -> WarmCacheBinding: + if not isinstance(value, dict) or set(value) != { + "schema_version", + "layout", + "gate9_acquisition_record_sha256", + "gate9_resource_envelope_sha256", + "source_commit", + "materialization_plan_sha256", + "materializer_sources_sha256", + "materialization_record_sha256", + "materialization_record_bytes", + "artifact_count", + "artifact_bytes", + "artifacts", + }: + raise Gate14LifecycleError("warm-cache binding schema is invalid") + expected = _GATE9_WARM_CACHE[platform] + raw_artifacts = value.get("artifacts") + if ( + type(value.get("schema_version")) is not int + or value.get("schema_version") != 1 + or value.get("layout") != "manifest-artifacts-v1" + or value.get("gate9_acquisition_record_sha256") != expected["gate9_acquisition_record_sha256"] + or value.get("gate9_resource_envelope_sha256") != expected["gate9_resource_envelope_sha256"] + or not isinstance(raw_artifacts, list) + ): + raise Gate14LifecycleError("warm-cache Gate 9 identity is invalid") + artifacts = tuple(_validate_cache_artifact(item) for item in raw_artifacts) + expected_artifacts = tuple(CacheArtifactBinding(*item) for item in expected["artifacts"]) + profile = acceptance.MODEL_PROFILES[model_id] + artifact_count = _bounded_integer( + value.get("artifact_count"), + "warm-cache artifact count", + 1, + 100, + ) + artifact_bytes = _bounded_integer( + value.get("artifact_bytes"), + "warm-cache artifact bytes", + 1, + acceptance.MAX_BYTES, + ) + if ( + artifacts != expected_artifacts + or artifact_count != len(artifacts) + or artifact_count != profile["selected_artifact_count"] + or artifact_bytes != sum(item.size_bytes for item in artifacts) + or artifact_bytes != profile["selected_artifact_bytes"] + or [item.path for item in artifacts] != sorted(item.path for item in artifacts) + or len({item.path.casefold() for item in artifacts}) != len(artifacts) + or manifest_digest != profile["manifest_digest"] + ): + raise Gate14LifecycleError("warm-cache artifact identity is invalid") + + record_path = _exact_staged_path( + staging_root / _MATERIALIZATION_RECORD_NAME, + staging_root / _MATERIALIZATION_RECORD_NAME, + "cache materialization record", + directory=False, + ) + binding = WarmCacheBinding( + gate9_acquisition_record_sha256=expected["gate9_acquisition_record_sha256"], + gate9_resource_envelope_sha256=expected["gate9_resource_envelope_sha256"], + source_commit=value.get("source_commit"), + materialization_plan_sha256=_digest_field( + value.get("materialization_plan_sha256"), + "cache materialization plan digest", + ), + materializer_sources_sha256=_digest_field( + value.get("materializer_sources_sha256"), + "cache materializer sources digest", + ), + materialization_record_sha256=_digest_field( + value.get("materialization_record_sha256"), + "cache materialization record digest", + ), + materialization_record_bytes=_bounded_integer( + value.get("materialization_record_bytes"), + "cache materialization record size", + 1, + MAX_MATERIALIZATION_RECORD_BYTES, + ), + materialization_record_path=record_path, + artifact_count=artifact_count, + artifact_bytes=artifact_bytes, + artifacts=artifacts, + binding_sha256=_digest(_canonical(value)), + ) + if binding.source_commit != source_commit: + raise Gate14LifecycleError("cache materialization source binding changed") + ownership_verifier(record_path, directory=False) + record = _regular_bytes(record_path, MAX_MATERIALIZATION_RECORD_BYTES) + if len(record) != binding.materialization_record_bytes or _digest(record) != binding.materialization_record_sha256: + raise Gate14LifecycleError("cache materialization record identity changed") + _validate_materialization_record( + record, + platform=platform, + model_id=model_id, + manifest_digest=manifest_digest, + warm_cache=binding, + ) + return binding + + +def load_config( + path: Path, + *, + ownership_verifier: Callable[..., None] | None = None, +) -> LifecycleConfig: + if ownership_verifier is None: + ownership_verifier = _assert_controller_owned + payload = _regular_bytes(Path(path), MAX_CONFIG_BYTES) + raw = _strict_json(payload, MAX_CONFIG_BYTES) + if ( + set(raw) != _CONFIG_FIELDS + or type(raw.get("schema_version")) is not int + or raw.get("schema_version") != SCHEMA_VERSION + or raw.get("scope") != SCOPE + ): + raise Gate14LifecycleError("lifecycle configuration schema is invalid") + + run_id = raw["run_id"] + platform = raw["platform"] + source_commit = raw["source_commit"] + package_sha256 = raw["package_sha256"] + model_id = raw["model_id"] + manifest_digest = raw["manifest_digest"] + gate13_digest = raw["gate13_evidence_sha256"] + release_metadata_sha256 = raw["release_metadata_sha256"] + if not isinstance(run_id, str) or _RUN_RE.fullmatch(run_id) is None: + raise Gate14LifecycleError("run ID is invalid") + if platform not in acceptance.EXPECTED_PLATFORM_MODELS: + raise Gate14LifecycleError("platform is invalid") + if ( + not isinstance(source_commit, str) + or _COMMIT_RE.fullmatch(source_commit) is None + or not isinstance(package_sha256, str) + or _DIGEST_RE.fullmatch(package_sha256) is None + or not isinstance(manifest_digest, str) + or _DIGEST_RE.fullmatch(manifest_digest) is None + or not isinstance(release_metadata_sha256, str) + or _DIGEST_RE.fullmatch(release_metadata_sha256) is None + or gate13_digest != acceptance.EXPECTED_GATE13_EVIDENCE_SHA256 + ): + raise Gate14LifecycleError("source or evidence binding is invalid") + + expected_model = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[expected_model] + if model_id != expected_model or manifest_digest != profile["manifest_digest"]: + raise Gate14LifecycleError("model binding is invalid") + + staging_root = _work_root( + _absolute_path(raw["staging_root"], "staging root"), + controller_owned=True, + ) + root = _work_root(_absolute_path(raw["work_root"], "work root")) + if staging_root == root: + raise Gate14LifecycleError("staging and work roots overlap") + config_path = _path_under_root( + Path(path), + staging_root, + "gate14-lifecycle.json", + "lifecycle configuration", + ) + if _digest(_regular_bytes(config_path, MAX_CONFIG_BYTES)) != _digest(payload): + raise Gate14LifecycleError("lifecycle configuration changed") + + package_path = _path_under_root( + _absolute_path(raw["package_path"], "package path"), + staging_root, + _PACKAGE_NAMES[platform], + "production package", + ) + if package_path.name != _PACKAGE_NAMES[platform]: + raise Gate14LifecycleError("production package filename is invalid") + package_bytes = _bounded_integer( + raw["package_bytes"], + "package size", + 1, + MAX_PACKAGE_BYTES, + ) + if _regular_metadata(package_path, MAX_PACKAGE_BYTES).st_size != package_bytes: + raise Gate14LifecycleError("package size changed") + release_metadata_path = _exact_staged_path( + _absolute_path(raw["release_metadata_path"], "release metadata path"), + staging_root / _RELEASE_AUDIT_DIRECTORY_NAME / "release-metadata.json", + "release metadata", + directory=False, + ) + metadata_payload = _regular_bytes( + release_metadata_path, + MAX_RELEASE_METADATA_BYTES, + ) + if ( + _digest(metadata_payload) != release_metadata_sha256 + or _strict_json(metadata_payload, MAX_RELEASE_METADATA_BYTES) != _RELEASE_METADATA + ): + raise Gate14LifecycleError("release metadata binding is invalid") + release_audit = _load_release_audit( + raw["release_audit"], + staging_root=staging_root, + ownership_verifier=ownership_verifier, + release_metadata_path=release_metadata_path, + release_metadata_sha256=release_metadata_sha256, + platform=platform, + source_commit=source_commit, + package_sha256=package_sha256, + package_bytes=package_bytes, + ) + warm_cache = _load_warm_cache( + raw["warm_cache"], + staging_root=staging_root, + ownership_verifier=ownership_verifier, + platform=platform, + source_commit=source_commit, + model_id=model_id, + manifest_digest=manifest_digest, + ) + + challenge_path = _path_under_root( + _absolute_path(raw["challenge_path"], "challenge path"), + staging_root, + _CHALLENGE_NAME, + "controller challenge", + ) + ownership_verifier(staging_root.parent, directory=True) + ownership_verifier(staging_root, directory=True) + for staged_path in (config_path, package_path, release_metadata_path): + ownership_verifier(staged_path, directory=False) + if challenge_path.exists(): + ownership_verifier(challenge_path, directory=False) + private = {field: _private_path(raw, field, root) for field in _OUTPUT_NAMES} + if len(set(private.values())) != len(private): + raise Gate14LifecycleError("private lifecycle paths overlap") + + disk_bytes = _bounded_integer( + raw["disk_bytes"], + "disk limit", + profile["selected_artifact_bytes"], + acceptance.MAX_BYTES, + ) + vram_bytes = _bounded_integer( + raw["vram_bytes"], + "VRAM limit", + 1, + 32 * 1024**3 - 1, + ) + bandwidth = _bounded_number( + raw["bandwidth_mbps"], + "bandwidth limit", + 0.001, + 1_000_000.0, + ) + power = _bounded_number( + raw["power_watts"], + "power limit", + 0.001, + 1_000.0, + ) + pause = _bounded_number( + raw["pause_timeout_seconds"], + "pause timeout", + 1.0, + 300.0, + ) + sample_interval = _bounded_number( + raw["sample_interval_seconds"], + "sample interval", + 0.05, + 30.0, + ) + challenge_wait = _bounded_number( + raw["max_challenge_wait_seconds"], + "challenge wait", + 1.0, + MAX_CHALLENGE_WAIT_SECONDS, + ) + + return LifecycleConfig( + run_id=run_id, + platform=platform, + attempt_ordinal=_bounded_integer( + raw["attempt_ordinal"], + "attempt ordinal", + 1, + 100, + ), + source_commit=source_commit, + config_sha256=_digest(payload), + config_path=config_path, + package_path=package_path, + package_sha256=package_sha256, + package_bytes=package_bytes, + release_metadata_path=release_metadata_path, + release_metadata_sha256=release_metadata_sha256, + release_audit=release_audit, + warm_cache=warm_cache, + model_id=model_id, + manifest_digest=manifest_digest, + gate13_evidence_sha256=gate13_digest, + staging_root=staging_root, + work_root=root, + challenge_path=challenge_path, + checkpoint_path=private["checkpoint_path"], + facts_path=private["facts_path"], + evidence_path=private["evidence_path"], + disk_bytes=disk_bytes, + vram_bytes=vram_bytes, + bandwidth_mbps=bandwidth, + power_watts=power, + pause_timeout_seconds=pause, + sample_interval_seconds=sample_interval, + max_challenge_wait_seconds=challenge_wait, + ) + + +def _hash_package(config: LifecycleConfig) -> None: + stream, metadata = _open_regular(config.package_path, MAX_PACKAGE_BYTES) + digest = hashlib.sha256() + try: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + after = os.fstat(stream.fileno()) + except OSError as exc: + raise Gate14LifecycleError("package is unreadable") from exc + finally: + stream.close() + if ( + metadata.st_size != config.package_bytes + or (after.st_dev, after.st_ino, after.st_size) != (metadata.st_dev, metadata.st_ino, metadata.st_size) + or "sha256:" + digest.hexdigest() != config.package_sha256 + ): + raise Gate14LifecycleError("package digest changed") + + +def _verify_staged_inputs(config: LifecycleConfig) -> None: + _assert_controller_owned(config.staging_root.parent, directory=True) + _assert_controller_owned(config.staging_root, directory=True) + for staged_path in ( + config.config_path, + config.package_path, + config.release_metadata_path, + ): + _assert_controller_owned(staged_path, directory=False) + if config.challenge_path.exists(): + _assert_controller_owned(config.challenge_path, directory=False) + config_payload = _regular_bytes(config.config_path, MAX_CONFIG_BYTES) + if _digest(config_payload) != config.config_sha256: + raise Gate14LifecycleError("lifecycle configuration binding changed") + _hash_package(config) + metadata = _regular_bytes( + config.release_metadata_path, + MAX_RELEASE_METADATA_BYTES, + ) + if _digest(metadata) != config.release_metadata_sha256 or not _exact_equal( + _strict_json(metadata, MAX_RELEASE_METADATA_BYTES), + _RELEASE_METADATA, + ): + raise Gate14LifecycleError("release metadata binding changed") + + audit_payloads = _audit_member_payloads(config.release_audit) + _validate_release_semantics( + audit_payloads, + platform=config.platform, + source_commit=config.source_commit, + package_sha256=config.package_sha256, + package_bytes=config.package_bytes, + ) + _assert_controller_owned( + config.warm_cache.materialization_record_path, + directory=False, + ) + materialization = _regular_bytes( + config.warm_cache.materialization_record_path, + MAX_MATERIALIZATION_RECORD_BYTES, + ) + if ( + len(materialization) != config.warm_cache.materialization_record_bytes + or _digest(materialization) != config.warm_cache.materialization_record_sha256 + ): + raise Gate14LifecycleError("cache materialization record identity changed") + _validate_materialization_record( + materialization, + platform=config.platform, + model_id=config.model_id, + manifest_digest=config.manifest_digest, + warm_cache=config.warm_cache, + ) + + +def validate_prepared( + value: Mapping[str, Any], + config: LifecycleConfig, +) -> Mapping[str, Any]: + if ( + not isinstance(value, dict) + or set(value) != _PREPARED_FIELDS + or type(value["schema_version"]) is not int + or value["schema_version"] != SCHEMA_VERSION + or value["scope"] != PREPARED_SCOPE + or value["run_id"] != config.run_id + or value["platform"] != config.platform + or type(value["attempt_ordinal"]) is not int + or value["attempt_ordinal"] != config.attempt_ordinal + or value["source_commit"] != config.source_commit + or value["package_sha256"] != config.package_sha256 + ): + raise Gate14LifecycleError("prepared observation schema or binding is invalid") + try: + model = acceptance._validate_model(value["model"], config.platform) + if model["id"] != config.model_id or model["manifest_digest"] != config.manifest_digest: + raise Gate14LifecycleError("prepared model binding changed") + acceptance._validate_cache( + value["cache"], + model["selected_artifact_bytes"], + ) + acceptance._validate_placement( + value["placement"], + model["total_blocks"], + ) + acceptance._validate_limits( + value["limits"], + model["selected_artifact_bytes"], + 32 * 1024**3, + ) + acceptance._validate_recovery(value["recovery"]) + acceptance._validate_pause(value["pause"]) + acceptance._validate_restart(value["restart"]) + acceptance._validate_unsupported(value["unsupported_telemetry"]) + except acceptance.Gate14EvidenceError as exc: + raise Gate14LifecycleError("prepared observation is invalid") from exc + + limits = value["limits"] + if ( + limits["disk_bytes"] != config.disk_bytes + or limits["vram_bytes"] != config.vram_bytes + or float(limits["bandwidth_mbps"]) != config.bandwidth_mbps + or float(limits["power_watts"]) != config.power_watts + or limits["schedule_timezone"] != "UTC" + ): + raise Gate14LifecycleError("prepared resource limits changed") + return dict(value) + + +def _checkpoint_value( + config: LifecycleConfig, + prepared: Mapping[str, Any], + created_at_unix: int, +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "scope": CHECKPOINT_SCOPE, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": config.attempt_ordinal, + "source_commit": config.source_commit, + "lifecycle_config_sha256": config.config_sha256, + "package_sha256": config.package_sha256, + "release_metadata_sha256": config.release_metadata_sha256, + "release_audit_sha256": config.release_audit.binding_sha256, + "warm_cache_sha256": config.warm_cache.binding_sha256, + "materialization_record_sha256": (config.warm_cache.materialization_record_sha256), + "prepared_facts_sha256": _digest(_canonical(prepared)), + "phase": "challenge-ready", + "created_at_unix": created_at_unix, + } + + +def _validate_checkpoint_shape(value: Any) -> None: + digest_fields = ( + "lifecycle_config_sha256", + "package_sha256", + "release_metadata_sha256", + "release_audit_sha256", + "warm_cache_sha256", + "materialization_record_sha256", + "prepared_facts_sha256", + ) + if ( + not isinstance(value, dict) + or set(value) != _CHECKPOINT_FIELDS + or type(value.get("schema_version")) is not int + or value.get("schema_version") != SCHEMA_VERSION + or value.get("scope") != CHECKPOINT_SCOPE + or not isinstance(value.get("run_id"), str) + or _RUN_RE.fullmatch(value["run_id"]) is None + or value.get("platform") not in {"windows", "linux"} + or type(value.get("attempt_ordinal")) is not int + or value.get("attempt_ordinal") < 1 + or not isinstance(value.get("source_commit"), str) + or _COMMIT_RE.fullmatch(value["source_commit"]) is None + or any( + not isinstance(value.get(field), str) or _DIGEST_RE.fullmatch(value[field]) is None + for field in digest_fields + ) + or value.get("phase") != "challenge-ready" + or type(value.get("created_at_unix")) is not int + or value.get("created_at_unix") < 0 + ): + raise Gate14LifecycleError("checkpoint schema is invalid") + + +def validate_checkpoint( + value: Mapping[str, Any], + config: LifecycleConfig, + prepared: Mapping[str, Any], + *, + now_unix: float, +) -> Mapping[str, Any]: + _validate_checkpoint_shape(value) + expected = _checkpoint_value( + config, + prepared, + value.get("created_at_unix"), + ) + if not _exact_equal(value, expected): + raise Gate14LifecycleError("checkpoint binding is invalid") + created = value["created_at_unix"] + if ( + type(created) is not int + or type(now_unix) not in (int, float) + or not math.isfinite(float(now_unix)) + or not 0 <= created <= float(now_unix) + ): + raise Gate14LifecycleError("checkpoint time is invalid") + return dict(value) + + +def checkpoint_digest(value: Mapping[str, Any]) -> str: + _validate_checkpoint_shape(value) + return _digest(_canonical(value)) + + +def load_checkpoint_for_controller( + path: Path, + *, + run_id: str, + platform: str, + source_commit: str, + package_sha256: str, + now_unix: float, +) -> Mapping[str, Any]: + value = _strict_json( + _regular_bytes(Path(path), MAX_PRIVATE_JSON_BYTES), + MAX_PRIVATE_JSON_BYTES, + ) + if ( + set(value) != _CHECKPOINT_FIELDS + or type(value.get("schema_version")) is not int + or value.get("schema_version") != SCHEMA_VERSION + or value.get("scope") != CHECKPOINT_SCOPE + or value.get("run_id") != run_id + or value.get("platform") != platform + or value.get("source_commit") != source_commit + or value.get("package_sha256") != package_sha256 + or value.get("phase") != "challenge-ready" + or type(value.get("attempt_ordinal")) is not int + or not 1 <= value["attempt_ordinal"] <= 100 + or not isinstance(value.get("lifecycle_config_sha256"), str) + or _DIGEST_RE.fullmatch(value["lifecycle_config_sha256"]) is None + or not isinstance(value.get("release_metadata_sha256"), str) + or _DIGEST_RE.fullmatch(value["release_metadata_sha256"]) is None + or not isinstance(value.get("release_audit_sha256"), str) + or _DIGEST_RE.fullmatch(value["release_audit_sha256"]) is None + or not isinstance(value.get("warm_cache_sha256"), str) + or _DIGEST_RE.fullmatch(value["warm_cache_sha256"]) is None + or not isinstance(value.get("materialization_record_sha256"), str) + or _DIGEST_RE.fullmatch(value["materialization_record_sha256"]) is None + or not isinstance(value.get("prepared_facts_sha256"), str) + or _DIGEST_RE.fullmatch(value["prepared_facts_sha256"]) is None + or type(value.get("created_at_unix")) is not int + or type(now_unix) not in (int, float) + or not math.isfinite(float(now_unix)) + or not 0 <= value["created_at_unix"] <= float(now_unix) + ): + raise Gate14LifecycleError("checkpoint controller binding is invalid") + return dict(value) + + +def _write_new(path: Path, value: Mapping[str, Any]) -> None: + if path.exists(): + raise Gate14LifecycleError("private lifecycle output already exists") + payload = _canonical(value) + os.linesep.encode("ascii") + if len(payload) > MAX_PRIVATE_JSON_BYTES: + raise Gate14LifecycleError("private lifecycle output is too large") + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, path) + except FileExistsError as exc: + raise Gate14LifecycleError("private lifecycle output already exists") from exc + temporary.unlink() + except BaseException: + try: + temporary.unlink() + except OSError: + pass + raise + + +def write_or_load_checkpoint( + config: LifecycleConfig, + prepared: Mapping[str, Any], + *, + now_unix: float, +) -> Mapping[str, Any]: + prepared = validate_prepared(prepared, config) + if config.checkpoint_path.exists(): + existing = _strict_json( + _regular_bytes( + config.checkpoint_path, + MAX_PRIVATE_JSON_BYTES, + ), + MAX_PRIVATE_JSON_BYTES, + ) + return validate_checkpoint( + existing, + config, + prepared, + now_unix=now_unix, + ) + if type(now_unix) not in (int, float) or not math.isfinite(float(now_unix)) or float(now_unix) < 0: + raise Gate14LifecycleError("checkpoint clock is invalid") + value = _checkpoint_value(config, prepared, int(now_unix)) + _write_new(config.checkpoint_path, value) + return value + + +def wait_for_challenge( + config: LifecycleConfig, + checkpoint: Mapping[str, Any], + *, + clock: Callable[[], float] = time.time, + monotonic: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, +) -> Mapping[str, Any]: + started = monotonic() + deadline = started + config.max_challenge_wait_seconds + while True: + now = clock() + if config.challenge_path.exists(): + try: + _assert_controller_owned( + config.challenge_path, + directory=False, + ) + value = challenge_contract.validate( + challenge_contract.load(config.challenge_path), + run_id=config.run_id, + platform=config.platform, + source_commit=config.source_commit, + package_sha256=config.package_sha256, + checkpoint_sha256=checkpoint_digest(checkpoint), + now_unix=now, + ) + except challenge_contract.Gate14ChallengeError as exc: + raise Gate14LifecycleError("calibration challenge is invalid") from exc + if value["issued_at_unix"] < checkpoint["created_at_unix"] or value[ + "checkpoint_sha256" + ] != checkpoint_digest(checkpoint): + raise Gate14LifecycleError("calibration challenge predates readiness") + return value + remaining = deadline - monotonic() + if remaining <= 0: + raise Gate14LifecycleError("calibration challenge timed out") + sleeper(min(config.sample_interval_seconds, remaining)) + + +def validate_suspensions( + value: Sequence[Mapping[str, Any]], + prepared: Mapping[str, Any], + challenge: Mapping[str, Any], +) -> list[Mapping[str, Any]]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise Gate14LifecycleError("calibration observations are invalid") + rendered = [dict(item) if isinstance(item, Mapping) else item for item in value] + challenge_summary = { + "challenge_sha256": challenge_contract.digest(challenge), + "controller_state_revision": challenge["controller_state_revision"], + "issued_at_unix": challenge["issued_at_unix"], + "expires_at_unix": challenge["expires_at_unix"], + } + try: + acceptance._validate_suspensions( + rendered, + prepared["limits"], + challenge_summary, + ) + except acceptance.Gate14EvidenceError as exc: + raise Gate14LifecycleError("calibration observations are invalid") from exc + return rendered + + +def validate_cleanup( + value: Mapping[str, Any], + config: LifecycleConfig, +) -> Mapping[str, Any]: + if ( + not isinstance(value, dict) + or set(value) != _CLEANUP_FIELDS + or type(value["schema_version"]) is not int + or value["schema_version"] != SCHEMA_VERSION + or value["scope"] != CLEANUP_SCOPE + or value["run_id"] != config.run_id + or value["platform"] != config.platform + or type(value["attempt_ordinal"]) is not int + or value["attempt_ordinal"] != config.attempt_ordinal + or any( + value[field] is not True + for field in ( + "processes_absent", + "credentials_removed", + "action_temporaries_removed", + ) + ) + ): + raise Gate14LifecycleError("lifecycle cleanup is incomplete") + return dict(value) + + +def _facts( + config: LifecycleConfig, + prepared: Mapping[str, Any], + suspensions: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "scope": host_probe.FACT_SCOPE, + "run_id": config.run_id, + "platform": config.platform, + "source_commit": config.source_commit, + "gate13_evidence_sha256": config.gate13_evidence_sha256, + "expected_package_sha256": config.package_sha256, + "model": prepared["model"], + "cache": prepared["cache"], + "placement": prepared["placement"], + "limits": prepared["limits"], + "suspensions": list(suspensions), + "recovery": prepared["recovery"], + "pause": prepared["pause"], + "restart": prepared["restart"], + "unsupported_telemetry": prepared["unsupported_telemetry"], + "qualification_temporaries_removed": True, + } + + +def _remove_output(path: Path) -> None: + try: + Path(path).unlink() + except FileNotFoundError: + return + except OSError as exc: + raise Gate14LifecycleError("private lifecycle output cleanup failed") from exc + + +def _load_platform_output(path: Path) -> Mapping[str, Any]: + return _strict_json( + _regular_bytes(path, host_probe.MAX_JSON_BYTES), + host_probe.MAX_JSON_BYTES, + ) + + +def run_lifecycle( + config: LifecycleConfig, + actions: LifecycleActions, + *, + hardware_probe: HardwareProbe = host_probe.probe_hardware, + clock: Callable[[], float] = time.time, + monotonic: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, +) -> Mapping[str, Any]: + """Run the shared ordering contract and emit one strict platform document.""" + + pending_path = config.work_root / _PENDING_EVIDENCE_NAME + owns_outputs = False + try: + if any( + path.exists() + for path in ( + config.challenge_path, + config.checkpoint_path, + config.facts_path, + config.evidence_path, + pending_path, + ) + ): + raise Gate14LifecycleError("fresh lifecycle outputs are required") + owns_outputs = True + _verify_staged_inputs(config) + prepared = validate_prepared(actions.prepare(config), config) + _verify_staged_inputs(config) + if config.challenge_path.exists(): + raise Gate14LifecycleError("calibration challenge arrived before readiness") + checkpoint = write_or_load_checkpoint( + config, + prepared, + now_unix=clock(), + ) + challenge = wait_for_challenge( + config, + checkpoint, + clock=clock, + monotonic=monotonic, + sleeper=sleeper, + ) + suspensions = validate_suspensions( + actions.calibrate(config, challenge), + prepared, + challenge, + ) + _verify_staged_inputs(config) + current_challenge = challenge_contract.validate( + challenge_contract.load(config.challenge_path), + run_id=config.run_id, + platform=config.platform, + source_commit=config.source_commit, + package_sha256=config.package_sha256, + checkpoint_sha256=checkpoint_digest(checkpoint), + now_unix=clock(), + ) + if current_challenge != challenge: + raise Gate14LifecycleError("calibration challenge changed") + validate_cleanup(actions.cleanup(config), config) + _verify_staged_inputs(config) + + _write_new(config.facts_path, _facts(config, prepared, suspensions)) + document = host_probe.run_probe( + platform_name=config.platform, + facts_path=config.facts_path, + challenge_path=config.challenge_path, + package_path=config.package_path, + release_metadata_path=config.release_metadata_path, + output_path=pending_path, + hardware_probe=hardware_probe, + now_unix=clock(), + ) + _remove_output(config.facts_path) + _verify_staged_inputs(config) + if challenge_contract.load(config.challenge_path) != challenge: + raise Gate14LifecycleError("calibration challenge changed") + acceptance.validate_platform_document(document) + persisted = _load_platform_output(pending_path) + acceptance.validate_platform_document(persisted) + if ( + _canonical(persisted) != _canonical(document) + or persisted["package"]["release_metadata_sha256"] != config.release_metadata_sha256 + ): + raise Gate14LifecycleError("persisted platform evidence changed") + _write_new(config.evidence_path, persisted) + _remove_output(pending_path) + published = _load_platform_output(config.evidence_path) + acceptance.validate_platform_document(published) + if _canonical(published) != _canonical(document): + raise Gate14LifecycleError("published platform evidence changed") + return published + except BaseException as exc: + removal_error = None + if owns_outputs: + for path in (config.facts_path, pending_path, config.evidence_path): + try: + _remove_output(path) + except Gate14LifecycleError as cleanup_exc: + removal_error = cleanup_exc + try: + validate_cleanup(actions.cleanup(config), config) + except BaseException as cleanup_exc: + raise Gate14LifecycleError("lifecycle failed and cleanup did not complete") from cleanup_exc + if removal_error is not None: + raise Gate14LifecycleError( + "lifecycle failed and private output cleanup did not complete" + ) from removal_error + raise diff --git a/scripts/gate14_run_controller.py b/scripts/gate14_run_controller.py new file mode 100644 index 000000000..f3f946cc6 --- /dev/null +++ b/scripts/gate14_run_controller.py @@ -0,0 +1,1080 @@ +"""Durable, source-bound controller for one bounded Gate 14 GCP run. + +The controller never invokes a provider. Every start, status, collect, or cleanup +operation first consumes an exact provider observation, persists its decision, and +returns one allowlisted action. A provider adapter may execute only that action. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import tempfile +import time +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping, Sequence + +import gate14_calibration_challenge as challenge_contract +import gate14_hardware_acceptance as acceptance +import gate14_packaged_lifecycle as packaged_lifecycle +import qualification_cost_guard as cost_guard + +SCHEMA_VERSION = 1 +STATE_SCHEMA_VERSION = 1 +MAX_JSON_BYTES = 262_144 +PROTECTED_INSTANCE = "communityai-bootstrap-1" +ALLOWED_CEILING_USD = 100.0 +CURRENT_EPOCH_ANCHOR_RUN_ID = "gate13-20260901-a" +CURRENT_EPOCH_ANCHOR_MAXIMUM_USD = Decimal("56.00") + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_NAME_RE = re.compile(r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") + +PHASES = { + "ABSENT", + "WINDOWS_RUNNING", + "WINDOWS_DELETING", + "LINUX_RUNNING", + "LINUX_DELETING", + "CLEANING_FAILED", + "CLEANED_PASS", + "CLEANED_FAILURE", +} +TERMINAL_PHASES = {"CLEANED_PASS", "CLEANED_FAILURE"} +ACTIONS = { + "start_windows", + "collect_windows", + "delete_windows", + "start_linux", + "collect_linux", + "delete_linux", + "cleanup_failure", + "none", +} +JOB_STATES = {"absent", "starting", "running", "passed", "failed", "ambiguous"} + +_AUTH_FIELDS = { + "schema_version", + "gate", + "result", + "run_id", + "source_commit", + "provider_plan_digest", + "provider_plan", + "authorization", + "prohibited", +} +_AUTHORIZATION_FIELDS = { + "combined_cloud_ceiling_usd", + "ledger_committed_before_run_usd", + "maximum_estimate_usd", + "remaining_after_run_maximum_usd", + "reservation_recorded", + "native_auth_revalidated", + "provisioning_authorized_after_fail_closed_preflight", +} +_PLAN_FIELDS = {"project", "zone", "clients", "sequencing"} +_CLIENT_PLAN_FIELDS = { + "platform", + "instance", + "disk", + "source_commit", + "termination_unix", + "package_sha256", + "model_id", + "manifest_digest", + "machine_type", + "image_project", + "image", + "boot_disk_gib", + "boot_disk_type", + "service_account_disabled", + "max_run_seconds", + "termination_action", +} +_SEQUENCING_FIELDS = { + "clients_may_run_concurrently", + "windows_first", + "fresh_host_per_platform", +} +_STATE_FIELDS = { + "schema_version", + "run_id", + "authorization_sha256", + "provider_plan_digest", + "revision", + "phase", + "failure_code", + "windows_evidence_digest", + "linux_evidence_digest", + "windows_challenge_sha256", + "linux_challenge_sha256", + "windows_challenge_consumed", + "linux_challenge_consumed", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + "next_action", +} +_OBSERVATION_FIELDS = { + "schema_version", + "run_id", + "observed_at_unix", + "instances", + "disks", + "clients", + "l4_usage", + "protected_bootstrap_running", +} +_INSTANCE_FIELDS = {"present", "run_id", "source_commit", "termination_unix"} +_CLIENT_FIELDS = {"job_state", "attempt_ordinal", "evidence_digest"} + + +class Gate14ControllerError(ValueError): + """The plan, state, observation, or transition failed closed.""" + + +@dataclass(frozen=True) +class ClientPlan: + platform: str + instance: str + disk: str + source_commit: str + termination_unix: int + package_sha256: str + model_id: str + manifest_digest: str + machine_type: str + image_project: str + image: str + boot_disk_gib: int + boot_disk_type: str + max_run_seconds: int + + +@dataclass(frozen=True) +class RunPlan: + run_id: str + authorization_sha256: str + provider_plan_digest: str + source_commit: str + ledger_state: str + project: str + zone: str + windows: ClientPlan + linux: ClientPlan + + @property + def instances(self) -> tuple[str, str]: + return (self.windows.instance, self.linux.instance) + + @property + def disks(self) -> tuple[str, str]: + return (self.windows.disk, self.linux.disk) + + +def _reject_constant(_value: str) -> None: + raise Gate14ControllerError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14ControllerError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path, maximum: int = MAX_JSON_BYTES) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14ControllerError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise Gate14ControllerError("required file is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise Gate14ControllerError("required file is unreadable") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_JSON_BYTES: + raise Gate14ControllerError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14ControllerError("invalid JSON") from exc + if not isinstance(value, dict): + raise Gate14ControllerError("JSON root is invalid") + return value + + +def _mapping(value: Any, fields: set[str]) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise Gate14ControllerError("schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str]) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise Gate14ControllerError("string is invalid") + return value + + +def _integer(value: Any, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise Gate14ControllerError("integer is invalid") + return value + + +def _canonical_digest(value: Mapping[str, Any]) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _client_plan(value: Any, platform: str, source_commit: str) -> ClientPlan: + raw = _mapping(value, _CLIENT_PLAN_FIELDS) + if raw["platform"] != platform or raw["source_commit"] != source_commit: + raise Gate14ControllerError("client source binding is invalid") + instance = _string(raw["instance"], _NAME_RE) + disk = _string(raw["disk"], _NAME_RE) + if instance == PROTECTED_INSTANCE or disk == PROTECTED_INSTANCE: + raise Gate14ControllerError("protected resource is targeted") + package_sha256 = _string(raw["package_sha256"], _DIGEST_RE) + expected_model = acceptance.EXPECTED_PLATFORM_MODELS[platform] + if raw["model_id"] != expected_model: + raise Gate14ControllerError("client model is invalid") + expected_manifest = acceptance.MODEL_PROFILES[expected_model]["manifest_digest"] + if raw["manifest_digest"] != expected_manifest: + raise Gate14ControllerError("client manifest is invalid") + if raw["machine_type"] != "g2-standard-8": + raise Gate14ControllerError("client machine type is invalid") + expected_image_project = "windows-cloud" if platform == "windows" else "ubuntu-os-cloud" + expected_image_pattern = ( + re.compile(r"windows-server-2022-dc-v[0-9]{8}") + if platform == "windows" + else re.compile(r"ubuntu-2404-noble-amd64-v[0-9]{8}") + ) + if raw["image_project"] != expected_image_project: + raise Gate14ControllerError("client image project is invalid") + image = _string(raw["image"], expected_image_pattern) + boot_disk_gib = _integer(raw["boot_disk_gib"], 100, 200) + if ( + raw["boot_disk_type"] != "pd-balanced" + or raw["service_account_disabled"] is not True + or raw["termination_action"] != "DELETE" + ): + raise Gate14ControllerError("client runtime boundary is invalid") + max_run_seconds = _integer(raw["max_run_seconds"], 1_800, 14_400) + return ClientPlan( + platform=platform, + instance=instance, + disk=disk, + source_commit=source_commit, + termination_unix=_integer(raw["termination_unix"], 1), + package_sha256=package_sha256, + model_id=expected_model, + manifest_digest=expected_manifest, + machine_type="g2-standard-8", + image_project=expected_image_project, + image=image, + boot_disk_gib=boot_disk_gib, + boot_disk_type="pd-balanced", + max_run_seconds=max_run_seconds, + ) + + +def load_plan(authorization_path: Path, ledger_path: Path) -> RunPlan: + authorization_payload = _regular_bytes(authorization_path) + raw = _mapping(_strict_json(authorization_payload), _AUTH_FIELDS) + if raw["schema_version"] != SCHEMA_VERSION or raw["gate"] != 14 or raw["result"] != "authorized": + raise Gate14ControllerError("authorization scope is invalid") + run_id = _string(raw["run_id"], _RUN_RE) + source_commit = _string(raw["source_commit"], _COMMIT_RE) + provider_plan = _mapping(raw["provider_plan"], _PLAN_FIELDS) + provider_digest = _canonical_digest(provider_plan) + if raw["provider_plan_digest"] != provider_digest: + raise Gate14ControllerError("provider plan digest changed") + project = _string(provider_plan["project"], re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]")) + zone = _string(provider_plan["zone"], re.compile(r"[a-z]+(?:-[a-z0-9]+)+-[a-z]")) + sequencing = _mapping(provider_plan["sequencing"], _SEQUENCING_FIELDS) + if ( + sequencing["clients_may_run_concurrently"] is not False + or sequencing["windows_first"] is not True + or sequencing["fresh_host_per_platform"] is not True + ): + raise Gate14ControllerError("client sequencing is invalid") + clients = provider_plan["clients"] + if not isinstance(clients, list) or len(clients) != 2: + raise Gate14ControllerError("client plan is invalid") + by_platform = { + item.get("platform"): item + for item in clients + if isinstance(item, dict) and isinstance(item.get("platform"), str) + } + if set(by_platform) != {"windows", "linux"}: + raise Gate14ControllerError("client platform plan is invalid") + windows = _client_plan(by_platform["windows"], "windows", source_commit) + linux = _client_plan(by_platform["linux"], "linux", source_commit) + if windows.instance == linux.instance or windows.disk == linux.disk: + raise Gate14ControllerError("client resources overlap") + + cost = _mapping(raw["authorization"], _AUTHORIZATION_FIELDS) + try: + ceiling = Decimal(str(cost["combined_cloud_ceiling_usd"])) + before = Decimal(str(cost["ledger_committed_before_run_usd"])) + maximum = Decimal(str(cost["maximum_estimate_usd"])) + remaining = Decimal(str(cost["remaining_after_run_maximum_usd"])) + except (InvalidOperation, TypeError, ValueError) as exc: + raise Gate14ControllerError("cost authorization is invalid") from exc + if ( + not all(value.is_finite() for value in (ceiling, before, maximum, remaining)) + or ceiling != Decimal(str(ALLOWED_CEILING_USD)) + or before < 0 + or maximum <= 0 + or before + maximum > ceiling + or ceiling - before - maximum != remaining + or cost["reservation_recorded"] is not True + or cost["native_auth_revalidated"] is not True + or cost["provisioning_authorized_after_fail_closed_preflight"] is not True + ): + raise Gate14ControllerError("cost authorization is inconsistent") + prohibited = raw["prohibited"] + if ( + not isinstance(prohibited, dict) + or set(prohibited) != {"credits", "macos", "fly_gpu"} + or any(type(value) is not int or value != 0 for value in prohibited.values()) + ): + raise Gate14ControllerError("prohibited work is present") + try: + entries = cost_guard.load_spend_ledger(ledger_path) + except cost_guard.CostGuardError as exc: + raise Gate14ControllerError("spend ledger is invalid") from exc + anchors = [entry for entry in entries if entry.run_id == CURRENT_EPOCH_ANCHOR_RUN_ID] + if len(anchors) != 1 or anchors[0].maximum_usd != CURRENT_EPOCH_ANCHOR_MAXIMUM_USD: + raise Gate14ControllerError("current accounting epoch anchor is invalid") + anchor_index = entries.index(anchors[0]) + historical_entries = entries[anchor_index + 1 :] + if any(entry.state not in {"CANCELED", "CLEANED-COMMITTED", "CLEANED-RELEASED"} for entry in historical_entries): + raise Gate14ControllerError("active reservation is hidden below the epoch anchor") + current_epoch_entries = entries[: anchor_index + 1] + matches = [entry for entry in current_epoch_entries if entry.run_id == run_id] + if ( + len(matches) != 1 + or matches[0].provider != "GCP" + or matches[0].maximum_usd != maximum + or matches[0].state != "RESERVED" + or provider_digest not in matches[0].purpose + ): + raise Gate14ControllerError("spend ledger reservation is invalid") + ledger_committed = sum( + (entry.committed_usd for entry in current_epoch_entries), + Decimal("0"), + ) + committed_before = ledger_committed - matches[0].committed_usd + if committed_before != before or ledger_committed > ceiling: + raise Gate14ControllerError("spend ledger exceeds the authorized ceiling") + return RunPlan( + run_id=run_id, + authorization_sha256="sha256:" + hashlib.sha256(authorization_payload).hexdigest(), + provider_plan_digest=provider_digest, + source_commit=source_commit, + ledger_state=matches[0].state, + project=project, + zone=zone, + windows=windows, + linux=linux, + ) + + +def initial_state(plan: RunPlan) -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "run_id": plan.run_id, + "authorization_sha256": plan.authorization_sha256, + "provider_plan_digest": plan.provider_plan_digest, + "revision": 0, + "phase": "ABSENT", + "failure_code": None, + "windows_evidence_digest": None, + "linux_evidence_digest": None, + "windows_challenge_sha256": None, + "linux_challenge_sha256": None, + "windows_challenge_consumed": False, + "linux_challenge_consumed": False, + "windows_consumed": False, + "linux_consumed": False, + "cleanup_verified": False, + "next_action": "none", + } + + +def validate_state(value: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + state = dict(_mapping(value, _STATE_FIELDS)) + if ( + state["schema_version"] != STATE_SCHEMA_VERSION + or state["run_id"] != plan.run_id + or state["authorization_sha256"] != plan.authorization_sha256 + or state["provider_plan_digest"] != plan.provider_plan_digest + or state["phase"] not in PHASES + or state["next_action"] not in ACTIONS + ): + raise Gate14ControllerError("state binding is invalid") + _integer(state["revision"]) + for field in ( + "windows_consumed", + "linux_consumed", + "windows_challenge_consumed", + "linux_challenge_consumed", + "cleanup_verified", + ): + if type(state[field]) is not bool: + raise Gate14ControllerError("state boolean is invalid") + for field in ( + "windows_evidence_digest", + "linux_evidence_digest", + "windows_challenge_sha256", + "linux_challenge_sha256", + ): + if state[field] is not None: + _string(state[field], _DIGEST_RE) + for platform in ("windows", "linux"): + if state[f"{platform}_challenge_consumed"] and state[f"{platform}_challenge_sha256"] is None: + raise Gate14ControllerError("consumed calibration challenge is missing") + if state["failure_code"] is not None: + _string(state["failure_code"], re.compile(r"[a-z0-9][a-z0-9-]{0,63}")) + allowed_actions = { + "ABSENT": {"none", "start_windows"}, + "WINDOWS_RUNNING": {"none", "collect_windows"}, + "WINDOWS_DELETING": {"delete_windows", "start_linux"}, + "LINUX_RUNNING": {"none", "collect_linux"}, + "LINUX_DELETING": {"delete_linux"}, + "CLEANING_FAILED": {"cleanup_failure"}, + "CLEANED_PASS": {"none"}, + "CLEANED_FAILURE": {"none"}, + } + if state["next_action"] not in allowed_actions[state["phase"]]: + raise Gate14ControllerError("state action is inconsistent") + if state["windows_evidence_digest"] is not None and not state["windows_consumed"]: + raise Gate14ControllerError("Windows evidence state is inconsistent") + if state["linux_evidence_digest"] is not None and not state["linux_consumed"]: + raise Gate14ControllerError("Linux evidence state is inconsistent") + for platform in ("windows", "linux"): + if state[f"{platform}_evidence_digest"] is not None and state[f"{platform}_challenge_sha256"] is None: + raise Gate14ControllerError("platform evidence lacks a calibration challenge") + phase = state["phase"] + failed_phase = phase in {"CLEANING_FAILED", "CLEANED_FAILURE"} + if failed_phase is (state["failure_code"] is None): + raise Gate14ControllerError("failure state is inconsistent") + if state["cleanup_verified"] is not (phase in TERMINAL_PHASES): + raise Gate14ControllerError("cleanup state is inconsistent") + windows_evidence = state["windows_evidence_digest"] is not None + linux_evidence = state["linux_evidence_digest"] is not None + if phase == "ABSENT" and any( + ( + state["windows_consumed"], + state["linux_consumed"], + windows_evidence, + linux_evidence, + state["windows_challenge_sha256"] is not None, + state["linux_challenge_sha256"] is not None, + ) + ): + raise Gate14ControllerError("initial state is inconsistent") + if phase == "WINDOWS_RUNNING" and ( + not state["windows_consumed"] + or state["linux_consumed"] + or linux_evidence + or (state["next_action"] == "collect_windows") is not windows_evidence + ): + raise Gate14ControllerError("Windows running state is inconsistent") + if phase == "WINDOWS_DELETING" and ( + not state["windows_consumed"] + or state["linux_consumed"] + or not windows_evidence + or linux_evidence + or not state["windows_challenge_consumed"] + ): + raise Gate14ControllerError("Windows deletion state is inconsistent") + if phase == "LINUX_RUNNING" and ( + not state["windows_consumed"] + or not state["linux_consumed"] + or not windows_evidence + or not state["windows_challenge_consumed"] + or (state["next_action"] == "collect_linux") is not linux_evidence + ): + raise Gate14ControllerError("Linux running state is inconsistent") + if phase in {"LINUX_DELETING", "CLEANED_PASS"} and ( + not state["windows_consumed"] + or not state["linux_consumed"] + or not windows_evidence + or not linux_evidence + or not state["windows_challenge_consumed"] + or not state["linux_challenge_consumed"] + ): + raise Gate14ControllerError("completed evidence state is inconsistent") + return state + + +def validate_observation(value: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + observation = dict(_mapping(value, _OBSERVATION_FIELDS)) + if observation["schema_version"] != SCHEMA_VERSION or observation["run_id"] != plan.run_id: + raise Gate14ControllerError("observation binding is invalid") + now = _integer(observation["observed_at_unix"], 1) + if observation["protected_bootstrap_running"] is not True: + raise Gate14ControllerError("protected bootstrap is not healthy") + _integer(observation["l4_usage"], 0, 1) + instances = observation["instances"] + disks = observation["disks"] + clients = observation["clients"] + if ( + not isinstance(instances, dict) + or set(instances) != set(plan.instances) + or not isinstance(disks, dict) + or set(disks) != set(plan.disks) + or not isinstance(clients, dict) + or set(clients) != {"windows", "linux"} + ): + raise Gate14ControllerError("provider inventory is not exact") + for client in (plan.windows, plan.linux): + instance = _mapping(instances[client.instance], _INSTANCE_FIELDS) + if type(instance["present"]) is not bool or type(disks[client.disk]) is not bool: + raise Gate14ControllerError("provider inventory type is invalid") + if instance["present"]: + if ( + instance["run_id"] != plan.run_id + or instance["source_commit"] != client.source_commit + or _integer(instance["termination_unix"], 1) != client.termination_unix + or disks[client.disk] is not True + ): + raise Gate14ControllerError("provider resource binding is invalid") + elif any( + value is not None + for value in ( + instance["run_id"], + instance["source_commit"], + instance["termination_unix"], + ) + ): + raise Gate14ControllerError("absent instance metadata is invalid") + job = _mapping(clients[client.platform], _CLIENT_FIELDS) + job_state = job["job_state"] + if job_state not in JOB_STATES: + raise Gate14ControllerError("host job state is invalid") + attempt = _integer(job["attempt_ordinal"], 0, 1) + evidence_digest = job["evidence_digest"] + if job_state == "absent": + if attempt != 0 or evidence_digest is not None: + raise Gate14ControllerError("absent host job evidence is inconsistent") + else: + if attempt != 1: + raise Gate14ControllerError("host job attempt is inconsistent") + if job_state == "passed": + _string(evidence_digest, _DIGEST_RE) + elif evidence_digest is not None: + raise Gate14ControllerError("unfinished host job exposed evidence") + expected_l4_usage = sum(int(instances[client.instance]["present"]) for client in (plan.windows, plan.linux)) + if observation["l4_usage"] != expected_l4_usage: + raise Gate14ControllerError("accelerator inventory is inconsistent") + return observation + + +def _next(state: Mapping[str, Any], **changes: Any) -> dict[str, Any]: + result = dict(state) + result.update(changes) + result["revision"] = int(state["revision"]) + 1 + return result + + +def _resources_absent(observation: Mapping[str, Any], plan: RunPlan) -> bool: + return ( + all(not observation["instances"][name]["present"] for name in plan.instances) + and all(observation["disks"][name] is False for name in plan.disks) + and observation["l4_usage"] == 0 + ) + + +def _observed_evidence_matches(state: Mapping[str, Any], observation: Mapping[str, Any], platform: str) -> bool: + job = observation["clients"][platform] + return job["job_state"] == "passed" and job["evidence_digest"] == state[f"{platform}_evidence_digest"] + + +def reconcile( + state_value: Mapping[str, Any], + observation_value: Mapping[str, Any], + plan: RunPlan, +) -> dict[str, Any]: + state = validate_state(state_value, plan) + observation = validate_observation(observation_value, plan) + phase = state["phase"] + windows_present = observation["instances"][plan.windows.instance]["present"] + linux_present = observation["instances"][plan.linux.instance]["present"] + windows_job = observation["clients"]["windows"]["job_state"] + linux_job = observation["clients"]["linux"]["job_state"] + observed_at = observation["observed_at_unix"] + + if phase in TERMINAL_PHASES: + if not _resources_absent(observation, plan): + raise Gate14ControllerError("resources returned after terminal cleanup") + if phase == "CLEANED_PASS" and not ( + _observed_evidence_matches(state, observation, "windows") + and _observed_evidence_matches(state, observation, "linux") + ): + raise Gate14ControllerError("terminal evidence binding is inconsistent") + return state + if phase != "CLEANING_FAILED": + deadline = ( + plan.windows.termination_unix if phase in {"ABSENT", "WINDOWS_RUNNING"} else plan.linux.termination_unix + ) + if observed_at >= deadline: + if _resources_absent(observation, plan): + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="run-expired", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="run-expired", + next_action="cleanup_failure", + ) + if phase == "CLEANING_FAILED": + if _resources_absent(observation, plan): + return _next( + state, + phase="CLEANED_FAILURE", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="cleanup_failure") + if phase == "ABSENT": + orphan_disk = (observation["disks"][plan.windows.disk] and not windows_present) or ( + observation["disks"][plan.linux.disk] and not linux_present + ) + if orphan_disk: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="orphaned-planned-disk", + next_action="cleanup_failure", + ) + if linux_present or linux_job != "absent": + if _resources_absent(observation, plan): + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="unexpected-linux-state", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="unexpected-linux-state", + next_action="cleanup_failure", + ) + if windows_present: + return _next( + state, + phase="WINDOWS_RUNNING", + windows_consumed=True, + next_action="none", + ) + if windows_job != "absent": + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="stale-windows-job", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="start_windows") + if phase == "WINDOWS_RUNNING": + if linux_present or observation["disks"][plan.linux.disk] or not windows_present: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="windows-inventory-lost", + next_action="cleanup_failure", + ) + if state["next_action"] == "collect_windows" and not _observed_evidence_matches(state, observation, "windows"): + raise Gate14ControllerError("reported Windows evidence changed") + if windows_job in {"starting", "running"}: + return _next(state, next_action="none") + if windows_job == "passed": + return _next( + state, + windows_evidence_digest=observation["clients"]["windows"]["evidence_digest"], + next_action="collect_windows", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="windows-job-failed", + next_action="cleanup_failure", + ) + if phase == "WINDOWS_DELETING": + if not _observed_evidence_matches(state, observation, "windows"): + raise Gate14ControllerError("validated Windows evidence is unavailable") + if windows_present or observation["disks"][plan.windows.disk]: + if linux_present: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="clients-overlapped", + next_action="cleanup_failure", + ) + return _next(state, next_action="delete_windows") + if linux_present: + return _next( + state, + phase="LINUX_RUNNING", + linux_consumed=True, + next_action="none", + ) + if observation["disks"][plan.linux.disk]: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="orphaned-linux-disk", + next_action="cleanup_failure", + ) + if linux_job != "absent": + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="stale-linux-job", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="start_linux") + if phase == "LINUX_RUNNING": + if observation["disks"][plan.windows.disk]: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="orphaned-windows-disk", + next_action="cleanup_failure", + ) + if not _observed_evidence_matches(state, observation, "windows"): + raise Gate14ControllerError("validated Windows evidence is unavailable") + if windows_present or not linux_present: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="linux-inventory-lost", + next_action="cleanup_failure", + ) + if state["next_action"] == "collect_linux" and not _observed_evidence_matches(state, observation, "linux"): + raise Gate14ControllerError("reported Linux evidence changed") + if linux_job in {"starting", "running"}: + return _next(state, next_action="none") + if linux_job == "passed": + return _next( + state, + linux_evidence_digest=observation["clients"]["linux"]["evidence_digest"], + next_action="collect_linux", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="linux-job-failed", + next_action="cleanup_failure", + ) + if phase == "LINUX_DELETING": + if windows_present or observation["disks"][plan.windows.disk]: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="windows-resources-returned", + next_action="cleanup_failure", + ) + if not ( + _observed_evidence_matches(state, observation, "windows") + and _observed_evidence_matches(state, observation, "linux") + ): + raise Gate14ControllerError("validated platform evidence is unavailable") + if not _resources_absent(observation, plan): + return _next(state, next_action="delete_linux") + if state["windows_evidence_digest"] is None or state["linux_evidence_digest"] is None: + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="evidence-missing", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANED_PASS", + cleanup_verified=True, + next_action="none", + ) + raise Gate14ControllerError("unhandled lifecycle phase") + + +def issue_calibration_challenge( + state_value: Mapping[str, Any], + plan: RunPlan, + platform: str, + challenge_path: Path, + checkpoint_path: Path, + *, + issued_at_unix: int | None = None, + nonce: str | None = None, +) -> tuple[dict[str, Any], Mapping[str, Any]]: + state = validate_state(state_value, plan) + expected_phase = "WINDOWS_RUNNING" if platform == "windows" else "LINUX_RUNNING" + if ( + state["phase"] != expected_phase + or state["next_action"] != "none" + or state[f"{platform}_challenge_sha256"] is not None + or state[f"{platform}_challenge_consumed"] + ): + raise Gate14ControllerError("calibration challenge is out of sequence") + client = plan.windows if platform == "windows" else plan.linux + issued = int(time.time()) if issued_at_unix is None else issued_at_unix + lifetime = min(challenge_contract.MAX_LIFETIME_SECONDS, client.termination_unix - issued) + if lifetime < challenge_contract.MIN_LIFETIME_SECONDS: + raise Gate14ControllerError("calibration challenge would outlive the host") + try: + checkpoint = packaged_lifecycle.load_checkpoint_for_controller( + checkpoint_path, + run_id=plan.run_id, + platform=platform, + source_commit=client.source_commit, + package_sha256=client.package_sha256, + now_unix=issued, + ) + checkpoint_sha256 = packaged_lifecycle.checkpoint_digest(checkpoint) + except packaged_lifecycle.Gate14LifecycleError as exc: + raise Gate14ControllerError("challenge-ready checkpoint is invalid") from exc + if Path(challenge_path).exists(): + value = challenge_contract.validate( + challenge_contract.load(challenge_path), + run_id=plan.run_id, + platform=platform, + source_commit=client.source_commit, + package_sha256=client.package_sha256, + checkpoint_sha256=checkpoint_sha256, + now_unix=issued, + ) + if value["controller_state_revision"] != state["revision"]: + raise Gate14ControllerError("existing calibration challenge is stale") + else: + value = challenge_contract.create( + run_id=plan.run_id, + platform=platform, + source_commit=client.source_commit, + package_sha256=client.package_sha256, + checkpoint_sha256=checkpoint_sha256, + controller_state_revision=state["revision"], + issued_at_unix=issued, + lifetime_seconds=lifetime, + nonce=nonce, + ) + challenge_contract.write_new(challenge_path, value) + challenge_sha256 = challenge_contract.digest(value) + return ( + _next( + state, + **{f"{platform}_challenge_sha256": challenge_sha256}, + ), + value, + ) + + +def collect_platform( + state_value: Mapping[str, Any], + plan: RunPlan, + platform: str, + evidence_path: Path, + challenge_path: Path, +) -> dict[str, Any]: + state = validate_state(state_value, plan) + expected_phase = "WINDOWS_RUNNING" if platform == "windows" else "LINUX_RUNNING" + expected_action = f"collect_{platform}" + if state["phase"] != expected_phase or state["next_action"] != expected_action: + raise Gate14ControllerError("collect is out of sequence") + payload = _regular_bytes(evidence_path) + summary = acceptance.validate_platform_document(_strict_json(payload)) + client = plan.windows if platform == "windows" else plan.linux + challenge_value = challenge_contract.validate( + challenge_contract.load(challenge_path), + run_id=plan.run_id, + platform=platform, + source_commit=client.source_commit, + package_sha256=client.package_sha256, + ) + challenge_sha256 = challenge_contract.digest(challenge_value) + if ( + challenge_value["controller_state_revision"] >= state["revision"] + or state[f"{platform}_challenge_consumed"] + or state[f"{platform}_challenge_sha256"] != challenge_sha256 + ): + raise Gate14ControllerError("calibration challenge state is invalid") + if summary["calibration_challenge_sha256"] != challenge_sha256: + raise Gate14ControllerError("platform evidence used a different calibration challenge") + if ( + summary["run_id"] != plan.run_id + or summary["platform"] != platform + or summary["source_commit"] != client.source_commit + or summary["package_sha256"] != client.package_sha256 + or summary["model_id"] != client.model_id + or summary["manifest_digest"] != client.manifest_digest + ): + raise Gate14ControllerError("platform evidence does not match the plan") + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + if digest != state[f"{platform}_evidence_digest"]: + raise Gate14ControllerError("collected evidence changed after host completion") + return _next( + state, + **{ + f"{platform}_evidence_digest": digest, + f"{platform}_challenge_consumed": True, + "phase": f"{platform.upper()}_DELETING", + "next_action": f"delete_{platform}", + }, + ) + + +def begin_cleanup( + state_value: Mapping[str, Any], + plan: RunPlan, + failure_code: str, +) -> dict[str, Any]: + state = validate_state(state_value, plan) + if state["phase"] in TERMINAL_PHASES: + return state + _string(failure_code, re.compile(r"[a-z0-9][a-z0-9-]{0,63}")) + return _next( + state, + phase="CLEANING_FAILED", + failure_code=failure_code, + next_action="cleanup_failure", + ) + + +def load_state(path: Path, plan: RunPlan) -> dict[str, Any]: + return validate_state(_strict_json(_regular_bytes(path)), plan) + + +def save_state(path: Path, state_value: Mapping[str, Any], plan: RunPlan) -> None: + state = validate_state(state_value, plan) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(state, sort_keys=True, separators=(",", ":")) + os.linesep).encode("utf-8") + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + temporary.unlink() + except OSError: + pass + raise + + +def _common_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("start", "status", "challenge", "collect", "cleanup")) + parser.add_argument("--authorization", type=Path, required=True) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--state", type=Path, required=True) + parser.add_argument("--observation", type=Path) + parser.add_argument("--platform", choices=("windows", "linux")) + parser.add_argument("--evidence", type=Path) + parser.add_argument("--challenge", type=Path) + parser.add_argument("--checkpoint", type=Path) + parser.add_argument("--failure-code", default="operator-cleanup") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _common_parser().parse_args(argv) + try: + plan = load_plan(args.authorization, args.ledger) + state = load_state(args.state, plan) if args.state.exists() else initial_state(plan) + result: Mapping[str, Any] + if args.operation in {"start", "status"}: + if args.observation is None: + raise Gate14ControllerError("observation is required") + state = reconcile( + state, + _strict_json(_regular_bytes(args.observation)), + plan, + ) + result = state + elif args.operation == "challenge": + if args.platform is None or args.challenge is None or args.checkpoint is None: + raise Gate14ControllerError("challenge platform, checkpoint, and output are required") + state, result = issue_calibration_challenge( + state, + plan, + args.platform, + args.challenge, + args.checkpoint, + ) + elif args.operation == "collect": + if args.platform is None or args.evidence is None or args.challenge is None: + raise Gate14ControllerError("platform evidence and challenge are required") + state = collect_platform( + state, + plan, + args.platform, + args.evidence, + args.challenge, + ) + result = state + else: + state = begin_cleanup(state, plan, args.failure_code) + if args.observation is not None: + state = reconcile( + state, + _strict_json(_regular_bytes(args.observation)), + plan, + ) + result = state + save_state(args.state, state, plan) + except ( + Gate14ControllerError, + challenge_contract.Gate14ChallengeError, + acceptance.Gate14EvidenceError, + ) as exc: + raise SystemExit(str(exc)) from exc + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_run_packaged_lifecycle.py b/scripts/gate14_run_packaged_lifecycle.py new file mode 100644 index 000000000..ae4abdd7a --- /dev/null +++ b/scripts/gate14_run_packaged_lifecycle.py @@ -0,0 +1,94 @@ +"""Run one source-bound Gate 14 packaged lifecycle on its native host.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate14_linux_action_transport as linux_transport +import gate14_packaged_lifecycle as lifecycle +import gate14_windows_action_transport as windows_transport + +ActionFactory = Callable[[lifecycle.LifecycleConfig], Any] + + +class Gate14LifecycleEntrypointError(ValueError): + """The native platform or action adapter failed closed.""" + + +def _canonical(value: Mapping[str, Any]) -> str: + return json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _native_platform() -> str: + if sys.platform == "win32": + return "windows" + if sys.platform.startswith("linux"): + return "linux" + raise Gate14LifecycleEntrypointError("unsupported lifecycle platform") + + +def _factory(platform_name: str) -> ActionFactory: + if platform_name == "windows": + return windows_transport.WindowsActionTransport + if platform_name == "linux": + return linux_transport.LinuxActionTransport + raise Gate14LifecycleEntrypointError("lifecycle platform is invalid") + + +def run_from_config( + path: Path, + *, + action_factory: ActionFactory | None = None, + native_platform: str | None = None, +) -> Mapping[str, Any]: + config = lifecycle.load_config(Path(path)) + observed_platform = _native_platform() if native_platform is None else native_platform + if observed_platform not in {"windows", "linux"} or config.platform != observed_platform: + raise Gate14LifecycleEntrypointError("lifecycle platform binding changed") + factory = action_factory or _factory(config.platform) + actions = factory(config) + if not ( + callable(getattr(actions, "prepare", None)) + and callable(getattr(actions, "calibrate", None)) + and callable(getattr(actions, "cleanup", None)) + and callable(getattr(actions, "close", None)) + ): + raise Gate14LifecycleEntrypointError("lifecycle action adapter is invalid") + try: + return lifecycle.run_lifecycle(config, actions) + finally: + actions.close() + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--config", required=True) + try: + arguments = parser.parse_args(sys.argv[1:] if argv is None else argv) + document = run_from_config(Path(arguments.config)) + print(_canonical(document)) + return 0 + except (Exception, SystemExit): + print( + _canonical( + { + "failure_code": "gate14_lifecycle_failed", + "result": "failed", + "schema_version": 1, + } + ) + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_windows_action_transport.py b/scripts/gate14_windows_action_transport.py new file mode 100644 index 000000000..612165d12 --- /dev/null +++ b/scripts/gate14_windows_action_transport.py @@ -0,0 +1,703 @@ +"""Persistent, bounded RPC transport for Gate 14 Windows lifecycle actions. + +The production action handlers live in one Windows PowerShell process so its +kill-on-close Job Object, control credential, and packaged process state survive +the controller-owned challenge wait. This module only owns source verification, +framing, sequencing, and fail-closed process cleanup. The source-bound +PowerShell product module implements concrete package/cache/control operations. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import queue +import re +import secrets +import shutil +import stat +import subprocess +import threading +from pathlib import Path +from typing import Any, BinaryIO, Callable, Mapping, Sequence + +import gate14_calibration_challenge as challenge_contract + +SCHEMA_VERSION = 1 +SCOPE = "gate14-windows-lifecycle-actions" +MAX_FRAME_BYTES = 262_144 +MAX_SOURCE_BYTES = 8 * 1024 * 1024 +DEFAULT_OPERATION_TIMEOUT_SECONDS = { + "prepare": 3_600.0, + "calibrate": 1_800.0, + "cleanup": 300.0, +} +DEFAULT_CLOSE_TIMEOUT_SECONDS = 30.0 + +_GATE13_LIFECYCLE_SHA256 = "aa549335b63f43ef2e68f40881635ab077e916878bc472b8674424aa087a6dda" +_GATE13_INFERENCE_SHA256 = "2d53424c886ff4a70367a3a0844e33a234bc6c290828a21b70a134b5bf115611" +_PRODUCT_ACTIONS_SHA256 = "3a29f13ecd855fbdb21d42b21ffd3e793e8a3c1086f816a28d20f9e8cfbb2e23" +_ACTION_HOST_SHA256 = "4ebf68d5fbeb3afad9cd52a7e062162de61da4f6ecee0cd113585a20c84fdab5" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_FAILURE_RE = re.compile(r"[a-z][a-z0-9-]{0,63}") +_FORBIDDEN_KEYS = { + "api_key", + "api_token", + "authorization", + "argv", + "command", + "control_key", + "control_token", + "credential", + "endpoint", + "environment", + "gpu_uuid", + "hostname", + "output", + "password", + "path", + "prompt", + "secret", + "token", + "url", + "username", +} +_RESPONSE_FIELDS = { + "failure_code", + "operation", + "payload", + "request_id", + "result", + "schema_version", + "scope", + "session_id", +} +_FRAME_FIELDS = { + "binding", + "operation", + "payload", + "request_id", + "schema_version", + "scope", + "session_id", +} + + +class Gate14ActionTransportError(ValueError): + """The action host source, RPC stream, or process failed closed.""" + + +ProcessFactory = Callable[..., subprocess.Popen[bytes]] + + +def _reject_constant(_value: str) -> None: + raise Gate14ActionTransportError("non-finite RPC value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14ActionTransportError("duplicate RPC field") + result[key] = value + return result + + +def _canonical(value: Mapping[str, Any]) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Gate14ActionTransportError("RPC value is not canonical JSON") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_FRAME_BYTES: + raise Gate14ActionTransportError("RPC frame size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14ActionTransportError("RPC frame is invalid") from exc + if not isinstance(value, dict): + raise Gate14ActionTransportError("RPC root is invalid") + if _canonical(value) != payload: + raise Gate14ActionTransportError("RPC frame is not canonical") + return value + + +def _open_locked_source(candidate: Path) -> BinaryIO: + if os.name != "nt": + return candidate.open("rb") + + import ctypes + import msvcrt + from ctypes import wintypes + + create_file = ctypes.WinDLL("kernel32", use_last_error=True).CreateFileW + create_file.argtypes = ( + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ) + create_file.restype = wintypes.HANDLE + close_handle = ctypes.WinDLL("kernel32", use_last_error=True).CloseHandle + close_handle.argtypes = (wintypes.HANDLE,) + close_handle.restype = wintypes.BOOL + + generic_read = 0x80000000 + share_read_only = 0x00000001 + open_existing = 3 + open_reparse_point = 0x00200000 + sequential_scan = 0x08000000 + raw_handle = create_file( + str(candidate), + generic_read, + share_read_only, + None, + open_existing, + open_reparse_point | sequential_scan, + None, + ) + invalid_handle = ctypes.c_void_p(-1).value + if raw_handle == invalid_handle: + error = ctypes.get_last_error() + raise OSError(error, "could not lock action source", str(candidate)) + try: + descriptor = msvcrt.open_osfhandle(int(raw_handle), os.O_RDONLY) + except BaseException: + close_handle(raw_handle) + raise + return os.fdopen(descriptor, "rb", closefd=True) + + +def _open_verified_source( + path: Path, + expected_sha256: str, +) -> tuple[Path, BinaryIO]: + candidate = Path(path) + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14ActionTransportError("action source is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or not 1 <= metadata.st_size <= MAX_SOURCE_BYTES + ): + raise Gate14ActionTransportError("action source is unsafe") + + handle: BinaryIO | None = None + try: + handle = _open_locked_source(candidate) + opened = os.fstat(handle.fileno()) + if not stat.S_ISREG(opened.st_mode) or (metadata.st_dev, metadata.st_ino, metadata.st_size,) != ( + opened.st_dev, + opened.st_ino, + opened.st_size, + ): + raise Gate14ActionTransportError("action source changed while opening") + payload = handle.read(MAX_SOURCE_BYTES + 1) + after = os.fstat(handle.fileno()) + if len(payload) != opened.st_size or (opened.st_dev, opened.st_ino, opened.st_size,) != ( + after.st_dev, + after.st_ino, + after.st_size, + ): + raise Gate14ActionTransportError("action source changed while reading") + normalized = payload.replace(b"\r\n", b"\n") + if b"\r" in normalized or hashlib.sha256(normalized).hexdigest() != expected_sha256: + raise Gate14ActionTransportError("action source digest changed") + handle.seek(0) + return candidate.resolve(), handle + except Gate14ActionTransportError: + if handle is not None: + handle.close() + raise + except OSError as exc: + if handle is not None: + handle.close() + raise Gate14ActionTransportError("action source is unreadable") from exc + + +def _normalized_source(path: Path, expected_sha256: str) -> Path: + verified, handle = _open_verified_source(path, expected_sha256) + handle.close() + return verified + + +def _assert_safe_payload(value: Any) -> None: + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str) or key.casefold() in _FORBIDDEN_KEYS: + raise Gate14ActionTransportError("action response contains private material") + _assert_safe_payload(item) + return + if isinstance(value, (list, tuple)): + for item in value: + _assert_safe_payload(item) + return + if isinstance(value, str) and ( + value.startswith("drift_control_") or "\r" in value or "\n" in value or "\x00" in value + ): + raise Gate14ActionTransportError("action response contains private material") + + +def _binding(config: Any) -> dict[str, Any]: + value = { + "attempt_ordinal": config.attempt_ordinal, + "lifecycle_config_sha256": config.config_sha256, + "package_sha256": config.package_sha256, + "platform": config.platform, + "run_id": config.run_id, + "source_commit": config.source_commit, + } + if ( + type(value["attempt_ordinal"]) is not int + or not 1 <= value["attempt_ordinal"] <= 100 + or value["platform"] != "windows" + or not isinstance(value["run_id"], str) + or _RUN_RE.fullmatch(value["run_id"]) is None + or not isinstance(value["source_commit"], str) + or _COMMIT_RE.fullmatch(value["source_commit"]) is None + or not isinstance(value["package_sha256"], str) + or _DIGEST_RE.fullmatch(value["package_sha256"]) is None + or not isinstance(value["lifecycle_config_sha256"], str) + or _DIGEST_RE.fullmatch(value["lifecycle_config_sha256"]) is None + ): + raise Gate14ActionTransportError("lifecycle action binding is invalid") + return value + + +def _operation_timeouts(value: Mapping[str, float] | None) -> dict[str, float]: + result = dict(DEFAULT_OPERATION_TIMEOUT_SECONDS if value is None else value) + if set(result) != set(DEFAULT_OPERATION_TIMEOUT_SECONDS): + raise Gate14ActionTransportError("action transport timeout schema is invalid") + caps = {"prepare": 7_200.0, "calibrate": 3_600.0, "cleanup": 600.0} + for operation, maximum in caps.items(): + timeout = result[operation] + if type(timeout) not in (int, float) or not 0.1 <= float(timeout) <= maximum: + raise Gate14ActionTransportError("action transport timeout is invalid") + result[operation] = float(timeout) + return result + + +def _challenge_payload(challenge: Mapping[str, Any]) -> dict[str, Any]: + return { + "challenge_sha256": challenge_contract.digest(challenge), + "controller_state_revision": challenge["controller_state_revision"], + "issued_at_unix": challenge["issued_at_unix"], + "expires_at_unix": challenge["expires_at_unix"], + } + + +def _open_verified_config( + path: Path, + expected_sha256: str, + maximum: int, +) -> tuple[Path, BinaryIO]: + candidate = Path(path) + try: + metadata = candidate.lstat() + except OSError as exc: + raise Gate14ActionTransportError("lifecycle configuration is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or candidate.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise Gate14ActionTransportError("lifecycle configuration is unsafe") + + handle: BinaryIO | None = None + try: + handle = _open_locked_source(candidate) + opened = os.fstat(handle.fileno()) + if not stat.S_ISREG(opened.st_mode) or (metadata.st_dev, metadata.st_ino, metadata.st_size,) != ( + opened.st_dev, + opened.st_ino, + opened.st_size, + ): + raise Gate14ActionTransportError("lifecycle configuration changed while opening") + payload = handle.read(maximum + 1) + after = os.fstat(handle.fileno()) + if len(payload) != opened.st_size or (opened.st_dev, opened.st_ino, opened.st_size,) != ( + after.st_dev, + after.st_ino, + after.st_size, + ): + raise Gate14ActionTransportError("lifecycle configuration changed while reading") + if "sha256:" + hashlib.sha256(payload).hexdigest() != expected_sha256: + raise Gate14ActionTransportError("lifecycle configuration binding changed") + handle.seek(0) + return candidate.resolve(), handle + except Gate14ActionTransportError: + if handle is not None: + handle.close() + raise + except OSError as exc: + if handle is not None: + handle.close() + raise Gate14ActionTransportError("lifecycle configuration is unreadable") from exc + + +class WindowsActionTransport: + """Own exactly one source-bound PowerShell action host.""" + + def __init__( + self, + config: Any, + *, + powershell: str | None = None, + process_factory: ProcessFactory = subprocess.Popen, + operation_timeouts: Mapping[str, float] | None = None, + close_timeout_seconds: float = DEFAULT_CLOSE_TIMEOUT_SECONDS, + transport_self_test: bool = False, + self_test_cleanup_marker: Path | None = None, + ) -> None: + if type(close_timeout_seconds) not in (int, float) or not 0.1 <= float(close_timeout_seconds) <= 60.0: + raise Gate14ActionTransportError("action transport timeout is invalid") + self._operation_timeouts = _operation_timeouts(operation_timeouts) + self._binding = _binding(config) + config_path = Path(config.config_path) + directory = Path(__file__).resolve().parent + self._source_handles: list[BinaryIO] = [] + try: + if config_path.name != "gate14-lifecycle.json": + raise Gate14ActionTransportError("lifecycle configuration binding changed") + self._config_path, handle = _open_verified_config( + config_path, + config.config_sha256, + 65_536, + ) + self._source_handles.append(handle) + self._host_path, handle = _open_verified_source( + directory / "gate14_windows_lifecycle_actions.ps1", + _ACTION_HOST_SHA256, + ) + self._source_handles.append(handle) + self._gate13_lifecycle_path, handle = _open_verified_source( + directory / "gate13_windows_packaged_lifecycle.ps1", + _GATE13_LIFECYCLE_SHA256, + ) + self._source_handles.append(handle) + self._gate13_inference_path, handle = _open_verified_source( + directory / "gate13_windows_localhost_inference.ps1", + _GATE13_INFERENCE_SHA256, + ) + self._source_handles.append(handle) + self._product_actions_path, handle = _open_verified_source( + directory / "gate14_windows_product_actions.ps1", + _PRODUCT_ACTIONS_SHA256, + ) + self._source_handles.append(handle) + if not ( + self._gate13_lifecycle_path.parent == self._gate13_inference_path.parent + and self._host_path.parent == self._gate13_lifecycle_path.parent + and self._product_actions_path.parent == self._gate13_lifecycle_path.parent + ): + raise Gate14ActionTransportError("action sources are not colocated") + + executable = powershell or shutil.which("powershell.exe") + if not executable: + raise Gate14ActionTransportError("Windows PowerShell 5.1 is unavailable") + except BaseException: + for source_handle in self._source_handles: + source_handle.close() + self._source_handles.clear() + raise + self._session_id = secrets.token_hex(32) + arguments = [ + executable, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + os.fspath(self._host_path), + "-SessionId", + self._session_id, + "-RunId", + self._binding["run_id"], + "-AttemptOrdinal", + str(self._binding["attempt_ordinal"]), + "-SourceCommit", + self._binding["source_commit"], + "-PackageSha256", + self._binding["package_sha256"], + "-Gate13Lifecycle", + os.fspath(self._gate13_lifecycle_path), + "-Gate13Inference", + os.fspath(self._gate13_inference_path), + "-Gate13LifecycleSha256", + _GATE13_LIFECYCLE_SHA256, + "-Gate13InferenceSha256", + _GATE13_INFERENCE_SHA256, + "-ProductActions", + os.fspath(self._product_actions_path), + "-ProductActionsSha256", + _PRODUCT_ACTIONS_SHA256, + "-LifecycleConfig", + os.fspath(self._config_path), + "-LifecycleConfigSha256", + config.config_sha256, + ] + if transport_self_test: + arguments.append("-TransportSelfTest") + if self_test_cleanup_marker is not None: + arguments.extend(("-SelfTestCleanupMarker", os.fspath(Path(self_test_cleanup_marker)))) + + creation_flags = getattr(subprocess, "CREATE_NO_WINDOW", 0) + try: + self._process = process_factory( + arguments, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + creationflags=creation_flags, + ) + except (OSError, ValueError) as exc: + self._release_source_locks() + raise Gate14ActionTransportError("action host could not start") from exc + if self._process.stdin is None or self._process.stdout is None or self._process.stderr is None: + self._terminate() + raise Gate14ActionTransportError("action host pipes are unavailable") + + self._close_timeout = float(close_timeout_seconds) + self._responses: queue.Queue[bytes | BaseException | None] = queue.Queue(maxsize=2) + self._stderr_bytes = 0 + self._stderr_overflow = False + self._next_request_id = 1 + self._phase = "new" + self._closed = False + self._reader = threading.Thread( + target=self._read_stdout, + name="gate14-windows-action-stdout", + daemon=True, + ) + self._stderr_reader = threading.Thread( + target=self._drain_stderr, + name="gate14-windows-action-stderr", + daemon=True, + ) + self._reader.start() + self._stderr_reader.start() + + def _read_stdout(self) -> None: + try: + while True: + line = self._process.stdout.readline(MAX_FRAME_BYTES + 2) + if not line: + self._responses.put(None) + return + if len(line) > MAX_FRAME_BYTES + 1 or not line.endswith(b"\n"): + self._responses.put(Gate14ActionTransportError("action response framing is invalid")) + return + frame = line[:-1] + if frame.endswith(b"\r"): + frame = frame[:-1] + self._responses.put(frame) + except BaseException as exc: + self._responses.put(exc) + + def _drain_stderr(self) -> None: + try: + while chunk := self._process.stderr.read(65_536): + self._stderr_bytes += len(chunk) + if self._stderr_bytes > MAX_FRAME_BYTES: + self._stderr_overflow = True + except BaseException: + self._stderr_overflow = True + + def _release_source_locks(self) -> None: + handles = getattr(self, "_source_handles", []) + for handle in handles: + try: + handle.close() + except OSError: + pass + handles.clear() + + def _terminate(self) -> None: + process = getattr(self, "_process", None) + if process is None: + self._release_source_locks() + return + try: + if process.stdin is not None: + process.stdin.close() + except OSError: + pass + try: + process.wait( + timeout=getattr( + self, + "_close_timeout", + DEFAULT_CLOSE_TIMEOUT_SECONDS, + ) + ) + except (subprocess.TimeoutExpired, OSError): + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=DEFAULT_CLOSE_TIMEOUT_SECONDS) + except (subprocess.TimeoutExpired, OSError): + pass + try: + ended = process.poll() is not None + except OSError: + ended = False + if ended: + self._release_source_locks() + + def _fail(self, message: str) -> None: + self._closed = True + self._terminate() + raise Gate14ActionTransportError(message) + + def request(self, operation: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: + if self._closed: + raise Gate14ActionTransportError("action transport is closed") + if operation not in {"prepare", "calibrate", "cleanup"}: + self._fail("action operation is invalid") + if not isinstance(payload, dict): + self._fail("action payload is invalid") + request_id = self._next_request_id + frame = { + "binding": self._binding, + "operation": operation, + "payload": payload, + "request_id": request_id, + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "session_id": self._session_id, + } + rendered = _canonical(frame) + b"\n" + if len(rendered) > MAX_FRAME_BYTES: + self._fail("action request is too large") + try: + self._process.stdin.write(rendered) + self._process.stdin.flush() + except (BrokenPipeError, OSError): + self._fail("action host ended before request") + + try: + response_item = self._responses.get(timeout=self._operation_timeouts[operation]) + except queue.Empty: + self._fail("action response timed out") + if response_item is None: + self._fail("action host ended before response") + if isinstance(response_item, BaseException): + self._fail("action response could not be read") + try: + response = _strict_json(response_item) + except Gate14ActionTransportError: + self._fail("action response is invalid") + if ( + set(response) != _RESPONSE_FIELDS + or type(response.get("schema_version")) is not int + or response.get("schema_version") != SCHEMA_VERSION + or response.get("scope") != SCOPE + or response.get("session_id") != self._session_id + or type(response.get("request_id")) is not int + or response.get("request_id") != request_id + or response.get("operation") != operation + or response.get("result") not in {"passed", "failed"} + ): + self._fail("action response binding is invalid") + self._next_request_id += 1 + if response["result"] == "failed": + failure_code = response["failure_code"] + if ( + response["payload"] is not None + or not isinstance(failure_code, str) + or _FAILURE_RE.fullmatch(failure_code) is None + ): + self._fail("action failure response is invalid") + raise Gate14ActionTransportError(f"action host rejected {operation}: {failure_code}") + if response["failure_code"] is not None or not isinstance(response["payload"], dict): + self._fail("action success response is invalid") + _assert_safe_payload(response["payload"]) + if self._stderr_overflow: + self._fail("action host diagnostics exceeded the bound") + return dict(response["payload"]) + + def prepare(self, _config: Any) -> Mapping[str, Any]: + if self._phase != "new": + self._fail("action operation order is invalid") + try: + result = self.request("prepare", {}) + except BaseException: + self._phase = "failed" + raise + self._phase = "prepared" + return result + + def calibrate( + self, + _config: Any, + challenge: Mapping[str, Any], + ) -> Sequence[Mapping[str, Any]]: + if self._phase != "prepared": + self._fail("action operation order is invalid") + try: + result = self.request( + "calibrate", + _challenge_payload(challenge), + ) + except BaseException: + self._phase = "failed" + raise + self._phase = "calibrated" + observations = result.get("suspensions") + if not isinstance(observations, list): + raise Gate14ActionTransportError("action calibration response is not an observation list") + return observations + + def cleanup(self, config: Any) -> Mapping[str, Any]: + if self._closed: + raise Gate14ActionTransportError("action transport is closed") + result = self.request("cleanup", {}) + expected = { + "action_temporaries_removed": True, + "attempt_ordinal": config.attempt_ordinal, + "credentials_removed": True, + "platform": "windows", + "processes_absent": True, + "run_id": config.run_id, + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + } + if result != expected: + self._fail("action cleanup response is invalid") + self._phase = "cleaned" + return result + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._terminate() + + def __enter__(self) -> "WindowsActionTransport": + return self + + def __exit__(self, _type, _value, _traceback) -> None: + self.close() diff --git a/scripts/gate14_windows_lifecycle_actions.ps1 b/scripts/gate14_windows_lifecycle_actions.ps1 new file mode 100644 index 000000000..a40cdba10 --- /dev/null +++ b/scripts/gate14_windows_lifecycle_actions.ps1 @@ -0,0 +1,380 @@ +param( + [Parameter(Mandatory = $true)][ValidatePattern("^[0-9a-f]{64}$")][string]$SessionId, + [Parameter(Mandatory = $true)][ValidatePattern("^[a-z0-9][a-z0-9-]{0,62}$")][string]$RunId, + [Parameter(Mandatory = $true)][ValidateRange(1, 100)][int]$AttemptOrdinal, + [Parameter(Mandatory = $true)][ValidatePattern("^[0-9a-f]{40}$")][string]$SourceCommit, + [Parameter(Mandatory = $true)][ValidatePattern("^sha256:[0-9a-f]{64}$")][string]$PackageSha256, + [Parameter(Mandatory = $true)][string]$Gate13Lifecycle, + [Parameter(Mandatory = $true)][string]$Gate13Inference, + [Parameter(Mandatory = $true)][ValidatePattern("^[0-9a-f]{64}$")][string]$Gate13LifecycleSha256, + [Parameter(Mandatory = $true)][ValidatePattern("^[0-9a-f]{64}$")][string]$Gate13InferenceSha256, + [Parameter(Mandatory = $true)][string]$ProductActions, + [Parameter(Mandatory = $true)][ValidatePattern("^[0-9a-f]{64}$")][string]$ProductActionsSha256, + [Parameter(Mandatory = $true)][string]$LifecycleConfig, + [Parameter(Mandatory = $true)][ValidatePattern("^sha256:[0-9a-f]{64}$")][string]$LifecycleConfigSha256, + [switch]$TransportSelfTest, + [string]$SelfTestCleanupMarker = "" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest +$ProgressPreference = "SilentlyContinue" +$VerbosePreference = "SilentlyContinue" +$DebugPreference = "SilentlyContinue" +$InformationPreference = "SilentlyContinue" + +$script:Gate14MaxFrameBytes = 262144 +$script:Gate14Scope = "gate14-windows-lifecycle-actions" +$script:Gate14Phase = "new" +$script:Gate14Binding = $null +$script:Gate14Cleaned = $false +$script:Gate14CleanupResult = $null +$script:Gate14StateNonce = [Guid]::NewGuid().ToString("N") + +function Get-Gate14NormalizedSha256 { + param([Parameter(Mandatory = $true)][string]$Path) + + $item = Get-Item -LiteralPath $Path -Force + if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 -or -not ($item -is [IO.FileInfo])) { + throw "action helper is unsafe" + } + if ($item.Length -lt 1 -or $item.Length -gt (8 * 1024 * 1024)) { + throw "action helper size is invalid" + } + $bytes = [IO.File]::ReadAllBytes($item.FullName) + $utf8 = New-Object Text.UTF8Encoding($false, $true) + $text = $utf8.GetString($bytes) + if ($text.Length -gt 0 -and $text[0] -eq [char]0xfeff) { + throw "action helper encoding is invalid" + } + $text = $text.Replace("`r`n", "`n") + if ($text.Contains("`r")) { + throw "action helper line endings are invalid" + } + $normalized = $utf8.GetBytes($text) + $hasher = [Security.Cryptography.SHA256]::Create() + try { + return ([BitConverter]::ToString($hasher.ComputeHash($normalized))).Replace("-", "").ToLowerInvariant() + } + finally { + $hasher.Dispose() + } +} + +function Assert-Gate14ExactProperties { + param( + [Parameter(Mandatory = $true)]$Value, + [Parameter(Mandatory = $true)][AllowEmptyCollection()][string[]]$Names + ) + + if ($null -eq $Value) { + throw "RPC object is absent" + } + $actual = @($Value.PSObject.Properties | ForEach-Object { $_.Name }) + if ($actual.Count -ne $Names.Count) { + throw "RPC object schema is invalid" + } + foreach ($name in $Names) { + if (-not ($actual -ccontains $name)) { + throw "RPC object schema is invalid" + } + } +} + +function Assert-Gate14Binding { + param([Parameter(Mandatory = $true)]$Binding) + + Assert-Gate14ExactProperties -Value $Binding -Names @( + "attempt_ordinal", + "lifecycle_config_sha256", + "package_sha256", + "platform", + "run_id", + "source_commit" + ) + if ( + $Binding.platform -isnot [string] -or + $Binding.platform -cne "windows" -or + $Binding.run_id -isnot [string] -or + $Binding.run_id -cne $RunId -or + $Binding.source_commit -isnot [string] -or + $Binding.source_commit -cne $SourceCommit -or + $Binding.package_sha256 -isnot [string] -or + $Binding.package_sha256 -cne $PackageSha256 -or + $Binding.lifecycle_config_sha256 -isnot [string] -or + $Binding.lifecycle_config_sha256 -cne $LifecycleConfigSha256 -or + $Binding.attempt_ordinal -isnot [int] -or + $Binding.attempt_ordinal -ne $AttemptOrdinal + ) { + throw "RPC binding is invalid" + } + $canonical = ConvertTo-Json -InputObject $Binding -Compress + if ($null -eq $script:Gate14Binding) { + $script:Gate14Binding = $canonical + } + elseif ($script:Gate14Binding -cne $canonical) { + throw "RPC binding changed" + } +} + +function Write-Gate14Response { + param( + [Parameter(Mandatory = $true)][int]$RequestId, + [Parameter(Mandatory = $true)][string]$Operation, + [Parameter(Mandatory = $true)][ValidateSet("passed", "failed")][string]$Result, + $Payload, + [AllowNull()]$FailureCode + ) + + $response = [ordered]@{ + failure_code = $FailureCode + operation = $Operation + payload = $Payload + request_id = $RequestId + result = $Result + schema_version = 1 + scope = $script:Gate14Scope + session_id = $SessionId + } + $rendered = ConvertTo-Json -InputObject $response -Compress -Depth 20 + if ([Text.Encoding]::UTF8.GetByteCount($rendered) -gt $script:Gate14MaxFrameBytes) { + throw "RPC response is too large" + } + [Console]::Out.WriteLine($rendered) + [Console]::Out.Flush() +} + +function Invoke-Gate14Cleanup { + if ($script:Gate14Cleaned) { + return $script:Gate14CleanupResult + } + if ($TransportSelfTest) { + if ($SelfTestCleanupMarker.Length -gt 0) { + [IO.File]::WriteAllText($SelfTestCleanupMarker, "cleaned", (New-Object Text.UTF8Encoding($false))) + } + $result = [ordered]@{ + action_temporaries_removed = $true + attempt_ordinal = [int]$AttemptOrdinal + credentials_removed = $true + platform = "windows" + processes_absent = $true + run_id = [string]$RunId + schema_version = 1 + scope = "gate14-host-lifecycle-cleanup" + } + } + else { + $result = Invoke-Gate14WindowsProductCleanup + } + $script:Gate14CleanupResult = $result + $script:Gate14Cleaned = $true + return $result +} + +$lifecyclePath = (Get-Item -LiteralPath $Gate13Lifecycle -Force).FullName +$inferencePath = (Get-Item -LiteralPath $Gate13Inference -Force).FullName +$productActionsPath = (Get-Item -LiteralPath $ProductActions -Force).FullName +if ( + [IO.Path]::GetFileName($lifecyclePath) -cne "gate13_windows_packaged_lifecycle.ps1" -or + [IO.Path]::GetFileName($inferencePath) -cne "gate13_windows_localhost_inference.ps1" -or + [IO.Path]::GetFileName($productActionsPath) -cne "gate14_windows_product_actions.ps1" -or + [IO.Path]::GetDirectoryName($lifecyclePath) -cne [IO.Path]::GetDirectoryName($inferencePath) -or + [IO.Path]::GetDirectoryName($lifecyclePath) -cne [IO.Path]::GetDirectoryName($productActionsPath) -or + (Get-Gate14NormalizedSha256 -Path $lifecyclePath) -cne $Gate13LifecycleSha256 -or + (Get-Gate14NormalizedSha256 -Path $inferencePath) -cne $Gate13InferenceSha256 -or + (Get-Gate14NormalizedSha256 -Path $productActionsPath) -cne $ProductActionsSha256 +) { + throw "action helper binding is invalid" +} +$configItem = Get-Item -LiteralPath $LifecycleConfig -Force +if ( + ($configItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 -or + -not ($configItem -is [IO.FileInfo]) -or + $configItem.Name -cne "gate14-lifecycle.json" -or + $configItem.Length -lt 1 -or + $configItem.Length -gt 65536 +) { + throw "lifecycle configuration is unsafe" +} +$configHasher = [Security.Cryptography.SHA256]::Create() +try { + $configDigest = "sha256:" + ([BitConverter]::ToString($configHasher.ComputeHash([IO.File]::ReadAllBytes($configItem.FullName)))).Replace("-", "").ToLowerInvariant() +} +finally { + $configHasher.Dispose() +} +if ($configDigest -cne $LifecycleConfigSha256) { + throw "lifecycle configuration binding is invalid" +} + +. $lifecyclePath +. $inferencePath +. $productActionsPath +if ( + $null -eq (Get-Command Force-Gate13ProductCleanup -ErrorAction SilentlyContinue) -or + $null -eq (Get-Command Invoke-Gate13LoopbackJson -ErrorAction SilentlyContinue) -or + $null -eq (Get-Command Initialize-Gate14WindowsProductActions -ErrorAction SilentlyContinue) -or + $null -eq (Get-Command Invoke-Gate14WindowsProductPrepare -ErrorAction SilentlyContinue) -or + $null -eq (Get-Command Invoke-Gate14WindowsProductCalibrate -ErrorAction SilentlyContinue) -or + $null -eq (Get-Command Invoke-Gate14WindowsProductCleanup -ErrorAction SilentlyContinue) +) { + throw "required action helpers are unavailable" +} +if (-not $TransportSelfTest) { + Initialize-Gate14WindowsProductActions ` + -LifecycleConfig $configItem.FullName ` + -RunId $RunId ` + -AttemptOrdinal $AttemptOrdinal ` + -SourceCommit $SourceCommit ` + -PackageSha256 $PackageSha256 ` + -ProductActionsPath $productActionsPath +} + +$expectedRequestId = 1 +try { + while ($true) { + $line = [Console]::In.ReadLine() + if ($null -eq $line) { + break + } + if ( + $line.Length -eq 0 -or + $line.Contains([char]0) -or + [Text.Encoding]::UTF8.GetByteCount($line) -gt $script:Gate14MaxFrameBytes + ) { + throw "RPC frame is invalid" + } + try { + $frame = ConvertFrom-Json -InputObject $line + if ((ConvertTo-Json -InputObject $frame -Compress -Depth 20) -cne $line) { + throw "RPC frame is not canonical" + } + Assert-Gate14ExactProperties -Value $frame -Names @( + "binding", + "operation", + "payload", + "request_id", + "schema_version", + "scope", + "session_id" + ) + if ( + $frame.schema_version -isnot [int] -or + $frame.schema_version -ne 1 -or + $frame.scope -isnot [string] -or + $frame.scope -cne $script:Gate14Scope -or + $frame.session_id -isnot [string] -or + $frame.session_id -cne $SessionId -or + $frame.request_id -isnot [int] -or + $frame.request_id -ne $expectedRequestId -or + $frame.operation -isnot [string] -or + $frame.operation -cnotin @("prepare", "calibrate", "cleanup") + ) { + throw "RPC frame binding is invalid" + } + Assert-Gate14Binding -Binding $frame.binding + $operation = [string]$frame.operation + $requestId = [int]$frame.request_id + $expectedRequestId += 1 + + if ($operation -ceq "prepare") { + if ($script:Gate14Phase -cne "new") { + throw "RPC operation order is invalid" + } + Assert-Gate14ExactProperties -Value $frame.payload -Names @() + if ($TransportSelfTest) { + $payload = [ordered]@{ + helpers_loaded = $true + host_process_id = [int]$PID + state_nonce = $script:Gate14StateNonce + } + } + else { + try { + $payload = Invoke-Gate14WindowsProductPrepare + } + catch { + $script:Gate14Phase = "failed" + Write-Gate14Response -RequestId $requestId -Operation $operation -Result "failed" -Payload $null -FailureCode "product-prepare-failed" + continue + } + } + $script:Gate14Phase = "prepared" + Write-Gate14Response -RequestId $requestId -Operation $operation -Result "passed" -FailureCode $null -Payload $payload + continue + } + + if ($operation -ceq "calibrate") { + if ($script:Gate14Phase -cne "prepared") { + throw "RPC operation order is invalid" + } + Assert-Gate14ExactProperties -Value $frame.payload -Names @( + "challenge_sha256", + "controller_state_revision", + "issued_at_unix", + "expires_at_unix" + ) + if ( + $frame.payload.challenge_sha256 -isnot [string] -or + $frame.payload.challenge_sha256 -cnotmatch "^sha256:[0-9a-f]{64}$" -or + $frame.payload.controller_state_revision -isnot [int] -or + $frame.payload.controller_state_revision -lt 0 -or + $frame.payload.issued_at_unix -isnot [int] -or + $frame.payload.expires_at_unix -isnot [int] -or + ($frame.payload.expires_at_unix - $frame.payload.issued_at_unix) -lt 60 -or + ($frame.payload.expires_at_unix - $frame.payload.issued_at_unix) -gt 900 + ) { + throw "RPC calibration binding is invalid" + } + if ($TransportSelfTest) { + $payload = [ordered]@{ + challenge_sha256 = [string]$frame.payload.challenge_sha256 + host_process_id = [int]$PID + state_nonce = $script:Gate14StateNonce + } + } + else { + try { + $payload = [ordered]@{ + suspensions = @(Invoke-Gate14WindowsProductCalibrate -Challenge $frame.payload) + } + } + catch { + $script:Gate14Phase = "failed" + Write-Gate14Response -RequestId $requestId -Operation $operation -Result "failed" -Payload $null -FailureCode "product-calibration-failed" + continue + } + } + $script:Gate14Phase = "calibrated" + Write-Gate14Response -RequestId $requestId -Operation $operation -Result "passed" -FailureCode $null -Payload $payload + continue + } + + if ($operation -ceq "cleanup") { + Assert-Gate14ExactProperties -Value $frame.payload -Names @() + $payload = Invoke-Gate14Cleanup + $script:Gate14Phase = "cleaned" + Write-Gate14Response -RequestId $requestId -Operation $operation -Result "passed" -FailureCode $null -Payload $payload + continue + } + } + catch { + try { + $discardedCleanup = Invoke-Gate14Cleanup + $discardedCleanup = $null + } + catch { + } + try { + Write-Gate14Response -RequestId $expectedRequestId -Operation "invalid" -Result "failed" -Payload $null -FailureCode "invalid-action-frame" + } + catch { + } + break + } + } +} +finally { + $discardedCleanup = Invoke-Gate14Cleanup + $discardedCleanup = $null +} diff --git a/scripts/gate14_windows_probe.ps1 b/scripts/gate14_windows_probe.ps1 new file mode 100644 index 000000000..02ccec104 --- /dev/null +++ b/scripts/gate14_windows_probe.ps1 @@ -0,0 +1,17 @@ +param( + [Parameter(Mandatory = $true)][string]$Python, + [Parameter(Mandatory = $true)][string]$Facts, + [Parameter(Mandatory = $true)][string]$Challenge, + [Parameter(Mandatory = $true)][string]$Package, + [Parameter(Mandatory = $true)][string]$ReleaseMetadata, + [Parameter(Mandatory = $true)][string]$Output +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$probe = Join-Path $PSScriptRoot "gate14_host_probe.py" +& $Python $probe --platform windows --facts $Facts --challenge $Challenge --package $Package --release-metadata $ReleaseMetadata --output $Output +if ($LASTEXITCODE -ne 0) { + throw "Gate 14 Windows probe failed" +} diff --git a/scripts/gate14_windows_product_actions.ps1 b/scripts/gate14_windows_product_actions.ps1 new file mode 100644 index 000000000..236b71c12 --- /dev/null +++ b/scripts/gate14_windows_product_actions.ps1 @@ -0,0 +1,1649 @@ +# Concrete Gate 14 Windows packaged-product actions. +# +# This module is dot-sourced only after the exact Gate 13 lifecycle and +# inference helpers plus this file have been source-digest verified by the +# persistent Gate 14 action host. Product, Job Object, credential, and cache +# state remain in that one PowerShell process across the controller challenge. + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" +$VerbosePreference = "SilentlyContinue" +$DebugPreference = "SilentlyContinue" +$InformationPreference = "SilentlyContinue" + +$script:Gate14ProductInitialized = $false +$script:Gate14ProductPrepared = $false +$script:Gate14ProductCleaned = $false +$script:Gate14ProductCredentialCreated = $false +$script:Gate14ProductConfig = $null +$script:Gate14ProductProfile = $null +$script:Gate14ProductContext = $null +$script:Gate14ProductInventory = $null +$script:Gate14ProductArtifacts = @() +$script:Gate14ProductExpectedPolicy = $null +$script:Gate14ProductWorkerPid = 0 +$script:Gate14ProductBaselineProcesses = 0 +$script:Gate14ProductActionRoot = "" +$script:Gate14ProductWarmCache = "" +$script:Gate14ProductSourcePath = "" +$script:Gate14ProductBurns = New-Object System.Collections.ArrayList +$script:Gate14ProductCacheLocks = New-Object System.Collections.ArrayList + +function Assert-Gate14WindowsProductHelpers { + foreach ($name in @( + "Assert-Gate13ExactProperties", + "Assert-Gate13NoTranscript", + "Assert-Gate13SameArtifactInventory", + "ConvertFrom-Gate13Json", + "Force-Gate13ProductCleanup", + "Get-Gate13CredentialCount", + "Get-Gate13ExactWorkerSnapshot", + "Get-Gate13FileCount", + "Get-Gate13ProductProcessCount", + "Get-Gate13Property", + "Get-Gate13SelectedManifestContext", + "Get-Gate13SelectedProfile", + "Get-Gate13StreamSha256", + "Initialize-Gate13CredentialInterop", + "Initialize-Gate13NativeHost", + "Install-Gate13VerifiedPackage", + "Invoke-Gate13Bootstrap", + "Invoke-Gate13Contained", + "Invoke-Gate13LoopbackJson", + "Read-Gate13ControlKey", + "Read-Gate13JsonFile", + "Stop-Gate13Product", + "Test-Gate13PackageAudit", + "Test-Gate13PackagedSelfTests", + "Test-Gate13SafeArtifactPath", + "Wait-Gate13ProductStatus" + )) { + if ($null -eq (Get-Command $name -CommandType Function -ErrorAction SilentlyContinue)) { + throw "Gate 13 product helper binding is incomplete" + } + } +} + +function Get-Gate14WindowsUnixSeconds { + return [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() / 1000.0 +} + +function Test-Gate14WindowsInteger { + param([object]$Value) + return ($Value -is [int] -or $Value -is [long]) +} + +function Test-Gate14WindowsActiveWorker { + param([Parameter(Mandatory = $true)][object]$Worker) + $state = Get-Gate13Property $Worker "state" + return ($state -cin @("starting", "running", "stopping")) +} + +function Assert-Gate14WindowsFreshDirectory { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string]$Label + ) + if (-not (Test-Path -LiteralPath $Path -PathType Container)) { + throw ($Label + " is unavailable") + } + $item = Get-Item -LiteralPath $Path -Force -ErrorAction Stop + if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw ($Label + " is unsafe") + } +} + +function Write-Gate14WindowsUtf8New { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string]$Text + ) + if (Test-Path -LiteralPath $Path) { + throw "temporary product input already exists" + } + $bytes = (New-Object Text.UTF8Encoding($false, $true)).GetBytes($Text) + $stream = $null + try { + $stream = New-Object IO.FileStream( + $Path, + [IO.FileMode]::CreateNew, + [IO.FileAccess]::Write, + [IO.FileShare]::None, + 4096, + [IO.FileOptions]::WriteThrough + ) + $stream.Write($bytes, 0, $bytes.Length) + $stream.Flush($true) + } + finally { + if ($null -ne $stream) { + $stream.Dispose() + } + [Array]::Clear($bytes, 0, $bytes.Length) + } +} + +function Write-Gate14WindowsAtomicBytes { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][byte[]]$Bytes + ) + $directory = [IO.Path]::GetDirectoryName([IO.Path]::GetFullPath($Path)) + $temporary = Join-Path $directory (".gate14-" + [Guid]::NewGuid().ToString("N") + ".tmp") + $stream = $null + try { + $stream = New-Object IO.FileStream( + $temporary, + [IO.FileMode]::CreateNew, + [IO.FileAccess]::Write, + [IO.FileShare]::None, + 4096, + [IO.FileOptions]::WriteThrough + ) + $stream.Write($Bytes, 0, $Bytes.Length) + $stream.Flush($true) + $stream.Dispose() + $stream = $null + Move-Item -LiteralPath $temporary -Destination $Path -Force -ErrorAction Stop + } + finally { + if ($null -ne $stream) { + $stream.Dispose() + } + if (Test-Path -LiteralPath $temporary) { + Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue + } + } +} + +function Get-Gate14WindowsPackageVersion { + $metricsPath = Join-Path $script:LifecycleAuditRoot "desktop-metrics.json" + $metrics = Read-Gate13JsonFile -Path $metricsPath + $node = Get-Gate13Property $metrics "node_sidecar" + $runtime = Get-Gate13Property $node "runtime" + $version = Get-Gate13Property $runtime "drift" + if (-not ($version -is [string]) -or $version -notmatch "^[0-9A-Za-z][0-9A-Za-z.+-]{0,63}$") { + throw "package version binding is invalid" + } + return $version +} + +function New-Gate14WindowsRunInput { + $value = [ordered]@{ + schema_version = 1 + run_id = [string](Get-Gate13Property $script:Gate14ProductConfig "run_id") + source_commit = [string](Get-Gate13Property $script:Gate14ProductConfig "source_commit") + package_version = Get-Gate14WindowsPackageVersion + package_sha256 = ([string](Get-Gate13Property $script:Gate14ProductConfig "package_sha256")).Substring(7) + package_bytes = [int64](Get-Gate13Property $script:Gate14ProductConfig "package_bytes") + model_id = [string](Get-Gate13Property $script:Gate14ProductConfig "model_id") + manifest_digest = ([string](Get-Gate13Property $script:Gate14ProductConfig "manifest_digest")).Substring(7) + } + $rendered = ConvertTo-Json -InputObject $value -Compress -Depth 8 + Write-Gate14WindowsUtf8New -Path $script:LifecycleRunInput -Text $rendered +} + +function Get-Gate14WindowsArtifactRecords { + $warm = Get-Gate13Property $script:Gate14ProductConfig "warm_cache" + $raw = @((Get-Gate13Property $warm "artifacts")) + if ($raw.Count -lt 1) { + throw "warm cache artifact inventory is absent" + } + $records = New-Object System.Collections.ArrayList + $seen = New-Object "System.Collections.Generic.HashSet[string]" ([StringComparer]::OrdinalIgnoreCase) + foreach ($item in $raw) { + Assert-Gate13ExactProperties -InputObject $item -Names @( + "path", "role", "sha256", "size_bytes" + ) + $relative = Get-Gate13Property $item "path" + $role = Get-Gate13Property $item "role" + $digest = Get-Gate13Property $item "sha256" + $size = Get-Gate13Property $item "size_bytes" + if ( + -not ($relative -is [string]) -or + -not (Test-Gate13SafeArtifactPath -Path $relative) -or + -not $seen.Add($relative) -or + -not ($role -is [string]) -or + $role -notmatch "^[a-z][a-z0-9_-]{0,31}$" -or + -not ($digest -is [string]) -or + $digest -notmatch "^sha256:[0-9a-f]{64}$" -or + -not (Test-Gate14WindowsInteger $size) -or + [int64]$size -lt 1 + ) { + throw "warm cache artifact inventory is invalid" + } + [void]$records.Add([pscustomobject]@{ + path = $relative + role = $role + sha256 = $digest.Substring(7) + size_bytes = [int64]$size + }) + } + return @($records) +} + +function New-Gate14WindowsProfile { + $modelId = [string](Get-Gate13Property $script:Gate14ProductConfig "model_id") + $manifest = [string](Get-Gate13Property $script:Gate14ProductConfig "manifest_digest") + if ($modelId -cne "Qwen3.5 2B" -or $manifest -cne "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33") { + throw "Windows product model binding changed" + } + $base = $script:Profiles[$modelId] + if ( + $null -eq $base -or + $base.ManifestDigest -cne $manifest.Substring(7) -or + [int]$base.SelectedCount -ne 8 -or + [int64]$base.SelectedBytes -ne 4571197320 + ) { + throw "Windows product profile binding changed" + } + return [pscustomobject]@{ + ModelId = $modelId + ManifestDigest = $base.ManifestDigest + RevisionCommit = $base.RevisionCommit + SelectedCount = [int]$base.SelectedCount + SelectedBytes = [int64]$base.SelectedBytes + TotalBlocks = 24 + Gate9EnvelopeSha256 = "sha256:cd68afb67d9b0f3cb8c82db0d3314ad89b558c20880998ea4d8c4493e9f4bc9f" + } +} + +function Initialize-Gate14WindowsProductActions { + param( + [Parameter(Mandatory = $true)][string]$LifecycleConfig, + [Parameter(Mandatory = $true)][string]$RunId, + [Parameter(Mandatory = $true)][int]$AttemptOrdinal, + [Parameter(Mandatory = $true)][string]$SourceCommit, + [Parameter(Mandatory = $true)][string]$PackageSha256, + [Parameter(Mandatory = $true)][string]$ProductActionsPath + ) + if ($script:Gate14ProductInitialized) { + throw "Windows product actions are already initialized" + } + Assert-Gate14WindowsProductHelpers + Assert-Gate13NoTranscript + Initialize-Gate13NativeHost + Initialize-Gate13CredentialInterop + $config = Read-Gate13JsonFile -Path $LifecycleConfig + foreach ($field in @( + "run_id", "attempt_ordinal", "source_commit", "package_sha256", "platform", + "model_id", "manifest_digest", "work_root", "staging_root", "package_path", + "package_bytes", "disk_bytes", "vram_bytes", "bandwidth_mbps", + "power_watts", "pause_timeout_seconds", "sample_interval_seconds", "warm_cache" + )) { + $discarded = Get-Gate13Property $config $field + $discarded = $null + } + if ( + (Get-Gate13Property $config "run_id") -cne $RunId -or + [int](Get-Gate13Property $config "attempt_ordinal") -ne $AttemptOrdinal -or + (Get-Gate13Property $config "source_commit") -cne $SourceCommit -or + (Get-Gate13Property $config "package_sha256") -cne $PackageSha256 -or + (Get-Gate13Property $config "platform") -cne "windows" + ) { + throw "Windows product action binding changed" + } + $sourcePath = [IO.Path]::GetFullPath($ProductActionsPath) + if ( + [IO.Path]::GetFileName($sourcePath) -cne "gate14_windows_product_actions.ps1" -or + -not (Test-Path -LiteralPath $sourcePath -PathType Leaf) + ) { + throw "Windows product action source identity is invalid" + } + + $workRoot = [IO.Path]::GetFullPath([string](Get-Gate13Property $config "work_root")) + $stagingRoot = [IO.Path]::GetFullPath([string](Get-Gate13Property $config "staging_root")) + $packagePath = [IO.Path]::GetFullPath([string](Get-Gate13Property $config "package_path")) + $actionRoot = [IO.Path]::GetFullPath((Join-Path $workRoot "gate14-product-action")) + $warmCache = [IO.Path]::GetFullPath((Join-Path $workRoot "gate14-warm-cache")) + if ( + $workRoot -ceq [IO.Path]::GetPathRoot($workRoot) -or + $actionRoot -ceq $workRoot -or + -not $actionRoot.StartsWith($workRoot + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase) -or + -not $warmCache.StartsWith($workRoot + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase) -or + $stagingRoot.StartsWith($workRoot + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase) -or + $workRoot.StartsWith($stagingRoot + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase) -or + -not (Test-Path -LiteralPath $packagePath -PathType Leaf) + ) { + throw "Windows product path binding is unsafe" + } + + $script:Gate14ProductConfig = $config + $script:Gate14ProductSourcePath = $sourcePath + $script:Gate14ProductActionRoot = $actionRoot + $script:Gate14ProductWarmCache = $warmCache + $script:LifecycleArchive = $packagePath + $script:LifecycleAuditRoot = Join-Path $stagingRoot "release-audit" + $script:LifecycleRunInput = Join-Path $actionRoot "gate13-windows-run.json" + $script:LifecycleController = $sourcePath + $script:LifecycleWorkRoot = $actionRoot + $script:LifecycleInstallRoot = Join-Path $actionRoot "install" + $script:LifecycleProductRoot = Join-Path $script:LifecycleInstallRoot "CommunityAI" + $script:LifecycleDesktopExe = Join-Path $script:LifecycleProductRoot "CommunityAI.exe" + $script:LifecycleNodeExe = Join-Path $script:LifecycleProductRoot "node\CommunityAI-Node.exe" + $script:LifecycleBootstrap = Join-Path $script:LifecycleProductRoot "_internal\bootstrap\catalog-bootstrap.json" + $script:LifecyclePersistentRoot = Join-Path $actionRoot "persistent" + $script:LifecycleNodeConfig = Join-Path $script:LifecyclePersistentRoot "node-config.json" + $script:LifecycleProcess = $null + $script:LifecycleOwnWorkRoot = $false + $script:LifecycleOwnPersistentRoot = $false + $script:Gate14ProductProfile = New-Gate14WindowsProfile + $script:Gate14ProductArtifacts = @(Get-Gate14WindowsArtifactRecords) + $script:Gate14ProductInitialized = $true +} + +function Get-Gate14WindowsProductArguments { + if ( + -not (Test-Path -LiteralPath $script:LifecycleNodeConfig -PathType Leaf) -or + -not (Test-Path -LiteralPath $script:LifecyclePersistentRoot -PathType Container) -or + -not (Test-Path -LiteralPath $script:LifecycleBootstrap -PathType Leaf) + ) { + throw "Windows product launch paths are unavailable" + } + return [string[]]@( + "--node-config", + $script:LifecycleNodeConfig, + "--node-data-dir", + $script:LifecyclePersistentRoot, + "--bootstrap-config", + $script:LifecycleBootstrap + ) +} + +function New-Gate14WindowsContainedProductProcess { + param( + [Parameter(Mandatory = $true)][string]$Executable, + [Parameter(Mandatory = $true)][string[]]$Arguments, + [Parameter(Mandatory = $true)][string]$WorkingDirectory + ) + return [Gate13.NativeHost]::Start($Executable, $Arguments, $WorkingDirectory) +} + +function Start-Gate14WindowsProduct { + if ($null -ne $script:LifecycleProcess) { + throw "product already running" + } + Initialize-Gate13NativeHost + $arguments = Get-Gate14WindowsProductArguments + try { + $script:LifecycleProcess = New-Gate14WindowsContainedProductProcess ` + -Executable $script:LifecycleDesktopExe ` + -Arguments $arguments ` + -WorkingDirectory $script:LifecycleProductRoot + } + finally { + $arguments = $null + } +} + +function Get-Gate14WindowsFullSchedule { + return [ordered]@{ + timezone = "UTC" + windows = @( + [ordered]@{ + days = @("mon", "tue", "wed", "thu", "fri", "sat", "sun") + start = "00:00" + end = "23:59" + } + ) + } +} + +function Get-Gate14WindowsClosedSchedule { + $names = @("sun", "mon", "tue", "wed", "thu", "fri", "sat") + $tomorrow = ([int][DateTime]::UtcNow.DayOfWeek + 1) % 7 + return [ordered]@{ + timezone = "UTC" + windows = @( + [ordered]@{ + days = @($names[$tomorrow]) + start = "00:00" + end = "00:01" + } + ) + } +} + +function Set-Gate14WindowsContributionPolicy { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [int64]$VramBytes = -1, + [object]$Schedule = $null + ) + if ($VramBytes -lt 0) { + $VramBytes = [int64](Get-Gate13Property $script:Gate14ProductConfig "vram_bytes") + } + if ($null -eq $Schedule) { + $Schedule = Get-Gate14WindowsFullSchedule + } + $snapshot = Invoke-Gate13LoopbackJson -Method "GET" -Path "/control/v1/contribution-policy" -BearerToken $ControlToken + Assert-Gate13ExactProperties -InputObject $snapshot -Names @( + "schema_version", "config_revision", "policy" + ) + $revision = Get-Gate13Property $snapshot "config_revision" + if ((Get-Gate13Property $snapshot "schema_version") -ne 1 -or -not ($revision -is [string])) { + throw "contribution policy snapshot is invalid" + } + $policy = [ordered]@{ + sharing_enabled = $true + allowed_models = @($script:Gate14ProductProfile.ModelId) + preferred_models = @($script:Gate14ProductProfile.ModelId) + denied_models = @() + max_disk_space = ([int64](Get-Gate13Property $script:Gate14ProductConfig "disk_bytes")).ToString() + "B" + max_vram = $VramBytes.ToString() + "B" + max_bandwidth_mbps = [double](Get-Gate13Property $script:Gate14ProductConfig "bandwidth_mbps") + max_power_watts = [double](Get-Gate13Property $script:Gate14ProductConfig "power_watts") + pause_timeout = [double](Get-Gate13Property $script:Gate14ProductConfig "pause_timeout_seconds") + schedule = $Schedule + } + $response = Invoke-Gate13LoopbackJson -Method "PUT" -Path "/control/v1/contribution-policy" -BearerToken $ControlToken -Body ([ordered]@{ + schema_version = 1 + expected_config_revision = $revision + policy = $policy + }) + Assert-Gate13ExactProperties -InputObject $response -Names @( + "schema_version", "config_revision", "policy" + ) + if ( + (Get-Gate13Property $response "schema_version") -ne 1 -or + ((Get-Gate13Property $response "policy") | ConvertTo-Json -Compress -Depth 16) -cne + ($policy | ConvertTo-Json -Compress -Depth 16) + ) { + throw "contribution policy update was not preserved" + } + $script:Gate14ProductExpectedPolicy = $policy + return $response +} + +function Get-Gate14WindowsExactWorker { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [switch]$Running + ) + $snapshot = Get-Gate13ExactWorkerSnapshot -ControlToken $ControlToken + $workers = @($snapshot.Workers) + $automatic = $snapshot.Automatic + $active = @($workers | Where-Object { Test-Gate14WindowsActiveWorker $_ }) + if ($Running -and ($active.Count -ne 1 -or $active[0] -ne $automatic)) { + throw "automatic worker identity is invalid" + } + if ($Running) { + $pidValue = Get-Gate13Property $automatic "pid" + if ( + (Get-Gate13Property $automatic "state") -cne "running" -or + (Get-Gate13Property $automatic "desired_running") -ne $true -or + (Get-Gate13Property $automatic "model") -cne $script:Gate14ProductProfile.ModelId -or + (Get-Gate13Property $automatic "intent_published") -ne $true -or + (Get-Gate13Property $automatic "remote_acknowledged") -ne $true -or + -not (Test-Gate14WindowsInteger $pidValue) -or + [int64]$pidValue -lt 1 -or + $null -eq $script:LifecycleProcess -or + -not $script:LifecycleProcess.ContainsProcessId([int]$pidValue) + ) { + throw "automatic worker is not running with acknowledged owned intent" + } + } + return $automatic +} + +function Get-Gate14WindowsPublicWorker { + param([Parameter(Mandatory = $true)][string]$ControlToken) + $status = Invoke-Gate13LoopbackJson -Method "GET" -Path "/control/v1/status" -BearerToken $ControlToken + $profile = Get-Gate13SelectedProfile -Status $status + if ( + $profile.ModelId -cne $script:Gate14ProductProfile.ModelId -or + $profile.ManifestDigest -cne $script:Gate14ProductProfile.ManifestDigest + ) { + throw "public product model identity changed" + } + $contribution = Get-Gate13Property $status "contribution" + if ((Get-Gate13Property $contribution "schema_version") -ne 3) { + throw "public contribution status is invalid" + } + $automatic = @(@((Get-Gate13Property $contribution "workers")) | Where-Object { + (Get-Gate13Property $_ "id") -ceq "automatic" + }) + if ($automatic.Count -ne 1) { + throw "public automatic worker status is invalid" + } + return $automatic[0] +} + +function Wait-Gate14WindowsRunning { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [int]$TimeoutSeconds = 300 + ) + $timer = [Diagnostics.Stopwatch]::StartNew() + $last = $null + while ($timer.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + try { + return Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running + } + catch { + $last = $_ + Start-Sleep -Milliseconds 250 + } + } + throw "automatic worker did not reach the required state" +} + +function Wait-Gate14WindowsInactive { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [switch]$Resource, + [switch]$Schedule, + [int]$PriorPid = 0, + [int]$TimeoutSeconds = 300 + ) + $timer = [Diagnostics.Stopwatch]::StartNew() + while ($timer.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + try { + $private = Get-Gate14WindowsExactWorker -ControlToken $ControlToken + $public = Get-Gate14WindowsPublicWorker -ControlToken $ControlToken + $valid = ( + (Get-Gate13Property $private "desired_running") -eq $true -and + $null -eq (Get-Gate13Property $private "pid") -and + -not (Test-Gate14WindowsActiveWorker $private) -and + (Get-Gate13Property $public "desired_running") -eq $true -and + -not (Test-Gate14WindowsActiveWorker $public) + ) + if ($Resource) { + $valid = $valid -and ((Get-Gate13Property $private "resource_suspended") -eq $true) + } + if ($Schedule) { + $valid = $valid -and ((Get-Gate13Property $private "schedule_suspended") -eq $true) + } + if ($PriorPid -gt 0 -and $null -ne (Get-Process -Id $PriorPid -ErrorAction SilentlyContinue)) { + $valid = $false + } + if ($valid) { + return [pscustomobject]@{ Private = $private; Public = $public } + } + } + catch { + } + Start-Sleep -Milliseconds 250 + } + throw "worker suspension did not reach the required state" +} + +function Invoke-Gate14WindowsLowVramProbe { + param([Parameter(Mandatory = $true)][string]$ControlToken) + $before = Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running + $priorPid = [int](Get-Gate13Property $before "pid") + $discarded = Set-Gate14WindowsContributionPolicy -ControlToken $ControlToken -VramBytes 1 + $discarded = $null + $timer = [Diagnostics.Stopwatch]::StartNew() + while ($timer.Elapsed.TotalSeconds -lt 300) { + $worker = Get-Gate14WindowsExactWorker -ControlToken $ControlToken + if ( + $null -eq (Get-Gate13Property $worker "pid") -and + (Get-Gate13Property $worker "resource_admitted") -eq $false -and + $null -eq (Get-Process -Id $priorPid -ErrorAction SilentlyContinue) + ) { + break + } + Start-Sleep -Milliseconds 250 + } + if ($timer.Elapsed.TotalSeconds -ge 300) { + throw "low VRAM policy was not rejected" + } + $discarded = Set-Gate14WindowsContributionPolicy -ControlToken $ControlToken + $discarded = $null + $worker = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $worker "pid") +} + +function Set-Gate14WindowsNodeConfigBytes { + param([Parameter(Mandatory = $true)][byte[]]$Bytes) + Write-Gate14WindowsAtomicBytes -Path $script:LifecycleNodeConfig -Bytes $Bytes +} + +function Invoke-Gate14WindowsCpuPowerProbe { + param([Parameter(Mandatory = $true)][string]$ControlToken) + $original = [IO.File]::ReadAllBytes($script:LifecycleNodeConfig) + try { + $utf8 = New-Object Text.UTF8Encoding($false, $true) + $config = ConvertFrom-Gate13Json -Payload $utf8.GetString($original) + $automatic = @(@((Get-Gate13Property $config "workers")) | Where-Object { + (Get-Gate13Property $_ "id") -ceq "automatic" + }) + if ($automatic.Count -ne 1) { + throw "automatic worker configuration is invalid" + } + $automatic[0].device = "cpu" + $changedText = ConvertTo-Json -InputObject $config -Compress -Depth 32 + $changed = $utf8.GetBytes($changedText) + try { + Set-Gate14WindowsNodeConfigBytes -Bytes $changed + } + finally { + [Array]::Clear($changed, 0, $changed.Length) + } + $timer = [Diagnostics.Stopwatch]::StartNew() + while ($timer.Elapsed.TotalSeconds -lt 300) { + $worker = Get-Gate14WindowsExactWorker -ControlToken $ControlToken + $reason = Get-Gate13Property $worker "resource_reason" + if ( + $null -eq (Get-Gate13Property $worker "pid") -and + (Get-Gate13Property $worker "resource_admitted") -eq $false -and + $reason -is [string] -and + $reason.Contains("power telemetry is unavailable") + ) { + break + } + Start-Sleep -Milliseconds 250 + } + if ($timer.Elapsed.TotalSeconds -ge 300) { + throw "CPU power telemetry was not rejected" + } + } + finally { + Set-Gate14WindowsNodeConfigBytes -Bytes $original + [Array]::Clear($original, 0, $original.Length) + } + $worker = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $worker "pid") + return [ordered]@{ + device = "cpu" + configured_limit = "power_watts" + start_rejected = $true + reason_code = "power-telemetry-unavailable" + private_detail_retained = $false + } +} + +function Invoke-Gate14WindowsCrashRecovery { + param([Parameter(Mandatory = $true)][string]$ControlToken) + $before = Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running + $oldPid = [int](Get-Gate13Property $before "pid") + $timer = [Diagnostics.Stopwatch]::StartNew() + $script:LifecycleProcess.KillMemberProcess($oldPid, 30000) + while ($timer.Elapsed.TotalSeconds -lt 300) { + try { + $after = Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running + $newPid = [int](Get-Gate13Property $after "pid") + if ($newPid -ne $oldPid -and $null -eq (Get-Process -Id $oldPid -ErrorAction SilentlyContinue)) { + $script:Gate14ProductWorkerPid = $newPid + return [ordered]@{ + worker_crash_observed = $true + worker_restarted = $true + restart_seconds = [Math]::Round($timer.Elapsed.TotalSeconds, 6) + previous_worker_absent = $true + manifest_unchanged = ((Get-Gate13Property $after "model") -ceq $script:Gate14ProductProfile.ModelId) + automatic_block_range_valid = ((Get-Gate13Property $after "block_indices") -is [string]) + desired_intent_preserved = ((Get-Gate13Property $after "desired_running") -eq $true) + } + } + } + catch { + } + Start-Sleep -Milliseconds 250 + } + throw "worker crash recovery did not complete" +} + +function Invoke-Gate14WindowsPause { + param([Parameter(Mandatory = $true)][string]$ControlToken) + $before = Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running + $oldPid = [int](Get-Gate13Property $before "pid") + $timer = [Diagnostics.Stopwatch]::StartNew() + $discarded = Invoke-Gate13LoopbackJson -Method "POST" -Path "/control/v1/workers/automatic/pause" -BearerToken $ControlToken + $discarded = $null + while ($timer.Elapsed.TotalSeconds -lt 300) { + $worker = Get-Gate14WindowsExactWorker -ControlToken $ControlToken + if ( + (Get-Gate13Property $worker "state") -ceq "paused" -and + (Get-Gate13Property $worker "desired_running") -eq $false -and + (Get-Gate13Property $worker "operator_paused") -eq $true -and + $null -eq (Get-Gate13Property $worker "pid") -and + $null -eq (Get-Process -Id $oldPid -ErrorAction SilentlyContinue) -and + $script:LifecycleProcess.ActiveProcessCount -eq $script:Gate14ProductBaselineProcesses + ) { + $result = [ordered]@{ + requested = $true + completed = $true + duration_seconds = [Math]::Round($timer.Elapsed.TotalSeconds, 6) + worker_count_after = 0 + descendant_count_after = 0 + } + $discarded = Invoke-Gate13LoopbackJson -Method "POST" -Path "/control/v1/workers/automatic/start" -BearerToken $ControlToken + $discarded = $null + $running = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $running "pid") + return $result + } + Start-Sleep -Milliseconds 250 + } + throw "automatic worker did not pause" +} + +function Invoke-Gate14WindowsRestart { + param([Parameter(Mandatory = $true)][string]$ControlToken) + $timer = [Diagnostics.Stopwatch]::StartNew() + $before = Get-Gate14WindowsExactCacheInventory -Root $script:Gate14ProductContext.CacheDir -Context $script:Gate14ProductContext + Stop-Gate13Product + Start-Gate14WindowsProduct + $status = Wait-Gate13ProductStatus -TimeoutSeconds 300 + $running = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $policy = Invoke-Gate13LoopbackJson -Method "GET" -Path "/control/v1/contribution-policy" -BearerToken $ControlToken + if ( + ((Get-Gate13Property $policy "policy") | ConvertTo-Json -Compress -Depth 16) -cne + ($script:Gate14ProductExpectedPolicy | ConvertTo-Json -Compress -Depth 16) + ) { + throw "restart did not preserve policy" + } + $after = Get-Gate14WindowsExactCacheInventory -Root $script:Gate14ProductContext.CacheDir -Context $script:Gate14ProductContext + Assert-Gate14WindowsSameCacheInventory -Expected $before -Actual $after + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $running "pid") + return [ordered]@{ + node_restarted = $true + policy_persisted = $true + desired_intent_persisted = ((Get-Gate13Property $running "desired_running") -eq $true) + worker_resumed = $true + duration_seconds = [Math]::Round($timer.Elapsed.TotalSeconds, 6) + cache_reused = $true + } +} + +function Invoke-Gate14WindowsExactCacheInventoryCore { + param( + [Parameter(Mandatory = $true)][string]$Root, + [Parameter(Mandatory = $true)][object]$Context, + [Parameter(Mandatory = $true)][AllowEmptyCollection()][Collections.ArrayList]$Opened + ) + $rootHandle = [Gate13.NativeHost]::OpenReadOnlyNoFollow( + [IO.Path]::GetFullPath($Root), + $true + ) + [void]$Opened.Add($rootHandle) + $manifestDigest = Get-Gate13Property $Context "ManifestDigest" + if (-not ($manifestDigest -is [string]) -or $manifestDigest -notmatch "^[0-9a-f]{64}$") { + throw "materialized cache manifest identity is invalid" + } + + $prefix = "manifest-artifacts/" + $manifestDigest + "/snapshot" + $expectedFiles = New-Object "System.Collections.Generic.Dictionary[string,object]" ( + [StringComparer]::Ordinal + ) + $expectedDirectories = New-Object "System.Collections.Generic.HashSet[string]" ( + [StringComparer]::Ordinal + ) + $expectedPaths = New-Object "System.Collections.Generic.HashSet[string]" ( + [StringComparer]::OrdinalIgnoreCase + ) + [int64]$expectedBytes = 0 + foreach ($record in @($script:Gate14ProductArtifacts)) { + $artifactRelative = Get-Gate13Property $record "path" + $cacheRelative = $prefix + "/" + $artifactRelative + if ($expectedFiles.ContainsKey($cacheRelative) -or -not $expectedPaths.Add($cacheRelative)) { + throw "materialized cache expected inventory collides" + } + $expectedFiles.Add($cacheRelative, $record) + $segments = $cacheRelative.Split("/") + for ($index = 1; $index -lt $segments.Count; $index++) { + $parent = [string]::Join("/", [string[]]$segments[0..($index - 1)]) + if ($expectedDirectories.Add($parent) -and -not $expectedPaths.Add($parent)) { + throw "materialized cache expected inventory collides" + } + } + $expectedBytes = [int64]($expectedBytes + [int64](Get-Gate13Property $record "size_bytes")) + } + if ( + $expectedFiles.Count -ne [int]$script:Gate14ProductProfile.SelectedCount -or + $expectedBytes -ne [int64]$script:Gate14ProductProfile.SelectedBytes + ) { + throw "materialized cache expected totals changed" + } + + $pending = New-Object Collections.Stack + $pending.Push([pscustomobject]@{ FullPath = [IO.Path]::GetFullPath($Root); RelativePath = "" }) + $seenPaths = New-Object "System.Collections.Generic.HashSet[string]" ( + [StringComparer]::OrdinalIgnoreCase + ) + $seenDirectories = New-Object "System.Collections.Generic.HashSet[string]" ( + [StringComparer]::Ordinal + ) + $seenFiles = New-Object "System.Collections.Generic.HashSet[string]" ( + [StringComparer]::Ordinal + ) + $entries = New-Object Collections.ArrayList + [int64]$actualBytes = 0 + + while ($pending.Count -gt 0) { + $current = $pending.Pop() + foreach ($item in @(Get-ChildItem -LiteralPath $current.FullPath -Force -ErrorAction Stop)) { + $itemIsDirectory = $item -is [IO.DirectoryInfo] + try { + $lockedItem = [Gate13.NativeHost]::OpenReadOnlyNoFollow( + $item.FullName, + $itemIsDirectory + ) + } + catch { + throw "materialized cache contains a reparse point or changed entry" + } + [void]$Opened.Add($lockedItem) + $relative = if ($current.RelativePath.Length -eq 0) { + $item.Name + } + else { + $current.RelativePath + "/" + $item.Name + } + if (-not (Test-Gate13SafeArtifactPath -Path $relative) -or -not $seenPaths.Add($relative)) { + throw "materialized cache path is unsafe or colliding" + } + + if ($lockedItem.IsDirectory) { + if (-not $expectedDirectories.Contains($relative) -or -not $seenDirectories.Add($relative)) { + throw "materialized cache contains an unexpected directory" + } + $pending.Push([pscustomobject]@{ + FullPath = $item.FullName + RelativePath = $relative + }) + continue + } + if ($itemIsDirectory -or $lockedItem.IsDirectory) { + throw "materialized cache contains a special entry" + } + if (-not $expectedFiles.ContainsKey($relative) -or -not $seenFiles.Add($relative)) { + throw "materialized cache contains an unexpected file" + } + + $record = $expectedFiles[$relative] + $expectedSize = [int64](Get-Gate13Property $record "size_bytes") + $expectedDigest = [string](Get-Gate13Property $record "sha256") + $openedLength = [int64]$lockedItem.Length + $actualDigest = $lockedItem.Sha256() + if ( + $openedLength -ne $expectedSize -or + $actualDigest -cne $expectedDigest + ) { + throw "materialized cache artifact verification failed" + } + [void]$entries.Add([pscustomobject]@{ + RelativePath = [string](Get-Gate13Property $record "path") + Size = $expectedSize + Digest = $expectedDigest + FileIdentity = $lockedItem.FileIdentity + LocalPath = $item.FullName + }) + $actualBytes = [int64]($actualBytes + $expectedSize) + } + } + + if ( + $seenFiles.Count -ne $expectedFiles.Count -or + $seenDirectories.Count -ne $expectedDirectories.Count -or + $actualBytes -ne $expectedBytes + ) { + throw "materialized cache inventory is incomplete" + } + return [pscustomobject]@{ + Entries = @($entries | Sort-Object RelativePath) + Count = $entries.Count + Bytes = $actualBytes + } +} + +function Close-Gate14WindowsCacheLocks { + foreach ($locked in @($script:Gate14ProductCacheLocks)) { + $locked.Dispose() + } + $script:Gate14ProductCacheLocks = New-Object System.Collections.ArrayList +} + +function Get-Gate14WindowsExactCacheInventory { + param( + [Parameter(Mandatory = $true)][string]$Root, + [Parameter(Mandatory = $true)][object]$Context, + [switch]$HoldLocks + ) + Assert-Gate14WindowsFreshDirectory -Path $Root -Label "materialized cache" + Initialize-Gate13NativeHost + $opened = New-Object Collections.ArrayList + try { + $result = Invoke-Gate14WindowsExactCacheInventoryCore ` + -Root $Root ` + -Context $Context ` + -Opened $opened + if ($HoldLocks) { + $previous = $script:Gate14ProductCacheLocks + $script:Gate14ProductCacheLocks = $opened + $opened = New-Object Collections.ArrayList + foreach ($locked in @($previous)) { + $locked.Dispose() + } + } + return $result + } + finally { + foreach ($locked in @($opened)) { + $locked.Dispose() + } + } +} + +function Assert-Gate14WindowsSameCacheInventory { + param( + [Parameter(Mandatory = $true)][object]$Expected, + [Parameter(Mandatory = $true)][object]$Actual + ) + Assert-Gate13SameArtifactInventory -Expected $Expected -Actual $Actual + for ($index = 0; $index -lt $Expected.Count; $index++) { + if ( + $Expected.Entries[$index].FileIdentity -cne + $Actual.Entries[$index].FileIdentity + ) { + throw "verified cache file identity changed" + } + } +} + +function Move-Gate14WindowsWarmCache { + Assert-Gate14WindowsFreshDirectory -Path $script:Gate14ProductWarmCache -Label "fresh materialized cache" + $destination = $script:Gate14ProductContext.CacheDir + $parent = [IO.Path]::GetDirectoryName($destination) + if (-not (Test-Path -LiteralPath $parent -PathType Container)) { + New-Item -ItemType Directory -Path $parent -Force -ErrorAction Stop | Out-Null + } + if (Test-Path -LiteralPath $destination) { + Assert-Gate14WindowsFreshDirectory -Path $destination -Label "selected cache" + if ((Get-Gate13FileCount $destination) -ne 0) { + throw "selected cache is not empty" + } + Remove-Item -LiteralPath $destination -Force -ErrorAction Stop + } + [IO.Directory]::Move($script:Gate14ProductWarmCache, $destination) +} + +function Invoke-Gate14WindowsPrepareCore { + Assert-Gate13NoTranscript + if ($script:Gate14ProductPrepared -or $script:Gate14ProductCleaned) { + throw "Windows product prepare order is invalid" + } + if (Test-Path -LiteralPath $script:Gate14ProductActionRoot) { + throw "Windows product action root is not fresh" + } + Assert-Gate14WindowsFreshDirectory -Path $script:Gate14ProductWarmCache -Label "fresh materialized cache" + if ((Get-Gate13CredentialCount) -ne 0 -or (Get-Gate13ProductProcessCount) -ne 0) { + throw "clean host product baseline is not empty" + } + + New-Item -ItemType Directory -Path $script:Gate14ProductActionRoot -ErrorAction Stop | Out-Null + $script:LifecycleOwnWorkRoot = $true + $script:LifecycleOwnPersistentRoot = $true + New-Gate14WindowsRunInput + $audit = Test-Gate13PackageAudit + if ( + $audit.SourceCommit -cne (Get-Gate13Property $script:Gate14ProductConfig "source_commit") -or + $audit.PackageDigest -cne ([string](Get-Gate13Property $script:Gate14ProductConfig "package_sha256")).Substring(7) + ) { + throw "package source binding changed" + } + Install-Gate13VerifiedPackage -Audit $audit + $selfTests = Test-Gate13PackagedSelfTests + if ((Get-Gate13CredentialCount) -ne 0) { + throw "packaged self-test retained credential" + } + $bootstrap = Invoke-Gate13Bootstrap + if ((Get-Gate13CredentialCount) -ne 0) { + throw "packaged bootstrap unexpectedly retained a credential" + } + $script:Gate14ProductContext = Get-Gate13SelectedManifestContext -Profile $script:Gate14ProductProfile + $warmInventory = Get-Gate14WindowsExactCacheInventory -Root $script:Gate14ProductWarmCache -Context $script:Gate14ProductContext + Move-Gate14WindowsWarmCache + $script:Gate14ProductInventory = Get-Gate14WindowsExactCacheInventory -Root $script:Gate14ProductContext.CacheDir -Context $script:Gate14ProductContext -HoldLocks + Assert-Gate14WindowsSameCacheInventory -Expected $warmInventory -Actual $script:Gate14ProductInventory + $warmInventory = $null + + Start-Gate14WindowsProduct + $status = Wait-Gate13ProductStatus -TimeoutSeconds 300 + if ((Get-Gate13CredentialCount) -ne 1) { + throw "packaged product credential was not created exactly once" + } + $script:Gate14ProductCredentialCreated = $true + if ( + $status.Profile.ModelId -cne $script:Gate14ProductProfile.ModelId -or + $status.Profile.ManifestDigest -cne $script:Gate14ProductProfile.ManifestDigest + ) { + throw "packaged product model identity changed" + } + + $control = [string]$status.ControlToken + $status.ControlToken = $null + try { + $discarded = Set-Gate14WindowsContributionPolicy -ControlToken $control + $discarded = $null + $script:Gate14ProductBaselineProcesses = [int]$script:LifecycleProcess.ActiveProcessCount + $discarded = Invoke-Gate13LoopbackJson -Method "POST" -Path "/control/v1/workers/automatic/start" -BearerToken $control + $discarded = $null + $worker = Wait-Gate14WindowsRunning -ControlToken $control -TimeoutSeconds 1800 + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $worker "pid") + $script:Gate14ProductBaselineProcesses = [int]$script:LifecycleProcess.ActiveProcessCount - 1 + if ($script:Gate14ProductBaselineProcesses -lt 1) { + throw "packaged product process baseline is invalid" + } + $public = Get-Gate14WindowsPublicWorker -ControlToken $control + $placement = Get-Gate13Property $public "placement" + $limits = Get-Gate13Property (Get-Gate13Property $public "resources") "limits" + $blockIndices = Get-Gate13Property $placement "block_indices" + if (-not ($blockIndices -is [string]) -or $blockIndices -notmatch "^([0-9]{1,3}):([0-9]{1,3})$") { + throw "automatic block placement is invalid" + } + $blockStart = [int]$Matches[1] + $blockEnd = [int]$Matches[2] + if ( + (Get-Gate13Property $placement "automatic") -ne $true -or + $blockEnd -le $blockStart -or + $blockEnd -gt $script:Gate14ProductProfile.TotalBlocks -or + [int64](Get-Gate13Property $limits "disk_bytes") -ne [int64](Get-Gate13Property $script:Gate14ProductConfig "disk_bytes") -or + [int64](Get-Gate13Property $limits "vram_bytes") -ne [int64](Get-Gate13Property $script:Gate14ProductConfig "vram_bytes") -or + [double](Get-Gate13Property $limits "bandwidth_mbps") -ne [double](Get-Gate13Property $script:Gate14ProductConfig "bandwidth_mbps") -or + [double](Get-Gate13Property $limits "power_watts") -ne [double](Get-Gate13Property $script:Gate14ProductConfig "power_watts") + ) { + throw "resolved contribution limits changed" + } + + Invoke-Gate14WindowsLowVramProbe -ControlToken $control + $unsupported = Invoke-Gate14WindowsCpuPowerProbe -ControlToken $control + $recovery = Invoke-Gate14WindowsCrashRecovery -ControlToken $control + $pause = Invoke-Gate14WindowsPause -ControlToken $control + $restart = Invoke-Gate14WindowsRestart -ControlToken $control + $after = Get-Gate14WindowsExactCacheInventory -Root $script:Gate14ProductContext.CacheDir -Context $script:Gate14ProductContext + Assert-Gate14WindowsSameCacheInventory -Expected $script:Gate14ProductInventory -Actual $after + $script:Gate14ProductPrepared = $true + return [ordered]@{ + schema_version = 1 + scope = "gate14-prepared-host-observations" + run_id = [string](Get-Gate13Property $script:Gate14ProductConfig "run_id") + platform = "windows" + attempt_ordinal = [int](Get-Gate13Property $script:Gate14ProductConfig "attempt_ordinal") + source_commit = [string](Get-Gate13Property $script:Gate14ProductConfig "source_commit") + package_sha256 = [string](Get-Gate13Property $script:Gate14ProductConfig "package_sha256") + model = [ordered]@{ + id = $script:Gate14ProductProfile.ModelId + manifest_digest = "sha256:" + $script:Gate14ProductProfile.ManifestDigest + revision_commit = $script:Gate14ProductProfile.RevisionCommit + gate9_envelope_sha256 = $script:Gate14ProductProfile.Gate9EnvelopeSha256 + selected_artifact_count = [int]$script:Gate14ProductProfile.SelectedCount + selected_artifact_bytes = [int64]$script:Gate14ProductProfile.SelectedBytes + total_blocks = [int]$script:Gate14ProductProfile.TotalBlocks + } + cache = [ordered]@{ + verified_bytes_before = [int64]$script:Gate14ProductInventory.Bytes + verified_bytes_after = [int64]$after.Bytes + transfer_bytes_during_gate = [int64]0 + digest_mismatch_count = 0 + forbidden_model_acquired = $false + } + placement = [ordered]@{ + automatic = $true + worker_count = 1 + block_start = $blockStart + block_end = $blockEnd + intent_published = $true + remote_acknowledged = $true + } + limits = [ordered]@{ + disk_bytes = [int64](Get-Gate13Property $script:Gate14ProductConfig "disk_bytes") + vram_bytes = [int64](Get-Gate13Property $script:Gate14ProductConfig "vram_bytes") + bandwidth_mbps = [double](Get-Gate13Property $script:Gate14ProductConfig "bandwidth_mbps") + power_watts = [double](Get-Gate13Property $script:Gate14ProductConfig "power_watts") + schedule_timezone = "UTC" + resource_limit_count = 5 + configured_and_resolved_match = $true + low_vram_rejected = $true + } + recovery = $recovery + pause = $pause + restart = $restart + unsupported_telemetry = $unsupported + } + } + finally { + $control = $null + } +} + +function Initialize-Gate14WindowsLoadInterop { + if ($null -ne ("Gate14.LoopbackLoad" -as [type])) { + return + } + Add-Type -TypeDefinition @' +using System; +using System.Net; +using System.Net.Sockets; +using System.Threading; + +namespace Gate14 +{ + public sealed class LoopbackLoad : IDisposable + { + private readonly ManualResetEvent stop = new ManualResetEvent(false); + private readonly TcpListener listener; + private readonly Thread receiver; + private readonly Thread sender; + private TcpClient accepted; + private TcpClient client; + + public LoopbackLoad() + { + listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(1); + int port = ((IPEndPoint)listener.LocalEndpoint).Port; + receiver = new Thread(delegate() + { + byte[] bytes = new byte[1048576]; + try + { + accepted = listener.AcceptTcpClient(); + NetworkStream stream = accepted.GetStream(); + while (!stop.WaitOne(0)) + { + if (stream.Read(bytes, 0, bytes.Length) == 0) + { + break; + } + } + } + catch + { + if (!stop.WaitOne(0)) + { + stop.Set(); + } + } + }); + sender = new Thread(delegate() + { + byte[] bytes = new byte[1048576]; + try + { + client = new TcpClient(); + client.Connect(IPAddress.Loopback, port); + NetworkStream stream = client.GetStream(); + while (!stop.WaitOne(0)) + { + stream.Write(bytes, 0, bytes.Length); + } + } + catch + { + if (!stop.WaitOne(0)) + { + stop.Set(); + } + } + }); + receiver.IsBackground = true; + sender.IsBackground = true; + receiver.Start(); + sender.Start(); + } + + public void Dispose() + { + stop.Set(); + try { if (client != null) client.Close(); } catch { } + try { if (accepted != null) accepted.Close(); } catch { } + try { listener.Stop(); } catch { } + receiver.Join(5000); + sender.Join(5000); + stop.Dispose(); + } + } +} +'@ -Language CSharp -ErrorAction Stop | Out-Null +} + +function Get-Gate14WindowsMeasurement { + param( + [Parameter(Mandatory = $true)][object]$Worker, + [Parameter(Mandatory = $true)][string]$Field + ) + $value = Get-Gate13Property $Worker $Field + if (-not ($value -is [int]) -and -not ($value -is [long]) -and -not ($value -is [double]) -and -not ($value -is [decimal])) { + throw "physical resource measurement is unavailable" + } + $number = [double]$value + if ([double]::IsNaN($number) -or [double]::IsInfinity($number) -or $number -lt 0) { + throw "physical resource measurement is unavailable" + } + return $number +} + +function New-Gate14WindowsCalibrationRecord { + param( + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][object]$Challenge, + [Parameter(Mandatory = $true)][double]$StartedAt, + [Parameter(Mandatory = $true)][double]$EndedAt, + [Parameter(Mandatory = $true)][double]$Baseline, + [Parameter(Mandatory = $true)][double]$Trigger, + [Parameter(Mandatory = $true)][double]$Resume, + [Parameter(Mandatory = $true)][string]$Source, + [Parameter(Mandatory = $true)][string]$Scope, + [Parameter(Mandatory = $true)][double]$Configured, + [Parameter(Mandatory = $true)][double]$Duration + ) + $interval = [double](Get-Gate13Property $script:Gate14ProductConfig "sample_interval_seconds") + if ($EndedAt - $StartedAt -lt 2 * $interval -or $EndedAt - $StartedAt -gt 120) { + throw "physical calibration sample window is invalid" + } + return [ordered]@{ + kind = $Kind + suspended = $true + resumed = $true + desired_intent_preserved = $true + worker_count_during = 0 + duration_seconds = [Math]::Round($Duration, 6) + calibration = [ordered]@{ + measurement_source = $Source + measurement_scope = $Scope + sample_count = 3 + sample_interval_seconds = $interval + baseline_value = $Baseline + configured_limit = $Configured + trigger_value = $Trigger + resume_value = $Resume + challenge_sha256 = [string](Get-Gate13Property $Challenge "challenge_sha256") + sample_started_at_unix = $StartedAt + sample_ended_at_unix = $EndedAt + } + } +} + +function Invoke-Gate14WindowsBandwidthCalibration { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [Parameter(Mandatory = $true)][object]$Challenge + ) + $limit = [double](Get-Gate13Property $script:Gate14ProductConfig "bandwidth_mbps") + $interval = [double](Get-Gate13Property $script:Gate14ProductConfig "sample_interval_seconds") + $startedAt = Get-Gate14WindowsUnixSeconds + $timer = [Diagnostics.Stopwatch]::StartNew() + $samples = @() + foreach ($index in 1..3) { + $samples += Get-Gate14WindowsMeasurement -Worker (Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running) -Field "current_bandwidth_mbps" + Start-Sleep -Milliseconds ([int]($interval * 1000)) + } + $baseline = [double]($samples | Measure-Object -Minimum).Minimum + if ($baseline -ge $limit) { + throw "bandwidth baseline already exceeds its limit" + } + Initialize-Gate14WindowsLoadInterop + $load = New-Object Gate14.LoopbackLoad + try { + $inactive = Wait-Gate14WindowsInactive -ControlToken $ControlToken -Resource -PriorPid $script:Gate14ProductWorkerPid + $trigger = Get-Gate14WindowsMeasurement -Worker $inactive.Private -Field "current_bandwidth_mbps" + } + finally { + $load.Dispose() + } + $running = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $resume = Get-Gate14WindowsMeasurement -Worker $running -Field "current_bandwidth_mbps" + $endedAt = Get-Gate14WindowsUnixSeconds + if (-not ($baseline -lt $limit -and $limit -lt $trigger -and $resume -lt $limit)) { + throw "bandwidth calibration did not cross its limit" + } + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $running "pid") + return New-Gate14WindowsCalibrationRecord -Kind "bandwidth" -Challenge $Challenge -StartedAt $startedAt -EndedAt $endedAt -Baseline $baseline -Trigger $trigger -Resume $resume -Source "host-network-counters" -Scope "aggregate-host-network" -Configured $limit -Duration $timer.Elapsed.TotalSeconds +} + +function Start-Gate14WindowsPowerBurn { + $arguments = [string[]]@( + "edge-benchmark", + $script:Gate14ProductContext.ManifestPath, + "--cache_dir", + $script:Gate14ProductContext.CacheDir, + "--allow_warm_cache", + "--prompt", + "CommunityAI Gate 14 calibration", + "--max_new_tokens", + "128", + "--supervisor_timeout", + "120" + ) + $burn = [Gate13.NativeHost]::Start( + $script:LifecycleNodeExe, + $arguments, + $script:LifecycleProductRoot + ) + [void]$script:Gate14ProductBurns.Add($burn) + return $burn +} + +function Stop-Gate14WindowsPowerBurn { + param([Parameter(Mandatory = $true)][object]$Burn) + $Burn.ForceAndVerify(30000) + if ($Burn.ActiveProcessCount -ne 0) { + throw "power calibration burn cleanup was not proved" + } + $burnIndex = $script:Gate14ProductBurns.IndexOf($Burn) + if ($burnIndex -lt 0) { + throw "power calibration burn tracking changed" + } + $script:Gate14ProductBurns.RemoveAt($burnIndex) +} + +function Invoke-Gate14WindowsPowerCalibration { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [Parameter(Mandatory = $true)][object]$Challenge + ) + $limit = [double](Get-Gate13Property $script:Gate14ProductConfig "power_watts") + $interval = [double](Get-Gate13Property $script:Gate14ProductConfig "sample_interval_seconds") + $startedAt = Get-Gate14WindowsUnixSeconds + $timer = [Diagnostics.Stopwatch]::StartNew() + $samples = @() + foreach ($index in 1..3) { + $samples += Get-Gate14WindowsMeasurement -Worker (Get-Gate14WindowsExactWorker -ControlToken $ControlToken -Running) -Field "current_power_watts" + Start-Sleep -Milliseconds ([int]($interval * 1000)) + } + $baseline = [double]($samples | Measure-Object -Minimum).Minimum + if ($baseline -ge $limit) { + throw "power baseline already exceeds its limit" + } + $burn = Start-Gate14WindowsPowerBurn + try { + $inactive = Wait-Gate14WindowsInactive -ControlToken $ControlToken -Resource -PriorPid $script:Gate14ProductWorkerPid + $trigger = Get-Gate14WindowsMeasurement -Worker $inactive.Private -Field "current_power_watts" + } + finally { + Stop-Gate14WindowsPowerBurn -Burn $burn + } + $running = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $resume = Get-Gate14WindowsMeasurement -Worker $running -Field "current_power_watts" + $endedAt = Get-Gate14WindowsUnixSeconds + if (-not ($baseline -lt $limit -and $limit -lt $trigger -and $resume -lt $limit)) { + throw "power calibration did not cross its limit" + } + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $running "pid") + return New-Gate14WindowsCalibrationRecord -Kind "power" -Challenge $Challenge -StartedAt $startedAt -EndedAt $endedAt -Baseline $baseline -Trigger $trigger -Resume $resume -Source "nvidia-nvml-device-power" -Scope "selected-nvidia-l4-device" -Configured $limit -Duration $timer.Elapsed.TotalSeconds +} + +function Invoke-Gate14WindowsScheduleCalibration { + param( + [Parameter(Mandatory = $true)][string]$ControlToken, + [Parameter(Mandatory = $true)][object]$Challenge + ) + $interval = [double](Get-Gate13Property $script:Gate14ProductConfig "sample_interval_seconds") + $startedAt = Get-Gate14WindowsUnixSeconds + $timer = [Diagnostics.Stopwatch]::StartNew() + Start-Sleep -Milliseconds ([int](2 * $interval * 1000)) + $discarded = Set-Gate14WindowsContributionPolicy -ControlToken $ControlToken -Schedule (Get-Gate14WindowsClosedSchedule) + $discarded = $null + try { + $inactive = Wait-Gate14WindowsInactive -ControlToken $ControlToken -Schedule -PriorPid $script:Gate14ProductWorkerPid + } + finally { + $discarded = Set-Gate14WindowsContributionPolicy -ControlToken $ControlToken -Schedule (Get-Gate14WindowsFullSchedule) + $discarded = $null + } + $running = Wait-Gate14WindowsRunning -ControlToken $ControlToken + $endedAt = Get-Gate14WindowsUnixSeconds + $script:Gate14ProductWorkerPid = [int](Get-Gate13Property $running "pid") + return New-Gate14WindowsCalibrationRecord -Kind "schedule" -Challenge $Challenge -StartedAt $startedAt -EndedAt $endedAt -Baseline 1.0 -Trigger 0.0 -Resume 1.0 -Source "utc-policy-clock" -Scope "utc-schedule-policy" -Configured 0.5 -Duration $timer.Elapsed.TotalSeconds +} + +function Get-Gate14WindowsCleanupRecord { + return [ordered]@{ + schema_version = 1 + scope = "gate14-host-lifecycle-cleanup" + run_id = [string](Get-Gate13Property $script:Gate14ProductConfig "run_id") + platform = "windows" + attempt_ordinal = [int](Get-Gate13Property $script:Gate14ProductConfig "attempt_ordinal") + processes_absent = $true + credentials_removed = $true + action_temporaries_removed = $true + } +} + +function Remove-Gate14WindowsExactTree { + param([Parameter(Mandatory = $true)][string]$Path) + $full = [IO.Path]::GetFullPath($Path) + if ( + $full -cne $script:Gate14ProductActionRoot -and + $full -cne $script:Gate14ProductWarmCache + ) { + throw "Gate 14 cleanup path rejected" + } + if (-not (Test-Path -LiteralPath $full)) { + return + } + + Initialize-Gate13NativeHost + $rootLock = $null + $pending = New-Object "System.Collections.Generic.Stack[string]" + $directories = New-Object Collections.ArrayList + $files = New-Object Collections.ArrayList + try { + try { + $rootLock = [Gate13.NativeHost]::OpenReadOnlyNoFollow($full, $true) + } + catch { + throw "Gate 14 cleanup root is unsafe" + } + + $pending.Push($full) + while ($pending.Count -gt 0) { + $current = $pending.Pop() + foreach ($child in @(Get-ChildItem -LiteralPath $current -Force -ErrorAction Stop)) { + $isDirectory = $child -is [IO.DirectoryInfo] + try { + $locked = [Gate13.NativeHost]::OpenReadOnlyNoFollow( + $child.FullName, + $isDirectory + ) + } + catch { + throw "Gate 14 cleanup descendant is unsafe: $($child.FullName)" + } + $entry = [pscustomobject]@{ + Path = [string]$child.FullName + Lock = $locked + } + if ($locked.IsDirectory) { + [void]$directories.Add($entry) + $pending.Push([string]$child.FullName) + } + elseif (-not $isDirectory) { + [void]$files.Add($entry) + } + else { + $locked.Dispose() + $entry.Lock = $null + throw "Gate 14 cleanup descendant has an unsupported type: $($child.FullName)" + } + } + } + + foreach ($file in @($files)) { + $file.Lock.Dispose() + $file.Lock = $null + [IO.File]::Delete([string]$file.Path) + } + foreach ($directory in @( + $directories | Sort-Object -Property { $_.Path.Length } -Descending + )) { + $directory.Lock.Dispose() + $directory.Lock = $null + [IO.Directory]::Delete([string]$directory.Path, $false) + } + $rootLock.Dispose() + $rootLock = $null + [IO.Directory]::Delete($full, $false) + } + finally { + foreach ($file in @($files)) { + if ($null -ne $file.Lock) { + $file.Lock.Dispose() + } + } + foreach ($directory in @($directories)) { + if ($null -ne $directory.Lock) { + $directory.Lock.Dispose() + } + } + if ($null -ne $rootLock) { + $rootLock.Dispose() + } + } +} + +function Invoke-Gate14WindowsProductCleanup { + if (-not $script:Gate14ProductInitialized) { + throw "Windows product actions are not initialized" + } + if ($script:Gate14ProductCleaned) { + return Get-Gate14WindowsCleanupRecord + } + + $processFailure = $false + foreach ($burn in @($script:Gate14ProductBurns)) { + try { + Stop-Gate14WindowsPowerBurn -Burn $burn + } + catch { + $processFailure = $true + } + } + try { + Force-Gate13ProductCleanup + } + catch { + $processFailure = $true + } + try { + if ( + $script:Gate14ProductBurns.Count -ne 0 -or + (Get-Gate13ProductProcessCount) -ne 0 + ) { + $processFailure = $true + } + } + catch { + $processFailure = $true + } + if ($processFailure) { + throw "Windows packaged product process cleanup was not proved" + } + + $credentialFailure = $false + try { + $credentialCount = Get-Gate13CredentialCount + if ($credentialCount -gt 0) { + if (-not (Test-Path -LiteralPath $script:LifecycleDesktopExe -PathType Leaf)) { + $credentialFailure = $true + } + else { + $discarded = Invoke-Gate13Contained -Executable $script:LifecycleDesktopExe -Arguments ([string[]]@("--delete-control-key")) -WorkingDirectory $script:LifecycleProductRoot -TimeoutSeconds 180 + $discarded = $null + } + } + if ((Get-Gate13CredentialCount) -ne 0) { + $credentialFailure = $true + } + } + catch { + $credentialFailure = $true + } + if ($credentialFailure) { + throw "Windows packaged product credential cleanup was not proved" + } + $script:Gate14ProductCredentialCreated = $false + + Close-Gate14WindowsCacheLocks + $rootFailure = $false + try { + Remove-Gate14WindowsExactTree -Path $script:Gate14ProductWarmCache + } + catch { + $rootFailure = $true + } + try { + Remove-Gate14WindowsExactTree -Path $script:Gate14ProductActionRoot + } + catch { + $rootFailure = $true + } + try { + if ((Get-Gate13CredentialCount) -ne 0 -or (Get-Gate13ProductProcessCount) -ne 0) { + $rootFailure = $true + } + } + catch { + $rootFailure = $true + } + if ( + (Test-Path -LiteralPath $script:Gate14ProductWarmCache) -or + (Test-Path -LiteralPath $script:Gate14ProductActionRoot) -or + $script:Gate14ProductBurns.Count -ne 0 -or + $rootFailure + ) { + throw "Windows packaged product cleanup was not proved" + } + $script:Gate14ProductCleaned = $true + return Get-Gate14WindowsCleanupRecord +} + +function Invoke-Gate14WindowsProductPrepare { + if (-not $script:Gate14ProductInitialized) { + throw "Windows product actions are not initialized" + } + try { + return Invoke-Gate14WindowsPrepareCore + } + catch { + $failure = $_ + try { + $discarded = Invoke-Gate14WindowsProductCleanup + $discarded = $null + } + catch { + throw "Windows product prepare failed and cleanup was incomplete" + } + throw $failure + } +} + +function Invoke-Gate14WindowsProductCalibrate { + param([Parameter(Mandatory = $true)][object]$Challenge) + if (-not $script:Gate14ProductPrepared -or $script:Gate14ProductCleaned) { + throw "Windows product calibration order is invalid" + } + Assert-Gate13ExactProperties -InputObject $Challenge -Names @( + "challenge_sha256", "controller_state_revision", "issued_at_unix", "expires_at_unix" + ) + $now = Get-Gate14WindowsUnixSeconds + if ( + -not ((Get-Gate13Property $Challenge "challenge_sha256") -is [string]) -or + (Get-Gate13Property $Challenge "challenge_sha256") -notmatch "^sha256:[0-9a-f]{64}$" -or + -not (Test-Gate14WindowsInteger (Get-Gate13Property $Challenge "controller_state_revision")) -or + -not (Test-Gate14WindowsInteger (Get-Gate13Property $Challenge "issued_at_unix")) -or + -not (Test-Gate14WindowsInteger (Get-Gate13Property $Challenge "expires_at_unix")) -or + [double](Get-Gate13Property $Challenge "issued_at_unix") -gt $now -or + [double](Get-Gate13Property $Challenge "expires_at_unix") -lt $now + ) { + throw "controller challenge is invalid or stale" + } + $control = Read-Gate13ControlKey + try { + $records = @( + Invoke-Gate14WindowsBandwidthCalibration -ControlToken $control -Challenge $Challenge + Invoke-Gate14WindowsPowerCalibration -ControlToken $control -Challenge $Challenge + Invoke-Gate14WindowsScheduleCalibration -ControlToken $control -Challenge $Challenge + ) + if ((Get-Gate14WindowsUnixSeconds) -gt [double](Get-Gate13Property $Challenge "expires_at_unix")) { + throw "controller challenge expired during calibration" + } + return @($records) + } + finally { + $control = $null + } +} diff --git a/scripts/gate16_catalog_channel.py b/scripts/gate16_catalog_channel.py new file mode 100644 index 000000000..d177af521 --- /dev/null +++ b/scripts/gate16_catalog_channel.py @@ -0,0 +1,394 @@ +"""Prepare an isolated signed canary channel and observe ordinary packaged refresh. + +Preparation and phase changes write only a new/local bundle. Publishing that +bundle to an existing HTTPS path is a separate operator action. No production key, +catalog, native credential, worker, GUI or cloud resource is modified here. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlsplit + +import httpx + +from drift.model_catalog import CatalogSigningKey, ModelCatalog, SignedModelCatalog +from drift.model_manifest import ModelManifest +from drift.node.catalog_bootstrap import CatalogBootstrapConfig, CatalogBootstrapInstaller +from drift.node.config import NodeConfig + +PHASES = ("baseline", "withdrawal", "restore") +MAX_SECONDS = 900 +ROOT = Path(__file__).resolve().parents[1] + + +class ChannelError(RuntimeError): + pass + + +def require(condition, code): + if not condition: + raise ChannelError(code) + + +def encode(value): + return (json.dumps(value, indent=2, allow_nan=False, sort_keys=True) + "\n").encode("utf-8") + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def load_bundle(path): + require(path.is_dir() and not path.is_symlink(), "unsafe_bundle") + index = json.loads((path / "bundle.json").read_text()) + require(index["schema_version"] == 1 and index["catalog_id"].startswith("communityai-canary-"), "not_canary_bundle") + bootstrap = CatalogBootstrapConfig.load(path / "catalog-bootstrap.json") + require(bootstrap.trust_root.catalog_id == index["catalog_id"], "canary_root_mismatch") + require(bootstrap.trust_root_digest == index["trust_root_digest"], "canary_root_digest_mismatch") + for phase in PHASES: + data = (path / "phases" / f"{phase}.signed.json").read_bytes() + require(sha(data) == index["phases"][phase]["sha256"], "phase_digest_mismatch") + catalog = SignedModelCatalog.from_json(data.decode()).verify(bootstrap.trust_root) + require(catalog.sequence == index["phases"][phase]["sequence"], "phase_sequence_mismatch") + return index, bootstrap + + +def prepare(args, *, now=None): + require(re.fullmatch(r"[a-z0-9][a-z0-9-]{0,47}", args.run_id) is not None, "run_id") + require(args.base_url.endswith("/"), "base_url_requires_trailing_slash") + original_bootstrap = CatalogBootstrapConfig.load(args.release / "catalog-bootstrap.json") + original = SignedModelCatalog.from_json((args.release / "catalog.signed.json").read_text()).verify( + original_bootstrap.trust_root + ) + local = tuple(model for model in original.models if model.execution == "local") + remote = tuple(model for model in original.models if model.execution == "distributed") + require(len(local) == len(remote) == 1, "requires_one_local_and_one_community_model") + key = CatalogSigningKey.generate() # Kept only in memory; all three phases are signed now. + catalog_id = "communityai-canary-" + args.run_id + bootstrap = CatalogBootstrapConfig.from_dict( + { + "schema_version": 1, + "trust_root": { + "schema_version": 1, + "catalog_id": catalog_id, + "threshold": 1, + "keys": [key.trusted_key.to_dict()], + }, + "catalog_mirrors": [args.base_url + "catalog.signed.json"], + "initial_peers": [args.initial_peer], + "max_loaded_models": original_bootstrap.max_loaded_models, + } + ) + models = tuple( + replace( + model, + manifest_urls=(args.base_url + "manifests/" + model.manifest_digest.removeprefix("sha256:") + ".json",), + ) + for model in original.models + ) + issued = int((time.time() if now is None else now) * 1000) - 1000 + baseline = replace( + original, + catalog_id=catalog_id, + sequence=1, + issued_at_ms=issued, + expires_at_ms=issued + 2 * 60 * 60 * 1000, + models=models, + ) + local_model = next(model for model in models if model.execution == "local") + withdrawal = replace( + baseline, + sequence=2, + models=(local_model,), + rungs=tuple(rung for rung in baseline.rungs if rung.rung_id == local_model.rung_id), + ) + restore = replace(baseline, sequence=3) + for catalog in (baseline, withdrawal, restore): + ModelCatalog.from_dict(catalog.to_dict()) + args.output.mkdir(parents=True, exist_ok=False) + (args.output / "phases").mkdir() + channel = args.output / "channel" + (channel / "manifests").mkdir(parents=True) + manifests = {} + for model in models: + name = model.manifest_digest.removeprefix("sha256:") + ".json" + manifest = ModelManifest.load(args.release / "manifests" / name) + require(manifest.digest_id == model.manifest_digest, "manifest_digest_mismatch") + rendered = manifest.canonical_json() + "\n" + (channel / "manifests" / name).write_text(rendered, encoding="utf-8") + manifests[model.manifest_urls[0]] = rendered + index = { + "schema_version": 1, + "scope": "isolated-catalog-canary-channel", + "complete_gate16": False, + "catalog_id": catalog_id, + "trust_root_digest": bootstrap.trust_root_digest, + "original_catalog_digest": original.digest, + "local_manifest_digest": local_model.manifest_digest, + "community_manifest_digest": remote[0].manifest_digest, + "phases": {}, + "expires_at_ms": baseline.expires_at_ms, + "private_signing_key_retained": False, + "published": False, + } + for phase, catalog in zip(PHASES, (baseline, withdrawal, restore)): + envelope = SignedModelCatalog(1, catalog, ()).add_signature(key) + data = encode(envelope.to_dict()) + (args.output / "phases" / f"{phase}.signed.json").write_bytes(data) + index["phases"][phase] = {"sequence": catalog.sequence, "sha256": sha(data), "catalog_digest": catalog.digest} + (args.output / "catalog-bootstrap.json").write_bytes(encode(bootstrap.to_dict())) + (args.output / "bundle.json").write_bytes(encode(index)) + (channel / "catalog.signed.json").write_bytes((args.output / "phases/baseline.signed.json").read_bytes()) + + # Prepare only this new private state offline. The frozen node will consume + # subsequent phases through its real HTTPS refresh service. Keeping local-only + # and sharing disabled prevents autonomous model probes/downloads on launch. + state = args.output / "private-node" + installer = CatalogBootstrapInstaller( + bootstrap, + data_dir=state, + config_path=state / "node-config.json", + now=(issued + 1000) / 1000, + fetch_text=lambda url, maximum: (channel / "catalog.signed.json").read_text() + if url in bootstrap.catalog_mirrors + else manifests[url], + ) + installer.install() + config = json.loads((state / "node-config.json").read_text()) + config["inference_mode"] = "local_only" + config["contribution_policy"]["sharing_enabled"] = False + (state / "node-config.json").write_bytes(encode(config)) + return {"result": "prepared", "complete_gate16": False, "catalog_id": catalog_id, "published": False} + + +def advance(args): + index, bootstrap = load_bundle(args.bundle) + target = args.bundle / "channel/catalog.signed.json" + require(target.is_file() and not target.is_symlink(), "unsafe_active_catalog") + current_bytes = target.read_bytes() + current = SignedModelCatalog.from_json(current_bytes.decode()).verify(bootstrap.trust_root) + expected = index["phases"][args.phase] + require(current.sequence + 1 == expected["sequence"], "phase_must_advance_exactly_once") + require(any(sha(current_bytes) == phase["sha256"] for phase in index["phases"].values()), "unknown_active_catalog") + temporary = target.with_name("catalog.next.private.json") + require(not temporary.exists(), "unfinished_local_activation") + data = (args.bundle / "phases" / f"{args.phase}.signed.json").read_bytes() + with temporary.open("xb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target) + return {"result": "local-phase-staged", "phase": args.phase, "sequence": expected["sequence"], "published": False} + + +def local_preferences(config): + local = [ + {key: value for key, value in entry.items() if key != "manifest"} + for entry in config["models"] + if entry.get("execution") == "local" + ] + return sha( + encode( + { + "models": local, + "contribution_policy": config["contribution_policy"], + "inference_mode": config["inference_mode"], + "workers": config.get("workers", []), + } + ) + ) + + +def observe(args, *, get_status=None, monotonic=time.monotonic, sleep=time.sleep): + index, bootstrap = load_bundle(args.bundle) + require(1 <= args.timeout <= MAX_SECONDS, "observation_timeout_bounds") + parsed = urlsplit(args.node_url) + require( + parsed.scheme == "http" + and parsed.hostname in ("127.0.0.1", "::1", "localhost") + and not parsed.username + and not parsed.password + and parsed.path in ("", "/") + and not parsed.query + and not parsed.fragment, + "node_url_must_be_loopback", + ) + previous = None + if args.phase != "baseline": + require(args.previous is not None, "previous_phase_evidence_required") + previous = json.loads(args.previous.read_text()) + require( + previous["result"] == "passed" and previous["trust_root_digest"] == bootstrap.trust_root_digest, + "previous_evidence_mismatch", + ) + require(PHASES.index(args.phase) == PHASES.index(previous["phase"]) + 1, "observation_phase_order") + args.output.mkdir(parents=True, exist_ok=False) + result = { + "schema_version": 1, + "result": "failed", + "scope": "ordinary-canary-catalog-refresh-observation", + "complete_gate16": False, + "phase": args.phase, + "trust_root_digest": bootstrap.trust_root_digest, + "driver_sha256": sha(Path(__file__).read_bytes()), + "started_at": datetime.now(timezone.utc).isoformat(), + } + began = monotonic() + client = None + try: + if get_status is None: + sys.path.insert(0, str(ROOT / "desktop/src")) + from communityai_desktop.credentials import NativeCredentialStore + + secret = NativeCredentialStore(args.credential_service, args.credential_account).get() + client = httpx.Client( + base_url=args.node_url, + timeout=5, + trust_env=False, + follow_redirects=False, + headers={"Authorization": "Bearer " + secret}, + ) + + def get_status(): + response = client.get("/control/v1/status") + response.raise_for_status() + return response.json() + + while True: + require(monotonic() - began <= args.timeout, "catalog_observation_deadline") + config_bytes = args.node_config.read_bytes() + config = NodeConfig.from_json(config_bytes.decode("utf-8"), base_dir=args.node_config.resolve().parent) + installed_root = CatalogBootstrapConfig.load(config.catalog_bootstrap_path) + require(installed_root.trust_root_digest == bootstrap.trust_root_digest, "consumer_not_on_canary_root") + installed = SignedModelCatalog.from_json(config.catalog_path.read_text()).verify(bootstrap.trust_root) + try: + status = get_status() + except (httpx.TransportError, httpx.TimeoutException): + sleep(0.5) + continue + if status.get("status") != "running": + sleep(0.5) + continue + started_at = status.get("started_at") + require(type(started_at) is int and started_at > 0, "node_start_identity_unavailable") + expected = index["phases"][args.phase] + if installed.digest != expected["catalog_digest"] or ( + previous and started_at <= previous["node_started_at"] + ): + sleep(0.5) + continue + require(installed.sequence == expected["sequence"], "installed_sequence_mismatch") + config_revision = "sha256:" + sha(config_bytes) + if ( + args.node_config.read_bytes() != config_bytes + or status["contribution"]["policy"].get("config_revision") != config_revision + ): + sleep(0.5) + continue + require( + status["inference_mode"] == "local_only" + and not status["contribution"]["policy"]["policy"]["sharing_enabled"], + "consumer_must_remain_non_contributing_local_only", + ) + require(all(worker["state"] == "paused" for worker in status["workers"]), "unexpected_running_worker") + require(status["runtime_budget"]["resident_models"] == 0, "unexpected_model_load") + document = json.loads(config_bytes) + preferences = local_preferences(document) + if previous: + require(preferences == previous["local_preferences_sha256"], "consumer_preferences_changed") + if args.phase == "withdrawal": + require( + index["community_manifest_digest"] not in config.auto_model_priority, + "withdrawn_model_still_automatic", + ) + require( + all(model.execution == "local" for model in installed.models), "withdrawal_still_approves_community" + ) + else: + require( + index["community_manifest_digest"] in config.auto_model_priority, "community_priority_not_restored" + ) + result.update( + result="passed", + node_started_at=started_at, + catalog_digest=installed.digest, + catalog_sequence=installed.sequence, + config_revision=config_revision, + local_preferences_sha256=preferences, + node_restarted=previous is not None, + no_model_load=True, + sharing_paused=True, + ) + break + except BaseException as exc: + result["error_type"] = type(exc).__name__ + if isinstance(exc, ChannelError): + result["error_code"] = str(exc) + finally: + closed = client is None + try: + if client is not None: + client.close() + closed = True + except BaseException as exc: + result["result"] = "failed" + result["cleanup_error_type"] = type(exc).__name__ + result["duration_seconds"] = round(monotonic() - began, 3) + result["cleanup"] = {"created_processes": 0, "created_credentials": 0, "owned_http_client_closed": closed} + result["limitations"] = [ + "Prepared private state and an existing HTTPS channel are required; publishing is outside this driver.", + "No active-generation drain, inference, worker health reconstruction, GUI visibility, or full canary cleanup is established.", + "The observer never stops the external desktop or deletes its native credential; its owning lifecycle must do so.", + "Authenticated active config revision and a later running node bind saved policy; the API exposes no active catalog digest.", + ] + (args.output / "result.json").write_bytes(encode(result)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_subparsers(dest="mode", required=True) + stage = modes.add_parser("prepare") + stage.add_argument("--release", type=Path, required=True) + stage.add_argument("--base-url", required=True) + stage.add_argument("--initial-peer", required=True) + stage.add_argument("--run-id", required=True) + stage.add_argument("--output", type=Path, required=True) + advance_parser = modes.add_parser("advance-local") + advance_parser.add_argument("--bundle", type=Path, required=True) + advance_parser.add_argument("--phase", choices=("withdrawal", "restore"), required=True) + watch = modes.add_parser("observe") + watch.add_argument("--bundle", type=Path, required=True) + watch.add_argument("--phase", choices=PHASES, required=True) + watch.add_argument("--previous", type=Path) + watch.add_argument("--node-config", type=Path, required=True) + watch.add_argument("--node-url", default="http://127.0.0.1:18116") + watch.add_argument("--credential-service", required=True) + watch.add_argument("--credential-account", default="control") + watch.add_argument("--timeout", type=int, default=420) + watch.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = ( + prepare(args) if args.mode == "prepare" else advance(args) if args.mode == "advance-local" else observe(args) + ) + print( + json.dumps( + {key: value for key, value in result.items() if key in ("result", "phase", "complete_gate16", "published")} + ) + ) + if result["result"] == "failed": + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/gate16_live_rpc.py b/scripts/gate16_live_rpc.py new file mode 100644 index 000000000..812f21bb8 --- /dev/null +++ b/scripts/gate16_live_rpc.py @@ -0,0 +1,477 @@ +"""Bounded non-compute RPC canary against one explicitly selected, already-owned worker. + +Run beside the worker's live public-health file. This script creates one temporary +local TLS p2pd client, never provisions a server, never loads weights, and sends no +valid inference tensor. It does not close Gate 16 or replace post-probe inference. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import math +import re +import time +from datetime import datetime, timezone +from pathlib import Path + +import psutil +from hivemind.p2p import P2P, P2PHandlerError, PeerID +from hivemind.proto import runtime_pb2 +from hivemind.utils.multiaddr import Multiaddr +from hivemind.utils.serializer import MSGPackSerializer + +from drift.model_manifest import ModelManifest +from drift.protocol_identity import TRANSPORT_SECURITY +from drift.server.admission import PUBLIC_OVERLOAD_MESSAGE, AdmissionPolicy +from drift.server.handler import MAX_INFERENCE_METADATA_BYTES, TransformerConnectionHandler +from drift.server.health import MAX_HEALTH_STATE_BYTES, build_public_worker_health + +MAX_RUN_SECONDS = 600 +MAX_RPC_CALLS = 20 +MAX_INPUT_BYTES = 128 * 1024 + + +class CanaryError(RuntimeError): + """A sanitized, fixed-code canary failure.""" + + +def require(condition, code): + if not condition: + raise CanaryError(code) + + +def read_json(path, maximum): + require(path.is_file() and not path.is_symlink(), "unsafe_input_file") + with path.open("rb") as stream: + data = stream.read(maximum + 1) + require(len(data) <= maximum, "input_size_limit") + + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, "duplicate_json_key") + result[key] = value + return result + + return json.loads(data, object_pairs_hook=unique) + + +def load_policy(path): + source = read_json(path, 4096) + require(set(source) == {"schema_version", "admission", "step_timeout", "session_timeout"}, "policy_schema") + require(type(source["schema_version"]) is int and source["schema_version"] == 1, "policy_version") + policy = AdmissionPolicy(**source["admission"]) + require(policy.max_active_sessions_per_peer == 1, "probe_requires_one_session_per_peer") + require(policy.max_active_sessions > 1, "probe_requires_spare_global_session") + require(policy.allow_training_rpcs is False, "training_must_be_disabled") + for field in ("step_timeout", "session_timeout"): + value = source[field] + require(type(value) in (float, int) and math.isfinite(value) and 0 < value <= 60, "timeout_bounds") + require(source["step_timeout"] < source["session_timeout"], "step_must_precede_session_timeout") + refill = max(policy.peer_session_burst / policy.peer_session_rate, 1 / policy.global_session_rate) + require(refill <= 30, "refill_exceeds_probe_budget") + require(refill + 2 < source["step_timeout"], "step_timeout_must_allow_refilled_admission_probe") + return source, policy, refill + + +def exact_peer(value): + require(isinstance(value, str) and len(value) <= 2048, "peer_address_bounds") + address = Multiaddr(value) + protocols = tuple(protocol.name for protocol in address.protocols()) + require(protocols in (("ip4", "tcp", "p2p"), ("ip6", "tcp", "p2p"), ("dns4", "tcp", "p2p")), "peer_protocols") + require(str(address) == value, "peer_address_not_canonical") + require(1 <= int(address.value_for_protocol("tcp")) <= 65535, "peer_port") + return PeerID.from_base58(address.value_for_protocol("p2p")) + + +def read_health(path, manifest_digest, block, *, maximum_age=15, now=None): + source = read_json(path, MAX_HEALTH_STATE_BYTES) + require( + set(source) + == { + "schema_version", + "scope", + "observed_at", + "worker_healthy", + "route", + "admission_available", + "admission", + "components", + }, + "health_schema", + ) + require(type(source["schema_version"]) is int and source["schema_version"] == 1, "health_version") + canonical = build_public_worker_health( + **source["route"], + admission_snapshot=source["admission"], + observed_at=source["observed_at"], + **source["components"], + ) + require(source == canonical and canonical["worker_healthy"], "health_unavailable") + require(source["route"]["manifest_digest"] == manifest_digest, "health_manifest_mismatch") + require(source["route"]["start_block"] <= block < source["route"]["end_block"], "health_block_mismatch") + stamp = datetime.fromisoformat(source["observed_at"].replace("Z", "+00:00")).timestamp() + age = (time.time() if now is None else now) - stamp + require(0 <= age <= maximum_age, "health_stale") + return source + + +def malformed_cases(uid, wire_digest): + common = {"manifest_digest": wire_digest, "max_length": 1} + wrong_digest = "0" * 64 if wire_digest != "0" * 64 else "1" * 64 + values = ( + ("oversized_metadata", b"x" * (MAX_INFERENCE_METADATA_BYTES + 1), "too large"), + ( + "manifest_mismatch", + MSGPackSerializer.dumps(dict(common, manifest_digest=wrong_digest)), + "Manifest digest mismatch", + ), + ("non_dictionary_metadata", MSGPackSerializer.dumps([]), "metadata is invalid"), + ( + "non_finite_allocation_timeout", + MSGPackSerializer.dumps(dict(common, alloc_timeout=float("nan"))), + "metadata is invalid", + ), + ("negative_maximum_length", MSGPackSerializer.dumps(dict(common, max_length=-1)), "Cannot allocate KV cache"), + ) + return [ + (name, runtime_pb2.ExpertRequest(uid=uid, metadata=metadata), expected) for name, metadata, expected in values + ] + + +class Probe: + def __init__(self, stub, health, manifest, block, policy, *, refill, step_timeout, clock=time.monotonic): + self.stub, self.health, self.manifest, self.block = stub, health, manifest, block + self.policy, self.refill, self.step_timeout = policy, refill, step_timeout + self.clock = clock + self.calls, self.input_bytes = 0, 0 + self.checks = [] + self.baseline = None + self.current_case = "baseline" + + def count(self, message=None): + self.calls += 1 + if message is not None: + self.input_bytes += message.ByteSize() + require(self.calls <= MAX_RPC_CALLS and self.input_bytes <= MAX_INPUT_BYTES, "rpc_budget_exceeded") + + async def wait_health(self, predicate, timeout=15): + deadline = self.clock() + timeout + while True: + value = self.health() + admission = value["admission"] + require(admission["active_sessions"] <= self.policy.max_active_sessions, "active_session_bound") + require(admission["tracked_peers"] <= self.policy.max_tracked_peers, "tracked_peer_bound") + require(admission["pending_pushes"] <= self.policy.max_pending_pushes, "pending_push_bound") + if self.baseline is not None: + require( + admission["accepted_sessions"] >= self.baseline["admission"]["accepted_sessions"], + "worker_counter_reset", + ) + require( + admission["rejected_sessions"] >= self.baseline["admission"]["rejected_sessions"], + "worker_counter_reset", + ) + if predicate(value): + return value + require(self.clock() < deadline, "health_observation_deadline") + await asyncio.sleep(0.2) + + async def info(self): + message = runtime_pb2.ExpertUID(uid=f"{self.manifest.dht_prefix}.{self.block}") + self.count(message) + reply = await asyncio.wait_for(self.stub.rpc_info(message), 10) + require(len(reply.serialized_info) <= 128 * 1024, "rpc_info_size") + value = MSGPackSerializer.loads(reply.serialized_info) + require(value["manifest_digest"] == self.manifest.digest, "rpc_manifest_mismatch") + require(value["transport_security"] == TRANSPORT_SECURITY, "transport_security_mismatch") + require(value["server_peer_id"] == self.stub._peer.to_base58(), "rpc_peer_mismatch") + count = value["cache_tokens_available"] + require(type(count) is int and count >= 0, "invalid_cache_counter") + return count + + async def reject(self, message, expected, *, training=False): + self.count(message) + stream = None + + async def requests(): + yield message + + try: + if training: + await asyncio.wait_for(self.stub.rpc_forward(message), 10) + else: + stream = await asyncio.wait_for(self.stub.rpc_inference(requests()), 10) + await asyncio.wait_for(anext(stream), 10) + except P2PHandlerError as exc: + # Keep transport errors private; only the fixed expected category is + # exported. Overload must not masquerade as malformed-input rejection. + require(expected in str(exc), "unexpected_rpc_rejection") + return + finally: + if stream is not None: + with contextlib.suppress(Exception): + await asyncio.wait_for(stream.aclose(), 2) + raise CanaryError("invalid_request_accepted") + + async def run(self): + self.baseline = await self.wait_health( + lambda v: v["admission"]["active_sessions"] == 0 and v["admission"]["pending_pushes"] == 0 + ) + available = await self.info() + self.current_case = "idle_stream_and_per_peer_admission" + gate = asyncio.Event() + + async def idle(): + await gate.wait() + if False: + yield runtime_pb2.ExpertRequest() + + self.count() + stream = await asyncio.wait_for(self.stub.rpc_inference(idle()), 10) + opened = self.clock() + next_reply = asyncio.create_task(anext(stream)) + try: + await self.wait_health(lambda v: v["admission"]["active_sessions"] == 1, timeout=min(15, self.step_timeout)) + # Refill both buckets before the second stream so rate limiting cannot + # masquerade as enforcement of the per-peer active-session ceiling. + await asyncio.sleep(self.refill + 0.1) + require(not next_reply.done(), "idle_lease_expired_before_admission_probe") + await self.wait_health(lambda v: v["admission"]["active_sessions"] == 1, timeout=1) + await self.reject(runtime_pb2.ExpertRequest(), PUBLIC_OVERLOAD_MESSAGE) + after = await self.wait_health( + lambda v: v["admission"]["active_sessions"] == 0, + timeout=max(0, self.step_timeout + 5 - (self.clock() - opened)), + ) + elapsed = self.clock() - opened + require(elapsed >= self.step_timeout, "idle_lease_released_before_timeout") + require(elapsed <= self.step_timeout + 5, "idle_timeout_exceeded") + require( + after["admission"]["rejected_sessions"] == self.baseline["admission"]["rejected_sessions"] + 1 + and after["admission"]["accepted_sessions"] == self.baseline["admission"]["accepted_sessions"] + 1, + "admission_counters_contaminated_or_missing", + ) + if next_reply.done(): + require( + not next_reply.cancelled() and isinstance(next_reply.exception(), StopAsyncIteration), + "idle_stream_closed_before_producer_release", + ) + # Hivemind waits for the request producer after the server closes. + # Finish it only after health proves the server released its lease, + # so a client end-of-stream cannot falsely establish idle expiry. + gate.set() + try: + await asyncio.wait_for(next_reply, 5) + except StopAsyncIteration: + closure = "end_of_stream" + except ConnectionResetError: + # Upstream Linux Hivemind can fail writing END_OF_STREAM to the + # expired server stream. Only this post-proof producer release + # may accept a reset; admission/malformed RPCs still fail on it. + closure = "connection_reset_after_idle_release" + else: + raise CanaryError("idle_stream_produced_output") + self.checks.append( + { + "case": "idle_stream_and_per_peer_admission", + "result": "passed", + "duration_seconds": round(elapsed, 3), + "transport_closure": closure, + } + ) + finally: + next_reply.cancel() + with contextlib.suppress(BaseException): + await next_reply + gate.set() + with contextlib.suppress(Exception): + await asyncio.wait_for(stream.aclose(), 2) + + uid = f"{self.manifest.dht_prefix}.{self.block}" + for name, message, expected in malformed_cases(uid, self.manifest.digest): + self.current_case = name + await asyncio.sleep(self.refill + 0.1) + before = await self.info() + began = self.clock() + await self.reject(message, expected) + await self.wait_health(lambda v: v["admission"]["active_sessions"] == 0) + require(await self.info() == before == available, "cache_capacity_changed") + self.checks.append({"case": name, "result": "passed", "duration_seconds": round(self.clock() - began, 3)}) + self.current_case = "training_forward_disabled" + await self.reject(runtime_pb2.ExpertRequest(), "training RPCs are disabled", training=True) + self.checks.append({"case": "training_forward_disabled", "result": "passed"}) + final = await self.wait_health( + lambda v: v["admission"]["active_sessions"] == 0 and v["admission"]["pending_pushes"] == 0 + ) + require(await self.info() == available, "final_cache_capacity_changed") + return { + "checks": self.checks, + "rpc_calls": self.calls, + "input_bytes": self.input_bytes, + "before": self.baseline, + "after": final, + } + + +async def run(args): + manifest = ModelManifest.load(args.manifest) + require(manifest.digest_id == args.expected_manifest_digest, "manifest_identity_mismatch") + require(0 <= args.block < manifest.model.num_blocks, "block_bounds") + peer = exact_peer(args.worker_multiaddr) + source, policy, refill = load_policy(args.policy) + require(re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", args.worker_label) is not None, "worker_label") + health = lambda: read_health(args.health, manifest.digest_id, args.block) + health() # Refuse missing/stale health before any network connection. + args.output.mkdir(parents=True, exist_ok=False) + result = { + "schema_version": 1, + "result": "failed", + "scope": "exact-worker-non-compute-rpc-canary", + "executed": args.execute, + "complete_gate16": False, + "worker_label": args.worker_label, + "target_binding_sha256": hashlib.sha256( + json.dumps( + { + "peer": args.worker_multiaddr, + "health": str(args.health.resolve()), + "block": args.block, + "manifest": manifest.digest_id, + }, + sort_keys=True, + ).encode() + ).hexdigest(), + "manifest_digest": manifest.digest_id, + "policy": source, + "policy_source": "operator-supplied effective launch settings; per-peer/idle behavior exercised below", + "limits": { + "operation_seconds": MAX_RUN_SECONDS, + "cleanup_seconds": 80, + "rpc_calls": MAX_RPC_CALLS, + "input_bytes": MAX_INPUT_BYTES, + "model_executions": 0, + }, + "started_at": datetime.now(timezone.utc).isoformat(), + "driver_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "limitations": [ + "No weights, valid inference tensors, inference recovery, UI disclosure, or catalog action in this RPC probe.", + "Runs beside a live worker health file; no SSH, provisioning, worker stop or public catalog mutation.", + "A client timeout is a failure; final worker health and owned-client cleanup are recorded separately.", + "The probe exercises the per-peer active limit and idle expiry, not global saturation or identity churn.", + "Health-to-peer pairing is operator supplied; public health does not include an authenticated peer identity.", + ], + } + if not args.execute: + result.update(result="preflight-passed", network_connections=0, executed=False) + (args.output / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return result + # The standalone driver owns every newly spawned child. Refuse pre-existing + # children so final cleanup cannot accidentally include another workload. + require(not psutil.Process().children(recursive=True), "driver_has_existing_children") + p2p = None + probe = None + began = time.monotonic() + try: + async with asyncio.timeout(MAX_RUN_SECONDS): + p2p = await P2P.create( + initial_peers=[args.worker_multiaddr], + host_maddrs=["/ip4/127.0.0.1/tcp/0"], + dht_mode="client", + auto_nat=False, + conn_manager=False, + nat_port_map=False, + use_relay=False, + use_ipfs=False, + tls=True, + startup_timeout=15, + persistent_conn_max_msg_size=128 * 1024, + ) + stub = TransformerConnectionHandler.get_stub(p2p, peer) + probe = Probe( + stub, health, manifest, args.block, policy, refill=refill, step_timeout=source["step_timeout"] + ) + result.update(await probe.run()) + result["result"] = "passed" + except BaseException as exc: + result["error_type"] = type(exc).__name__ + if isinstance(exc, CanaryError): + result["error_code"] = str(exc) + if probe is not None: + result.update(checks=probe.checks, rpc_calls=probe.calls, input_bytes=probe.input_bytes) + result["failed_case"] = probe.current_case + finally: + cleanup = {"owned_client_stopped": False, "worker_sessions_released": False} + try: + owned_children = psutil.Process().children(recursive=True) + if p2p is not None: + try: + await asyncio.wait_for(p2p.shutdown(), 10) + except BaseException: + result["result"] = "failed" + # These handles refer only to this standalone driver's descendants; + # psutil's signal methods reject PID reuse. Never select by image name. + for child in owned_children: + with contextlib.suppress(psutil.NoSuchProcess): + child.terminate() + _, alive = psutil.wait_procs(owned_children, timeout=2) + for child in alive: + with contextlib.suppress(psutil.NoSuchProcess): + child.kill() + _, alive = psutil.wait_procs(alive, timeout=3) + cleanup["owned_client_stopped"] = not alive and not psutil.Process().children(recursive=True) + except BaseException as exc: + result["result"] = "failed" + result["client_cleanup_error_type"] = type(exc).__name__ + try: + until = time.monotonic() + source["session_timeout"] + 5 + while True: + last = health() + if last["admission"]["active_sessions"] == 0 and last["admission"]["pending_pushes"] == 0: + cleanup["worker_sessions_released"] = True + break + require(time.monotonic() < until, "cleanup_session_deadline") + await asyncio.sleep(0.2) + except BaseException: + result["result"] = "failed" + if not all(cleanup.values()): + result["result"] = "failed" + result["cleanup"] = cleanup + result["duration_seconds"] = round(time.monotonic() - began, 3) + result["privacy"] = { + "raw_rpc_errors_exported": False, + "peer_identities_exported": False, + "credentials_created": False, + "prompts_or_outputs_exported": False, + } + (args.output / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n", encoding="utf-8") + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--expected-manifest-digest", required=True) + parser.add_argument("--worker-multiaddr", required=True) + parser.add_argument("--worker-label", required=True) + parser.add_argument("--block", type=int, required=True) + parser.add_argument("--health", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--execute", + action="store_true", + help="Send the bounded probe to the explicitly selected owned worker; default only validates local preflight", + ) + args = parser.parse_args() + result = asyncio.run(run(args)) + print(json.dumps({"result": result["result"], "complete_gate16": False})) + if result["result"] not in ("passed", "preflight-passed"): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/gate16_local_preflight.py b/scripts/gate16_local_preflight.py new file mode 100644 index 000000000..067d13790 --- /dev/null +++ b/scripts/gate16_local_preflight.py @@ -0,0 +1,309 @@ +"""Exercise the frozen node's local boundary without loading models or joining a swarm. + +This is a prerequisite to Gate 16, not a public canary or inference acceptance. +All generated keys are private files below a new, operator-selected output directory. +The runner removes those exact files and observes its owned process tree on exit. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import socket +import subprocess +import time +from pathlib import Path + +import httpx +import psutil + + +def digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def require(condition: bool, code: str) -> None: + if not condition: + raise RuntimeError(code) + + +def run(args) -> dict: + node, manifest, output = args.node.resolve(), args.manifest.resolve(), args.output.resolve() + require(node.is_file() and manifest.is_file(), "input_missing") + node_hash = digest(node) + require(node_hash == args.expected_node_sha256, "node_digest_mismatch") + source = json.loads(manifest.read_text(encoding="utf-8")) + require(source["name"] == "Qwen3.5-0.8B-Local", "unexpected_local_manifest") + output.mkdir(parents=True, exist_ok=False) + data = output / "data" + config_path = output / "node-config.json" + config_path.write_text( + json.dumps( + { + "schema_version": 1, + "inference_mode": "local_only", + "models": [ + { + "manifest": str(manifest), + "initial_peers": [], + "execution": "local", + "local_device": "cpu", + "cache_dir": str(output / "unused-model-cache"), + "request_timeout": 2.0, + "max_retries": 1, + } + ], + "contribution_policy": {"sharing_enabled": False}, + } + ), + encoding="utf-8", + ) + # Refuse a used endpoint before creating the node. An authenticated status + # response must also match the new, private control key produced by this run. + with socket.socket() as probe: + probe.bind(("127.0.0.1", args.port)) + started = time.monotonic() + result = { + "schema_version": 1, + "result": "failed", + "scope": "frozen-node-local-api-preflight", + "complete_gate16": False, + "node_sha256": node_hash, + "manifest_sha256": digest(manifest), + "packaged": True, + "checks": {}, + "limitations": [ + "No public swarm, model load, inference, GPU work, or public mutation was performed.", + "HTTP deadlines bound this probe, not stalled generation or worker RPC execution.", + "Local inference mode persistence is not signed catalog withdrawal or remote route disable.", + "File credentials isolate this headless-node probe; native desktop credential lifecycle is covered separately.", + ], + } + process = None + owned = [] + secrets_to_check = [] + prompt = "gate16-private-content-sentinel-do-not-export" + try: + with (output / "node.log").open("wb") as log: + process = subprocess.Popen( + [str(node), "--config", str(config_path), "--data_dir", str(data), "--port", str(args.port)], + stdout=log, + stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + owned.append(psutil.Process(process.pid)) + with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=5, trust_env=False) as api: + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + require(process.poll() is None, "node_exited_during_startup") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(0.25) + else: + raise RuntimeError("node_startup_deadline") + control = (data / "control-api.key").read_text().strip() + client = (data / "local-api.key").read_text().strip() + secrets_to_check += [control, client] + control_auth = {"Authorization": "Bearer " + control} + client_auth = {"Authorization": "Bearer " + client} + + def check(name, method, path, code, **kwargs): + beginning = time.monotonic() + reply = api.request(method, path, **kwargs) + require(reply.status_code == code, name + "_unexpected_status_" + str(reply.status_code)) + result["checks"][name] = { + "status_code": reply.status_code, + "duration_seconds": round(time.monotonic() - beginning, 4), + } + return reply + + check("control_requires_auth", "GET", "/control/v1/status", 401) + check("client_cannot_control", "GET", "/control/v1/status", 401, headers=client_auth) + check("control_cannot_infer", "GET", "/v1/models", 401, headers=control_auth) + check("client_requires_auth", "GET", "/v1/models", 401) + check("client_models", "GET", "/v1/models", 200, headers=client_auth) + status = check("control_status", "GET", "/control/v1/status", 200, headers=control_auth).json() + require(status["runtime_budget"]["resident_models"] == 0, "unexpected_model_load") + require(status["contribution"]["workers"] == [], "unexpected_worker") + check( + "malformed_json", + "POST", + "/v1/completions", + 422, + headers={**client_auth, "Content-Type": "application/json"}, + content=b"{", + ) + check( + "unknown_model_bounded_rejection", + "POST", + "/v1/completions", + 404, + headers=client_auth, + json={"model": "gate16-nonexistent-model", "prompt": prompt, "max_tokens": 1}, + ) + check( + "malformed_chat", + "POST", + "/v1/chat/completions", + 422, + headers=client_auth, + json={"model": "gate16-nonexistent-model", "messages": "invalid"}, + ) + check( + "invalid_inference_mode", + "PUT", + "/control/v1/inference-mode", + 422, + headers=control_auth, + json={"inference_mode": "invalid", "expected_config_revision": "sha256:" + "0" * 64}, + ) + check( + "control_policy_requires_json", + "PUT", + "/control/v1/contribution-policy", + 415, + headers={**control_auth, "Content-Type": "text/plain"}, + content=b"{}", + ) + check( + "oversized_control_policy", + "PUT", + "/control/v1/contribution-policy", + 413, + headers={**control_auth, "Content-Type": "application/json"}, + content=b" " * (256 * 1024 + 1), + ) + check( + "stale_policy_revision", + "PUT", + "/control/v1/inference-mode", + 412, + headers=control_auth, + json={"inference_mode": "auto", "expected_config_revision": "sha256:" + "0" * 64}, + ) + for mode in ("auto", "local_only"): + current = api.get("/control/v1/contribution-policy", headers=control_auth) + current.raise_for_status() + check( + "mode_" + mode, + "PUT", + "/control/v1/inference-mode", + 200, + headers=control_auth, + json={ + "inference_mode": mode, + "expected_config_revision": current.json()["config_revision"], + }, + ) + require(json.loads(config_path.read_text())["inference_mode"] == mode, "mode_not_persisted") + created = check( + "create_disposable_client_key", + "POST", + "/control/v1/keys", + 201, + headers=control_auth, + json={"label": "Gate 16 disposable preflight"}, + ).json() + secrets_to_check.append(created["secret"]) + disposable_auth = {"Authorization": "Bearer " + created["secret"]} + check("disposable_key_works", "GET", "/v1/models", 200, headers=disposable_auth) + check( + "revoke_disposable_key", + "DELETE", + "/control/v1/keys/" + created["key"]["id"], + 200, + headers=control_auth, + ) + check("revoked_key_rejected", "GET", "/v1/models", 401, headers=disposable_auth) + check("original_key_preserved", "GET", "/v1/models", 200, headers=client_auth) + status = api.get("/control/v1/status", headers=control_auth).json() + require(status["runtime_budget"]["resident_models"] == 0, "unexpected_final_model_load") + require(status["contribution"]["workers"] == [], "unexpected_final_worker") + require(not (output / "unused-model-cache").exists(), "unexpected_model_cache") + owned.extend(owned[0].children(recursive=True)) + # Windows creates a console host even with CREATE_NO_WINDOW. + # Count and identify the owned executable images, + # without retaining command lines, paths or process identities. + images = [Path(child.exe()).name for child in owned] + result["checks"]["owned_process_images"] = sorted(images) + allowed_images = {node.name.casefold(), "conhost.exe"} + require(all(image.casefold() in allowed_images for image in images), "unexpected_child_image") + result["checks"]["no_model_or_worker_loaded"] = True + result["checks"]["owned_runtime_process_count"] = len(owned) + result["result"] = "passed" + except BaseException as exc: + # Exception messages can contain request data, secrets, or host paths. + result["error_type"] = type(exc).__name__ + if type(exc) is RuntimeError: + result["error_code"] = str(exc) + raise + finally: + if owned and owned[0].is_running(): + known = {(child.pid, child.create_time()) for child in owned} + try: + descendants = owned[0].children(recursive=True) + except psutil.NoSuchProcess: + descendants = [] + for child in descendants: + if (child.pid, child.create_time()) not in known: + owned.append(child) + identities = [{"pid": child.pid, "created_at": child.create_time()} for child in owned] + (output / "process-identities.private.json").write_text(json.dumps(identities), encoding="utf-8") + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + if owned: + psutil.wait_procs(owned, timeout=5) + result["cleanup"] = {"owned_processes_stopped": all(not p.is_running() for p in owned)} + # These paths are generated only by this run. No recursive cleanup or + # native credential mutation is necessary for the file-mode probe. + private_files = [data / name for name in ("control-api.key", "local-api.key", "api-keys.json")] + for path in private_files: + if path.is_file() and not path.is_symlink(): + path.unlink() + result["cleanup"]["credential_files_removed"] = all(not p.exists() for p in private_files) + log_path = output / "node.log" + contents = log_path.read_text(encoding="utf-8", errors="replace") if log_path.is_file() else "" + result["privacy"] = { + "secrets_absent_from_log": all(secret not in contents for secret in secrets_to_check), + "synthetic_prompt_absent_from_log": prompt not in contents, + "raw_log_exported": False, + "prompt_or_output_exported": False, + } + if not all(result["cleanup"].values()) or not all( + result["privacy"][name] for name in ("secrets_absent_from_log", "synthetic_prompt_absent_from_log") + ): + result["result"] = "failed" + result["duration_seconds"] = round(time.monotonic() - started, 3) + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--node", required=True, type=Path) + parser.add_argument("--expected-node-sha256", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--port", type=int, default=18116) + args = parser.parse_args() + result = run(args) + print(json.dumps({"result": result["result"], "complete_gate16": False, "checks": len(result["checks"])})) + if result["result"] != "passed": + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/gateq38_gcp_adapter.py b/scripts/gateq38_gcp_adapter.py new file mode 100644 index 000000000..ac5e94f6e --- /dev/null +++ b/scripts/gateq38_gcp_adapter.py @@ -0,0 +1,1336 @@ +"""Source-bound GCP adapter for the durable Qwen3.8 complete-route controller. + +The adapter compiles exact private start specifications, observes run-scoped resources, +and performs retry-safe cleanup. Paid start and route collection remain fail-closed +until the protected Qwen3.8 host runtime and status transport are source-bound. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Mapping, Sequence + +from scripts import gateq38_linux_host_transport as transport, gateq38_route_controller as controller + +MAX_OUTPUT_BYTES = 1_048_576 +MAX_JSON_BYTES = controller.MAX_JSON_BYTES +SCOPE_LABEL = "q38-complete-route" +ROUTE_TAG_PREFIX = "communityai-q38-" +ROUTE_TCP_RULE = "tcp:31330-31339" +IAP_TCP_RULE = "tcp:22" +IAP_SOURCE_RANGE = "35.235.240.0/20" +GUEST_ATTRIBUTE_NAMESPACE = "communityai-q38" +GUEST_ATTRIBUTE_KEY = "status-v1" +GUEST_ATTRIBUTE_QUERY_PATH = f"{GUEST_ATTRIBUTE_NAMESPACE}/{GUEST_ATTRIBUTE_KEY}" +MAX_GUEST_ATTRIBUTE_OUTPUT_BYTES = transport.MAX_ENVELOPE_BYTES * 2 + 4_096 +REMOTE_HOST_RUNTIME = "/var/lib/communityai-q38/input/source/scripts/gateq38_linux_host_runtime.py" +DELIVERY_TIMEOUT_SECONDS = 300 +RUNTIME_ACTIONS_BLOCKED = "source-bound Qwen3.8 host runtime is not plan-bound" + + +class Q38GcpAdapterError(RuntimeError): + """A provider observation or exact action failed closed.""" + + +@dataclass(frozen=True) +class CommandResult: + returncode: int + stdout: bytes + stderr: bytes + + +Runner = Callable[[Sequence[str], int], CommandResult] +DeliveryRunner = Callable[[Sequence[str], bytes, int], CommandResult] +StatusKeyResolver = Callable[[str, str], bytes] +StatusCheckpointResolver = Callable[[str, str], tuple[str | None, int]] + + +def _default_delivery_runner( + argv: Sequence[str], + payload: bytes, + timeout: int, +) -> CommandResult: + if ( + not argv + or any(not isinstance(item, str) or not item for item in argv) + or not isinstance(payload, bytes) + or not 1 <= len(payload) <= transport.MAX_DELIVERY_BYTES + ): + raise Q38GcpAdapterError("instance delivery command is invalid") + try: + result = subprocess.run( + list(argv), + input=payload, + check=False, + capture_output=True, + timeout=timeout, + shell=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise Q38GcpAdapterError("instance delivery command failed") from exc + if len(result.stdout) > MAX_OUTPUT_BYTES or len(result.stderr) > MAX_OUTPUT_BYTES: + raise Q38GcpAdapterError("instance delivery output exceeded its bound") + return CommandResult(result.returncode, result.stdout, result.stderr) + + +def _default_runner(argv: Sequence[str], timeout: int) -> CommandResult: + if not argv or any(not isinstance(item, str) or not item for item in argv): + raise Q38GcpAdapterError("provider command is invalid") + try: + result = subprocess.run( + list(argv), + check=False, + capture_output=True, + timeout=timeout, + shell=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise Q38GcpAdapterError("provider command failed") from exc + if len(result.stdout) > MAX_OUTPUT_BYTES or len(result.stderr) > MAX_OUTPUT_BYTES: + raise Q38GcpAdapterError("provider command output exceeded its bound") + return CommandResult(result.returncode, result.stdout, result.stderr) + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Q38GcpAdapterError("duplicate provider JSON field") + result[key] = value + return result + + +def _reject_constant(_value: str) -> None: + raise Q38GcpAdapterError("non-finite provider JSON value") + + +def _json_bytes(payload: bytes, label: str, *, maximum: int = MAX_OUTPUT_BYTES) -> Any: + if not 1 <= len(payload) <= maximum: + raise Q38GcpAdapterError(f"{label} output is invalid") + try: + return json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Q38GcpAdapterError(f"{label} returned invalid JSON") from exc + + +def _basename(value: Any) -> str: + return value.rsplit("/", 1)[-1] if isinstance(value, str) else "" + + +def _route_tag(plan: controller.RoutePlan) -> str: + identity = f"{plan.run_id}\0{plan.plan_digest}".encode("utf-8") + return ROUTE_TAG_PREFIX + hashlib.sha256(identity).hexdigest()[:20] + + +def _labels(plan: controller.RoutePlan) -> dict[str, str]: + return { + "communityai-run": plan.run_id, + "communityai-scope": SCOPE_LABEL, + "communityai-source": plan.source_commit, + } + + +def _label_argument(value: Mapping[str, str]) -> str: + return ",".join(f"{key}={value[key]}" for key in sorted(value)) + + +def _resource_binding( + plan: controller.RoutePlan, + resource: controller.ResourcePlan, +) -> dict[str, Any]: + return { + "schema_version": controller.SCHEMA_VERSION, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "deadline_unix": plan.deadline_unix, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "resource_name": resource.name, + "kind": resource.kind, + "worker_id": resource.worker_id, + } + + +def _description(plan: controller.RoutePlan, resource: controller.ResourcePlan) -> str: + return json.dumps( + _resource_binding(plan, resource), + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def _metadata(value: Mapping[str, Any]) -> dict[str, str]: + raw = value.get("metadata") + items = raw.get("items") if isinstance(raw, dict) else None + if not isinstance(items, list): + raise Q38GcpAdapterError("instance metadata is invalid") + result: dict[str, str] = {} + for item in items: + if not isinstance(item, dict) or set(item) != {"key", "value"}: + raise Q38GcpAdapterError("instance metadata item is invalid") + key, field = item["key"], item["value"] + if not isinstance(key, str) or not isinstance(field, str) or key in result: + raise Q38GcpAdapterError("instance metadata item is ambiguous") + result[key] = field + return result + + +def _instance_metadata( + plan: controller.RoutePlan, + resource: controller.ResourcePlan, +) -> dict[str, str]: + result = {key.replace("_", "-"): str(value) for key, value in _resource_binding(plan, resource).items()} + result["worker-id"] = resource.worker_id or "none" + result["worker-plan-digest"] = plan.worker_plan_digest + result["manifest-digest"] = plan.manifest_digest + if resource.worker_id is not None: + worker = plan.worker_by_id[resource.worker_id] + result.update( + { + "machine-id": worker.machine_id, + "span": worker.span, + "artifact-bytes": str(worker.artifact_bytes), + "artifact-set-digest": worker.artifact_set_digest, + "cache-root": worker.cache_root, + } + ) + return result + + +def _metadata_argument(value: Mapping[str, str]) -> str: + if any("," in item or "=" in item for item in value.values()): + raise Q38GcpAdapterError("instance metadata cannot be represented safely") + return ",".join(f"{key}={value[key]}" for key in sorted(value)) + + +def _assert_source_bound(plan: controller.RoutePlan, source_root: Path) -> None: + root = source_root.resolve() + expected = { + controller.GCP_ADAPTER_SOURCE_PATH: Path(__file__).resolve(), + "scripts/gateq38_route_controller.py": Path(controller.__file__).resolve(), + } + bindings = {item["relative_path"]: item for item in plan.source_bindings} + for relative, imported in expected.items(): + candidate = (root / Path(*PurePosixPath(relative).parts)).resolve() + binding = bindings.get(relative) + if binding is None or imported != candidate: + raise Q38GcpAdapterError("imported provider adapter sources are not source-bound") + payload = controller._regular_bytes(candidate) + if len(payload) != binding["byte_size"] or "sha256:" + hashlib.sha256(payload).hexdigest() != binding["sha256"]: + raise Q38GcpAdapterError("provider adapter source binding changed") + + +def _absent_worker() -> dict[str, Any]: + return {field: ("absent" if field == "state" else None) for field in controller._OBS_WORKER_FIELDS} + + +def _starting_worker(plan: controller.RoutePlan, worker: controller.WorkerPlan) -> dict[str, Any]: + return { + "state": "starting", + "machine_id": worker.machine_id, + "peer_id": None, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "span": worker.span, + "manifest_digest": plan.manifest_digest, + "artifact_bytes": worker.artifact_bytes, + "artifact_set_digest": worker.artifact_set_digest, + "cache_root": worker.cache_root, + } + + +def _absent_job() -> dict[str, Any]: + return {field: ("absent" if field == "state" else None) for field in controller._ROUTE_JOB_FIELDS} + + +def blank_host_status(plan: controller.RoutePlan) -> dict[str, Any]: + return { + "schema_version": controller.SCHEMA_VERSION, + "run_id": plan.run_id, + "workers": {worker.worker_id: _absent_worker() for worker in plan.workers}, + "route_job": _absent_job(), + } + + +def validate_host_status(value: Mapping[str, Any], plan: controller.RoutePlan) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != {"schema_version", "run_id", "workers", "route_job"}: + raise Q38GcpAdapterError("host status schema is invalid") + if value["schema_version"] != controller.SCHEMA_VERSION or value["run_id"] != plan.run_id: + raise Q38GcpAdapterError("host status identity is invalid") + workers = value["workers"] + if not isinstance(workers, dict) or set(workers) != set(plan.worker_by_id): + raise Q38GcpAdapterError("host worker inventory is not exact") + for worker_id, worker in workers.items(): + if not isinstance(worker, dict) or set(worker) != controller._OBS_WORKER_FIELDS: + raise Q38GcpAdapterError(f"host worker status is invalid: {worker_id}") + route_job = value["route_job"] + if not isinstance(route_job, dict) or set(route_job) != controller._ROUTE_JOB_FIELDS: + raise Q38GcpAdapterError("host route status is invalid") + return dict(value) + + +def load_host_status(path: Path | None, plan: controller.RoutePlan, source_root: Path) -> dict[str, Any]: + if path is None or not path.exists(): + return blank_host_status(plan) + controller._assert_protected_path(path, plan, source_root, directory=False) + return validate_host_status( + controller._strict_json(controller._regular_bytes(path)), + plan, + ) + + +class GcpAdapter: + def __init__( + self, + plan: controller.RoutePlan, + source_root: Path, + *, + runner: Runner = _default_runner, + delivery_runner: DeliveryRunner = _default_delivery_runner, + clock: Callable[[], float] = time.time, + status_key_resolver: StatusKeyResolver | None = None, + status_checkpoint_resolver: StatusCheckpointResolver | None = None, + ) -> None: + if (status_key_resolver is None) != (status_checkpoint_resolver is None): + raise Q38GcpAdapterError("authenticated host status requires key and checkpoint resolvers") + self.plan = plan + self.source_root = source_root.resolve() + self.runner = runner + self.delivery_runner = delivery_runner + self.clock = clock + self.status_key_resolver = status_key_resolver + self.status_checkpoint_resolver = status_checkpoint_resolver + _assert_source_bound(plan, self.source_root) + + def _gcloud( + self, + *arguments: str, + timeout: int = 300, + check: bool = True, + ) -> CommandResult: + result = self.runner(("gcloud", *arguments, "--quiet"), timeout) + if check and result.returncode != 0: + raise Q38GcpAdapterError("gcloud action failed") + return result + + def _gcloud_json(self, *arguments: str, timeout: int = 300) -> Any: + result = self._gcloud(*arguments, "--format=json", timeout=timeout) + return _json_bytes(result.stdout, "gcloud") + + def _check_auth(self) -> None: + result = self._gcloud( + "auth", + "list", + "--filter=status:ACTIVE", + "--format=value(account)", + timeout=60, + ) + accounts = [line for line in result.stdout.decode("utf-8", "strict").splitlines() if line.strip()] + if len(accounts) != 1: + raise Q38GcpAdapterError("exactly one active native gcloud account is required") + project = self._gcloud_json("projects", "describe", controller.EXPECTED_PROJECT, timeout=60) + if not isinstance(project, dict) or project.get("lifecycleState") != "ACTIVE": + raise Q38GcpAdapterError("authorized GCP project is unavailable") + + def _describe(self, resource: controller.ResourcePlan) -> Mapping[str, Any] | None: + if resource.kind.endswith("firewall"): + arguments = ( + "compute", + "firewall-rules", + "describe", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + ) + else: + kind = "instances" if resource.kind.endswith("instance") else "disks" + arguments = ( + "compute", + kind, + "describe", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + ) + result = self._gcloud(*arguments, "--format=json", timeout=60, check=False) + if result.returncode != 0: + message = result.stderr.decode("utf-8", "replace").casefold() + if "not found" in message or "was not found" in message: + return None + raise Q38GcpAdapterError("provider inventory failed") + value = _json_bytes(result.stdout, "resource inventory") + if not isinstance(value, dict): + raise Q38GcpAdapterError("provider inventory item is invalid") + return value + + def _validate_description( + self, + resource: controller.ResourcePlan, + value: Mapping[str, Any], + ) -> None: + description = value.get("description") + if not isinstance(description, str) or len(description.encode("utf-8")) > 4_096: + raise Q38GcpAdapterError("planned resource binding is invalid") + parsed = _json_bytes(description.encode("utf-8"), "resource description", maximum=4_096) + if parsed != _resource_binding(self.plan, resource): + raise Q38GcpAdapterError("planned resource binding changed") + + def _validate_disk( + self, + resource: controller.ResourcePlan, + value: Mapping[str, Any], + *, + cleanup: bool = False, + ) -> None: + if value.get("name") != resource.name: + raise Q38GcpAdapterError("planned disk identity is invalid") + if not cleanup and value.get("status") not in {"READY", "CREATING"}: + raise Q38GcpAdapterError("planned disk state is invalid") + if value.get("labels") != _labels(self.plan): + raise Q38GcpAdapterError("planned disk ownership is invalid") + source_image = value.get("sourceImage") + api_prefix = "https://www.googleapis.com/compute/v1/" + if isinstance(source_image, str) and source_image.startswith(api_prefix): + source_image = source_image.removeprefix(api_prefix) + if ( + _basename(value.get("type")) != controller.EXPECTED_DISK_TYPE + or str(value.get("sizeGb")) != str(controller.EXPECTED_DISK_SIZE_GB) + or source_image + != f"projects/{controller.EXPECTED_SOURCE_IMAGE.split('/', 1)[0]}/global/images/{controller.EXPECTED_SOURCE_IMAGE.split('/', 1)[1]}" + ): + raise Q38GcpAdapterError("planned disk shape is invalid") + self._validate_description(resource, value) + + def _disk_for_instance(self, resource: controller.ResourcePlan) -> controller.ResourcePlan: + kind = "worker_disk" if resource.worker_id is not None else "bootstrap_disk" + matches = [item for item in self.plan.resources if item.kind == kind and item.worker_id == resource.worker_id] + if len(matches) != 1: + raise Q38GcpAdapterError("instance disk plan is ambiguous") + return matches[0] + + def _validate_instance( + self, + resource: controller.ResourcePlan, + value: Mapping[str, Any], + *, + cleanup: bool = False, + ) -> None: + if value.get("name") != resource.name: + raise Q38GcpAdapterError("planned instance identity is invalid") + try: + controller.instance_generation_digest( + resource.name, + value.get("id"), + value.get("creationTimestamp"), + ) + except controller.RouteControllerError as exc: + raise Q38GcpAdapterError("planned instance generation is invalid") from exc + if not cleanup and value.get("status") not in { + "PROVISIONING", + "STAGING", + "RUNNING", + "STOPPING", + }: + raise Q38GcpAdapterError("planned instance state is invalid") + if value.get("labels") != _labels(self.plan): + raise Q38GcpAdapterError("planned instance ownership is invalid") + spec = controller._expected_resource_spec(resource) + disks = value.get("disks") + interfaces = value.get("networkInterfaces") + accelerators = value.get("guestAccelerators", []) + tags = value.get("tags") + scheduling = value.get("scheduling") + expected_disk = self._disk_for_instance(resource) + if "serviceAccounts" in value and value["serviceAccounts"] not in (None, []): + raise Q38GcpAdapterError("planned instance has a service account") + if ( + _basename(value.get("machineType")) != spec["machine_type"] + or not isinstance(disks, list) + or len(disks) != 1 + or _basename(disks[0].get("source") if isinstance(disks[0], dict) else None) != expected_disk.name + or disks[0].get("boot") is not True + or disks[0].get("autoDelete") is not True + or not isinstance(interfaces, list) + or len(interfaces) != 1 + or _basename(interfaces[0].get("network") if isinstance(interfaces[0], dict) else None) + != controller.EXPECTED_NETWORK + or _basename(interfaces[0].get("subnetwork") if isinstance(interfaces[0], dict) else None) + != controller.EXPECTED_SUBNET + or bool(interfaces[0].get("accessConfigs")) + or bool(interfaces[0].get("ipv6AccessConfigs")) + or interfaces[0].get("externalIpv6") not in (None, "") + or interfaces[0].get("stackType") != "IPV4_ONLY" + or not isinstance(tags, dict) + or tags.get("items") != [_route_tag(self.plan)] + or value.get("canIpForward") not in (None, False) + or value.get("deletionProtection") not in (None, False) + or not isinstance(scheduling, dict) + or scheduling.get("automaticRestart") is not True + or scheduling.get("provisioningModel") != "STANDARD" + or scheduling.get("onHostMaintenance") != "TERMINATE" + or scheduling.get("instanceTerminationAction") != "DELETE" + or scheduling.get("maxRunDuration") + != {"seconds": str(controller.EXPECTED_MAX_LIFETIME_SECONDS), "nanos": 0} + or _metadata(value) != _instance_metadata(self.plan, resource) + ): + raise Q38GcpAdapterError("planned instance shape is invalid") + if resource.kind == "worker_instance": + if ( + not isinstance(accelerators, list) + or len(accelerators) != 1 + or _basename(accelerators[0].get("acceleratorType")) != controller.EXPECTED_ACCELERATOR_TYPE + or accelerators[0].get("acceleratorCount") != 1 + ): + raise Q38GcpAdapterError("planned worker accelerator is invalid") + elif accelerators not in (None, []): + raise Q38GcpAdapterError("bootstrap instance has an accelerator") + + def _validate_firewall( + self, + resource: controller.ResourcePlan, + value: Mapping[str, Any], + ) -> None: + if resource.kind != "firewall" or value.get("name") != resource.name: + raise Q38GcpAdapterError("planned firewall identity is invalid") + allowed = value.get("allowed") + if ( + value.get("direction") != "INGRESS" + or _basename(value.get("network")) != controller.EXPECTED_NETWORK + or value.get("sourceTags") != [_route_tag(self.plan)] + or value.get("targetTags") != [_route_tag(self.plan)] + or value.get("sourceRanges") not in (None, []) + or allowed != [{"IPProtocol": "tcp", "ports": ["31330-31339"]}] + or value.get("disabled") not in (None, False) + ): + raise Q38GcpAdapterError("planned firewall policy is invalid") + self._validate_description(resource, value) + + def _validate_iap_firewall( + self, + resource: controller.ResourcePlan, + value: Mapping[str, Any], + ) -> None: + if resource.kind != "iap_firewall" or value.get("name") != resource.name: + raise Q38GcpAdapterError("planned IAP firewall identity is invalid") + allowed = value.get("allowed") + if ( + value.get("direction") != "INGRESS" + or _basename(value.get("network")) != controller.EXPECTED_NETWORK + or value.get("sourceTags") not in (None, []) + or value.get("targetTags") != [_route_tag(self.plan)] + or value.get("sourceRanges") != [IAP_SOURCE_RANGE] + or allowed != [{"IPProtocol": "tcp", "ports": ["22"]}] + or value.get("disabled") not in (None, False) + ): + raise Q38GcpAdapterError("planned IAP firewall policy is invalid") + self._validate_description(resource, value) + + def _validate_resource( + self, + resource: controller.ResourcePlan, + value: Mapping[str, Any], + *, + cleanup: bool = False, + ) -> None: + if resource.kind.endswith("disk"): + self._validate_disk(resource, value, cleanup=cleanup) + elif resource.kind.endswith("instance"): + self._validate_instance(resource, value, cleanup=cleanup) + elif resource.kind == "firewall": + self._validate_firewall(resource, value) + elif resource.kind == "iap_firewall": + self._validate_iap_firewall(resource, value) + else: + raise Q38GcpAdapterError("planned resource kind is invalid") + + def _listed_names(self, kind: str) -> set[str]: + command_kind = "firewall-rules" if kind == "firewall" else kind + arguments = ( + "compute", + command_kind, + "list", + f"--project={controller.EXPECTED_PROJECT}", + f"--filter=name~'^{self.plan.run_id}-'", + ) + value = self._gcloud_json(*arguments, timeout=60) + if not isinstance(value, list): + raise Q38GcpAdapterError("run-scoped provider inventory is invalid") + result: set[str] = set() + for item in value: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise Q38GcpAdapterError("run-scoped provider inventory item is invalid") + if item["name"] in result: + raise Q38GcpAdapterError("run-scoped provider inventory is ambiguous") + result.add(item["name"]) + return result + + def _protected_bootstrap_running(self) -> bool: + result = self._gcloud( + "compute", + "instances", + "describe", + controller.PROTECTED_INSTANCE, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + "--format=json", + timeout=60, + check=False, + ) + if result.returncode != 0: + return False + value = _json_bytes(result.stdout, "protected bootstrap inventory") + return ( + isinstance(value, dict) + and value.get("name") == controller.PROTECTED_INSTANCE + and value.get("status") == "RUNNING" + ) + + def _provider_inventory( + self, + *, + cleanup: bool = False, + ) -> tuple[dict[str, Mapping[str, Any] | None], bool]: + self._check_auth() + values: dict[str, Mapping[str, Any] | None] = {} + for resource in self.plan.resources: + value = self._describe(resource) + if value is not None: + self._validate_resource(resource, value, cleanup=cleanup) + values[resource.name] = value + expected_by_kind = { + "instances": { + item.name + for item in self.plan.resources + if item.kind.endswith("instance") and values[item.name] is not None + }, + "disks": { + item.name + for item in self.plan.resources + if item.kind.endswith("disk") and values[item.name] is not None + }, + "firewall": { + item.name + for item in self.plan.resources + if item.kind.endswith("firewall") and values[item.name] is not None + }, + } + for kind, expected in expected_by_kind.items(): + if self._listed_names(kind) != expected: + raise Q38GcpAdapterError("run-scoped provider inventory is not exact") + return values, self._protected_bootstrap_running() + + def _resource_observations( + self, + provider: Mapping[str, Mapping[str, Any] | None], + ) -> dict[str, dict[str, Any]]: + resources: dict[str, dict[str, Any]] = {} + for resource in self.plan.resources: + value = provider[resource.name] + present = value is not None + instance_id: str | None = None + creation_timestamp: str | None = None + instance_generation_digest: str | None = None + if present and resource.kind.endswith("instance"): + instance_id = value["id"] + creation_timestamp = value["creationTimestamp"] + instance_generation_digest = controller.instance_generation_digest( + resource.name, + instance_id, + creation_timestamp, + ) + resources[resource.name] = { + "present": present, + "kind": resource.kind, + "provider": resource.provider, + "region": resource.region, + "run_id": self.plan.run_id if present else None, + "source_commit": self.plan.source_commit if present else None, + "deadline_unix": self.plan.deadline_unix if present else None, + "plan_digest": self.plan.plan_digest if present else None, + "start_action_id": (controller._action_id(self.plan, "start_route") if present else None), + "worker_id": resource.worker_id if present else None, + "instance_id": instance_id, + "creation_timestamp": creation_timestamp, + "instance_generation_digest": instance_generation_digest, + } + return resources + + def _guest_attribute(self, resource: controller.ResourcePlan) -> bytes | None: + if not resource.kind.endswith("instance"): + raise Q38GcpAdapterError("guest attribute target is not an instance") + result = self._gcloud( + "compute", + "instances", + "get-guest-attributes", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + f"--query-path={GUEST_ATTRIBUTE_QUERY_PATH}", + "--format=json", + timeout=60, + check=False, + ) + if result.returncode != 0: + raise Q38GcpAdapterError("guest attribute read failed") + value = _json_bytes( + result.stdout, + "guest attribute", + maximum=MAX_GUEST_ATTRIBUTE_OUTPUT_BYTES, + ) + allowed_fields = {"kind", "queryPath", "queryValue", "selfLink"} + if ( + not isinstance(value, dict) + or not set(value).issubset(allowed_fields) + or value.get("queryPath") != GUEST_ATTRIBUTE_QUERY_PATH + or value.get("kind", "compute#guestAttributes") != "compute#guestAttributes" + ): + raise Q38GcpAdapterError("guest attribute response is invalid") + query_value = value.get("queryValue") + if not isinstance(query_value, dict) or set(query_value) != {"items"}: + raise Q38GcpAdapterError("guest attribute response is invalid") + items = query_value["items"] + if not isinstance(items, list) or len(items) > 1: + raise Q38GcpAdapterError("guest attribute response is ambiguous") + if not items: + return None + item = items[0] + if ( + not isinstance(item, dict) + or set(item) != {"namespace", "key", "value"} + or item["namespace"] != GUEST_ATTRIBUTE_NAMESPACE + or item["key"] != GUEST_ATTRIBUTE_KEY + or not isinstance(item["value"], str) + ): + raise Q38GcpAdapterError("guest attribute item is invalid") + try: + payload = item["value"].encode("ascii") + except UnicodeEncodeError as exc: + raise Q38GcpAdapterError("guest attribute value is invalid") from exc + if not 1 <= len(payload) <= transport.MAX_ENVELOPE_BYTES: + raise Q38GcpAdapterError("guest attribute value exceeded its size bound") + return payload + + def _authenticated_host_status( + self, + pre_provider: Mapping[str, Mapping[str, Any] | None], + pre_protected: bool, + *, + now_unix: int, + ) -> tuple[dict[str, Any], dict[str, Mapping[str, Any] | None], bool,]: + key_resolver = self.status_key_resolver + checkpoint_resolver = self.status_checkpoint_resolver + if key_resolver is None or checkpoint_resolver is None: + raise Q38GcpAdapterError("authenticated host status is not configured") + host = blank_host_status(self.plan) + pre_resources = self._resource_observations(pre_provider) + pre_digest = controller.observation_instance_generations_digest( + pre_resources, + self.plan, + ) + for resource in self.plan.resources: + if not resource.kind.endswith("instance"): + continue + observed = pre_resources[resource.name] + generation = observed["instance_generation_digest"] + if generation is None: + continue + payload = self._guest_attribute(resource) + if payload is None: + continue + try: + key = key_resolver(resource.name, generation) + checkpoint = checkpoint_resolver(resource.name, generation) + except Exception as exc: + raise Q38GcpAdapterError("protected host status material is unavailable") from exc + if ( + not isinstance(checkpoint, tuple) + or len(checkpoint) != 2 + or checkpoint[0] is not None + and not isinstance(checkpoint[0], str) + or not isinstance(checkpoint[1], int) + or isinstance(checkpoint[1], bool) + or checkpoint[1] < 0 + or checkpoint[1] > 0 + and checkpoint[0] is None + ): + raise Q38GcpAdapterError("protected host status checkpoint is invalid") + try: + envelope = transport.validate_status_envelope( + transport.decode_status_envelope(payload), + self.plan, + key=key, + now_unix=now_unix, + expected_resource_name=resource.name, + expected_generation_digest=generation, + expected_boot_id=checkpoint[0], + minimum_revision=checkpoint[1], + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38GcpAdapterError("authenticated guest attribute is invalid") from exc + context = envelope["context"] + if ( + context["instance_id"] != observed["instance_id"] + or context["creation_timestamp"] != observed["creation_timestamp"] + ): + raise Q38GcpAdapterError("authenticated guest attribute generation changed") + if resource.kind == "worker_instance": + host["workers"][resource.worker_id] = envelope["payload"] + else: + host["route_job"] = envelope["payload"] + post_provider, post_protected = self._provider_inventory() + post_resources = self._resource_observations(post_provider) + post_digest = controller.observation_instance_generations_digest( + post_resources, + self.plan, + ) + if pre_digest != post_digest or not pre_protected or not post_protected or pre_protected != post_protected: + raise Q38GcpAdapterError("provider generation changed during authenticated host-status read") + return validate_host_status(host, self.plan), post_provider, post_protected + + def inventory( + self, + host_status: Mapping[str, Any], + *, + manifest_path: Path | None, + artifact_root: Path | None, + evidence_root: Path | None = None, + cleanup_only: bool = False, + ) -> dict[str, Any]: + if cleanup_only: + host = blank_host_status(self.plan) + else: + host = validate_host_status(host_status, self.plan) + if host != blank_host_status(self.plan): + raise Q38GcpAdapterError("protected Qwen3.8 host status transport is not plan-bound") + provider, protected = self._provider_inventory() + now = int(self.clock()) + if not cleanup_only and self.status_key_resolver is not None: + host, provider, protected = self._authenticated_host_status( + provider, + protected, + now_unix=now, + ) + if cleanup_only: + revalidation: Mapping[str, Any] | None = None + else: + if manifest_path is None or artifact_root is None: + raise Q38GcpAdapterError("production artifact inputs are required") + revalidation = controller.revalidate_production_artifact_plan( + self.plan, + manifest_path, + artifact_root, + self.source_root, + verified_at_unix=now, + ) + resources = self._resource_observations(provider) + workers: dict[str, Any] = {} + for worker in self.plan.workers: + instance_value = provider[worker.instance] + disk_value = provider[worker.disk] + instance_present = instance_value is not None + supplied = host["workers"][worker.worker_id] + if not instance_present: + if supplied["state"] != "absent": + raise Q38GcpAdapterError("host status survived an absent worker instance") + workers[worker.worker_id] = _absent_worker() + elif supplied["state"] == "absent": + inferred = _starting_worker(self.plan, worker) + if instance_value.get("status") == "STOPPING": + inferred["state"] = "failed" + workers[worker.worker_id] = inferred + else: + if supplied["state"] == "ready" and ( + instance_value.get("status") != "RUNNING" + or disk_value is None + or disk_value.get("status") != "READY" + ): + raise Q38GcpAdapterError("ready host status lacks ready provider resources") + workers[worker.worker_id] = dict(supplied) + bootstrap = next(item for item in self.plan.resources if item.kind == "bootstrap_instance") + bootstrap_disk = next(item for item in self.plan.resources if item.kind == "bootstrap_disk") + route_job = dict(host["route_job"]) + if provider[bootstrap.name] is None: + if route_job["state"] != "absent": + raise Q38GcpAdapterError("route job survived an absent bootstrap instance") + route_job = _absent_job() + elif route_job["state"] != "absent" and ( + provider[bootstrap.name].get("status") != "RUNNING" + or provider[bootstrap_disk.name] is None + or provider[bootstrap_disk.name].get("status") != "READY" + ): + raise Q38GcpAdapterError("route job lacks ready provider resources") + observation = { + "schema_version": controller.SCHEMA_VERSION, + "run_id": self.plan.run_id, + "observed_at_unix": now, + "protected_bootstrap_running": protected, + "artifact_plan_revalidation": revalidation, + "instance_generations_digest": controller.observation_instance_generations_digest( + resources, + self.plan, + ), + "resources": resources, + "workers": workers, + "route_job": route_job, + } + validated = controller.validate_observation( + observation, + self.plan, + cleanup_only=cleanup_only, + ) + if route_job["state"] == "passed": + if evidence_root is None: + raise Q38GcpAdapterError("protected route evidence is required") + controller.revalidate_route_evidence( + self.plan, + validated, + evidence_root, + self.source_root, + ) + return validated + + def _disk_create_arguments(self, resource: controller.ResourcePlan) -> tuple[str, ...]: + image_project, image_name = controller.EXPECTED_SOURCE_IMAGE.split("/", 1) + return ( + "compute", + "disks", + "create", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + f"--type={controller.EXPECTED_DISK_TYPE}", + f"--size={controller.EXPECTED_DISK_SIZE_GB}GB", + f"--image={image_name}", + f"--image-project={image_project}", + f"--labels={_label_argument(_labels(self.plan))}", + f"--description={_description(self.plan, resource)}", + ) + + def _firewall_create_arguments(self, resource: controller.ResourcePlan) -> tuple[str, ...]: + return ( + "compute", + "firewall-rules", + "create", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--network={controller.EXPECTED_NETWORK}", + "--direction=INGRESS", + "--action=ALLOW", + f"--rules={ROUTE_TCP_RULE}", + f"--source-tags={_route_tag(self.plan)}", + f"--target-tags={_route_tag(self.plan)}", + f"--description={_description(self.plan, resource)}", + "--no-enable-logging", + ) + + def _iap_firewall_create_arguments(self, resource: controller.ResourcePlan) -> tuple[str, ...]: + if resource.kind != "iap_firewall": + raise Q38GcpAdapterError("IAP firewall resource kind is invalid") + return ( + "compute", + "firewall-rules", + "create", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--network={controller.EXPECTED_NETWORK}", + "--direction=INGRESS", + "--action=ALLOW", + f"--rules={IAP_TCP_RULE}", + f"--source-ranges={IAP_SOURCE_RANGE}", + f"--target-tags={_route_tag(self.plan)}", + f"--description={_description(self.plan, resource)}", + "--no-enable-logging", + ) + + def _instance_create_arguments(self, resource: controller.ResourcePlan) -> tuple[str, ...]: + spec = controller._expected_resource_spec(resource) + disk = self._disk_for_instance(resource) + return ( + "compute", + "instances", + "create", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + f"--machine-type={spec['machine_type']}", + f"--disk=name={disk.name},boot=yes,auto-delete=yes", + f"--network={controller.EXPECTED_NETWORK}", + f"--subnet={controller.EXPECTED_SUBNET}", + "--stack-type=IPV4_ONLY", + f"--tags={_route_tag(self.plan)}", + f"--labels={_label_argument(_labels(self.plan))}", + "--no-address", + "--no-service-account", + "--maintenance-policy=TERMINATE", + "--restart-on-failure", + f"--max-run-duration={controller.EXPECTED_MAX_LIFETIME_SECONDS}s", + "--instance-termination-action=DELETE", + f"--metadata={_metadata_argument(_instance_metadata(self.plan, resource))}", + ) + + def compiled_start_commands(self) -> tuple[tuple[str, ...], ...]: + disks = tuple( + ("gcloud", *self._disk_create_arguments(resource), "--quiet") + for resource in self.plan.resources + if resource.kind.endswith("disk") + ) + route_firewall = next(item for item in self.plan.resources if item.kind == "firewall") + iap_firewall = next(item for item in self.plan.resources if item.kind == "iap_firewall") + firewalls = ( + ("gcloud", *self._firewall_create_arguments(route_firewall), "--quiet"), + ("gcloud", *self._iap_firewall_create_arguments(iap_firewall), "--quiet"), + ) + instances = tuple( + ("gcloud", *self._instance_create_arguments(resource), "--quiet") + for resource in self.plan.resources + if resource.kind.endswith("instance") + ) + return (*disks, *firewalls, *instances) + + def compiled_instance_delivery_command( + self, + delivery: transport.InstanceDelivery, + *, + now_unix: int | None = None, + ) -> tuple[str, ...]: + verification_time = int(self.clock()) if now_unix is None else now_unix + try: + record, _context, _key = transport.validate_instance_delivery( + delivery, + self.plan, + now_unix=verification_time, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38GcpAdapterError("instance delivery is invalid") from exc + resource = self.plan.resource_by_name.get(record["resource_name"]) + if resource is None or not resource.kind.endswith("instance"): + raise Q38GcpAdapterError("instance delivery target is not planned") + if not any(item.kind == "iap_firewall" for item in self.plan.resources): + raise Q38GcpAdapterError("instance delivery lacks an IAP firewall") + remote_command = ( + f"/usr/bin/sudo --non-interactive /usr/bin/python3 {REMOTE_HOST_RUNTIME} " + f"install-delivery --resource-name {resource.name} " + f"--instance-generation-digest {record['instance_generation_digest']}" + ) + return ( + "gcloud", + "compute", + "ssh", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + "--tunnel-through-iap", + "--ssh-flag=-T", + "--ssh-flag=-oBatchMode=yes", + f"--command={remote_command}", + "--quiet", + ) + + def deliver_instance( + self, + delivery: transport.InstanceDelivery, + *, + now_unix: int | None = None, + ) -> dict[str, Any]: + """Deliver one bundle only across generation-stable exact provider reads.""" + + verification_time = int(self.clock()) if now_unix is None else now_unix + try: + record, _context, _key = transport.validate_instance_delivery( + delivery, + self.plan, + now_unix=verification_time, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38GcpAdapterError("instance delivery is invalid") from exc + pre_provider, pre_protected = self._provider_inventory() + pre_resources = self._resource_observations(pre_provider) + pre_digest = controller.observation_instance_generations_digest( + pre_resources, + self.plan, + ) + resource_name = record["resource_name"] + observed = pre_resources[resource_name] + iap_firewall = next(item for item in self.plan.resources if item.kind == "iap_firewall") + provider_instance = pre_provider[resource_name] + if ( + not pre_protected + or pre_digest is None + or observed["instance_generation_digest"] != record["instance_generation_digest"] + or provider_instance is None + or provider_instance.get("status") != "RUNNING" + or pre_provider[iap_firewall.name] is None + ): + raise Q38GcpAdapterError("instance delivery provider inventory is not ready") + command = self.compiled_instance_delivery_command( + delivery, + now_unix=verification_time, + ) + result = self.delivery_runner( + command, + delivery.payload, + DELIVERY_TIMEOUT_SECONDS, + ) + if result.returncode != 0: + raise Q38GcpAdapterError("instance delivery failed") + receipt_value = _json_bytes( + result.stdout, + "instance delivery receipt", + maximum=transport.MAX_DELIVERY_HEADER_BYTES, + ) + try: + receipt = transport.validate_instance_delivery_receipt( + receipt_value, + delivery, + self.plan, + now_unix=verification_time, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38GcpAdapterError("instance delivery receipt is invalid") from exc + post_provider, post_protected = self._provider_inventory() + post_resources = self._resource_observations(post_provider) + post_digest = controller.observation_instance_generations_digest( + post_resources, + self.plan, + ) + if ( + not post_protected + or pre_protected != post_protected + or pre_digest != post_digest + or post_resources[resource_name]["instance_generation_digest"] != record["instance_generation_digest"] + or post_provider[iap_firewall.name] is None + ): + raise Q38GcpAdapterError("provider generation changed during instance delivery") + return receipt + + def _delete_resource(self, resource: controller.ResourcePlan) -> None: + value = self._describe(resource) + if value is None: + return + self._validate_resource(resource, value, cleanup=True) + if resource.kind.endswith("instance"): + self._gcloud( + "compute", + "instances", + "delete", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + "--keep-disks=all", + timeout=900, + ) + elif resource.kind.endswith("disk"): + self._gcloud( + "compute", + "disks", + "delete", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + f"--zone={controller.EXPECTED_ZONE}", + timeout=900, + ) + else: + self._gcloud( + "compute", + "firewall-rules", + "delete", + resource.name, + f"--project={controller.EXPECTED_PROJECT}", + timeout=900, + ) + + def _cleanup_route(self) -> None: + errors = 0 + order = ( + [item for item in self.plan.resources if item.kind.endswith("instance")] + + [item for item in self.plan.resources if item.kind.endswith("disk")] + + [item for item in self.plan.resources if item.kind.endswith("firewall")] + ) + for resource in order: + try: + self._delete_resource(resource) + except Q38GcpAdapterError: + errors += 1 + remaining, protected = self._provider_inventory(cleanup=True) + if errors or any(value is not None for value in remaining.values()) or not protected: + raise Q38GcpAdapterError("provider cleanup is incomplete") + + def execute( + self, + state_value: Mapping[str, Any], + decision_value: Mapping[str, Any], + host_status: Mapping[str, Any], + *, + manifest_path: Path | None, + artifact_root: Path | None, + evidence_root: Path | None = None, + ) -> dict[str, Any]: + state = controller.validate_state(state_value, self.plan) + expected_decision = controller.action_record(state, self.plan) + if dict(decision_value) != expected_decision: + raise Q38GcpAdapterError("controller decision is stale or unbound") + action = state["next_action"] + if action in {"start_route", "collect_route"}: + raise Q38GcpAdapterError(RUNTIME_ACTIONS_BLOCKED) + if action == "cleanup_route": + # Cleanup deliberately skips aggregate preflight. Each deletion performs + # its own exact binding check so one foreign resource cannot strand the + # rest of the independently verified, paid run inventory. + self._cleanup_route() + return self.inventory( + blank_host_status(self.plan), + manifest_path=None, + artifact_root=None, + cleanup_only=True, + ) + if action != "none": + raise Q38GcpAdapterError("controller action is unsupported") + cleanup_only = state["phase"] == "CLEANING" or int(self.clock()) >= self.plan.deadline_unix + observation = self.inventory( + host_status, + manifest_path=manifest_path, + artifact_root=artifact_root, + evidence_root=evidence_root, + cleanup_only=cleanup_only, + ) + reconciled = controller.reconcile( + "status", + state, + observation, + self.plan, + now_unix=int(self.clock()), + route_evidence_validated=False, + start_was_issued=False, + ) + if controller.action_record(reconciled, self.plan) != expected_decision: + raise Q38GcpAdapterError("controller action is no longer current") + return observation + + +def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: + payload = (json.dumps(value, allow_nan=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + if len(payload) > MAX_JSON_BYTES: + raise Q38GcpAdapterError("output exceeded its size bound") + path.parent.mkdir(parents=True, exist_ok=True) + parent = path.parent.lstat() + parent_reparse = bool(getattr(parent, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if parent_reparse or path.parent.is_symlink() or not stat.S_ISDIR(parent.st_mode): + raise Q38GcpAdapterError("output parent is unsafe") + if path.exists() or path.is_symlink(): + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise Q38GcpAdapterError("output target is unsafe") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + +def _load_mapping(path: Path) -> Mapping[str, Any]: + return controller._strict_json(controller._regular_bytes(path)) + + +def _assert_distinct_paths(paths: Sequence[Path | None]) -> None: + observed: set[Path] = set() + for path in paths: + if path is None: + continue + resolved = path.resolve() + if resolved in observed: + raise Q38GcpAdapterError("adapter input and output paths must be distinct") + observed.add(resolved) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("observe", "step")) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path) + parser.add_argument("--artifact-root", type=Path) + parser.add_argument("--state", type=Path) + parser.add_argument("--decision", type=Path) + parser.add_argument("--host-status", type=Path) + parser.add_argument("--evidence-root", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cleanup-only", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + _assert_distinct_paths( + ( + args.plan, + args.manifest, + args.state, + args.decision, + args.host_status, + args.output, + ) + ) + plan = controller.load_plan(args.plan, args.source_root) + adapter = GcpAdapter(plan, args.source_root) + if args.state is not None: + controller._assert_protected_path(args.state, plan, args.source_root, directory=False) + if args.decision is not None: + controller._assert_protected_path(args.decision, plan, args.source_root, directory=False) + host_status = load_host_status(args.host_status, plan, args.source_root) + if args.operation == "observe": + result = adapter.inventory( + host_status, + manifest_path=args.manifest, + artifact_root=args.artifact_root, + evidence_root=args.evidence_root, + cleanup_only=args.cleanup_only, + ) + else: + if args.state is None or args.decision is None: + raise Q38GcpAdapterError("step inputs are required") + result = adapter.execute( + _load_mapping(args.state), + _load_mapping(args.decision), + host_status, + manifest_path=args.manifest, + artifact_root=args.artifact_root, + evidence_root=args.evidence_root, + ) + _atomic_json(args.output, result) + except (Q38GcpAdapterError, controller.RouteControllerError) as exc: + raise SystemExit(str(exc)) from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gateq38_linux_host_runtime.py b/scripts/gateq38_linux_host_runtime.py new file mode 100644 index 000000000..56fa184b9 --- /dev/null +++ b/scripts/gateq38_linux_host_runtime.py @@ -0,0 +1,2554 @@ +"""Privileged Linux package preparation for the Qwen3.8 complete-route attempt. + +This host-only helper consumes the controller-protected final plan and exact provider +action. It verifies and extracts the plan-bound packaged node, protects it from the +ordinary qualification identity, performs one offline help preflight, and emits a +digest-only prepared record. It never provisions resources or downloads model data. +""" + +from __future__ import annotations + +import argparse +import hashlib +import http.client +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, BinaryIO, Callable, Iterator, Mapping, Sequence + +from scripts import gateq38_linux_host_transport as transport, gateq38_route_controller as controller + +SCHEMA_VERSION = 1 +PREPARED_SCOPE = "qwen3.8-linux-host-runtime-prepared" +CLEANUP_SCOPE = "qwen3.8-linux-host-runtime-cleanup-terminal" +PUBLICATION_SCOPE = "qwen3.8-linux-host-status-publication" +DELIVERY_INSTALL_SCOPE = "qwen3.8-linux-instance-delivery-install" +QUALIFICATION_USER = "communityai-q38" +MAX_JSON_BYTES = controller.MAX_JSON_BYTES +MAX_PROVENANCE_BYTES = controller.MAX_RELEASE_PROVENANCE_BYTES +MAX_CHECKSUMS_BYTES = controller.MAX_RELEASE_CHECKSUMS_BYTES +MAX_METRICS_BYTES = controller.MAX_RELEASE_METRICS_BYTES +MAX_OUTPUT_BYTES = 1_048_576 +MAX_ARCHIVE_ENTRIES = 100_000 +MAX_ARCHIVE_BYTES = 8 * 1024**3 +MAX_EXPANDED_BYTES = 16 * 1024**3 +HASH_CHUNK_BYTES = 1_048_576 +KILL_SIGNAL = getattr(signal, "SIGKILL", 9) +RUNTIME_BASE = Path("/opt/communityai-q38") +INPUT_BASE = Path("/var/lib/communityai-q38/input") +WORK_BASE = Path("/var/lib/communityai-q38/work") +STATE_BASE = Path("/var/lib/communityai-q38/state") +BOOT_ID_PATH = Path("/proc/sys/kernel/random/boot_id") +METADATA_HOST = "169.254.169.254" +METADATA_PORT = 80 +GUEST_ATTRIBUTE_PATH = "/computeMetadata/v1/instance/guest-attributes/communityai-q38/status-v1" +METADATA_TIMEOUT_SECONDS = 5.0 +MAX_METADATA_RESPONSE_BYTES = 4_096 +_HEX_DIGEST_RE = re.compile(r"[0-9a-f]{64}") +_PREPARED_FIELDS = { + "schema_version", + "scope", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "start_action_id", + "instance_context_digest", + "resource_name", + "resource_kind", + "worker_id", + "instance_generation_digest", + "boot_id", + "runtime_package_digest", + "release_archive_sha256", + "node_executable_sha256", + "node_runtime_inventory_digest", + "node_runtime_entry_count", + "node_runtime_bytes", + "qualification_user", + "qualification_uid", + "qualification_gid", + "preflight_returncode", + "preflight_stdout_sha256", + "preflight_stdout_bytes", + "preflight_stderr_sha256", + "preflight_stderr_bytes", + "prepared_record_digest", +} + + +class Q38LinuxHostRuntimeError(RuntimeError): + """The protected package, host action, preflight, or cleanup failed closed.""" + + +@dataclass(frozen=True) +class HostPaths: + plan: Path + start_action: Path + cleanup_action: Path + source_root: Path + release_root: Path + manifest: Path + runtime_base: Path + work_base: Path + prepared_record: Path + instance_context: Path + transport_key: Path + status_envelope: Path + boot_id: Path + transport_bundle: Path | None = None + + @classmethod + def production(cls) -> "HostPaths": + return cls( + plan=INPUT_BASE / "route-plan.json", + start_action=INPUT_BASE / "start-action.json", + cleanup_action=INPUT_BASE / "cleanup-action.json", + source_root=INPUT_BASE / "source", + release_root=INPUT_BASE / "release", + manifest=INPUT_BASE / "manifest.json", + runtime_base=RUNTIME_BASE, + work_base=WORK_BASE, + prepared_record=STATE_BASE / "prepared.json", + instance_context=INPUT_BASE / "instance-context.json", + transport_key=INPUT_BASE / "host-status.key", + status_envelope=STATE_BASE / "host-status.json", + boot_id=BOOT_ID_PATH, + transport_bundle=INPUT_BASE / "instance-delivery.bin", + ) + + +@dataclass(frozen=True) +class Artifact: + path: str + kind: str + sha256: str + size_bytes: int + mode: int | None + link_target: str | None + + +@dataclass(frozen=True) +class QualificationIdentity: + name: str + uid: int + gid: int + + +@dataclass(frozen=True) +class PreflightResult: + returncode: int + stdout: bytes + stderr: bytes + + +@dataclass(frozen=True) +class TransportInputs: + context: dict[str, Any] + key: bytes + boot_id: str + + +def _reject_constant(_value: str) -> None: + raise Q38LinuxHostRuntimeError("non-finite JSON value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Q38LinuxHostRuntimeError("duplicate JSON field") + result[key] = value + return result + + +def _strict_json(payload: bytes, *, maximum: int = MAX_JSON_BYTES) -> dict[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= maximum: + raise Q38LinuxHostRuntimeError("JSON payload exceeded its bound") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Q38LinuxHostRuntimeError("JSON payload is invalid") from exc + if not isinstance(value, dict): + raise Q38LinuxHostRuntimeError("JSON root must be an object") + return value + + +def _canonical_bytes(value: Any) -> bytes: + try: + return ( + json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Q38LinuxHostRuntimeError("value is not canonical JSON") from exc + + +def _sha256(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _sha256_stream(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(HASH_CHUNK_BYTES), b""): + digest.update(chunk) + except OSError as exc: + raise Q38LinuxHostRuntimeError("runtime file is unreadable") from exc + return "sha256:" + digest.hexdigest() + + +def _same_json_value(actual: Any, expected: Any) -> bool: + if type(actual) is not type(expected): + return False + if isinstance(actual, dict): + return set(actual) == set(expected) and all(_same_json_value(actual[key], expected[key]) for key in actual) + if isinstance(actual, list): + return len(actual) == len(expected) and all( + _same_json_value(left, right) for left, right in zip(actual, expected) + ) + return actual == expected + + +def _file_identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + stat.S_IFMT(metadata.st_mode), + metadata.st_size, + getattr(metadata, "st_mtime_ns", int(metadata.st_mtime * 1_000_000_000)), + ) + + +def _is_reparse(path: Path, metadata: os.stat_result) -> bool: + return ( + bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + or path.is_symlink() + ) + + +@contextmanager +def _verified_file( + path: Path, + *, + expected_size: int | None = None, + expected_digest: str | None = None, + maximum: int | None = None, +) -> Iterator[tuple[BinaryIO, bytes | None]]: + descriptor: int | None = None + try: + before = path.lstat() + if ( + _is_reparse(path, before) + or not stat.S_ISREG(before.st_mode) + or (expected_size is not None and before.st_size != expected_size) + or (maximum is not None and not 1 <= before.st_size <= maximum) + ): + raise Q38LinuxHostRuntimeError(f"required file is unsafe: {path.name}") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _file_identity(opened) != _file_identity(before): + raise Q38LinuxHostRuntimeError("required file identity changed while opened") + stream = os.fdopen(descriptor, "rb", closefd=False) + payload: bytes | None = None + if maximum is not None: + payload = stream.read(maximum + 1) + if len(payload) != opened.st_size: + raise Q38LinuxHostRuntimeError("required file changed while read") + stream.seek(0) + if expected_digest is not None: + digest = hashlib.sha256() + total = 0 + for chunk in iter(lambda: stream.read(HASH_CHUNK_BYTES), b""): + total += len(chunk) + if expected_size is not None and total > expected_size: + raise Q38LinuxHostRuntimeError("required file grew while hashed") + digest.update(chunk) + if ( + expected_size is not None + and total != expected_size + or "sha256:" + digest.hexdigest() != expected_digest + ): + raise Q38LinuxHostRuntimeError("required file binding changed") + stream.seek(0) + after = os.fstat(descriptor) + final = path.lstat() + if ( + _is_reparse(path, final) + or not stat.S_ISREG(final.st_mode) + or _file_identity(after) != _file_identity(opened) + or _file_identity(final) != _file_identity(opened) + ): + raise Q38LinuxHostRuntimeError("required file identity changed while verified") + yield stream, payload + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError(f"required file is unreadable: {path.name}") from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +def _regular_bytes(path: Path, *, maximum: int = MAX_JSON_BYTES) -> bytes: + with _verified_file(path, maximum=maximum) as (_stream, payload): + if payload is None: + raise AssertionError("bounded read omitted its payload") + return payload + + +def _assert_root_private_file(path: Path) -> None: + _assert_root_managed(path, directory=False) + if os.name != "posix": + return + metadata = path.lstat() + if ( + _is_reparse(path, metadata) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != 0 + or metadata.st_gid != 0 + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise Q38LinuxHostRuntimeError(f"private host input is unsafe: {path.name}") + + +def _read_boot_id(path: Path) -> str: + descriptor: int | None = None + try: + before = path.lstat() + if _is_reparse(path, before) or not stat.S_ISREG(before.st_mode): + raise Q38LinuxHostRuntimeError("Linux boot identity source is unsafe") + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _file_identity(opened) != _file_identity(before): + raise Q38LinuxHostRuntimeError("Linux boot identity changed while opened") + payload = os.read(descriptor, 129) + after = os.fstat(descriptor) + final = path.lstat() + if ( + _is_reparse(path, final) + or _file_identity(after) != _file_identity(opened) + or _file_identity(final) != _file_identity(opened) + ): + raise Q38LinuxHostRuntimeError("Linux boot identity changed while read") + try: + value = payload.decode("ascii") + except UnicodeDecodeError as exc: + raise Q38LinuxHostRuntimeError("Linux boot identity is invalid") from exc + if len(payload) != 37 or not value.endswith("\n") or transport._BOOT_ID_RE.fullmatch(value[:-1]) is None: + raise Q38LinuxHostRuntimeError("Linux boot identity is invalid") + return value[:-1] + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError("Linux boot identity is unreadable") from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +def _load_authenticated_context( + plan: controller.RoutePlan, + paths: HostPaths, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, + allow_expired_for_cleanup: bool, +) -> tuple[dict[str, Any], bytes]: + bundle_path = paths.transport_bundle + if bundle_path is not None: + if bundle_path.parent != paths.plan.parent: + raise Q38LinuxHostRuntimeError("transport bundle is outside the protected input boundary") + _assert_root_managed(bundle_path.parent, directory=True) + _assert_root_private_file(bundle_path) + try: + payload = _regular_bytes( + bundle_path, + maximum=transport.MAX_DELIVERY_BYTES, + ) + _record, context, key = transport.decode_instance_delivery( + payload, + plan, + now_unix=now_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + allow_expired_for_cleanup=allow_expired_for_cleanup, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + return context, key + if ( + paths.instance_context.parent != paths.transport_key.parent + or paths.instance_context.parent != paths.plan.parent + ): + raise Q38LinuxHostRuntimeError("transport inputs are outside the protected input boundary") + _assert_root_managed(paths.instance_context.parent, directory=True) + _assert_root_private_file(paths.instance_context) + _assert_root_private_file(paths.transport_key) + try: + with _verified_file(paths.instance_context, maximum=transport.MAX_ENVELOPE_BYTES,) as ( + _context_stream, + context_payload, + ), _verified_file(paths.transport_key, expected_size=transport.KEY_BYTES,) as (key_stream, _key_payload): + if context_payload is None: + raise AssertionError("instance context payload was not read") + key = key_stream.read(transport.KEY_BYTES + 1) + if len(key) != transport.KEY_BYTES: + raise Q38LinuxHostRuntimeError("transport key is invalid") + context = transport.decode_instance_context(context_payload) + validated = transport.validate_instance_context( + context, + plan, + key=key, + now_unix=now_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + _allow_expired_for_cleanup=allow_expired_for_cleanup, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + return validated, key + + +def _load_transport_inputs( + plan: controller.RoutePlan, + paths: HostPaths, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, +) -> TransportInputs: + context, key = _load_authenticated_context( + plan, + paths, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=now_unix, + allow_expired_for_cleanup=False, + ) + return TransportInputs(context=context, key=key, boot_id=_read_boot_id(paths.boot_id)) + + +def _safe_member_path(raw: Any, *, allow_root: bool = False) -> str: + if not isinstance(raw, str) or not raw or "\\" in raw or any(ord(character) < 32 for character in raw): + raise Q38LinuxHostRuntimeError("package path is unsafe") + normalized = raw[:-1] if raw.endswith("/") else raw + pure = PurePosixPath(normalized) + if ( + pure.is_absolute() + or pure.as_posix() != normalized + or any(part in {"", ".", ".."} for part in pure.parts) + or not pure.parts + or pure.parts[0] != "CommunityAI" + or (len(pure.parts) == 1 and not allow_root) + ): + raise Q38LinuxHostRuntimeError("package path is unsafe") + return normalized + + +def _canonical_link_target(member_path: str, raw_target: Any) -> str: + if ( + not isinstance(raw_target, str) + or not raw_target + or raw_target.startswith("/") + or "\\" in raw_target + or any(ord(character) < 32 for character in raw_target) + ): + raise Q38LinuxHostRuntimeError("package symlink is unsafe") + parts: list[str] = [] + for part in (PurePosixPath(member_path).parent / PurePosixPath(raw_target)).parts: + if part in {"", "."}: + continue + if part == "..": + if not parts: + raise Q38LinuxHostRuntimeError("package symlink escapes its root") + parts.pop() + else: + parts.append(part) + return _safe_member_path(PurePosixPath(*parts).as_posix()) + + +def _digest_field(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.startswith("sha256:") or _HEX_DIGEST_RE.fullmatch(value[7:]) is None: + raise Q38LinuxHostRuntimeError(f"{field} is invalid") + return value + + +def _positive_integer(value: Any, field: str) -> int: + if type(value) is not int or value <= 0: + raise Q38LinuxHostRuntimeError(f"{field} is invalid") + return value + + +def _artifact(raw: Any) -> Artifact: + if not isinstance(raw, dict): + raise Q38LinuxHostRuntimeError("release artifact is invalid") + kind = raw.get("kind") + expected = ( + {"path", "kind", "mode", "sha256", "size_bytes"} + if kind == "file" + else {"path", "kind", "link_target", "sha256", "size_bytes"} + if kind == "symlink" + else set() + ) + if not expected or set(raw) != expected: + raise Q38LinuxHostRuntimeError("release artifact schema is invalid") + path = _safe_member_path(raw["path"]) + digest = raw["sha256"] + if not isinstance(digest, str) or _HEX_DIGEST_RE.fullmatch(digest) is None: + raise Q38LinuxHostRuntimeError("release artifact digest is invalid") + size = _positive_integer(raw["size_bytes"], "release artifact size") + if kind == "file": + mode = raw["mode"] + if type(mode) is not int or mode not in {0o644, 0o755}: + raise Q38LinuxHostRuntimeError("release artifact mode is unsafe") + return Artifact(path, kind, digest, size, mode, None) + target = _safe_member_path(raw["link_target"]) + return Artifact(path, kind, digest, size, None, target) + + +def _assert_root_managed(path: Path, *, directory: bool) -> None: + if not path.is_absolute(): + raise Q38LinuxHostRuntimeError("protected path is not absolute") + try: + target = path.lstat() + except OSError as exc: + raise Q38LinuxHostRuntimeError("protected path is unavailable") from exc + if _is_reparse(path, target): + raise Q38LinuxHostRuntimeError("protected path is linked") + expected_type = stat.S_ISDIR if directory else stat.S_ISREG + if not expected_type(target.st_mode): + raise Q38LinuxHostRuntimeError("protected path type is invalid") + if target.st_uid != 0 or target.st_gid != 0 or stat.S_IMODE(target.st_mode) & 0o022: + raise Q38LinuxHostRuntimeError("protected path is writable by the qualification identity") + current = path.parent + while current != current.parent: + metadata = current.lstat() + if ( + _is_reparse(current, metadata) + or not stat.S_ISDIR(metadata.st_mode) + or metadata.st_uid != 0 + or metadata.st_gid != 0 + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + raise Q38LinuxHostRuntimeError("protected path parent is unsafe") + current = current.parent + + +def _assert_qualification_traversal(path: Path) -> None: + current = path + while current != current.parent: + metadata = current.lstat() + if ( + _is_reparse(current, metadata) + or not stat.S_ISDIR(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) & 0o001 == 0 + ): + raise Q38LinuxHostRuntimeError("preflight parent is not traversable by the qualification identity") + current = current.parent + + +def _assert_source_bound(plan: controller.RoutePlan, source_root: Path) -> None: + root = source_root.resolve(strict=True) + bindings = {item["relative_path"]: item for item in plan.source_bindings} + imported = { + controller.LINUX_HOST_RUNTIME_SOURCE_PATH: Path(__file__).resolve(), + controller.LINUX_HOST_TRANSPORT_SOURCE_PATH: Path(transport.__file__).resolve(), + "scripts/gateq38_route_controller.py": Path(controller.__file__).resolve(), + } + _assert_root_managed(root, directory=True) + for relative, module in imported.items(): + expected = (root / Path(*PurePosixPath(relative).parts)).resolve(strict=True) + binding = bindings.get(relative) + if module != expected or binding is None: + raise Q38LinuxHostRuntimeError("imported host sources are not plan-bound") + _assert_root_managed(expected, directory=False) + payload = _regular_bytes(expected) + if len(payload) != binding["byte_size"] or _sha256(payload) != binding["sha256"]: + raise Q38LinuxHostRuntimeError("host source binding changed") + + +def _load_plan_and_action( + plan_path: Path, + action_path: Path, + source_root: Path, + *, + expected_action: str, + now_unix: int, +) -> tuple[controller.RoutePlan, dict[str, Any]]: + if expected_action not in {"start_route", "cleanup_route"}: + raise Q38LinuxHostRuntimeError("host action is invalid") + initial_plan = _regular_bytes(plan_path) + try: + plan = controller.load_plan(plan_path, source_root) + except controller.RouteControllerError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + _assert_root_managed(plan_path.parent, directory=True) + _assert_root_managed(plan_path, directory=False) + if _regular_bytes(plan_path) != initial_plan: + raise Q38LinuxHostRuntimeError("route plan changed while loaded") + _assert_source_bound(plan, source_root) + if type(now_unix) is not int or now_unix <= 0: + raise Q38LinuxHostRuntimeError("trusted current time is invalid") + if expected_action == "start_route" and now_unix >= plan.deadline_unix: + raise Q38LinuxHostRuntimeError("route plan is expired") + if expected_action == "start_route" and any( + plan.authorization[field] is not True + for field in ( + "reservation_recorded", + "native_auth_revalidated", + "inventory_revalidated", + "pricing_revalidated", + "provisioning_authorized", + ) + ): + raise Q38LinuxHostRuntimeError("route start is not fully authorized") + + action_payload = _regular_bytes(action_path) + _assert_root_managed(action_path.parent, directory=True) + _assert_root_managed(action_path, directory=False) + if _regular_bytes(action_path) != action_payload: + raise Q38LinuxHostRuntimeError("host action changed while loaded") + action = _strict_json(action_payload) + revision = action.get("revision") + if type(revision) is not int or revision < 0: + raise Q38LinuxHostRuntimeError("host action revision is invalid") + expected = controller.action_record( + {"revision": revision, "next_action": expected_action}, + plan, + ) + if not _same_json_value(action, expected): + raise Q38LinuxHostRuntimeError("host action is not the exact controller action") + return plan, action + + +def _load_release_inventory( + plan: controller.RoutePlan, + paths: HostPaths, +) -> tuple[list[Artifact], list[Artifact]]: + package = plan.runtime_package + _assert_root_managed(paths.release_root, directory=True) + bound_files = { + "SHA256SUMS": ( + package["checksums_bytes"], + package["checksums_sha256"], + MAX_CHECKSUMS_BYTES, + ), + "provenance.json": ( + package["provenance_bytes"], + package["provenance_sha256"], + MAX_PROVENANCE_BYTES, + ), + "desktop-metrics.json": ( + package["desktop_metrics_bytes"], + package["desktop_metrics_sha256"], + MAX_METRICS_BYTES, + ), + } + payloads: dict[str, bytes] = {} + for name, (size, digest, maximum) in bound_files.items(): + candidate = paths.release_root / name + _assert_root_managed(candidate, directory=False) + with _verified_file(candidate, expected_size=size, expected_digest=digest, maximum=maximum) as ( + _stream, + payload, + ): + if payload is None: + raise AssertionError("release companion payload was not read") + payloads[name] = payload + _assert_root_managed(paths.manifest.parent, directory=True) + _assert_root_managed(paths.manifest, directory=False) + with _verified_file( + paths.manifest, + expected_size=package["manifest_bytes"], + expected_digest=package["manifest_sha256"], + maximum=MAX_JSON_BYTES, + ) as (_stream, manifest_payload): + if manifest_payload is None: + raise AssertionError("manifest payload was not read") + manifest = _strict_json(manifest_payload) + source = manifest.get("source") + model = manifest.get("model") + if ( + type(manifest.get("schema_version")) is not int + or manifest["schema_version"] != 1 + or not isinstance(source, dict) + or source.get("revision") != plan.model_revision + or not isinstance(model, dict) + or model.get("num_blocks") != 64 + ): + raise Q38LinuxHostRuntimeError("manifest physical identity is inconsistent") + + provenance = _strict_json(payloads["provenance.json"], maximum=MAX_PROVENANCE_BYTES) + if provenance.get("source_commit") != plan.source_commit or provenance.get("source_tree") != package["source_tree"]: + raise Q38LinuxHostRuntimeError("release provenance source identity changed") + raw_artifacts = provenance.get("artifacts") + if not isinstance(raw_artifacts, list) or not raw_artifacts: + raise Q38LinuxHostRuntimeError("release artifact inventory is invalid") + artifacts = [_artifact(value) for value in raw_artifacts] + paths_list = [item.path for item in artifacts] + if paths_list != sorted(paths_list) or len({item.casefold() for item in paths_list}) != len(paths_list): + raise Q38LinuxHostRuntimeError("release artifact paths are not canonical") + artifact_map = {item.path: item for item in artifacts} + for item in artifacts: + if item.kind == "symlink": + if item.link_target not in artifact_map or artifact_map[item.link_target].kind != "file": + raise Q38LinuxHostRuntimeError("release symlink target is invalid") + expected_checksums = "".join(f"{item.sha256} {item.path}\n" for item in artifacts).encode("utf-8") + if payloads["SHA256SUMS"] != expected_checksums: + raise Q38LinuxHostRuntimeError("release checksum inventory changed") + + node_prefix = package["node_root"] + "/" + node = [item for item in artifacts if item.path == package["node_executable"] or item.path.startswith(node_prefix)] + raw_node = [value for value in raw_artifacts if value.get("path") in {item.path for item in node}] + raw_node.sort(key=lambda item: item["path"]) + if ( + len(node) != package["node_runtime_entry_count"] + or sum(item.size_bytes for item in node) != package["node_runtime_bytes"] + or _sha256(_canonical_bytes(raw_node)) != package["node_runtime_inventory_digest"] + ): + raise Q38LinuxHostRuntimeError("node runtime inventory changed") + executable = artifact_map.get(package["node_executable"]) + if ( + executable is None + or executable.kind != "file" + or executable.sha256 != package["node_executable_sha256"][7:] + or executable.size_bytes != package["node_executable_bytes"] + or executable.mode != 0o755 + ): + raise Q38LinuxHostRuntimeError("node executable identity changed") + return artifacts, node + + +def _audit_members( + source: tarfile.TarFile, + artifacts: Sequence[Artifact], +) -> dict[str, tarfile.TarInfo]: + artifact_map = {item.path: item for item in artifacts} + members: dict[str, tarfile.TarInfo] = {} + folded_paths: set[str] = set() + total_bytes = 0 + for member in source: + if len(members) >= MAX_ARCHIVE_ENTRIES: + raise Q38LinuxHostRuntimeError("archive entry count exceeded its bound") + path = _safe_member_path(member.name, allow_root=True) + folded = path.casefold() + if folded in folded_paths: + raise Q38LinuxHostRuntimeError("archive contains duplicate members") + folded_paths.add(folded) + if ( + member.isdev() + or member.isfifo() + or getattr(member, "sparse", None) + or not (member.isdir() or member.isfile() or member.issym() or member.islnk()) + ): + raise Q38LinuxHostRuntimeError("archive member type is unsafe") + if stat.S_IMODE(member.mode) & 0o7000 or (not member.isfile() and member.size != 0): + raise Q38LinuxHostRuntimeError("archive member mode or size is unsafe") + effective_size = member.size + if member.islnk(): + target = _safe_member_path(member.linkname) + prior = members.get(target) + if member.linkname != target or prior is None or not prior.isfile(): + raise Q38LinuxHostRuntimeError("archive hardlink target is not a prior regular member") + effective_size = prior.size + members[path] = member + if member.isfile() or member.islnk(): + total_bytes += effective_size + if total_bytes > MAX_EXPANDED_BYTES: + raise Q38LinuxHostRuntimeError("archive expanded size exceeded its bound") + payload_members = {path: member for path, member in members.items() if not member.isdir()} + if set(payload_members) != set(artifact_map): + raise Q38LinuxHostRuntimeError("archive payload inventory changed") + if total_bytes != sum(item.size_bytes for item in artifacts if item.kind == "file"): + raise Q38LinuxHostRuntimeError("archive expanded size changed") + verified_regular: set[str] = set() + for path, member in payload_members.items(): + artifact = artifact_map[path] + if artifact.kind == "file": + effective_size = member.size + if member.islnk(): + target = artifact_map.get(member.linkname) + if ( + member.linkname not in verified_regular + or target is None + or target.kind != "file" + or (target.sha256, target.size_bytes, target.mode) + != (artifact.sha256, artifact.size_bytes, artifact.mode) + ): + raise Q38LinuxHostRuntimeError("archive hardlink target is not a verified identical file") + effective_size = target.size_bytes + if ( + not (member.isfile() or member.islnk()) + or effective_size != artifact.size_bytes + or stat.S_IMODE(member.mode) != artifact.mode + ): + raise Q38LinuxHostRuntimeError("archive file identity changed") + stream = source.extractfile(member) + if stream is None: + raise Q38LinuxHostRuntimeError("archive file is unreadable") + digest = hashlib.sha256() + for chunk in iter(lambda: stream.read(HASH_CHUNK_BYTES), b""): + digest.update(chunk) + if digest.hexdigest() != artifact.sha256: + raise Q38LinuxHostRuntimeError("archive file digest changed") + if member.isfile(): + verified_regular.add(path) + elif not member.issym() or _canonical_link_target(path, member.linkname) != artifact.link_target: + raise Q38LinuxHostRuntimeError("archive symlink identity changed") + return members + + +def _remove_tree_strict(path: Path, message: str) -> None: + try: + if path.is_symlink(): + raise Q38LinuxHostRuntimeError(message) + if path.exists(): + shutil.rmtree(path) + if path.exists() or path.is_symlink(): + raise Q38LinuxHostRuntimeError(message) + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError(message) from exc + + +def _extract_verified_archive( + archive: Path, + package: Mapping[str, Any], + artifacts: Sequence[Artifact], + node: Sequence[Artifact], + destination: Path, +) -> None: + if destination.exists() or destination.is_symlink(): + raise Q38LinuxHostRuntimeError("runtime staging destination is not empty") + if ( + type(package["release_archive_bytes"]) is not int + or not 1 <= package["release_archive_bytes"] <= MAX_ARCHIVE_BYTES + or sum(item.size_bytes for item in artifacts) > MAX_EXPANDED_BYTES + ): + raise Q38LinuxHostRuntimeError("archive size exceeds the host-stage bound") + _assert_root_managed(archive, directory=False) + destination.mkdir(mode=0o700, parents=False) + node_paths = {item.path for item in node} + try: + with _verified_file( + archive, + expected_size=package["release_archive_bytes"], + expected_digest=package["release_archive_sha256"], + ) as (handle, _payload): + with tarfile.open(fileobj=handle, mode="r:gz") as source: + _audit_members(source, artifacts) + handle.seek(0) + node_map = {item.path: item for item in node} + pending_links: list[tuple[Artifact, str]] = [] + with tarfile.open(fileobj=handle, mode="r:gz") as source: + for member in source: + path = _safe_member_path(member.name, allow_root=True) + artifact = node_map.get(path) + if member.isdir() and ( + path == "CommunityAI" + or path == package["node_root"] + or package["node_root"].startswith(path + "/") + ): + target = destination.joinpath(*PurePosixPath(path).parts) + target.mkdir(mode=0o755, parents=True, exist_ok=True) + elif artifact is not None and artifact.kind == "file": + target = destination.joinpath(*PurePosixPath(path).parts) + target.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + if member.islnk(): + if member.linkname not in node_paths: + raise Q38LinuxHostRuntimeError("node hardlink leaves the runtime inventory") + prior = destination.joinpath(*PurePosixPath(member.linkname).parts) + if not stat.S_ISREG(prior.lstat().st_mode): + raise Q38LinuxHostRuntimeError("node hardlink target is not a regular file") + os.link(prior, target, follow_symlinks=False) + continue + stream = source.extractfile(member) + if stream is None: + raise Q38LinuxHostRuntimeError("archive file is unreadable") + with target.open("xb") as output: + shutil.copyfileobj(stream, output, length=HASH_CHUNK_BYTES) + output.flush() + os.fsync(output.fileno()) + os.chmod(target, artifact.mode or 0o644) + elif artifact is not None and artifact.kind == "symlink": + if artifact.link_target not in node_paths: + raise Q38LinuxHostRuntimeError("node symlink leaves the runtime inventory") + pending_links.append((artifact, member.linkname)) + for artifact, linkname in pending_links: + target = destination.joinpath(*PurePosixPath(artifact.path).parts) + target.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + target.symlink_to(linkname) + except BaseException: + _remove_tree_strict(destination, "runtime staging cleanup is incomplete") + raise + + +def _verify_runtime_tree( + root: Path, + node: Sequence[Artifact], + *, + protected: bool, +) -> tuple[int, int]: + product = root / "CommunityAI" + runtime = product / "node" + for directory in (root, product, runtime): + if not directory.is_dir() or directory.is_symlink(): + raise Q38LinuxHostRuntimeError("installed node runtime is unsafe") + metadata = directory.lstat() + if protected and (metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) != 0o755): + raise Q38LinuxHostRuntimeError("runtime ancestor protection changed") + if {item.name for item in root.iterdir()} != {"CommunityAI"} or {item.name for item in product.iterdir()} != { + "node" + }: + raise Q38LinuxHostRuntimeError("runtime ancestor inventory changed") + expected = {item.path: item for item in node} + observed: set[str] = set() + folded_observed: set[str] = set() + resolved_root = runtime.resolve(strict=True) + for directory, names, files in os.walk(runtime, topdown=True, followlinks=False): + base = Path(directory) + metadata = base.lstat() + if base.is_symlink() or not stat.S_ISDIR(metadata.st_mode): + raise Q38LinuxHostRuntimeError("runtime directory is unsafe") + if protected and (metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) != 0o755): + raise Q38LinuxHostRuntimeError("runtime directory protection changed") + for name in names: + child = base / name + child_metadata = child.lstat() + if child.is_symlink() or not stat.S_ISDIR(child_metadata.st_mode): + raise Q38LinuxHostRuntimeError("runtime contains an unsafe directory") + for name in files: + candidate = base / name + relative = "CommunityAI/node/" + candidate.relative_to(runtime).as_posix() + artifact = expected.get(relative) + folded = relative.casefold() + if artifact is None or folded in folded_observed: + raise Q38LinuxHostRuntimeError("runtime contains an extra or duplicate entry") + observed.add(relative) + folded_observed.add(folded) + metadata = candidate.lstat() + if artifact.kind == "file": + if ( + candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_size != artifact.size_bytes + or _sha256_stream(candidate) != "sha256:" + artifact.sha256 + or protected + and ( + metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) != artifact.mode + ) + ): + raise Q38LinuxHostRuntimeError("runtime file identity changed") + else: + if not candidate.is_symlink(): + raise Q38LinuxHostRuntimeError("runtime symlink identity changed") + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(resolved_root) + except (OSError, RuntimeError, ValueError) as exc: + raise Q38LinuxHostRuntimeError("runtime symlink escaped its root") from exc + if ( + _canonical_link_target(relative, os.readlink(candidate)) != artifact.link_target + or _sha256_stream(resolved) != "sha256:" + artifact.sha256 + or protected + and (metadata.st_uid != 0 or metadata.st_gid != 0) + ): + raise Q38LinuxHostRuntimeError("runtime symlink target changed") + if observed != set(expected): + raise Q38LinuxHostRuntimeError("runtime entry inventory is incomplete") + return len(observed), sum(item.size_bytes for item in node) + + +def _protect_runtime(root: Path, node: Sequence[Artifact]) -> None: + if not hasattr(os, "geteuid") or os.geteuid() != 0: + raise Q38LinuxHostRuntimeError("runtime protection requires root") + artifact_map = {item.path: item for item in node} + for directory, names, files in os.walk(root, topdown=False, followlinks=False): + base = Path(directory) + for name in files: + candidate = base / name + relative = candidate.relative_to(root).as_posix() + if candidate.is_symlink(): + os.chown(candidate, 0, 0, follow_symlinks=False) + else: + artifact = artifact_map.get(relative) + if artifact is None or artifact.mode is None: + raise Q38LinuxHostRuntimeError("runtime protection target is unbound") + os.chown(candidate, 0, 0) + os.chmod(candidate, artifact.mode) + for name in names: + candidate = base / name + if candidate.is_symlink(): + raise Q38LinuxHostRuntimeError("runtime contains a directory symlink") + os.chown(candidate, 0, 0) + os.chmod(candidate, 0o755) + os.chown(base, 0, 0) + os.chmod(base, 0o755) + + +def _runtime_key(plan: controller.RoutePlan) -> str: + package_digest = plan.runtime_package["runtime_package_digest"] + if ( + not isinstance(package_digest, str) + or not package_digest.startswith("sha256:") + or _HEX_DIGEST_RE.fullmatch(package_digest[7:]) is None + ): + raise Q38LinuxHostRuntimeError("runtime package destination digest is invalid") + return hashlib.sha256(f"{plan.plan_digest}\0{package_digest}".encode("ascii")).hexdigest() + + +def _runtime_destination(plan: controller.RoutePlan, paths: HostPaths) -> Path: + return paths.runtime_base / _runtime_key(plan) + + +def _qualification_identity() -> QualificationIdentity: + try: + import pwd + + account = pwd.getpwnam(QUALIFICATION_USER) + except (ImportError, KeyError) as exc: + raise Q38LinuxHostRuntimeError("qualification identity is unavailable") from exc + if account.pw_uid <= 0 or account.pw_gid <= 0: + raise Q38LinuxHostRuntimeError("qualification identity must be non-root") + return QualificationIdentity(QUALIFICATION_USER, account.pw_uid, account.pw_gid) + + +def _empty_tree(path: Path) -> bool: + return path.is_dir() and not any(path.rglob("*")) + + +def _assert_executable_handle( + supplied: os.stat_result, + opened: os.stat_result, +) -> None: + if ( + _file_identity(opened) != _file_identity(supplied) + or opened.st_uid != 0 + or opened.st_gid != 0 + or stat.S_IMODE(opened.st_mode) != 0o755 + ): + raise Q38LinuxHostRuntimeError("packaged node executable protection changed") + + +def _preflight_child(identity: QualificationIdentity) -> Callable[[], None]: + def configure() -> None: + import resource + + os.setsid() + os.setgroups([]) + os.setgid(identity.gid) + os.setuid(identity.uid) + os.umask(0o077) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_FSIZE, (MAX_OUTPUT_BYTES, MAX_OUTPUT_BYTES)) + + return configure + + +def _kill_process_group(process_group: int) -> None: + try: + os.killpg(process_group, KILL_SIGNAL) + except ProcessLookupError: + return + except OSError as exc: + raise Q38LinuxHostRuntimeError("preflight process group could not be killed") from exc + + +def _prove_process_group_empty( + process_group: int, + *, + timeout: float = 5.0, + clock: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, +) -> None: + deadline = clock() + timeout + while clock() < deadline: + try: + os.killpg(process_group, 0) + except ProcessLookupError: + return + except OSError as exc: + raise Q38LinuxHostRuntimeError("preflight process group could not be inspected") from exc + sleeper(0.05) + raise Q38LinuxHostRuntimeError("preflight process group cleanup is incomplete") + + +def _cleanup_started_process(process: Any, *, leader_reaped: bool) -> None: + _kill_process_group(process.pid) + if not leader_reaped: + try: + process.wait(timeout=30) + except BaseException as exc: + raise Q38LinuxHostRuntimeError("packaged preflight leader could not be reaped") from exc + _prove_process_group_empty(process.pid) + + +def _remove_preflight_work(work: Path) -> None: + _remove_tree_strict(work, "preflight work cleanup is incomplete") + + +def _create_preflight_work( + work: Path, + identity: QualificationIdentity, +) -> tuple[Path, Path, Path]: + created = False + try: + work.mkdir(mode=0o711, parents=False, exist_ok=False) + created = True + os.chown(work, 0, 0) + os.chmod(work, 0o711) + _assert_root_managed(work, directory=True) + _assert_qualification_traversal(work) + home = work / "home" + cache = work / "cache" + temporary = work / "tmp" + for candidate in (home, cache, temporary): + candidate.mkdir(mode=0o700, parents=False, exist_ok=False) + os.chown(candidate, identity.uid, identity.gid) + os.chmod(candidate, 0o700) + return home, cache, temporary + except BaseException: + if created: + _remove_preflight_work(work) + raise + + +def _run_packaged_preflight( + plan: controller.RoutePlan, + runtime: Path, + node: Sequence[Artifact], + paths: HostPaths, + identity: QualificationIdentity, + *, + popen_factory: Callable[..., Any] = subprocess.Popen, +) -> PreflightResult: + _verify_runtime_tree(runtime, node, protected=True) + executable = runtime.joinpath(*PurePosixPath(plan.runtime_package["node_executable"]).parts) + work = paths.work_base / _runtime_key(plan) + if work.exists() or work.is_symlink(): + raise Q38LinuxHostRuntimeError("preflight work root is not empty") + paths.work_base.mkdir(mode=0o711, parents=True, exist_ok=True) + _assert_root_managed(paths.work_base, directory=True) + os.chown(paths.work_base, 0, 0) + os.chmod(paths.work_base, 0o711) + _assert_qualification_traversal(paths.work_base) + home, cache, temporary = _create_preflight_work(work, identity) + environment = { + "HOME": str(home), + "XDG_CACHE_HOME": str(cache), + "TMPDIR": str(temporary), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "NO_PROXY": "*", + "no_proxy": "*", + "PATH": "/usr/bin:/bin", + "LANG": "C.UTF-8", + } + descriptor: int | None = None + process: Any | None = None + leader_reaped = False + group_clean = False + try: + metadata = executable.lstat() + if executable.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise Q38LinuxHostRuntimeError("packaged node executable is unsafe") + descriptor = os.open(executable, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + opened = os.fstat(descriptor) + _assert_executable_handle(metadata, opened) + digest = hashlib.sha256() + for chunk in iter(lambda: os.read(descriptor, HASH_CHUNK_BYTES), b""): + digest.update(chunk) + if "sha256:" + digest.hexdigest() != plan.runtime_package["node_executable_sha256"]: + raise Q38LinuxHostRuntimeError("packaged node executable digest changed") + os.lseek(descriptor, 0, os.SEEK_SET) + with tempfile.TemporaryFile(dir=work) as stdout_file, tempfile.TemporaryFile(dir=work) as stderr_file: + argv = (str(executable), "edge-acquire", "--help") + process = popen_factory( + argv, + executable=f"/proc/self/fd/{descriptor}", + cwd=runtime / "CommunityAI" / "node", + env=environment, + stdin=subprocess.DEVNULL, + stdout=stdout_file, + stderr=stderr_file, + shell=False, + close_fds=True, + pass_fds=(descriptor,), + preexec_fn=_preflight_child(identity), + ) + try: + returncode = process.wait(timeout=180) + except subprocess.TimeoutExpired as exc: + raise Q38LinuxHostRuntimeError("packaged preflight timed out") from exc + leader_reaped = True + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + group_clean = True + else: + raise Q38LinuxHostRuntimeError("packaged preflight left descendants") + stdout_file.seek(0, os.SEEK_END) + stderr_file.seek(0, os.SEEK_END) + stdout_bytes = stdout_file.tell() + stderr_bytes = stderr_file.tell() + if stdout_bytes > MAX_OUTPUT_BYTES or stderr_bytes > MAX_OUTPUT_BYTES: + raise Q38LinuxHostRuntimeError("packaged preflight output exceeded its bound") + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read() + stderr = stderr_file.read() + if returncode != 0 or b"edge-acquire" not in stdout.lower(): + raise Q38LinuxHostRuntimeError("packaged edge-acquire help preflight failed") + if not _empty_tree(home) or not _empty_tree(cache) or not _empty_tree(temporary): + raise Q38LinuxHostRuntimeError("packaged preflight wrote to its isolated state") + _verify_runtime_tree(runtime, node, protected=True) + return PreflightResult(returncode, stdout, stderr) + except OSError as exc: + raise Q38LinuxHostRuntimeError("packaged preflight could not start") from exc + finally: + try: + if process is not None and not group_clean: + _cleanup_started_process(process, leader_reaped=leader_reaped) + finally: + try: + if descriptor is not None: + os.close(descriptor) + finally: + _remove_preflight_work(work) + + +def _prepared_record( + plan: controller.RoutePlan, + action: Mapping[str, Any], + identity: QualificationIdentity, + result: PreflightResult, + context: Mapping[str, Any], + boot_id: str, +) -> dict[str, Any]: + record: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": PREPARED_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": action["action_id"], + "instance_context_digest": context["context_digest"], + "resource_name": context["resource_name"], + "resource_kind": context["resource_kind"], + "worker_id": context["worker_id"], + "instance_generation_digest": context["instance_generation_digest"], + "boot_id": boot_id, + "runtime_package_digest": plan.runtime_package["runtime_package_digest"], + "release_archive_sha256": plan.runtime_package["release_archive_sha256"], + "node_executable_sha256": plan.runtime_package["node_executable_sha256"], + "node_runtime_inventory_digest": plan.runtime_package["node_runtime_inventory_digest"], + "node_runtime_entry_count": plan.runtime_package["node_runtime_entry_count"], + "node_runtime_bytes": plan.runtime_package["node_runtime_bytes"], + "qualification_user": identity.name, + "qualification_uid": identity.uid, + "qualification_gid": identity.gid, + "preflight_returncode": result.returncode, + "preflight_stdout_sha256": _sha256(result.stdout), + "preflight_stdout_bytes": len(result.stdout), + "preflight_stderr_sha256": _sha256(result.stderr), + "preflight_stderr_bytes": len(result.stderr), + "prepared_record_digest": "", + } + record["prepared_record_digest"] = _prepared_digest(record) + return record + + +def _prepared_digest(record: Mapping[str, Any]) -> str: + unsigned = dict(record) + unsigned.pop("prepared_record_digest", None) + return _sha256(_canonical_bytes(unsigned)) + + +def validate_prepared_record( + value: Any, + plan: controller.RoutePlan, + *, + instance_context: Mapping[str, Any] | None = None, + boot_id: str | None = None, +) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _PREPARED_FIELDS: + raise Q38LinuxHostRuntimeError("prepared record schema is invalid") + resource = plan.resource_by_name.get(value.get("resource_name")) + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != SCHEMA_VERSION + or value["scope"] != PREPARED_SCOPE + or value["run_id"] != plan.run_id + or value["source_commit"] != plan.source_commit + or value["plan_digest"] != plan.plan_digest + or value["execution_inventory_digest"] != plan.execution_inventory_digest + or value["start_action_id"] != controller._action_id(plan, "start_route") + or resource is None + or not resource.kind.endswith("instance") + or value["resource_kind"] != resource.kind + or value["worker_id"] != resource.worker_id + or not isinstance(value["boot_id"], str) + or transport._BOOT_ID_RE.fullmatch(value["boot_id"]) is None + or value["runtime_package_digest"] != plan.runtime_package["runtime_package_digest"] + or value["release_archive_sha256"] != plan.runtime_package["release_archive_sha256"] + or value["node_executable_sha256"] != plan.runtime_package["node_executable_sha256"] + or value["node_runtime_inventory_digest"] != plan.runtime_package["node_runtime_inventory_digest"] + or value["node_runtime_entry_count"] != plan.runtime_package["node_runtime_entry_count"] + or value["node_runtime_bytes"] != plan.runtime_package["node_runtime_bytes"] + or value["qualification_user"] != QUALIFICATION_USER + or type(value["qualification_uid"]) is not int + or value["qualification_uid"] <= 0 + or type(value["qualification_gid"]) is not int + or value["qualification_gid"] <= 0 + or type(value["preflight_returncode"]) is not int + or value["preflight_returncode"] != 0 + or type(value["preflight_stdout_bytes"]) is not int + or not 1 <= value["preflight_stdout_bytes"] <= MAX_OUTPUT_BYTES + or type(value["preflight_stderr_bytes"]) is not int + or not 0 <= value["preflight_stderr_bytes"] <= MAX_OUTPUT_BYTES + ): + raise Q38LinuxHostRuntimeError("prepared record identity is invalid") + for field in ( + "instance_context_digest", + "instance_generation_digest", + "preflight_stdout_sha256", + "preflight_stderr_sha256", + "prepared_record_digest", + ): + _digest_field(value[field], field) + if instance_context is not None: + expected_context = { + "instance_context_digest": instance_context.get("context_digest"), + "resource_name": instance_context.get("resource_name"), + "resource_kind": instance_context.get("resource_kind"), + "worker_id": instance_context.get("worker_id"), + "instance_generation_digest": instance_context.get("instance_generation_digest"), + } + if any(value[field] != expected for field, expected in expected_context.items()): + raise Q38LinuxHostRuntimeError("prepared record instance binding changed") + if boot_id is not None and value["boot_id"] != boot_id: + raise Q38LinuxHostRuntimeError("prepared record boot identity changed") + if value["prepared_record_digest"] != _prepared_digest(value): + raise Q38LinuxHostRuntimeError("prepared record digest changed") + return dict(value) + + +def _load_prepared_record( + path: Path, + plan: controller.RoutePlan, + *, + instance_context: Mapping[str, Any], + boot_id: str, +) -> dict[str, Any]: + _assert_root_private_file(path) + value = _strict_json(_regular_bytes(path)) + return validate_prepared_record( + value, + plan, + instance_context=instance_context, + boot_id=boot_id, + ) + + +def build_prepared_status_envelope( + prepared_record: Mapping[str, Any], + inputs: TransportInputs, + plan: controller.RoutePlan, + *, + expected_resource_name: str, + expected_generation_digest: str, + revision: int, + published_at_unix: int, +) -> dict[str, Any]: + try: + context = transport.validate_instance_context( + inputs.context, + plan, + key=inputs.key, + now_unix=published_at_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + ) + prepared = validate_prepared_record( + dict(prepared_record), + plan, + instance_context=context, + boot_id=inputs.boot_id, + ) + envelope = transport.build_status_envelope( + context, + transport.initial_status_payload(context, plan), + plan, + key=inputs.key, + boot_id=inputs.boot_id, + revision=revision, + published_at_unix=published_at_unix, + prepared_record_digest=prepared["prepared_record_digest"], + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + if envelope["prepared_record_digest"] != prepared["prepared_record_digest"]: + raise Q38LinuxHostRuntimeError("host status does not bind the protected prepared record") + return envelope + + +@contextmanager +def _prepared_state_lock(parent: Path) -> Iterator[None]: + if os.name != "posix": + yield + return + import fcntl + + lock = parent / ".prepared.lock" + descriptor: int | None = None + try: + descriptor = os.open( + lock, + os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + os.fchown(descriptor, 0, 0) + os.fchmod(descriptor, 0o600) + opened = os.fstat(descriptor) + current = lock.lstat() + if ( + not stat.S_ISREG(opened.st_mode) + or _file_identity(opened) != _file_identity(current) + or opened.st_uid != 0 + or opened.st_gid != 0 + or stat.S_IMODE(opened.st_mode) != 0o600 + ): + raise Q38LinuxHostRuntimeError("prepared state lock is unsafe") + fcntl.flock(descriptor, fcntl.LOCK_EX) + except Q38LinuxHostRuntimeError: + if descriptor is not None: + os.close(descriptor) + raise + except OSError as exc: + if descriptor is not None: + os.close(descriptor) + raise Q38LinuxHostRuntimeError("prepared state lock is unavailable") from exc + try: + yield + finally: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +def _remove_stale_prepared_temporaries(parent: Path) -> None: + for candidate in parent.iterdir(): + if not candidate.name.startswith(".prepared.") or not candidate.name.endswith(".tmp"): + continue + metadata = candidate.lstat() + if ( + candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or os.name == "posix" + and (metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) != 0o600) + ): + raise Q38LinuxHostRuntimeError("stale prepared temporary is unsafe") + candidate.unlink() + if any( + candidate.name.startswith(".prepared.") and candidate.name.endswith(".tmp") for candidate in parent.iterdir() + ): + raise Q38LinuxHostRuntimeError("stale prepared cleanup is incomplete") + + +def _accept_existing_prepared( + path: Path, + value: Mapping[str, Any], + plan: controller.RoutePlan, +) -> None: + metadata = path.lstat() + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise Q38LinuxHostRuntimeError("prepared record target is unsafe") + _assert_root_managed(path, directory=False) + existing = validate_prepared_record( + _strict_json(_regular_bytes(path)), + plan, + ) + if not _same_json_value(existing, dict(value)): + raise Q38LinuxHostRuntimeError("prepared record already binds another result") + + +def _fsync_directory(path: Path) -> None: + if os.name != "posix": + return + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), + ) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _atomic_prepared_locked( + path: Path, + value: Mapping[str, Any], + plan: controller.RoutePlan, +) -> bool: + payload = _canonical_bytes(value) + _remove_stale_prepared_temporaries(path.parent) + if path.exists() or path.is_symlink(): + _accept_existing_prepared(path, value, plan) + return False + descriptor, raw = tempfile.mkstemp(prefix=".prepared.", suffix=".tmp", dir=path.parent) + temporary = Path(raw) + linked = False + try: + with os.fdopen(descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.chown(temporary, 0, 0) + os.chmod(temporary, 0o600) + try: + os.link(temporary, path) + except FileExistsError: + _accept_existing_prepared(path, value, plan) + return False + linked = True + _accept_existing_prepared(path, value, plan) + temporary.unlink() + linked = False + return True + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError("prepared record could not be committed") from exc + finally: + temporary.unlink(missing_ok=True) + if linked: + _remove_exact_published_file(path, payload, "prepared record") + + +def _atomic_prepared( + path: Path, + value: Mapping[str, Any], + plan: controller.RoutePlan, +) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + _assert_root_managed(path.parent, directory=True) + with _prepared_state_lock(path.parent): + _atomic_prepared_locked(path, value, plan) + _fsync_directory(path.parent) + + +def _remove_stale_status_temporaries(parent: Path) -> None: + for candidate in parent.iterdir(): + if not candidate.name.startswith(".status.") or not candidate.name.endswith(".tmp"): + continue + metadata = candidate.lstat() + if ( + candidate.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or os.name == "posix" + and (metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) != 0o600) + ): + raise Q38LinuxHostRuntimeError("stale status temporary is unsafe") + candidate.unlink() + if any(candidate.name.startswith(".status.") and candidate.name.endswith(".tmp") for candidate in parent.iterdir()): + raise Q38LinuxHostRuntimeError("stale status cleanup is incomplete") + + +def _accept_existing_status( + path: Path, + intended: Mapping[str, Any], + plan: controller.RoutePlan, + inputs: TransportInputs, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, +) -> dict[str, Any]: + _assert_root_private_file(path) + try: + existing = transport.decode_status_envelope(_regular_bytes(path, maximum=transport.MAX_ENVELOPE_BYTES)) + validated = transport.validate_status_envelope( + existing, + plan, + key=inputs.key, + now_unix=now_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + expected_boot_id=inputs.boot_id, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + stable_fields = ( + "context", + "boot_id", + "revision", + "prepared_record_digest", + "payload", + "payload_digest", + ) + if any(not _same_json_value(validated[field], intended[field]) for field in stable_fields): + raise Q38LinuxHostRuntimeError("host status already binds another result") + return validated + + +def _atomic_status_locked( + path: Path, + value: Mapping[str, Any], + plan: controller.RoutePlan, + inputs: TransportInputs, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, +) -> tuple[dict[str, Any], bool]: + payload = transport.encode_status_envelope(value) + _remove_stale_status_temporaries(path.parent) + if path.exists() or path.is_symlink(): + return ( + _accept_existing_status( + path, + value, + plan, + inputs, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=now_unix, + ), + False, + ) + descriptor, raw = tempfile.mkstemp(prefix=".status.", suffix=".tmp", dir=path.parent) + temporary = Path(raw) + linked = False + try: + with os.fdopen(descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.chown(temporary, 0, 0) + os.chmod(temporary, 0o600) + try: + os.link(temporary, path) + except FileExistsError: + return ( + _accept_existing_status( + path, + value, + plan, + inputs, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=now_unix, + ), + False, + ) + linked = True + result = _accept_existing_status( + path, + value, + plan, + inputs, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=now_unix, + ) + temporary.unlink() + linked = False + return result, True + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError("host status could not be committed") from exc + finally: + temporary.unlink(missing_ok=True) + if linked: + _remove_exact_published_file(path, payload, "host status") + + +def _atomic_status( + path: Path, + value: Mapping[str, Any], + plan: controller.RoutePlan, + inputs: TransportInputs, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, +) -> dict[str, Any]: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + _assert_root_managed(path.parent, directory=True) + with _prepared_state_lock(path.parent): + result, _created = _atomic_status_locked( + path, + value, + plan, + inputs, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=now_unix, + ) + _fsync_directory(path.parent) + return result + + +def _remove_exact_published_file(path: Path, expected: bytes, label: str) -> None: + if not path.exists() and not path.is_symlink(): + return + _assert_root_private_file(path) + if _regular_bytes(path, maximum=max(len(expected), 1)) != expected: + raise Q38LinuxHostRuntimeError(f"{label} rollback target changed") + path.unlink() + + +def _publish_prepared_status_locked( + paths: HostPaths, + record: Mapping[str, Any], + plan: controller.RoutePlan, + inputs: TransportInputs, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, +) -> dict[str, Any]: + parent = paths.prepared_record.parent + prepared_payload = _canonical_bytes(record) + prepared_created = False + status_created = False + status_payload: bytes | None = None + try: + prepared_created = _atomic_prepared_locked(paths.prepared_record, record, plan) + persisted = _load_prepared_record( + paths.prepared_record, + plan, + instance_context=inputs.context, + boot_id=inputs.boot_id, + ) + envelope = build_prepared_status_envelope( + persisted, + inputs, + plan, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + revision=1, + published_at_unix=now_unix, + ) + status_payload = transport.encode_status_envelope(envelope) + _status, status_created = _atomic_status_locked( + paths.status_envelope, + envelope, + plan, + inputs, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=now_unix, + ) + _fsync_directory(parent) + return persisted + except BaseException: + if status_created and status_payload is not None: + _remove_exact_published_file(paths.status_envelope, status_payload, "host status") + if prepared_created: + _remove_exact_published_file(paths.prepared_record, prepared_payload, "prepared record") + _remove_stale_status_temporaries(parent) + _remove_stale_prepared_temporaries(parent) + _fsync_directory(parent) + raise + + +def _state_parent(paths: HostPaths) -> Path: + if paths.status_envelope.parent != paths.prepared_record.parent: + raise Q38LinuxHostRuntimeError("host status is outside the protected state boundary") + parent = paths.prepared_record.parent + created = False + try: + parent.mkdir(mode=0o700, parents=True, exist_ok=False) + created = True + except FileExistsError: + pass + except OSError as exc: + raise Q38LinuxHostRuntimeError("protected host state is unavailable") from exc + if created: + os.chown(parent, 0, 0) + os.chmod(parent, 0o700) + _assert_root_managed(parent, directory=True) + return parent + + +def _cleanup_marker_value( + plan: controller.RoutePlan, + context: Mapping[str, Any], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "scope": CLEANUP_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "cleanup_action_id": controller._action_id(plan, "cleanup_route"), + "instance_context_digest": context["context_digest"], + "resource_name": context["resource_name"], + "resource_kind": context["resource_kind"], + "worker_id": context["worker_id"], + "instance_generation_digest": context["instance_generation_digest"], + "runtime_key": _runtime_key(plan), + } + + +def _cleanup_marker_path_for_generation( + paths: HostPaths, + generation_digest: str, +) -> Path: + if not isinstance(generation_digest, str) or controller._DIGEST_RE.fullmatch(generation_digest) is None: + raise Q38LinuxHostRuntimeError("cleanup generation digest is invalid") + return paths.prepared_record.parent / f"cleaned-{generation_digest.removeprefix('sha256:')}.json" + + +def _cleanup_marker_path(paths: HostPaths, context: Mapping[str, Any]) -> Path: + return _cleanup_marker_path_for_generation( + paths, + str(context["instance_generation_digest"]), + ) + + +def _validate_cleanup_marker_generation( + path: Path, + plan: controller.RoutePlan, + *, + expected_resource_name: str, + expected_generation_digest: str, +) -> dict[str, Any]: + resource = plan.resource_by_name.get(expected_resource_name) + if resource is None or not resource.kind.endswith("instance"): + raise Q38LinuxHostRuntimeError("cleanup marker resource is invalid") + _assert_root_private_file(path) + value = _strict_json(_regular_bytes(path)) + fixed = { + "schema_version": SCHEMA_VERSION, + "scope": CLEANUP_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "cleanup_action_id": controller._action_id(plan, "cleanup_route"), + "resource_name": resource.name, + "resource_kind": resource.kind, + "worker_id": resource.worker_id, + "instance_generation_digest": expected_generation_digest, + "runtime_key": _runtime_key(plan), + } + if set(value) != {*fixed, "instance_context_digest"} or any( + value[field] != expected for field, expected in fixed.items() + ): + raise Q38LinuxHostRuntimeError("cleanup marker binds another host generation") + context_digest = value["instance_context_digest"] + if not isinstance(context_digest, str) or controller._DIGEST_RE.fullmatch(context_digest) is None: + raise Q38LinuxHostRuntimeError("cleanup marker context is invalid") + return value + + +def _remove_stale_cleanup_temporaries(parent: Path) -> None: + for candidate in parent.iterdir(): + if not candidate.name.startswith(".cleanup.") or not candidate.name.endswith(".tmp"): + continue + _assert_root_private_file(candidate) + candidate.unlink() + if any( + candidate.name.startswith(".cleanup.") and candidate.name.endswith(".tmp") for candidate in parent.iterdir() + ): + raise Q38LinuxHostRuntimeError("stale cleanup marker cleanup is incomplete") + + +def _accept_existing_cleanup_marker(path: Path, value: Mapping[str, Any]) -> None: + _assert_root_private_file(path) + if not _same_json_value(_strict_json(_regular_bytes(path)), dict(value)): + raise Q38LinuxHostRuntimeError("cleanup marker binds another host generation") + + +def _atomic_cleanup_marker_locked(path: Path, value: Mapping[str, Any]) -> None: + payload = _canonical_bytes(value) + _remove_stale_cleanup_temporaries(path.parent) + if path.exists() or path.is_symlink(): + _accept_existing_cleanup_marker(path, value) + return + descriptor, raw = tempfile.mkstemp(prefix=".cleanup.", suffix=".tmp", dir=path.parent) + temporary = Path(raw) + linked = False + try: + with os.fdopen(descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.chown(temporary, 0, 0) + os.chmod(temporary, 0o600) + try: + os.link(temporary, path) + except FileExistsError: + _accept_existing_cleanup_marker(path, value) + return + linked = True + _accept_existing_cleanup_marker(path, value) + temporary.unlink() + linked = False + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError("cleanup marker could not be committed") from exc + finally: + temporary.unlink(missing_ok=True) + if linked: + _remove_exact_published_file(path, payload, "cleanup marker") + + +def _reject_cleaned_generation_locked(path: Path, value: Mapping[str, Any]) -> None: + _remove_stale_cleanup_temporaries(path.parent) + if not path.exists() and not path.is_symlink(): + return + _accept_existing_cleanup_marker(path, value) + raise Q38LinuxHostRuntimeError("host generation cleanup is terminal") + + +def _remove_exact_tree(path: Path, parent: Path) -> None: + if path.parent != parent or not path.is_absolute() or path in {Path("/"), Path.home()}: + raise Q38LinuxHostRuntimeError("cleanup target is outside the runtime boundary") + if not path.exists() and not path.is_symlink(): + return + metadata = path.lstat() + if path.is_symlink() or not stat.S_ISDIR(metadata.st_mode): + raise Q38LinuxHostRuntimeError("cleanup target is unsafe") + shutil.rmtree(path) + if path.exists() or path.is_symlink(): + raise Q38LinuxHostRuntimeError("runtime cleanup is incomplete") + + +def _remove_stale_delivery_temporaries(parent: Path) -> None: + for candidate in parent.iterdir(): + if not candidate.name.startswith(".delivery.") or not candidate.name.endswith(".tmp"): + continue + _assert_root_private_file(candidate) + candidate.unlink() + if any( + candidate.name.startswith(".delivery.") and candidate.name.endswith(".tmp") for candidate in parent.iterdir() + ): + raise Q38LinuxHostRuntimeError("stale delivery cleanup is incomplete") + + +def _delivery_path(paths: HostPaths) -> Path: + path = paths.transport_bundle + if path is None or path.parent != paths.plan.parent or path.name != "instance-delivery.bin": + raise Q38LinuxHostRuntimeError("transport delivery target is invalid") + _assert_root_managed(path.parent, directory=True) + return path + + +def _decode_delivery( + payload: bytes, + plan: controller.RoutePlan, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int, +) -> tuple[dict[str, Any], dict[str, Any], bytes]: + try: + return transport.decode_instance_delivery( + payload, + plan, + now_unix=now_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + + +def install_instance_delivery( + paths: HostPaths, + payload: bytes, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int | None = None, +) -> dict[str, Any]: + """Atomically install one authenticated context/key bundle under the lifecycle lock.""" + + installation_time = int(time.time()) if now_unix is None else now_unix + plan, _action = _load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=installation_time, + ) + record, context, _key = _decode_delivery( + payload, + plan, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=installation_time, + ) + delivery = transport.InstanceDelivery(record, payload) + target = _delivery_path(paths) + state_parent = _state_parent(paths) + cleanup_marker = _cleanup_marker_path(paths, context) + cleanup_value = _cleanup_marker_value(plan, context) + with _prepared_state_lock(state_parent): + _reject_cleaned_generation_locked(cleanup_marker, cleanup_value) + _remove_stale_delivery_temporaries(target.parent) + if target.exists() or target.is_symlink(): + _assert_root_private_file(target) + current_payload = _regular_bytes( + target, + maximum=transport.MAX_DELIVERY_BYTES, + ) + current_record, _current_context, _current_key = _decode_delivery( + current_payload, + plan, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=installation_time, + ) + if current_payload == payload: + return transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=installation_time, + ) + if ( + record["key_epoch"] != current_record["key_epoch"] + 1 + or record["previous_key_record_digest"] != current_record["key_record_digest"] + ): + raise Q38LinuxHostRuntimeError("instance delivery rotation is stale or discontinuous") + elif record["key_epoch"] != 1 or record["previous_key_record_digest"] is not None: + raise Q38LinuxHostRuntimeError("initial instance delivery does not begin at key epoch one") + descriptor, raw = tempfile.mkstemp( + prefix=".delivery.", + suffix=".tmp", + dir=target.parent, + ) + temporary = Path(raw) + replaced = False + try: + with os.fdopen(descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.chown(temporary, 0, 0) + os.chmod(temporary, 0o600) + _assert_root_private_file(temporary) + os.replace(temporary, target) + replaced = True + _assert_root_private_file(target) + installed_payload = _regular_bytes( + target, + maximum=transport.MAX_DELIVERY_BYTES, + ) + installed_record, _installed_context, _installed_key = _decode_delivery( + installed_payload, + plan, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=installation_time, + ) + if installed_payload != payload or installed_record != record: + raise Q38LinuxHostRuntimeError("installed instance delivery changed") + _fsync_directory(target.parent) + except Q38LinuxHostRuntimeError: + raise + except OSError as exc: + raise Q38LinuxHostRuntimeError("instance delivery could not be installed") from exc + finally: + if not replaced: + temporary.unlink(missing_ok=True) + return transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=installation_time, + ) + + +def prepare( + paths: HostPaths, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int | None = None, + identity: QualificationIdentity | None = None, + protector: Callable[[Path, Sequence[Artifact]], None] = _protect_runtime, + preflight: Callable[ + [controller.RoutePlan, Path, Sequence[Artifact], HostPaths, QualificationIdentity], + PreflightResult, + ] = _run_packaged_preflight, +) -> dict[str, Any]: + entry_time = int(time.time()) if now_unix is None else now_unix + plan, action = _load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=entry_time, + ) + inputs = _load_transport_inputs( + plan, + paths, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=entry_time, + ) + artifacts, node = _load_release_inventory(plan, paths) + state_parent = _state_parent(paths) + cleanup_marker = _cleanup_marker_path(paths, inputs.context) + cleanup_value = _cleanup_marker_value(plan, inputs.context) + with _prepared_state_lock(state_parent): + _reject_cleaned_generation_locked(cleanup_marker, cleanup_value) + paths.runtime_base.mkdir(mode=0o755, parents=True, exist_ok=True) + paths.work_base.mkdir(mode=0o711, parents=True, exist_ok=True) + _assert_root_managed(paths.runtime_base, directory=True) + _assert_root_managed(paths.work_base, directory=True) + os.chown(paths.runtime_base, 0, 0) + os.chmod(paths.runtime_base, 0o755) + _assert_qualification_traversal(paths.runtime_base) + os.chown(paths.work_base, 0, 0) + os.chmod(paths.work_base, 0o711) + destination = _runtime_destination(plan, paths) + created = False + if destination.exists() or destination.is_symlink(): + if destination.is_symlink() or not destination.is_dir(): + raise Q38LinuxHostRuntimeError("runtime destination is foreign") + _verify_runtime_tree(destination, node, protected=True) + else: + temporary = Path(tempfile.mkdtemp(prefix=f".{destination.name}.", dir=paths.runtime_base)) + shutil.rmtree(temporary) + try: + _extract_verified_archive( + paths.release_root / plan.runtime_package["release_archive_name"], + plan.runtime_package, + artifacts, + node, + temporary, + ) + protector(temporary, node) + _verify_runtime_tree(temporary, node, protected=True) + os.replace(temporary, destination) + created = True + finally: + if temporary.exists() or temporary.is_symlink(): + _remove_tree_strict(temporary, "runtime staging cleanup is incomplete") + try: + resolved_identity = _qualification_identity() if identity is None else identity + result = preflight(plan, destination, node, paths, resolved_identity) + publication_time = int(time.time()) if now_unix is None else now_unix + refreshed = _load_transport_inputs( + plan, + paths, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=publication_time, + ) + if ( + not _same_json_value(refreshed.context, inputs.context) + or refreshed.key != inputs.key + or refreshed.boot_id != inputs.boot_id + ): + raise Q38LinuxHostRuntimeError("transport inputs changed during host preparation") + record = _prepared_record( + plan, + action, + resolved_identity, + result, + refreshed.context, + refreshed.boot_id, + ) + validate_prepared_record( + record, + plan, + instance_context=refreshed.context, + boot_id=refreshed.boot_id, + ) + return _publish_prepared_status_locked( + paths, + record, + plan, + refreshed, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=publication_time, + ) + except BaseException: + if created: + _remove_exact_tree(destination, paths.runtime_base) + raise + + +def _publish_guest_attribute( + payload: bytes, + *, + connection_factory: Callable[..., Any] = http.client.HTTPConnection, +) -> None: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= transport.MAX_ENVELOPE_BYTES: + raise Q38LinuxHostRuntimeError("host status publication payload is invalid") + connection: Any | None = None + try: + connection = connection_factory( + METADATA_HOST, + METADATA_PORT, + timeout=METADATA_TIMEOUT_SECONDS, + ) + connection.request( + "PUT", + GUEST_ATTRIBUTE_PATH, + body=payload, + headers={ + "Metadata-Flavor": "Google", + "Content-Type": "application/octet-stream", + "Content-Length": str(len(payload)), + "Connection": "close", + }, + ) + response = connection.getresponse() + declared_length = response.getheader("Content-Length") + declared: int | None = None + if declared_length is not None: + if not isinstance(declared_length, str) or re.fullmatch(r"[0-9]+", declared_length) is None: + raise Q38LinuxHostRuntimeError("metadata publication response length is invalid") + if len(declared_length) > len(str(MAX_METADATA_RESPONSE_BYTES)): + raise Q38LinuxHostRuntimeError("metadata publication response exceeded its size bound") + declared = int(declared_length, 10) + if declared > MAX_METADATA_RESPONSE_BYTES: + raise Q38LinuxHostRuntimeError("metadata publication response exceeded its size bound") + response_payload = response.read(MAX_METADATA_RESPONSE_BYTES + 1) + if not isinstance(response_payload, bytes) or len(response_payload) > MAX_METADATA_RESPONSE_BYTES: + raise Q38LinuxHostRuntimeError("metadata publication response exceeded its size bound") + if declared is not None and len(response_payload) != declared: + raise Q38LinuxHostRuntimeError("metadata publication response length changed") + response_flavor = response.getheader("Metadata-Flavor") + if ( + type(response.status) is not int + or response.status != 200 + or not isinstance(response_flavor, str) + or response_flavor != "Google" + ): + raise Q38LinuxHostRuntimeError("metadata publication was not acknowledged") + except Q38LinuxHostRuntimeError: + raise + except (OSError, http.client.HTTPException) as exc: + raise Q38LinuxHostRuntimeError("metadata publication failed") from exc + finally: + if connection is not None: + try: + connection.close() + except OSError: + pass + + +def publish_status( + paths: HostPaths, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int | None = None, + sender: Callable[[bytes], None] = _publish_guest_attribute, +) -> dict[str, Any]: + if paths.status_envelope.parent != paths.prepared_record.parent: + raise Q38LinuxHostRuntimeError("host status is outside the protected state boundary") + state_parent = paths.prepared_record.parent + if state_parent.is_symlink() or not state_parent.is_dir(): + raise Q38LinuxHostRuntimeError("protected host state is unavailable") + _assert_root_managed(state_parent, directory=True) + with _prepared_state_lock(state_parent): + verification_time = int(time.time()) if now_unix is None else now_unix + plan, _action = _load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=verification_time, + ) + inputs = _load_transport_inputs( + plan, + paths, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=verification_time, + ) + cleanup_marker = _cleanup_marker_path(paths, inputs.context) + _reject_cleaned_generation_locked( + cleanup_marker, + _cleanup_marker_value(plan, inputs.context), + ) + if not paths.prepared_record.exists() or paths.prepared_record.is_symlink(): + raise Q38LinuxHostRuntimeError("protected prepared record is unavailable") + prepared = _load_prepared_record( + paths.prepared_record, + plan, + instance_context=inputs.context, + boot_id=inputs.boot_id, + ) + _assert_root_private_file(paths.status_envelope) + raw_envelope = _regular_bytes( + paths.status_envelope, + maximum=transport.MAX_ENVELOPE_BYTES, + ) + try: + envelope = transport.validate_status_envelope( + transport.decode_status_envelope(raw_envelope), + plan, + key=inputs.key, + now_unix=verification_time, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + expected_boot_id=inputs.boot_id, + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + if ( + envelope["prepared_record_digest"] != prepared["prepared_record_digest"] + or transport.encode_status_envelope(envelope) != raw_envelope + ): + raise Q38LinuxHostRuntimeError("host status does not bind the protected prepared record") + sender(raw_envelope) + return { + "schema_version": SCHEMA_VERSION, + "scope": PUBLICATION_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "resource_name": expected_resource_name, + "instance_generation_digest": expected_generation_digest, + "context_digest": inputs.context["context_digest"], + "boot_id": inputs.boot_id, + "revision": envelope["revision"], + "prepared_record_digest": prepared["prepared_record_digest"], + "envelope_sha256": "sha256:" + hashlib.sha256(raw_envelope).hexdigest(), + "envelope_bytes": len(raw_envelope), + } + + +def cleanup( + paths: HostPaths, + *, + expected_resource_name: str, + expected_generation_digest: str, + now_unix: int | None = None, +) -> None: + verification_time = int(time.time()) if now_unix is None else now_unix + plan, _action = _load_plan_and_action( + paths.plan, + paths.cleanup_action, + paths.source_root, + expected_action="cleanup_route", + now_unix=verification_time, + ) + destination = _runtime_destination(plan, paths) + work = paths.work_base / _runtime_key(plan) + state_parent = _state_parent(paths) + cleanup_marker = _cleanup_marker_path_for_generation( + paths, + expected_generation_digest, + ) + bundle_path = paths.transport_bundle + with _prepared_state_lock(state_parent): + _remove_stale_prepared_temporaries(state_parent) + _remove_stale_status_temporaries(state_parent) + _remove_stale_cleanup_temporaries(state_parent) + if bundle_path is not None: + _delivery_path(paths) + _remove_stale_delivery_temporaries(bundle_path.parent) + if bundle_path is not None and not bundle_path.exists() and not bundle_path.is_symlink(): + if not cleanup_marker.exists() and not cleanup_marker.is_symlink(): + raise Q38LinuxHostRuntimeError("transport delivery is unavailable before terminal cleanup") + _validate_cleanup_marker_generation( + cleanup_marker, + plan, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + ) + _remove_exact_tree(destination, paths.runtime_base) + _remove_exact_tree(work, paths.work_base) + for target in (paths.status_envelope, paths.prepared_record): + if target.exists() or target.is_symlink(): + _assert_root_private_file(target) + target.unlink() + _fsync_directory(state_parent) + else: + context, _key = _load_authenticated_context( + plan, + paths, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + now_unix=verification_time, + allow_expired_for_cleanup=True, + ) + cleanup_value = _cleanup_marker_value(plan, context) + if cleanup_marker.exists() or cleanup_marker.is_symlink(): + _accept_existing_cleanup_marker(cleanup_marker, cleanup_value) + prepared: dict[str, Any] | None = None + if paths.prepared_record.exists() or paths.prepared_record.is_symlink(): + _assert_root_private_file(paths.prepared_record) + prepared = validate_prepared_record( + _strict_json(_regular_bytes(paths.prepared_record)), + plan, + instance_context=context, + ) + if paths.status_envelope.exists() or paths.status_envelope.is_symlink(): + if prepared is None: + raise Q38LinuxHostRuntimeError("host status has no protected prepared record") + _assert_root_private_file(paths.status_envelope) + try: + status = transport.decode_status_envelope( + _regular_bytes( + paths.status_envelope, + maximum=transport.MAX_ENVELOPE_BYTES, + ) + ) + except transport.Q38LinuxHostTransportError as exc: + raise Q38LinuxHostRuntimeError(str(exc)) from exc + status_context = status.get("context") + if ( + not isinstance(status_context, dict) + or status_context.get("context_digest") != context["context_digest"] + or status_context.get("resource_name") != expected_resource_name + or status_context.get("instance_generation_digest") != expected_generation_digest + or status.get("boot_id") != prepared["boot_id"] + or status.get("prepared_record_digest") != prepared["prepared_record_digest"] + ): + raise Q38LinuxHostRuntimeError("host status cleanup generation changed") + _atomic_cleanup_marker_locked(cleanup_marker, cleanup_value) + _fsync_directory(state_parent) + _remove_exact_tree(destination, paths.runtime_base) + _remove_exact_tree(work, paths.work_base) + if paths.status_envelope.exists(): + paths.status_envelope.unlink() + if prepared is not None: + paths.prepared_record.unlink() + if bundle_path is not None: + _assert_root_private_file(bundle_path) + installed = _regular_bytes( + bundle_path, + maximum=transport.MAX_DELIVERY_BYTES, + ) + _record, installed_context, _installed_key = transport.decode_instance_delivery( + installed, + plan, + now_unix=verification_time, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + allow_expired_for_cleanup=True, + ) + if installed_context["context_digest"] != context["context_digest"]: + raise Q38LinuxHostRuntimeError("transport delivery cleanup generation changed") + bundle_path.unlink() + _fsync_directory(bundle_path.parent) + _fsync_directory(state_parent) + stale = [ + *state_parent.glob(".prepared.*.tmp"), + *state_parent.glob(".status.*.tmp"), + *state_parent.glob(".cleanup.*.tmp"), + ] + if bundle_path is not None: + stale.extend(bundle_path.parent.glob(".delivery.*.tmp")) + if ( + destination.exists() + or work.exists() + or paths.prepared_record.exists() + or paths.status_envelope.exists() + or bundle_path is not None + and (bundle_path.exists() or bundle_path.is_symlink()) + or stale + ): + raise Q38LinuxHostRuntimeError("host runtime cleanup is incomplete") + _validate_cleanup_marker_generation( + cleanup_marker, + plan, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + ) + + +def _require_linux_root() -> None: + if not sys.platform.startswith("linux") or not hasattr(os, "geteuid") or os.geteuid() != 0: + raise Q38LinuxHostRuntimeError("Qwen3.8 host runtime requires Linux root") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Install, prepare, publish, or clean the protected Qwen3.8 Linux runtime" + ) + parser.add_argument( + "operation", + choices=("install-delivery", "prepare", "publish-status", "cleanup"), + ) + parser.add_argument("--resource-name", required=True) + parser.add_argument("--instance-generation-digest", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + _require_linux_root() + paths = HostPaths.production() + arguments = { + "expected_resource_name": args.resource_name, + "expected_generation_digest": args.instance_generation_digest, + } + if args.operation == "install-delivery": + payload = sys.stdin.buffer.read(transport.MAX_DELIVERY_BYTES + 1) + receipt = install_instance_delivery(paths, payload, **arguments) + print( + json.dumps( + receipt, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + ) + elif args.operation == "prepare": + record = prepare(paths, **arguments) + print(json.dumps(record, allow_nan=False, sort_keys=True, separators=(",", ":"))) + elif args.operation == "publish-status": + receipt = publish_status(paths, **arguments) + print(json.dumps(receipt, allow_nan=False, sort_keys=True, separators=(",", ":"))) + else: + cleanup(paths, **arguments) + except ( + Q38LinuxHostRuntimeError, + controller.RouteControllerError, + transport.Q38LinuxHostTransportError, + ) as exc: + raise SystemExit(f"Qwen3.8 Linux host runtime failed: {exc}") from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gateq38_linux_host_transport.py b/scripts/gateq38_linux_host_transport.py new file mode 100644 index 000000000..0d3153276 --- /dev/null +++ b/scripts/gateq38_linux_host_transport.py @@ -0,0 +1,914 @@ +"""Authenticated per-instance status envelopes for the Qwen3.8 Linux hosts. + +Guest attributes or another host-to-controller carrier are transport bytes, not a +trust root. This module authenticates a strict status envelope with a +controller-generated, per-instance key and binds it to the exact provider +generation and route plan. It never invokes a provider or starts a paid host. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import re +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, Mapping, Sequence + +from scripts import gateq38_route_controller as controller + +SCHEMA_VERSION = 1 +CONTEXT_SCOPE = "qwen3.8-linux-instance-context" +ENVELOPE_SCOPE = "qwen3.8-linux-host-status" +DELIVERY_SCOPE = "qwen3.8-linux-instance-delivery" +DELIVERY_RECEIPT_SCOPE = "qwen3.8-linux-instance-delivery-receipt" +DELIVERY_MAGIC = b"Q38DELIVERY1\n" +KEY_BYTES = 32 +MAX_CONTEXT_SECONDS = controller.EXPECTED_MAX_LIFETIME_SECONDS +MAX_STATUS_AGE_SECONDS = 300 +MAX_DELIVERY_RECEIPT_AGE_SECONDS = 300 +MAX_FUTURE_SKEW_SECONDS = 30 +MAX_ENVELOPE_BYTES = 65_536 +MAX_DELIVERY_HEADER_BYTES = 16_384 +MAX_DELIVERY_BYTES = MAX_ENVELOPE_BYTES + MAX_DELIVERY_HEADER_BYTES + KEY_BYTES + len(DELIVERY_MAGIC) +MAX_REVISION = 2**63 - 1 + +_CONTEXT_FIELDS = { + "schema_version", + "scope", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "worker_plan_digest", + "start_action_id", + "collect_action_id", + "project", + "zone", + "resource_name", + "resource_kind", + "role", + "worker_id", + "instance_id", + "creation_timestamp", + "instance_generation_digest", + "issued_at_unix", + "expires_at_unix", + "context_digest", + "context_hmac", +} +_ENVELOPE_FIELDS = { + "schema_version", + "scope", + "context", + "boot_id", + "revision", + "published_at_unix", + "prepared_record_digest", + "payload", + "payload_digest", + "envelope_hmac", +} +_DELIVERY_FIELDS = { + "schema_version", + "scope", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "start_action_id", + "resource_name", + "resource_kind", + "instance_id", + "creation_timestamp", + "instance_generation_digest", + "key_epoch", + "key_record_digest", + "previous_key_record_digest", + "key_sha256", + "key_bytes", + "context_digest", + "context_sha256", + "context_bytes", + "delivery_digest", + "delivery_hmac", +} +_DELIVERY_RECEIPT_FIELDS = { + "schema_version", + "scope", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "start_action_id", + "resource_name", + "instance_generation_digest", + "key_epoch", + "key_record_digest", + "key_sha256", + "context_digest", + "delivery_digest", + "delivery_payload_sha256", + "delivery_bytes", + "installed_at_unix", + "receipt_digest", + "receipt_hmac", +} +_HMAC_RE = re.compile(r"hmac-sha256:[0-9a-f]{64}") +_BOOT_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}") + + +class Q38LinuxHostTransportError(RuntimeError): + """An instance context, delivery, or authenticated host status failed closed.""" + + +@dataclass(frozen=True) +class InstanceDelivery: + """A secret-free delivery record plus its protected wire payload.""" + + record: Mapping[str, Any] + payload: bytes = field(repr=False) + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise Q38LinuxHostTransportError("duplicate transport JSON field") + value[key] = item + return value + + +def _reject_constant(_value: str) -> None: + raise Q38LinuxHostTransportError("non-finite transport JSON value") + + +def _canonical(value: Any) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + except (TypeError, ValueError, UnicodeEncodeError, RecursionError) as exc: + raise Q38LinuxHostTransportError("transport value is not canonical") from exc + + +def _sha256(value: Any) -> str: + return "sha256:" + hashlib.sha256(_canonical(value)).hexdigest() + + +def _key(value: bytes | bytearray) -> bytes: + if not isinstance(value, (bytes, bytearray)) or len(value) != KEY_BYTES: + raise Q38LinuxHostTransportError("transport key is invalid") + return bytes(value) + + +def _mac(domain: bytes, value: Any, key: bytes | bytearray) -> str: + return "hmac-sha256:" + hmac.new(_key(key), domain + b"\0" + _canonical(value), hashlib.sha256).hexdigest() + + +def _integer(value: Any, label: str, *, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if not isinstance(value, int) or isinstance(value, bool) or not minimum <= value <= maximum: + raise Q38LinuxHostTransportError(f"{label} is invalid") + return value + + +def _digest(value: Any, label: str) -> str: + if not isinstance(value, str) or controller._DIGEST_RE.fullmatch(value) is None: + raise Q38LinuxHostTransportError(f"{label} is invalid") + return value + + +def _resource(plan: controller.RoutePlan, resource_name: str) -> controller.ResourcePlan: + if not isinstance(resource_name, str): + raise Q38LinuxHostTransportError("instance resource name is invalid") + resource = plan.resource_by_name.get(resource_name) + if resource is None or not resource.kind.endswith("instance"): + raise Q38LinuxHostTransportError("instance resource is not planned") + return resource + + +def _context_unsigned(value: Mapping[str, Any]) -> dict[str, Any]: + unsigned = dict(value) + unsigned.pop("context_hmac", None) + return unsigned + + +def _context_digest_value(value: Mapping[str, Any]) -> str: + unsigned = _context_unsigned(value) + unsigned.pop("context_digest", None) + return _sha256(unsigned) + + +def build_instance_context( + plan: controller.RoutePlan, + resource_name: str, + instance_id: str, + creation_timestamp: str, + *, + issued_at_unix: int, + expires_at_unix: int, + key: bytes | bytearray, +) -> dict[str, Any]: + """Create the exact post-provisioning context installed root-only on one VM.""" + + resource = _resource(plan, resource_name) + issued = _integer(issued_at_unix, "context issue time") + expires = _integer(expires_at_unix, "context expiry") + if expires <= issued or expires - issued > MAX_CONTEXT_SECONDS or expires > plan.deadline_unix: + raise Q38LinuxHostTransportError("instance context time window is invalid") + try: + generation = controller.instance_generation_digest( + resource.name, + instance_id, + creation_timestamp, + ) + except controller.RouteControllerError as exc: + raise Q38LinuxHostTransportError("instance generation is invalid") from exc + role = "worker" if resource.kind == "worker_instance" else "bootstrap" + value: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": CONTEXT_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "collect_action_id": controller._action_id(plan, "collect_route"), + "project": controller.EXPECTED_PROJECT, + "zone": controller.EXPECTED_ZONE, + "resource_name": resource.name, + "resource_kind": resource.kind, + "role": role, + "worker_id": resource.worker_id, + "instance_id": instance_id, + "creation_timestamp": creation_timestamp, + "instance_generation_digest": generation, + "issued_at_unix": issued, + "expires_at_unix": expires, + "context_digest": "", + "context_hmac": "", + } + value["context_digest"] = _context_digest_value(value) + value["context_hmac"] = _mac(b"gateq38-instance-context-v1", _context_unsigned(value), key) + return value + + +def validate_instance_context( + value: Any, + plan: controller.RoutePlan, + *, + key: bytes | bytearray, + now_unix: int, + expected_resource_name: str | None = None, + expected_generation_digest: str | None = None, + _allow_expired_for_cleanup: bool = False, +) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _CONTEXT_FIELDS: + raise Q38LinuxHostTransportError("instance context schema is invalid") + now = _integer(now_unix, "context verification time") + resource = _resource(plan, value.get("resource_name")) + role = "worker" if resource.kind == "worker_instance" else "bootstrap" + fixed = { + "schema_version": SCHEMA_VERSION, + "scope": CONTEXT_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "collect_action_id": controller._action_id(plan, "collect_route"), + "project": controller.EXPECTED_PROJECT, + "zone": controller.EXPECTED_ZONE, + "resource_name": resource.name, + "resource_kind": resource.kind, + "role": role, + "worker_id": resource.worker_id, + } + if any(value[field] != expected for field, expected in fixed.items()): + raise Q38LinuxHostTransportError("instance context plan binding is invalid") + if expected_resource_name is not None and resource.name != expected_resource_name: + raise Q38LinuxHostTransportError("instance context resource changed") + issued = _integer(value["issued_at_unix"], "context issue time") + expires = _integer(value["expires_at_unix"], "context expiry") + if ( + expires <= issued + or expires - issued > MAX_CONTEXT_SECONDS + or expires > plan.deadline_unix + or issued > now + MAX_FUTURE_SKEW_SECONDS + or not _allow_expired_for_cleanup + and now >= expires + ): + raise Q38LinuxHostTransportError("instance context is stale") + try: + generation = controller.instance_generation_digest( + resource.name, + value["instance_id"], + value["creation_timestamp"], + ) + except (controller.RouteControllerError, TypeError) as exc: + raise Q38LinuxHostTransportError("instance context generation is invalid") from exc + if generation != value["instance_generation_digest"]: + raise Q38LinuxHostTransportError("instance context generation binding changed") + if expected_generation_digest is not None and generation != expected_generation_digest: + raise Q38LinuxHostTransportError("instance context uses another provider generation") + if value["context_digest"] != _context_digest_value(value): + raise Q38LinuxHostTransportError("instance context digest changed") + supplied_hmac = value["context_hmac"] + if not isinstance(supplied_hmac, str) or _HMAC_RE.fullmatch(supplied_hmac) is None: + raise Q38LinuxHostTransportError("instance context authentication is invalid") + expected_hmac = _mac(b"gateq38-instance-context-v1", _context_unsigned(value), key) + if not hmac.compare_digest(supplied_hmac, expected_hmac): + raise Q38LinuxHostTransportError("instance context authentication failed") + return dict(value) + + +def encode_instance_context(value: Mapping[str, Any]) -> bytes: + payload = _canonical(value) + b"\n" + if not 1 <= len(payload) <= MAX_ENVELOPE_BYTES: + raise Q38LinuxHostTransportError("instance context exceeded its size bound") + return payload + + +def decode_instance_context(payload: bytes) -> dict[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_ENVELOPE_BYTES: + raise Q38LinuxHostTransportError("instance context transport bytes are invalid") + if not payload.endswith(b"\n") or payload.count(b"\n") != 1: + raise Q38LinuxHostTransportError("instance context transport framing is invalid") + try: + value = json.loads( + payload[:-1].decode("ascii"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise Q38LinuxHostTransportError("instance context transport JSON is invalid") from exc + if not isinstance(value, dict) or payload != _canonical(value) + b"\n": + raise Q38LinuxHostTransportError("instance context transport is not canonical") + return value + + +def _delivery_unsigned(value: Mapping[str, Any]) -> dict[str, Any]: + unsigned = dict(value) + unsigned.pop("delivery_hmac", None) + return unsigned + + +def _delivery_digest_value(value: Mapping[str, Any]) -> str: + unsigned = _delivery_unsigned(value) + unsigned.pop("delivery_digest", None) + return _sha256(unsigned) + + +def build_instance_delivery( + plan: controller.RoutePlan, + material: controller.InstanceGenerationKey, + *, + now_unix: int, +) -> InstanceDelivery: + """Build one exact-generation bundle for protected stdin delivery.""" + + if not isinstance(material, controller.InstanceGenerationKey): + raise Q38LinuxHostTransportError("instance delivery key material is invalid") + try: + record = controller.validate_instance_generation_key_record(material.record, plan) + except controller.RouteControllerError as exc: + raise Q38LinuxHostTransportError("instance delivery key record is invalid") from exc + key = _key(material.key) + if record["key_sha256"] != "sha256:" + hashlib.sha256(key).hexdigest(): + raise Q38LinuxHostTransportError("instance delivery key digest changed") + now = _integer(now_unix, "instance delivery time") + issued = record["issued_at_unix"] + expires = min(record["expires_at_unix"], issued + MAX_CONTEXT_SECONDS) + if issued > now + MAX_FUTURE_SKEW_SECONDS or now >= expires: + raise Q38LinuxHostTransportError("instance delivery key is stale") + context = build_instance_context( + plan, + record["resource_name"], + record["instance_id"], + record["creation_timestamp"], + issued_at_unix=issued, + expires_at_unix=expires, + key=key, + ) + context_payload = encode_instance_context(context) + value: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": DELIVERY_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "resource_name": record["resource_name"], + "resource_kind": record["resource_kind"], + "instance_id": record["instance_id"], + "creation_timestamp": record["creation_timestamp"], + "instance_generation_digest": record["instance_generation_digest"], + "key_epoch": record["key_epoch"], + "key_record_digest": record["record_digest"], + "previous_key_record_digest": record["previous_record_digest"], + "key_sha256": record["key_sha256"], + "key_bytes": KEY_BYTES, + "context_digest": context["context_digest"], + "context_sha256": "sha256:" + hashlib.sha256(context_payload).hexdigest(), + "context_bytes": len(context_payload), + "delivery_digest": "", + "delivery_hmac": "", + } + value["delivery_digest"] = _delivery_digest_value(value) + value["delivery_hmac"] = _mac( + b"gateq38-instance-delivery-v1", + _delivery_unsigned(value), + key, + ) + payload = DELIVERY_MAGIC + _canonical(value) + b"\n" + context_payload + key + if not 1 <= len(payload) <= MAX_DELIVERY_BYTES: + raise Q38LinuxHostTransportError("instance delivery exceeded its size bound") + validated, _context, _key_value = decode_instance_delivery( + payload, + plan, + now_unix=now, + expected_resource_name=record["resource_name"], + expected_generation_digest=record["instance_generation_digest"], + ) + return InstanceDelivery(MappingProxyType(validated), payload) + + +def _validate_delivery_record( + value: Any, + context: Mapping[str, Any], + context_payload: bytes, + key: bytes, + plan: controller.RoutePlan, +) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _DELIVERY_FIELDS: + raise Q38LinuxHostTransportError("instance delivery schema is invalid") + resource = _resource(plan, value.get("resource_name")) + fixed = { + "schema_version": SCHEMA_VERSION, + "scope": DELIVERY_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "resource_name": context["resource_name"], + "resource_kind": resource.kind, + "instance_id": context["instance_id"], + "creation_timestamp": context["creation_timestamp"], + "instance_generation_digest": context["instance_generation_digest"], + "key_sha256": "sha256:" + hashlib.sha256(key).hexdigest(), + "key_bytes": KEY_BYTES, + "context_digest": context["context_digest"], + "context_sha256": "sha256:" + hashlib.sha256(context_payload).hexdigest(), + "context_bytes": len(context_payload), + } + if any(value[field] != expected for field, expected in fixed.items()): + raise Q38LinuxHostTransportError("instance delivery binding is invalid") + epoch = _integer(value["key_epoch"], "instance delivery key epoch", minimum=1) + if epoch > 99_999_999: + raise Q38LinuxHostTransportError("instance delivery key epoch is invalid") + _digest(value["key_record_digest"], "instance delivery key record digest") + previous = value["previous_key_record_digest"] + if previous is not None: + _digest(previous, "previous instance delivery key record digest") + if epoch == 1 and previous is not None or epoch > 1 and previous is None: + raise Q38LinuxHostTransportError("instance delivery key chain is invalid") + _digest(value["delivery_digest"], "instance delivery digest") + if value["delivery_digest"] != _delivery_digest_value(value): + raise Q38LinuxHostTransportError("instance delivery digest changed") + supplied_hmac = value["delivery_hmac"] + if not isinstance(supplied_hmac, str) or _HMAC_RE.fullmatch(supplied_hmac) is None: + raise Q38LinuxHostTransportError("instance delivery authentication is invalid") + expected_hmac = _mac( + b"gateq38-instance-delivery-v1", + _delivery_unsigned(value), + key, + ) + if not hmac.compare_digest(supplied_hmac, expected_hmac): + raise Q38LinuxHostTransportError("instance delivery authentication failed") + return dict(value) + + +def decode_instance_delivery( + payload: bytes, + plan: controller.RoutePlan, + *, + now_unix: int, + expected_resource_name: str | None = None, + expected_generation_digest: str | None = None, + allow_expired_for_cleanup: bool = False, +) -> tuple[dict[str, Any], dict[str, Any], bytes]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_DELIVERY_BYTES: + raise Q38LinuxHostTransportError("instance delivery transport bytes are invalid") + if not payload.startswith(DELIVERY_MAGIC): + raise Q38LinuxHostTransportError("instance delivery framing is invalid") + header_start = len(DELIVERY_MAGIC) + header_end = payload.find(b"\n", header_start) + if header_end < header_start or header_end - header_start > MAX_DELIVERY_HEADER_BYTES: + raise Q38LinuxHostTransportError("instance delivery header is invalid") + header_payload = payload[header_start:header_end] + try: + value = json.loads( + header_payload.decode("ascii"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise Q38LinuxHostTransportError("instance delivery header JSON is invalid") from exc + if not isinstance(value, dict) or header_payload != _canonical(value): + raise Q38LinuxHostTransportError("instance delivery header is not canonical") + if set(value) != _DELIVERY_FIELDS: + raise Q38LinuxHostTransportError("instance delivery schema is invalid") + context_size = _integer( + value["context_bytes"], + "instance delivery context size", + minimum=1, + maximum=MAX_ENVELOPE_BYTES, + ) + key_size = _integer( + value["key_bytes"], + "instance delivery key size", + minimum=KEY_BYTES, + maximum=KEY_BYTES, + ) + body = payload[header_end + 1 :] + if len(body) != context_size + key_size: + raise Q38LinuxHostTransportError("instance delivery framing is invalid") + context_payload = body[:context_size] + key = _key(body[context_size:]) + context = decode_instance_context(context_payload) + validated_context = validate_instance_context( + context, + plan, + key=key, + now_unix=now_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + _allow_expired_for_cleanup=allow_expired_for_cleanup, + ) + record = _validate_delivery_record(value, validated_context, context_payload, key, plan) + if expected_resource_name is not None and record["resource_name"] != expected_resource_name: + raise Q38LinuxHostTransportError("instance delivery resource changed") + if expected_generation_digest is not None and record["instance_generation_digest"] != expected_generation_digest: + raise Q38LinuxHostTransportError("instance delivery provider generation changed") + return record, validated_context, key + + +def validate_instance_delivery( + delivery: InstanceDelivery, + plan: controller.RoutePlan, + *, + now_unix: int, + expected_resource_name: str | None = None, + expected_generation_digest: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any], bytes]: + if not isinstance(delivery, InstanceDelivery): + raise Q38LinuxHostTransportError("instance delivery is invalid") + record, context, key = decode_instance_delivery( + delivery.payload, + plan, + now_unix=now_unix, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + ) + if dict(delivery.record) != record: + raise Q38LinuxHostTransportError("instance delivery record changed") + return record, context, key + + +def _receipt_unsigned(value: Mapping[str, Any]) -> dict[str, Any]: + unsigned = dict(value) + unsigned.pop("receipt_hmac", None) + return unsigned + + +def _receipt_digest_value(value: Mapping[str, Any]) -> str: + unsigned = _receipt_unsigned(value) + unsigned.pop("receipt_digest", None) + return _sha256(unsigned) + + +def build_instance_delivery_receipt( + delivery: InstanceDelivery, + plan: controller.RoutePlan, + *, + installed_at_unix: int, +) -> dict[str, Any]: + installed = _integer(installed_at_unix, "instance delivery installation time") + record, _context, key = validate_instance_delivery( + delivery, + plan, + now_unix=installed, + ) + value: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": DELIVERY_RECEIPT_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "resource_name": record["resource_name"], + "instance_generation_digest": record["instance_generation_digest"], + "key_epoch": record["key_epoch"], + "key_record_digest": record["key_record_digest"], + "key_sha256": record["key_sha256"], + "context_digest": record["context_digest"], + "delivery_digest": record["delivery_digest"], + "delivery_payload_sha256": "sha256:" + hashlib.sha256(delivery.payload).hexdigest(), + "delivery_bytes": len(delivery.payload), + "installed_at_unix": installed, + "receipt_digest": "", + "receipt_hmac": "", + } + value["receipt_digest"] = _receipt_digest_value(value) + value["receipt_hmac"] = _mac( + b"gateq38-instance-delivery-receipt-v1", + _receipt_unsigned(value), + key, + ) + return value + + +def validate_instance_delivery_receipt( + value: Any, + delivery: InstanceDelivery, + plan: controller.RoutePlan, + *, + now_unix: int, +) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _DELIVERY_RECEIPT_FIELDS: + raise Q38LinuxHostTransportError("instance delivery receipt schema is invalid") + now = _integer(now_unix, "instance delivery receipt verification time") + record, _context, key = validate_instance_delivery(delivery, plan, now_unix=now) + _digest(value["receipt_digest"], "instance delivery receipt digest") + if value["receipt_digest"] != _receipt_digest_value(value): + raise Q38LinuxHostTransportError("instance delivery receipt digest changed") + supplied_hmac = value["receipt_hmac"] + if not isinstance(supplied_hmac, str) or _HMAC_RE.fullmatch(supplied_hmac) is None: + raise Q38LinuxHostTransportError("instance delivery receipt authentication is invalid") + expected_hmac = _mac( + b"gateq38-instance-delivery-receipt-v1", + _receipt_unsigned(value), + key, + ) + if not hmac.compare_digest(supplied_hmac, expected_hmac): + raise Q38LinuxHostTransportError("instance delivery receipt authentication failed") + installed = _integer(value["installed_at_unix"], "instance delivery installation time") + fixed = { + "schema_version": SCHEMA_VERSION, + "scope": DELIVERY_RECEIPT_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "resource_name": record["resource_name"], + "instance_generation_digest": record["instance_generation_digest"], + "key_epoch": record["key_epoch"], + "key_record_digest": record["key_record_digest"], + "key_sha256": record["key_sha256"], + "context_digest": record["context_digest"], + "delivery_digest": record["delivery_digest"], + "delivery_payload_sha256": "sha256:" + hashlib.sha256(delivery.payload).hexdigest(), + "delivery_bytes": len(delivery.payload), + "installed_at_unix": installed, + } + if any(value[field] != expected for field, expected in fixed.items()): + raise Q38LinuxHostTransportError("instance delivery receipt binding is invalid") + if installed > now + MAX_FUTURE_SKEW_SECONDS: + raise Q38LinuxHostTransportError("instance delivery receipt is future-dated") + if now - installed > MAX_DELIVERY_RECEIPT_AGE_SECONDS: + raise Q38LinuxHostTransportError("instance delivery receipt is stale") + return dict(value) + + +def _worker_payload(value: Any, plan: controller.RoutePlan, worker_id: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != controller._OBS_WORKER_FIELDS: + raise Q38LinuxHostTransportError("worker status payload schema is invalid") + worker = plan.worker_by_id.get(worker_id) + if worker is None: + raise Q38LinuxHostTransportError("worker status identity is invalid") + if value["state"] not in {"starting", "ready", "failed"}: + raise Q38LinuxHostTransportError("worker status state is invalid") + expected = { + "machine_id": worker.machine_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "span": worker.span, + "manifest_digest": plan.manifest_digest, + "artifact_bytes": worker.artifact_bytes, + "artifact_set_digest": worker.artifact_set_digest, + "cache_root": worker.cache_root, + } + if any(value[field] != expected_item for field, expected_item in expected.items()): + raise Q38LinuxHostTransportError("worker status plan binding is invalid") + peer_id = value["peer_id"] + if value["state"] == "ready": + if not isinstance(peer_id, str) or controller._PEER_RE.fullmatch(peer_id) is None: + raise Q38LinuxHostTransportError("ready worker status lacks an exact peer") + elif peer_id is not None: + raise Q38LinuxHostTransportError("unfinished worker exposed a peer identity") + return dict(value) + + +def _bootstrap_payload(value: Any, plan: controller.RoutePlan) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != controller._ROUTE_JOB_FIELDS: + raise Q38LinuxHostTransportError("route-job status payload schema is invalid") + state = value["state"] + if state not in {"absent", "running", "passed", "failed"}: + raise Q38LinuxHostTransportError("route-job status state is invalid") + if state == "absent": + if any(item is not None for field, item in value.items() if field != "state"): + raise Q38LinuxHostTransportError("absent route-job status exposed metadata") + return dict(value) + expected = { + "job_id": plan.route_job_id, + "collect_action_id": controller._action_id(plan, "collect_route"), + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "worker_plan_digest": plan.worker_plan_digest, + } + if any(value[field] != expected_item for field, expected_item in expected.items()): + raise Q38LinuxHostTransportError("route-job status plan binding is invalid") + if state == "passed": + record = value["route_record"] + evidence_digest = value["evidence_digest"] + if not isinstance(record, dict) or evidence_digest != controller._canonical_digest(record): + raise Q38LinuxHostTransportError("passed route-job status evidence is invalid") + elif value["route_record"] is not None or value["evidence_digest"] is not None: + raise Q38LinuxHostTransportError("non-passed route-job status exposed evidence") + return dict(value) + + +def initial_status_payload(context: Mapping[str, Any], plan: controller.RoutePlan) -> dict[str, Any]: + resource = _resource(plan, context.get("resource_name")) + if context.get("resource_kind") != resource.kind or context.get("worker_id") != resource.worker_id: + raise Q38LinuxHostTransportError("instance context resource binding is invalid") + if resource.kind == "bootstrap_instance": + return {field: ("absent" if field == "state" else None) for field in controller._ROUTE_JOB_FIELDS} + if resource.worker_id is None: + raise Q38LinuxHostTransportError("worker context lacks a worker identity") + worker = plan.worker_by_id[resource.worker_id] + return { + "state": "starting", + "machine_id": worker.machine_id, + "peer_id": None, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": controller._action_id(plan, "start_route"), + "span": worker.span, + "manifest_digest": plan.manifest_digest, + "artifact_bytes": worker.artifact_bytes, + "artifact_set_digest": worker.artifact_set_digest, + "cache_root": worker.cache_root, + } + + +def _payload(value: Any, context: Mapping[str, Any], plan: controller.RoutePlan) -> dict[str, Any]: + if context["role"] == "worker": + worker_id = context["worker_id"] + if not isinstance(worker_id, str): + raise Q38LinuxHostTransportError("worker context lacks a worker identity") + return _worker_payload(value, plan, worker_id) + if context["worker_id"] is not None: + raise Q38LinuxHostTransportError("bootstrap context exposed a worker identity") + return _bootstrap_payload(value, plan) + + +def _envelope_unsigned(value: Mapping[str, Any]) -> dict[str, Any]: + unsigned = dict(value) + unsigned.pop("envelope_hmac", None) + return unsigned + + +def build_status_envelope( + context: Mapping[str, Any], + payload: Mapping[str, Any], + plan: controller.RoutePlan, + *, + key: bytes | bytearray, + boot_id: str, + revision: int, + published_at_unix: int, + prepared_record_digest: str, +) -> dict[str, Any]: + published = _integer(published_at_unix, "status publication time") + validated_context = validate_instance_context(context, plan, key=key, now_unix=published) + if not isinstance(boot_id, str) or _BOOT_ID_RE.fullmatch(boot_id) is None: + raise Q38LinuxHostTransportError("Linux boot identity is invalid") + current_revision = _integer(revision, "status revision", minimum=1, maximum=MAX_REVISION) + prepared = _digest(prepared_record_digest, "prepared record digest") + validated_payload = _payload(dict(payload), validated_context, plan) + value: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": ENVELOPE_SCOPE, + "context": validated_context, + "boot_id": boot_id, + "revision": current_revision, + "published_at_unix": published, + "prepared_record_digest": prepared, + "payload": validated_payload, + "payload_digest": _sha256(validated_payload), + "envelope_hmac": "", + } + value["envelope_hmac"] = _mac(b"gateq38-host-status-v1", _envelope_unsigned(value), key) + if len(_canonical(value)) + 1 > MAX_ENVELOPE_BYTES: + raise Q38LinuxHostTransportError("host status envelope exceeded its size bound") + return value + + +def validate_status_envelope( + value: Any, + plan: controller.RoutePlan, + *, + key: bytes | bytearray, + now_unix: int, + expected_resource_name: str, + expected_generation_digest: str, + minimum_revision: int = 0, + expected_boot_id: str | None = None, +) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _ENVELOPE_FIELDS: + raise Q38LinuxHostTransportError("host status envelope schema is invalid") + now = _integer(now_unix, "status verification time") + context = validate_instance_context( + value["context"], + plan, + key=key, + now_unix=now, + expected_resource_name=expected_resource_name, + expected_generation_digest=expected_generation_digest, + ) + boot_id = value["boot_id"] + if not isinstance(boot_id, str) or _BOOT_ID_RE.fullmatch(boot_id) is None: + raise Q38LinuxHostTransportError("Linux boot identity is invalid") + if expected_boot_id is not None and boot_id != expected_boot_id: + raise Q38LinuxHostTransportError("Linux boot identity changed") + revision = _integer(value["revision"], "status revision", minimum=1, maximum=MAX_REVISION) + floor = _integer(minimum_revision, "minimum status revision", maximum=MAX_REVISION) + if revision <= floor: + raise Q38LinuxHostTransportError("host status revision is stale") + published = _integer(value["published_at_unix"], "status publication time") + if ( + published < context["issued_at_unix"] + or published > now + MAX_FUTURE_SKEW_SECONDS + or now - published > MAX_STATUS_AGE_SECONDS + ): + raise Q38LinuxHostTransportError("host status publication is stale") + _digest(value["prepared_record_digest"], "prepared record digest") + payload = _payload(value["payload"], context, plan) + if value["payload_digest"] != _sha256(payload): + raise Q38LinuxHostTransportError("host status payload digest changed") + supplied_hmac = value["envelope_hmac"] + if not isinstance(supplied_hmac, str) or _HMAC_RE.fullmatch(supplied_hmac) is None: + raise Q38LinuxHostTransportError("host status authentication is invalid") + expected_hmac = _mac(b"gateq38-host-status-v1", _envelope_unsigned(value), key) + if not hmac.compare_digest(supplied_hmac, expected_hmac): + raise Q38LinuxHostTransportError("host status authentication failed") + if len(_canonical(value)) + 1 > MAX_ENVELOPE_BYTES: + raise Q38LinuxHostTransportError("host status envelope exceeded its size bound") + result = dict(value) + result["context"] = context + result["payload"] = json.loads(_canonical(payload).decode("ascii")) + return result + + +def encode_status_envelope(value: Mapping[str, Any]) -> bytes: + payload = _canonical(value) + b"\n" + if not 1 <= len(payload) <= MAX_ENVELOPE_BYTES: + raise Q38LinuxHostTransportError("host status envelope exceeded its size bound") + return payload + + +def decode_status_envelope(payload: bytes) -> dict[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_ENVELOPE_BYTES: + raise Q38LinuxHostTransportError("host status transport bytes are invalid") + if not payload.endswith(b"\n") or payload.count(b"\n") != 1: + raise Q38LinuxHostTransportError("host status transport framing is invalid") + try: + value = json.loads( + payload[:-1].decode("ascii"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise Q38LinuxHostTransportError("host status transport JSON is invalid") from exc + if not isinstance(value, dict): + raise Q38LinuxHostTransportError("host status transport JSON is invalid") + if payload != _canonical(value) + b"\n": + raise Q38LinuxHostTransportError("host status transport is not canonical") + return value diff --git a/scripts/gateq38_route_controller.py b/scripts/gateq38_route_controller.py new file mode 100644 index 000000000..a02fc057d --- /dev/null +++ b/scripts/gateq38_route_controller.py @@ -0,0 +1,3883 @@ +"""Durable no-provider controller for the Qwen3.8 complete-route attempt. + +The controller never invokes a provider. Each operation consumes an exact bounded +observation, persists a source/plan-bound state, and emits at most one allowlisted +action for a separate provider adapter. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import secrets +import shutil +import stat +import subprocess +import tempfile +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime +from decimal import ROUND_CEILING, Decimal, InvalidOperation +from pathlib import Path, PurePosixPath +from types import MappingProxyType +from typing import Any, Iterator, Mapping, Sequence + +SCHEMA_VERSION = 1 +STATE_SCHEMA_VERSION = 1 +INSTANCE_KEY_SCHEMA_VERSION = 1 +INSTANCE_KEY_SCOPE = "qwen3.8-instance-generation-key" +INSTANCE_KEY_ACTIVE_SCOPE = "qwen3.8-active-instance-generation-key" +INSTANCE_KEY_TOMBSTONE_SCOPE = "qwen3.8-revoked-instance-generation-key" +INSTANCE_KEY_BYTES = 32 +MAX_INSTANCE_KEY_RECORD_BYTES = 16_384 +_WINDOWS_INSTANCE_KEY_DIRECTORY_ACL = r"""param( + [Parameter(Mandatory = $true)][string]$path, + [Parameter(Mandatory = $true)][bool]$apply +) +$ErrorActionPreference = 'Stop' +$sid = [System.Security.Principal.WindowsIdentity]::GetCurrent().User +if ($apply) { + $security = New-Object System.Security.AccessControl.DirectorySecurity + $security.SetAccessRuleProtection($true, $false) + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + $sid, + 'FullControl', + 'ContainerInherit,ObjectInherit', + 'None', + 'Allow' + ) + $security.SetAccessRule($rule) + [System.IO.Directory]::SetAccessControl($path, $security) +} +$verified = [System.IO.Directory]::GetAccessControl( + $path, + [System.Security.AccessControl.AccessControlSections]::Access +) +$rules = @($verified.GetAccessRules( + $true, + $true, + [System.Security.Principal.SecurityIdentifier] +)) +if ( + -not $verified.AreAccessRulesProtected -or + $rules.Count -ne 1 -or + $rules[0].IdentityReference.Value -ne $sid.Value -or + $rules[0].AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow -or + $rules[0].IsInherited -or + $rules[0].FileSystemRights -ne [System.Security.AccessControl.FileSystemRights]::FullControl +) { + throw 'private directory ACL verification failed' +} +""" +_WINDOWS_INSTANCE_KEY_FILE_ACL = r"""param( + [Parameter(Mandatory = $true)][string]$path, + [Parameter(Mandatory = $true)][bool]$apply +) +$ErrorActionPreference = 'Stop' +$sid = [System.Security.Principal.WindowsIdentity]::GetCurrent().User +if ($apply) { + $security = New-Object System.Security.AccessControl.FileSecurity + $security.SetAccessRuleProtection($true, $false) + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + $sid, + 'FullControl', + 'Allow' + ) + $security.SetAccessRule($rule) + [System.IO.File]::SetAccessControl($path, $security) +} +$verified = [System.IO.File]::GetAccessControl( + $path, + [System.Security.AccessControl.AccessControlSections]::Access +) +$rules = @($verified.GetAccessRules( + $true, + $true, + [System.Security.Principal.SecurityIdentifier] +)) +if ( + -not $verified.AreAccessRulesProtected -or + $rules.Count -ne 1 -or + $rules[0].IdentityReference.Value -ne $sid.Value -or + $rules[0].AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow -or + $rules[0].IsInherited -or + $rules[0].FileSystemRights -ne [System.Security.AccessControl.FileSystemRights]::FullControl +) { + throw 'private file ACL verification failed' +} +""" +GATE = "qwen3.8-complete-route" +MAX_JSON_BYTES = 262_144 +PROTECTED_INSTANCE = "communityai-bootstrap-1" +EXPECTED_PROVIDER = "gcp" +EXPECTED_PROJECT = "community-ai-506321" +EXPECTED_REGION = "us-central1" +EXPECTED_ZONE = "us-central1-b" +EXPECTED_WORKER_MACHINE_TYPE = "g2-standard-8" +EXPECTED_BOOTSTRAP_MACHINE_TYPE = "e2-standard-2" +EXPECTED_ACCELERATOR_TYPE = "nvidia-l4" +EXPECTED_SOURCE_IMAGE = "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831" +EXPECTED_DISK_TYPE = "pd-balanced" +EXPECTED_DISK_SIZE_GB = 50 +EXPECTED_NETWORK = "communityai-discovery" +EXPECTED_SUBNET = "communityai-us-central1" +EXPECTED_MAX_LIFETIME_SECONDS = 39_600 +EXPECTED_PRICED_DURATION_HOURS = Decimal("11.00") +EXPECTED_MANIFEST_DIGEST = "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" +EXPECTED_MODEL_REVISION = "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a" +EXPECTED_INDEX_DIGEST = "sha256:f0838c766951bdfe76d6afbdb2771a8f67aaa2231dedb3d33cebd817729843a2" +EXPECTED_BLOCK_PREFIX = "model.language_model.layers" +EXPECTED_ARTIFACTS_PER_SPAN = 18 +VERIFIER_SOURCE_PATH = "src/drift/model_manifest.py" +READINESS_LEDGER_PATH = "docs/RELEASE_READINESS.md" +PROTECTION_SOURCE_PATH = "scripts/gate14_packaged_lifecycle.py" +GCP_ADAPTER_SOURCE_PATH = "scripts/gateq38_gcp_adapter.py" +DESKTOP_RELEASE_VERIFIER_SOURCE_PATH = "desktop/build_desktop.py" +STAGE_PACKAGE_SOURCE_PATH = "scripts/gateq38_stage_package.py" +LINUX_HOST_RUNTIME_SOURCE_PATH = "scripts/gateq38_linux_host_runtime.py" +LINUX_HOST_TRANSPORT_SOURCE_PATH = "scripts/gateq38_linux_host_transport.py" +MAX_RELEASE_PROVENANCE_BYTES = 16 * 1024 * 1024 +MAX_RELEASE_CHECKSUMS_BYTES = 16 * 1024 * 1024 +MAX_RELEASE_METRICS_BYTES = 16 * 1024 * 1024 +RUNTIME_PACKAGE_SCHEMA_VERSION = 1 +RUNTIME_PACKAGE_SCOPE = "qwen3.8-linux-runtime-package" +RUNTIME_PACKAGE_PLATFORM = "linux" +RUNTIME_PACKAGE_ARCHIVE = "communityai-desktop-linux.tar.gz" +RUNTIME_PACKAGE_NODE_ROOT = "CommunityAI/node" +RUNTIME_PACKAGE_NODE_EXECUTABLE = "CommunityAI/node/CommunityAI-Node" +REQUIRED_SOURCE_PATHS = { + "desktop/build_desktop.py", + "docs/RELEASE_READINESS.md", + "scripts/gate14_packaged_lifecycle.py", + "scripts/gateq38_gcp_adapter.py", + "scripts/gateq38_linux_host_runtime.py", + "scripts/gateq38_linux_host_transport.py", + "scripts/gateq38_route_controller.py", + "scripts/gateq38_stage_package.py", + "scripts/qualify_model_multimachine.py", + "src/drift/model_manifest.py", + "src/drift/server/server.py", +} +MAX_PLAN_REVALIDATION_AGE_SECONDS = 300 +EXPECTED_SPANS = { + "0:16": ( + 6_095_829_165, + "sha256:70c0c950845c0c53dc0269d525c755bc72e661cf4ded8a78a7b5f99d8d195d89", + ), + "16:32": ( + 6_095_829_389, + "sha256:01d4ca6e77a9564e6896343b0c8558619fcda78819eeafb0d49393a955460866", + ), + "32:48": ( + 6_095_829_389, + "sha256:4b3ac15527d87d2dbd089fc4ba4ab0dec4610a5e9870df1401473159b55138e5", + ), + "48:64": ( + 6_095_829_389, + "sha256:2e779c52ab2eb5156aa3cfba60e5d08b4dd691e0302101cbc1a39c24d45745e1", + ), +} +EXPECTED_RESOURCE_KINDS = { + "bootstrap_instance": 1, + "bootstrap_disk": 1, + "worker_instance": 4, + "worker_disk": 4, + "firewall": 1, + "iap_firewall": 1, +} +ACTIONS = {"start_route", "collect_route", "cleanup_route", "none"} +PHASES = { + "ABSENT", + "STARTING", + "READY", + "COLLECTING", + "CLEANING", + "CLEANED_PASS", + "CLEANED_FAILURE", +} +TERMINAL_PHASES = {"CLEANED_PASS", "CLEANED_FAILURE"} +WORKER_STATES = {"absent", "starting", "ready", "failed"} +JOB_STATES = {"absent", "running", "passed", "failed"} + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{2,62}") +_LABEL_RE = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}") +_GCP_RESOURCE_RE = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_REVISION_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_PEER_RE = re.compile(r"[A-Za-z0-9]{20,128}") +_INSTANCE_ID_RE = re.compile(r"[1-9][0-9]{0,19}") +_CREATION_TIMESTAMP_RE = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?[+-][0-9]{2}:[0-9]{2}" +) + +_PLAN_FIELDS = { + "schema_version", + "gate", + "run_id", + "route_job_id", + "source_commit", + "manifest_digest", + "model_revision", + "deadline_unix", + "authorization", + "source_bindings", + "runtime_package", + "resources", + "workers", +} +_RUNTIME_PACKAGE_FIELDS = { + "schema_version", + "scope", + "platform", + "source_commit", + "source_tree", + "source_bindings_digest", + "release_archive_name", + "release_archive_sha256", + "release_archive_bytes", + "checksums_sha256", + "checksums_bytes", + "provenance_sha256", + "provenance_bytes", + "desktop_metrics_sha256", + "desktop_metrics_bytes", + "manifest_digest", + "manifest_sha256", + "manifest_bytes", + "node_root", + "node_executable", + "node_executable_sha256", + "node_executable_bytes", + "node_runtime_entry_count", + "node_runtime_bytes", + "node_runtime_inventory_digest", + "runtime_package_digest", +} +_AUTH_FIELDS = { + "combined_cloud_ceiling_usd", + "ledger_committed_before_run_usd", + "maximum_estimate_usd", + "reservation_recorded", + "native_auth_revalidated", + "inventory_revalidated", + "pricing_revalidated", + "provisioning_authorized", + "reservation_id", + "reservation_record_path", + "reservation_record_sha256", + "reservation_record_byte_size", + "preflight_record_path", + "preflight_record_sha256", + "preflight_record_byte_size", + "readiness_ledger_sha256", +} +_BINDING_FIELDS = {"relative_path", "sha256", "byte_size"} +_RESOURCE_FIELDS = {"name", "kind", "provider", "region", "worker_id"} +_WORKER_FIELDS = { + "worker_id", + "machine_id", + "instance", + "disk", + "span", + "artifact_bytes", + "artifact_set_digest", + "cache_root", +} +_JOURNAL_FIELDS = { + "schema_version", + "run_id", + "plan_digest", + "start_action_id", + "status", + "issued_at_unix", + "completed_at_unix", + "terminal_phase", + "terminal_revision", + "failure_code", + "evidence_digest", + "cleanup_verified", +} +_STATE_FIELDS = { + "schema_version", + "run_id", + "plan_digest", + "revision", + "phase", + "failure_code", + "evidence_digest", + "instance_generations_digest", + "cleanup_verified", + "next_action", +} +_INSTANCE_KEY_RECORD_FIELDS = { + "schema_version", + "scope", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "start_action_id", + "resource_name", + "resource_kind", + "instance_id", + "creation_timestamp", + "instance_generation_digest", + "key_epoch", + "issued_at_unix", + "expires_at_unix", + "key_sha256", + "key_bytes", + "previous_record_digest", + "record_digest", +} +_INSTANCE_KEY_ACTIVE_FIELDS = { + "schema_version", + "scope", + "run_id", + "plan_digest", + "resource_name", + "instance_generation_digest", + "key_epoch", + "record_digest", + "active_digest", +} +_INSTANCE_KEY_TOMBSTONE_FIELDS = { + "schema_version", + "scope", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "start_action_id", + "resource_name", + "instance_generation_digest", + "revoked_at_unix", + "last_key_epoch", + "last_record_digest", + "tombstone_digest", +} +_OBSERVATION_FIELDS = { + "schema_version", + "run_id", + "observed_at_unix", + "protected_bootstrap_running", + "artifact_plan_revalidation", + "instance_generations_digest", + "resources", + "workers", + "route_job", +} +_REVALIDATION_FIELDS = { + "verified_at_unix", + "source_commit", + "manifest_digest", + "model_revision", + "index_digest", + "block_prefix", + "worker_plan_digest", + "verifier_source_sha256", +} +_OBS_RESOURCE_FIELDS = { + "present", + "kind", + "provider", + "region", + "run_id", + "source_commit", + "deadline_unix", + "plan_digest", + "start_action_id", + "worker_id", + "instance_id", + "creation_timestamp", + "instance_generation_digest", +} +_OBS_WORKER_FIELDS = { + "state", + "machine_id", + "peer_id", + "source_commit", + "plan_digest", + "worker_plan_digest", + "start_action_id", + "span", + "manifest_digest", + "artifact_bytes", + "artifact_set_digest", + "cache_root", +} +_ROUTE_JOB_FIELDS = { + "state", + "job_id", + "collect_action_id", + "run_id", + "plan_digest", + "source_commit", + "manifest_digest", + "worker_plan_digest", + "evidence_digest", + "route_record", +} +_ROUTE_RECORD_FIELDS = { + "schema_version", + "result", + "run_id", + "job_id", + "collect_action_id", + "plan_digest", + "source_commit", + "manifest_digest", + "worker_plan_digest", + "route_span", + "session_id", + "route_rpc_evidence_digest", + "cleanup_ready", + "worker_results", +} +_ROUTE_WORKER_RESULT_FIELDS = { + "worker_id", + "machine_id", + "peer_id", + "span", + "source_commit", + "manifest_digest", + "artifact_bytes", + "artifact_set_digest", + "cache_root", + "worker_evidence_digest", +} +_RESERVATION_FIELDS = { + "schema_version", + "reservation_id", + "run_id", + "combined_cloud_ceiling_usd", + "ledger_committed_before_run_usd", + "maximum_estimate_usd", + "deadline_unix", + "plan_digest", + "execution_inventory_digest", + "worker_plan_digest", + "resource_costs", + "readiness_ledger_sha256", + "recorded_at_unix", + "expires_at_unix", + "reservation_recorded", +} +_COST_FIELDS = { + "resource_name", + "resource_spec_digest", + "unit_rate_usd", + "quantity", + "duration_hours", + "maximum_usd", +} +_RESOURCE_SPEC_FIELDS = { + "resource_name", + "kind", + "provider", + "region", + "project", + "zone", + "machine_type", + "accelerator_type", + "accelerator_count", + "source_image", + "disk_type", + "disk_size_gb", + "network", + "subnet", + "max_lifetime_seconds", +} + + +_RPC_EVIDENCE_FIELDS = { + "schema_version", + "result", + "run_id", + "job_id", + "collect_action_id", + "plan_digest", + "source_commit", + "manifest_digest", + "worker_plan_digest", + "route_span", + "session_id", +} +_WORKER_EVIDENCE_FIELDS = { + "schema_version", + "result", + "run_id", + "job_id", + "collect_action_id", + "plan_digest", + "source_commit", + "manifest_digest", + "worker_plan_digest", + "start_action_id", + "worker_id", + "machine_id", + "peer_id", + "span", + "artifact_bytes", + "artifact_set_digest", + "cache_root", +} + + +_PREFLIGHT_FIELDS = { + "schema_version", + "run_id", + "source_commit", + "plan_digest", + "execution_inventory_digest", + "worker_plan_digest", + "provider", + "resource_names", + "resource_specs", + "pricing_source", + "pricing_currency", + "pricing_checked_at_unix", + "gpu_quota_limit", + "gpu_quota_usage", + "required_gpu_count", + "checked_at_unix", + "native_auth_revalidated", + "inventory_revalidated", + "pricing_revalidated", + "provisioning_authorized", + "protected_bootstrap_running", + "reservation_record_sha256", +} + + +class RouteControllerError(ValueError): + """The route plan, observation, state, or transition failed closed.""" + + +@dataclass(frozen=True) +class InstanceGenerationKey: + """Controller-private key material with a digest-only public record.""" + + record: Mapping[str, Any] + key: bytes = field(repr=False) + + +@dataclass(frozen=True) +class WorkerPlan: + worker_id: str + machine_id: str + instance: str + disk: str + span: str + artifact_bytes: int + artifact_set_digest: str + cache_root: str + + +@dataclass(frozen=True) +class ResourcePlan: + name: str + kind: str + provider: str + region: str + worker_id: str | None + + +@dataclass(frozen=True) +class RoutePlan: + run_id: str + route_job_id: str + source_commit: str + manifest_digest: str + model_revision: str + deadline_unix: int + authorization: Mapping[str, Any] + source_bindings: tuple[Mapping[str, Any], ...] + runtime_package: Mapping[str, Any] + resources: tuple[ResourcePlan, ...] + workers: tuple[WorkerPlan, ...] + plan_digest: str + + @property + def resource_by_name(self) -> dict[str, ResourcePlan]: + return {resource.name: resource for resource in self.resources} + + @property + def worker_by_id(self) -> dict[str, WorkerPlan]: + return {worker.worker_id: worker for worker in self.workers} + + @property + def worker_plan_digest(self) -> str: + return _worker_plan_digest(self.workers) + + @property + def execution_inventory_digest(self) -> str: + return _canonical_digest( + { + "run_id": self.run_id, + "route_job_id": self.route_job_id, + "source_commit": self.source_commit, + "manifest_digest": self.manifest_digest, + "model_revision": self.model_revision, + "deadline_unix": self.deadline_unix, + "plan_digest": self.plan_digest, + "source_bindings": [dict(binding) for binding in self.source_bindings], + "runtime_package": dict(self.runtime_package), + "worker_plan_digest": self.worker_plan_digest, + "resources": [ + { + "name": resource.name, + "kind": resource.kind, + "provider": resource.provider, + "region": resource.region, + "worker_id": resource.worker_id, + "resource_spec_digest": _canonical_digest(_expected_resource_spec(resource)), + } + for resource in self.resources + ], + } + ) + + +def _expected_resource_spec(resource: ResourcePlan) -> dict[str, Any]: + is_worker_instance = resource.kind == "worker_instance" + is_bootstrap_instance = resource.kind == "bootstrap_instance" + is_instance = is_worker_instance or is_bootstrap_instance + is_disk = resource.kind in {"bootstrap_disk", "worker_disk"} + return { + "resource_name": resource.name, + "kind": resource.kind, + "provider": EXPECTED_PROVIDER, + "region": EXPECTED_REGION, + "project": EXPECTED_PROJECT, + "zone": EXPECTED_ZONE, + "machine_type": ( + EXPECTED_WORKER_MACHINE_TYPE + if is_worker_instance + else EXPECTED_BOOTSTRAP_MACHINE_TYPE + if is_bootstrap_instance + else "none" + ), + "accelerator_type": EXPECTED_ACCELERATOR_TYPE if is_worker_instance else "none", + "accelerator_count": 1 if is_worker_instance else 0, + "source_image": EXPECTED_SOURCE_IMAGE if is_instance else "none", + "disk_type": EXPECTED_DISK_TYPE if is_disk else "none", + "disk_size_gb": EXPECTED_DISK_SIZE_GB if is_disk else 0, + "network": EXPECTED_NETWORK, + "subnet": EXPECTED_SUBNET, + "max_lifetime_seconds": EXPECTED_MAX_LIFETIME_SECONDS, + } + + +def _reject_constant(_value: str) -> None: + raise RouteControllerError("invalid JSON constant") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise RouteControllerError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path, maximum: int = MAX_JSON_BYTES) -> bytes: + descriptor: int | None = None + try: + metadata = path.lstat() + except OSError as exc: + raise RouteControllerError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise RouteControllerError("required file is unsafe") + try: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + identity = lambda item: ( + item.st_dev, + item.st_ino, + item.st_size, + getattr(item, "st_mtime_ns", int(item.st_mtime * 1_000_000_000)), + ) + if not stat.S_ISREG(opened.st_mode) or identity(opened) != identity(metadata): + raise RouteControllerError("required file identity changed while opening") + with os.fdopen(descriptor, "rb", closefd=False) as handle: + payload = handle.read(maximum + 1) + after = os.fstat(descriptor) + if identity(after) != identity(opened) or len(payload) != opened.st_size: + raise RouteControllerError("required file changed while reading") + return payload + except OSError as exc: + raise RouteControllerError("required file is unreadable") from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not 1 <= len(payload) <= MAX_JSON_BYTES: + raise RouteControllerError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RouteControllerError("invalid JSON") from exc + if not isinstance(value, dict): + raise RouteControllerError("JSON root must be an object") + return value + + +def _mapping(value: Any, fields: set[str], field: str) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise RouteControllerError(f"{field} schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str], field: str) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise RouteControllerError(f"{field} is invalid") + return value + + +def _integer(value: Any, field: str, minimum: int = 0) -> int: + if type(value) is not int or value < minimum: + raise RouteControllerError(f"{field} is invalid") + return value + + +def _boolean(value: Any, field: str) -> bool: + if type(value) is not bool: + raise RouteControllerError(f"{field} is invalid") + return value + + +def _money(value: Any, field: str) -> Decimal: + if not isinstance(value, str): + raise RouteControllerError(f"{field} is invalid") + try: + result = Decimal(value) + except InvalidOperation as exc: + raise RouteControllerError(f"{field} is invalid") from exc + if not result.is_finite() or result < 0 or result.quantize(Decimal("0.01")) != result: + raise RouteControllerError(f"{field} is invalid") + return result + + +def _canonical_digest(value: Mapping[str, Any]) -> str: + payload = json.dumps(value, allow_nan=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _source_bindings_digest(source_bindings: Sequence[Mapping[str, Any]]) -> str: + return _canonical_digest({"source_bindings": [dict(binding) for binding in source_bindings]}) + + +def _runtime_package_digest(value: Mapping[str, Any]) -> str: + try: + payload = ( + json.dumps( + {key: value[key] for key in sorted(value) if key != "runtime_package_digest"}, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + except (KeyError, TypeError, ValueError) as exc: + raise RouteControllerError("runtime package record is not canonical JSON") from exc + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def validate_runtime_package_record( + value: Any, + *, + expected_source_commit: str | None = None, + expected_manifest_digest: str | None = None, + expected_source_bindings: Sequence[Mapping[str, Any]] | None = None, +) -> Mapping[str, Any]: + record = dict(_mapping(value, _RUNTIME_PACKAGE_FIELDS, "runtime_package")) + if ( + type(record["schema_version"]) is not int + or record["schema_version"] != RUNTIME_PACKAGE_SCHEMA_VERSION + or record["scope"] != RUNTIME_PACKAGE_SCOPE + or record["platform"] != RUNTIME_PACKAGE_PLATFORM + or record["release_archive_name"] != RUNTIME_PACKAGE_ARCHIVE + or record["node_root"] != RUNTIME_PACKAGE_NODE_ROOT + or record["node_executable"] != RUNTIME_PACKAGE_NODE_EXECUTABLE + ): + raise RouteControllerError("runtime package identity is invalid") + source_commit = _string(record["source_commit"], _COMMIT_RE, "runtime package source commit") + _string(record["source_tree"], _COMMIT_RE, "runtime package source tree") + manifest_digest = _string(record["manifest_digest"], _DIGEST_RE, "runtime package manifest digest") + for field in ( + "source_bindings_digest", + "release_archive_sha256", + "checksums_sha256", + "provenance_sha256", + "desktop_metrics_sha256", + "manifest_sha256", + "node_executable_sha256", + "node_runtime_inventory_digest", + "runtime_package_digest", + ): + _string(record[field], _DIGEST_RE, f"runtime package {field}") + for field in ( + "release_archive_bytes", + "checksums_bytes", + "provenance_bytes", + "desktop_metrics_bytes", + "manifest_bytes", + "node_executable_bytes", + "node_runtime_entry_count", + "node_runtime_bytes", + ): + _integer(record[field], f"runtime package {field}", 1) + if expected_source_commit is not None and source_commit != expected_source_commit: + raise RouteControllerError("runtime package source commit changed") + if expected_manifest_digest is not None and manifest_digest != expected_manifest_digest: + raise RouteControllerError("runtime package manifest binding changed") + if expected_source_bindings is not None and record["source_bindings_digest"] != _source_bindings_digest( + expected_source_bindings + ): + raise RouteControllerError("runtime package source bindings changed") + if record["runtime_package_digest"] != _runtime_package_digest(record): + raise RouteControllerError("runtime package record digest changed") + return MappingProxyType(record) + + +def _stable_plan_digest(raw: Mapping[str, Any]) -> str: + """Bind the exact plan without introducing record-hash self-reference.""" + + authorization = dict(raw["authorization"]) + for field in ( + "reservation_record_sha256", + "reservation_record_byte_size", + "preflight_record_sha256", + "preflight_record_byte_size", + ): + authorization.pop(field) + stable = dict(raw) + stable["authorization"] = authorization + return _canonical_digest(stable) + + +def _worker_plan_digest(workers: Sequence[WorkerPlan]) -> str: + value = { + "manifest_digest": EXPECTED_MANIFEST_DIGEST, + "model_revision": EXPECTED_MODEL_REVISION, + "block_prefix": EXPECTED_BLOCK_PREFIX, + "workers": [ + { + "worker_id": worker.worker_id, + "machine_id": worker.machine_id, + "span": worker.span, + "artifact_bytes": worker.artifact_bytes, + "artifact_set_digest": worker.artifact_set_digest, + "cache_root": worker.cache_root, + } + for worker in sorted(workers, key=lambda item: item.span) + ], + } + return _canonical_digest(value) + + +def _action_id(plan: RoutePlan, action: str) -> str: + if action not in ACTIONS - {"none"}: + raise RouteControllerError("action identity is invalid") + return _canonical_digest( + { + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "action": action, + } + ) + + +def _relative_path(value: Any) -> str: + if not isinstance(value, str) or not value or len(value) > 256 or "\\" in value: + raise RouteControllerError("source binding path is invalid") + path = PurePosixPath(value) + if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts): + raise RouteControllerError("source binding path is invalid") + return value + + +def _cache_root(value: Any) -> str: + if not isinstance(value, str) or len(value) > 512 or not value.startswith("/") or "\\" in value or "//" in value: + raise RouteControllerError("worker cache root is invalid") + path = PurePosixPath(value) + if any(part in {"", ".", ".."} for part in path.parts[1:]): + raise RouteControllerError("worker cache root is invalid") + return str(path) + + +def _validate_source_bindings(value: Any, source_root: Path) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, list) or not 1 <= len(value) <= 32: + raise RouteControllerError("source bindings must be a bounded list") + bindings: list[Mapping[str, Any]] = [] + previous = "" + resolved_root = source_root.resolve() + for index, item in enumerate(value): + binding = _mapping(item, _BINDING_FIELDS, f"source_bindings[{index}]") + relative = _relative_path(binding["relative_path"]) + if relative <= previous: + raise RouteControllerError("source bindings must be strictly sorted") + previous = relative + expected_size = _integer(binding["byte_size"], "source binding byte size", 1) + expected_digest = _string(binding["sha256"], _DIGEST_RE, "source binding digest") + candidate = (resolved_root / Path(*PurePosixPath(relative).parts)).resolve() + try: + candidate.relative_to(resolved_root) + except ValueError as exc: + raise RouteControllerError("source binding escapes source root") from exc + payload = _regular_bytes(candidate) + if len(payload) != expected_size: + raise RouteControllerError("source binding size changed") + observed_digest = "sha256:" + hashlib.sha256(payload).hexdigest() + if observed_digest != expected_digest: + raise RouteControllerError("source binding digest changed") + bindings.append(MappingProxyType(dict(binding))) + return tuple(bindings) + + +def load_plan(path: Path, source_root: Path) -> RoutePlan: + raw_bytes = _regular_bytes(path) + raw = _mapping(_strict_json(raw_bytes), _PLAN_FIELDS, "plan") + if raw["schema_version"] != SCHEMA_VERSION or raw["gate"] != GATE: + raise RouteControllerError("plan identity is invalid") + run_id = _string(raw["run_id"], _RUN_RE, "run_id") + route_job_id = _string(raw["route_job_id"], _LABEL_RE, "route_job_id") + source_commit = _string(raw["source_commit"], _COMMIT_RE, "source_commit") + manifest_digest = _string(raw["manifest_digest"], _DIGEST_RE, "manifest_digest") + model_revision = _string(raw["model_revision"], _REVISION_RE, "model_revision") + if manifest_digest != EXPECTED_MANIFEST_DIGEST or model_revision != EXPECTED_MODEL_REVISION: + raise RouteControllerError("Qwen3.8 model binding is invalid") + deadline_unix = _integer(raw["deadline_unix"], "deadline_unix", 1) + + authorization = dict(_mapping(raw["authorization"], _AUTH_FIELDS, "authorization")) + ceiling = _money(authorization["combined_cloud_ceiling_usd"], "combined cloud ceiling") + committed = _money(authorization["ledger_committed_before_run_usd"], "ledger committed amount") + maximum = _money(authorization["maximum_estimate_usd"], "maximum estimate") + if ( + ceiling != Decimal("100.00") + or committed != Decimal("56.00") + or maximum > Decimal("44.00") + or committed + maximum > ceiling + ): + raise RouteControllerError("authorization exceeds the current combined cloud ledger") + for field in ( + "reservation_recorded", + "native_auth_revalidated", + "inventory_revalidated", + "pricing_revalidated", + "provisioning_authorized", + ): + _boolean(authorization[field], f"authorization.{field}") + _string(authorization["reservation_id"], _LABEL_RE, "authorization.reservation_id") + _relative_path(authorization["reservation_record_path"]) + _integer( + authorization["reservation_record_byte_size"], + "authorization.reservation_record_byte_size", + 1, + ) + _string( + authorization["reservation_record_sha256"], + _DIGEST_RE, + "authorization.reservation_record_sha256", + ) + _relative_path(authorization["preflight_record_path"]) + _integer( + authorization["preflight_record_byte_size"], + "authorization.preflight_record_byte_size", + 1, + ) + _string( + authorization["preflight_record_sha256"], + _DIGEST_RE, + "authorization.preflight_record_sha256", + ) + _string( + authorization["readiness_ledger_sha256"], + _DIGEST_RE, + "authorization.readiness_ledger_sha256", + ) + if authorization["provisioning_authorized"] and maximum == 0: + raise RouteControllerError("authorized provisioning requires a positive bounded estimate") + + source_bindings = _validate_source_bindings(raw["source_bindings"], source_root) + if {binding["relative_path"] for binding in source_bindings} != REQUIRED_SOURCE_PATHS: + raise RouteControllerError("exact route execution sources are not bound") + ledger_binding = next(binding for binding in source_bindings if binding["relative_path"] == READINESS_LEDGER_PATH) + if ledger_binding["sha256"] != authorization["readiness_ledger_sha256"]: + raise RouteControllerError("authorization is not bound to the readiness ledger") + + runtime_package = validate_runtime_package_record( + raw["runtime_package"], + expected_source_commit=source_commit, + expected_manifest_digest=manifest_digest, + expected_source_bindings=source_bindings, + ) + + raw_workers = raw["workers"] + if not isinstance(raw_workers, list) or len(raw_workers) != 4: + raise RouteControllerError("plan must contain exactly four workers") + workers: list[WorkerPlan] = [] + for index, item in enumerate(raw_workers): + worker = _mapping(item, _WORKER_FIELDS, f"workers[{index}]") + span = worker["span"] + if span not in EXPECTED_SPANS: + raise RouteControllerError("worker span is not canonical") + expected_bytes, expected_digest = EXPECTED_SPANS[span] + artifact_bytes = _integer(worker["artifact_bytes"], "artifact_bytes", 1) + artifact_digest = _string(worker["artifact_set_digest"], _DIGEST_RE, "artifact_set_digest") + if artifact_bytes != expected_bytes or artifact_digest != expected_digest: + raise RouteControllerError("worker artifact plan changed") + workers.append( + WorkerPlan( + worker_id=_string(worker["worker_id"], _LABEL_RE, "worker_id"), + machine_id=_string(worker["machine_id"], _LABEL_RE, "machine_id"), + instance=_string(worker["instance"], _GCP_RESOURCE_RE, "instance"), + disk=_string(worker["disk"], _GCP_RESOURCE_RE, "disk"), + span=span, + artifact_bytes=artifact_bytes, + artifact_set_digest=artifact_digest, + cache_root=_cache_root(worker["cache_root"]), + ) + ) + if [worker.span for worker in workers] != list(EXPECTED_SPANS): + raise RouteControllerError("worker spans must be the canonical exact route") + for field, values in ( + ("worker_id", [worker.worker_id for worker in workers]), + ("machine_id", [worker.machine_id for worker in workers]), + ("instance", [worker.instance for worker in workers]), + ("disk", [worker.disk for worker in workers]), + ("cache_root", [worker.cache_root for worker in workers]), + ): + if len(set(values)) != len(values): + raise RouteControllerError(f"workers must have unique {field}") + if PROTECTED_INSTANCE in {value for worker in workers for value in (worker.instance, worker.disk)}: + raise RouteControllerError("protected bootstrap is targeted") + + raw_resources = raw["resources"] + if not isinstance(raw_resources, list) or len(raw_resources) != 12: + raise RouteControllerError("resource inventory must contain exactly 12 resources") + resources: list[ResourcePlan] = [] + kind_counts = {kind: 0 for kind in EXPECTED_RESOURCE_KINDS} + worker_by_id = {worker.worker_id: worker for worker in workers} + for index, item in enumerate(raw_resources): + resource = _mapping(item, _RESOURCE_FIELDS, f"resources[{index}]") + kind = resource["kind"] + if kind not in EXPECTED_RESOURCE_KINDS: + raise RouteControllerError("resource kind is invalid") + worker_id = resource["worker_id"] + if kind.startswith("worker_"): + worker_id = _string(worker_id, _LABEL_RE, "resource worker_id") + if worker_id not in worker_by_id: + raise RouteControllerError("resource references an unknown worker") + elif worker_id is not None: + raise RouteControllerError("non-worker resource has a worker_id") + name = _string(resource["name"], _GCP_RESOURCE_RE, "resource name") + if not name.startswith(f"{run_id}-"): + raise RouteControllerError("resource name is not run-scoped") + if name == PROTECTED_INSTANCE: + raise RouteControllerError("protected bootstrap is targeted") + resources.append( + ResourcePlan( + name=name, + kind=kind, + provider=_string(resource["provider"], _LABEL_RE, "resource provider"), + region=_string(resource["region"], _LABEL_RE, "resource region"), + worker_id=worker_id, + ) + ) + kind_counts[kind] += 1 + if kind_counts != EXPECTED_RESOURCE_KINDS: + raise RouteControllerError("resource kind inventory is invalid") + if len({resource.name for resource in resources}) != len(resources): + raise RouteControllerError("resource names must be unique") + if [resource.name for resource in resources] != sorted(resource.name for resource in resources): + raise RouteControllerError("resource inventory must be sorted by name") + if {resource.provider for resource in resources} != {EXPECTED_PROVIDER} or { + resource.region for resource in resources + } != {EXPECTED_REGION}: + raise RouteControllerError("route resources must use one exact provider and region") + for worker in workers: + expected = { + ("worker_instance", worker.instance), + ("worker_disk", worker.disk), + } + observed = {(resource.kind, resource.name) for resource in resources if resource.worker_id == worker.worker_id} + if observed != expected: + raise RouteControllerError("worker resource inventory is inconsistent") + + return RoutePlan( + run_id=run_id, + route_job_id=route_job_id, + source_commit=source_commit, + manifest_digest=manifest_digest, + model_revision=model_revision, + deadline_unix=deadline_unix, + authorization=MappingProxyType(authorization), + source_bindings=source_bindings, + runtime_package=runtime_package, + resources=tuple(resources), + workers=tuple(workers), + plan_digest=_stable_plan_digest(raw), + ) + + +def _bound_record_bytes( + root: Path, + relative_path: str, + expected_size: int, + expected_digest: str, +) -> bytes: + resolved_root = root.resolve() + candidate = (resolved_root / Path(*PurePosixPath(relative_path).parts)).resolve() + try: + candidate.relative_to(resolved_root) + except ValueError as exc: + raise RouteControllerError("authorization record escapes its root") from exc + payload = _regular_bytes(candidate) + if len(payload) != expected_size: + raise RouteControllerError("authorization record size changed") + if "sha256:" + hashlib.sha256(payload).hexdigest() != expected_digest: + raise RouteControllerError("authorization record digest changed") + return payload + + +def _assert_protected_path_from_bindings( + path: Path, + source_bindings: Sequence[Mapping[str, Any]], + source_root: Path, + *, + directory: bool, +) -> None: + """Require the controller-owned protection implementation and its native checks.""" + + try: + from scripts import gate14_packaged_lifecycle as lifecycle + except ImportError as exc: + raise RouteControllerError("controller protection verifier is unavailable") from exc + expected_module = (source_root.resolve() / Path(*PurePosixPath(PROTECTION_SOURCE_PATH).parts)).resolve() + imported_module = Path(lifecycle.__file__).resolve() + protection_binding = next( + (binding for binding in source_bindings if binding["relative_path"] == PROTECTION_SOURCE_PATH), + None, + ) + expected_payload = _regular_bytes(expected_module) + if ( + protection_binding is None + or imported_module != expected_module + or "sha256:" + hashlib.sha256(expected_payload).hexdigest() != protection_binding["sha256"] + ): + raise RouteControllerError("controller protection verifier is not source-bound") + try: + lifecycle._assert_controller_owned(path, directory=directory) + except (OSError, lifecycle.Gate14LifecycleError) as exc: + raise RouteControllerError("controller-managed input is not protected") from exc + + +def _assert_protected_path( + path: Path, + plan: RoutePlan, + source_root: Path, + *, + directory: bool, +) -> None: + _assert_protected_path_from_bindings( + path, + plan.source_bindings, + source_root, + directory=directory, + ) + + +def _protected_record_bytes( + plan: RoutePlan, + source_root: Path, + root: Path, + relative_path: str, + expected_size: int, + expected_digest: str, +) -> bytes: + _assert_protected_path(root, plan, source_root, directory=True) + resolved_root = root.resolve() + candidate = (resolved_root / Path(*PurePosixPath(relative_path).parts)).resolve() + try: + candidate.relative_to(resolved_root) + except ValueError as exc: + raise RouteControllerError("protected record escapes its root") from exc + _assert_protected_path(candidate, plan, source_root, directory=False) + return _bound_record_bytes( + root, + relative_path, + expected_size, + expected_digest, + ) + + +def _ledger_reservation_marker(plan: RoutePlan) -> bytes: + authorization = plan.authorization + return ( + "Q38_ROUTE_RESERVATION " + f"run_id={plan.run_id} " + f"reservation_id={authorization['reservation_id']} " + f"maximum_usd={authorization['maximum_estimate_usd']} " + f"deadline_unix={plan.deadline_unix}" + ).encode("ascii") + + +def revalidate_authorization_evidence( + plan: RoutePlan, + authorization_root: Path, + source_root: Path | None = None, + *, + now_unix: int, +) -> None: + """Open and validate exact ledger, pricing, quota, and provider records.""" + + now_unix = _integer(now_unix, "trusted current time", 1) + if source_root is None: + source_root = authorization_root.parent / "source" + authorization = plan.authorization + readiness_path = source_root.resolve() / Path(*PurePosixPath(READINESS_LEDGER_PATH).parts) + readiness_payload = _regular_bytes(readiness_path) + readiness_binding = next( + binding for binding in plan.source_bindings if binding["relative_path"] == READINESS_LEDGER_PATH + ) + if ( + len(readiness_payload) != readiness_binding["byte_size"] + or "sha256:" + hashlib.sha256(readiness_payload).hexdigest() != readiness_binding["sha256"] + ): + raise RouteControllerError("readiness ledger source binding changed") + if _ledger_reservation_marker(plan) not in readiness_payload.splitlines(): + raise RouteControllerError("readiness ledger does not contain the exact reservation") + reservation_payload = _protected_record_bytes( + plan, + source_root, + authorization_root, + authorization["reservation_record_path"], + authorization["reservation_record_byte_size"], + authorization["reservation_record_sha256"], + ) + reservation = _mapping( + _strict_json(reservation_payload), + _RESERVATION_FIELDS, + "reservation record", + ) + recorded_at = _integer( + reservation["recorded_at_unix"], + "reservation recorded_at_unix", + 1, + ) + expires_at = _integer( + reservation["expires_at_unix"], + "reservation expires_at_unix", + 1, + ) + resource_costs = reservation["resource_costs"] + if not isinstance(resource_costs, list) or len(resource_costs) != len(plan.resources): + raise RouteControllerError("reservation cost inventory is invalid") + expected_resource_names = [resource.name for resource in plan.resources] + observed_cost_names: list[str] = [] + observed_cost_spec_digests: list[str] = [] + total_cost = Decimal("0.00") + for index, value in enumerate(resource_costs): + item = _mapping(value, _COST_FIELDS, f"resource_costs[{index}]") + observed_cost_names.append(_string(item["resource_name"], _GCP_RESOURCE_RE, "cost resource name")) + observed_cost_spec_digests.append( + _string(item["resource_spec_digest"], _DIGEST_RE, "cost resource spec digest") + ) + unit_rate = _money(item["unit_rate_usd"], "resource unit rate") + quantity = _money(item["quantity"], "resource quantity") + duration = _money(item["duration_hours"], "resource duration") + maximum = _money(item["maximum_usd"], "resource maximum cost") + recomputed = (unit_rate * quantity * duration).quantize( + Decimal("0.01"), + rounding=ROUND_CEILING, + ) + if maximum != recomputed: + raise RouteControllerError("reservation resource cost was not recomputed") + resource = plan.resources[index] + if ( + quantity != Decimal("1.00") + or duration != EXPECTED_PRICED_DURATION_HOURS + or (resource.kind.endswith("firewall") and maximum != Decimal("0.00")) + or (not resource.kind.endswith("firewall") and maximum <= Decimal("0.00")) + ): + raise RouteControllerError("reservation pricing horizon or quantity is invalid") + total_cost += maximum + if observed_cost_names != expected_resource_names: + raise RouteControllerError("reservation cost inventory is not exact") + if total_cost != _money( + authorization["maximum_estimate_usd"], + "maximum estimate", + ): + raise RouteControllerError("reservation cost total changed") + if ( + reservation["schema_version"] != SCHEMA_VERSION + or reservation["reservation_id"] != authorization["reservation_id"] + or reservation["run_id"] != plan.run_id + or reservation["combined_cloud_ceiling_usd"] != authorization["combined_cloud_ceiling_usd"] + or reservation["ledger_committed_before_run_usd"] != authorization["ledger_committed_before_run_usd"] + or reservation["maximum_estimate_usd"] != authorization["maximum_estimate_usd"] + or reservation["deadline_unix"] != plan.deadline_unix + or reservation["plan_digest"] != plan.plan_digest + or reservation["execution_inventory_digest"] != plan.execution_inventory_digest + or reservation["worker_plan_digest"] != plan.worker_plan_digest + or reservation["readiness_ledger_sha256"] != authorization["readiness_ledger_sha256"] + or reservation["reservation_recorded"] is not True + or recorded_at > now_unix + or expires_at < plan.deadline_unix + or now_unix >= expires_at + ): + raise RouteControllerError("reservation record is invalid or expired") + + preflight_payload = _protected_record_bytes( + plan, + source_root, + authorization_root, + authorization["preflight_record_path"], + authorization["preflight_record_byte_size"], + authorization["preflight_record_sha256"], + ) + preflight = _mapping( + _strict_json(preflight_payload), + _PREFLIGHT_FIELDS, + "preflight record", + ) + checked_at = _integer( + preflight["checked_at_unix"], + "preflight checked_at_unix", + 1, + ) + pricing_checked_at = _integer( + preflight["pricing_checked_at_unix"], + "preflight pricing_checked_at_unix", + 1, + ) + _string(preflight["pricing_source"], _LABEL_RE, "preflight pricing_source") + if preflight["pricing_currency"] != "USD": + raise RouteControllerError("provider pricing currency is invalid") + resource_specs = preflight["resource_specs"] + if not isinstance(resource_specs, list) or len(resource_specs) != len(plan.resources): + raise RouteControllerError("provider resource specification inventory is invalid") + observed_spec_names: list[str] = [] + observed_spec_digests: list[str] = [] + for index, value in enumerate(resource_specs): + spec = _mapping(value, _RESOURCE_SPEC_FIELDS, f"resource_specs[{index}]") + name = _string(spec["resource_name"], _GCP_RESOURCE_RE, "spec resource name") + observed_spec_names.append(name) + resource = plan.resource_by_name.get(name) + if ( + resource is None + or spec["kind"] != resource.kind + or spec["provider"] != resource.provider + or spec["region"] != resource.region + ): + raise RouteControllerError("provider resource specification binding is invalid") + if dict(spec) != _expected_resource_spec(resource): + raise RouteControllerError("provider resource specification is not the exact launch profile") + if checked_at + EXPECTED_MAX_LIFETIME_SECONDS > plan.deadline_unix: + raise RouteControllerError("provider resource lifetime exceeds the route deadline") + observed_spec_digests.append(_canonical_digest(spec)) + if observed_spec_names != expected_resource_names: + raise RouteControllerError("provider resource specification inventory is not exact") + if observed_cost_spec_digests != observed_spec_digests: + raise RouteControllerError("reservation costs are not bound to the resource specifications") + quota_limit = _integer( + preflight["gpu_quota_limit"], + "preflight gpu_quota_limit", + ) + quota_usage = _integer( + preflight["gpu_quota_usage"], + "preflight gpu_quota_usage", + ) + required_gpu_count = _integer( + preflight["required_gpu_count"], + "preflight required_gpu_count", + 1, + ) + providers = {resource.provider for resource in plan.resources} + if ( + preflight["schema_version"] != SCHEMA_VERSION + or preflight["run_id"] != plan.run_id + or preflight["source_commit"] != plan.source_commit + or preflight["plan_digest"] != plan.plan_digest + or preflight["execution_inventory_digest"] != plan.execution_inventory_digest + or preflight["worker_plan_digest"] != plan.worker_plan_digest + or preflight["provider"] != next(iter(providers)) + or preflight["resource_names"] != expected_resource_names + or required_gpu_count != len(plan.workers) + or quota_usage > quota_limit + or quota_limit - quota_usage < required_gpu_count + or preflight["reservation_record_sha256"] != authorization["reservation_record_sha256"] + or checked_at > now_unix + or pricing_checked_at > now_unix + or now_unix - checked_at > MAX_PLAN_REVALIDATION_AGE_SECONDS + or now_unix - pricing_checked_at > MAX_PLAN_REVALIDATION_AGE_SECONDS + or preflight["protected_bootstrap_running"] is not True + or any( + preflight[field] is not True or authorization[field] is not True + for field in ( + "native_auth_revalidated", + "inventory_revalidated", + "pricing_revalidated", + "provisioning_authorized", + ) + ) + ): + raise RouteControllerError("provider preflight record is invalid or stale") + + +def revalidate_production_artifact_plan( + plan: RoutePlan, + manifest_path: Path, + artifact_root: Path, + source_root: Path, + *, + verified_at_unix: int, +) -> dict[str, Any]: + """Rederive the exact four span plans through the source-bound production verifier.""" + + try: + from drift import model_manifest as manifest_module + except ImportError as exc: + raise RouteControllerError("production artifact planner is unavailable") from exc + expected_module = (source_root.resolve() / Path(*PurePosixPath(VERIFIER_SOURCE_PATH).parts)).resolve() + imported_module = Path(manifest_module.__file__).resolve() + verifier_binding = next( + binding for binding in plan.source_bindings if binding["relative_path"] == VERIFIER_SOURCE_PATH + ) + expected_module_payload = _regular_bytes(expected_module) + if ( + imported_module != expected_module + or "sha256:" + hashlib.sha256(expected_module_payload).hexdigest() != verifier_binding["sha256"] + ): + raise RouteControllerError("imported artifact planner is not the source-bound verifier") + + ManifestArtifactVerifier = manifest_module.ManifestArtifactVerifier + ManifestError = manifest_module.ManifestError + ModelManifest = manifest_module.ModelManifest + try: + manifest = ModelManifest.load(manifest_path) + if ( + manifest.digest_id != plan.manifest_digest + or manifest.source.revision != plan.model_revision + or manifest.model.num_blocks != 64 + ): + raise RouteControllerError("production manifest identity is invalid") + indices = manifest.artifacts_for_roles({"weight_index"}) + if len(indices) != 1 or "sha256:" + indices[0].sha256 != EXPECTED_INDEX_DIGEST: + raise RouteControllerError("production checkpoint index identity is invalid") + metadata_paths = [artifact.path for artifact in manifest.artifacts_for_roles({"config", "weight_index"})] + verifier = ManifestArtifactVerifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=False, + artifact_root=artifact_root, + allowed_paths=metadata_paths, + ) + for worker in plan.workers: + start, end = (int(value) for value in worker.span.split(":", 1)) + derived = verifier.plan_block_artifacts( + block_prefix=EXPECTED_BLOCK_PREFIX, + start_block=start, + end_block=end, + ) + if ( + derived.artifact_bytes != worker.artifact_bytes + or "sha256:" + derived.artifact_set_digest != worker.artifact_set_digest + or len(derived.artifacts) != EXPECTED_ARTIFACTS_PER_SPAN + ): + raise RouteControllerError("production artifact plan differs from the route plan") + except RouteControllerError: + raise + except (ManifestError, OSError, UnicodeError, ValueError) as exc: + raise RouteControllerError("production artifact plan could not be revalidated") from exc + + return { + "verified_at_unix": _integer(verified_at_unix, "verified_at_unix", 1), + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "model_revision": plan.model_revision, + "index_digest": EXPECTED_INDEX_DIGEST, + "block_prefix": EXPECTED_BLOCK_PREFIX, + "worker_plan_digest": plan.worker_plan_digest, + "verifier_source_sha256": verifier_binding["sha256"], + } + + +def instance_generation_digest( + resource_name: str, + instance_id: str, + creation_timestamp: str, +) -> str: + _string(resource_name, _GCP_RESOURCE_RE, "instance generation resource_name") + _string(instance_id, _INSTANCE_ID_RE, "instance generation id") + if int(instance_id) > 2**64 - 1: + raise RouteControllerError("instance generation id is outside uint64") + _string( + creation_timestamp, + _CREATION_TIMESTAMP_RE, + "instance generation creation_timestamp", + ) + try: + parsed_timestamp = datetime.fromisoformat(creation_timestamp) + except ValueError as exc: + raise RouteControllerError("instance generation creation_timestamp is invalid") from exc + if parsed_timestamp.tzinfo is None or parsed_timestamp.utcoffset() is None: + raise RouteControllerError("instance generation creation_timestamp lacks an offset") + return _canonical_digest( + { + "project": EXPECTED_PROJECT, + "zone": EXPECTED_ZONE, + "resource_name": resource_name, + "instance_id": instance_id, + "creation_timestamp": creation_timestamp, + } + ) + + +def _instance_key_resource(plan: RoutePlan, resource_name: str) -> ResourcePlan: + resource = plan.resource_by_name.get(resource_name) + if resource is None or resource.kind not in {"bootstrap_instance", "worker_instance"}: + raise RouteControllerError("instance key resource is not planned") + return resource + + +def _instance_key_record_digest(value: Mapping[str, Any]) -> str: + unsigned = dict(value) + unsigned.pop("record_digest", None) + return _canonical_digest(unsigned) + + +def _instance_key_active_digest(value: Mapping[str, Any]) -> str: + unsigned = dict(value) + unsigned.pop("active_digest", None) + return _canonical_digest(unsigned) + + +def _instance_key_tombstone_digest(value: Mapping[str, Any]) -> str: + unsigned = dict(value) + unsigned.pop("tombstone_digest", None) + return _canonical_digest(unsigned) + + +def validate_instance_generation_key_record( + value: Any, + plan: RoutePlan, + *, + expected_resource_name: str | None = None, + expected_generation_digest: str | None = None, +) -> dict[str, Any]: + record = dict(_mapping(value, _INSTANCE_KEY_RECORD_FIELDS, "instance key record")) + resource = _instance_key_resource(plan, record.get("resource_name")) + try: + generation = instance_generation_digest( + resource.name, + record["instance_id"], + record["creation_timestamp"], + ) + except (KeyError, TypeError, RouteControllerError) as exc: + raise RouteControllerError("instance key generation is invalid") from exc + fixed = { + "schema_version": INSTANCE_KEY_SCHEMA_VERSION, + "scope": INSTANCE_KEY_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": _action_id(plan, "start_route"), + "resource_name": resource.name, + "resource_kind": resource.kind, + "instance_generation_digest": generation, + "key_bytes": INSTANCE_KEY_BYTES, + } + if any(record[field] != expected for field, expected in fixed.items()): + raise RouteControllerError("instance key record binding is invalid") + if expected_resource_name is not None and resource.name != expected_resource_name: + raise RouteControllerError("instance key resource changed") + if expected_generation_digest is not None and generation != expected_generation_digest: + raise RouteControllerError("instance key provider generation changed") + epoch = _integer(record["key_epoch"], "instance key epoch", 1) + if epoch > 99_999_999: + raise RouteControllerError("instance key epoch is invalid") + issued = _integer(record["issued_at_unix"], "instance key issue time", 1) + expires = _integer(record["expires_at_unix"], "instance key expiry", 1) + if expires <= issued or expires > plan.deadline_unix: + raise RouteControllerError("instance key time window is invalid") + _string(record["key_sha256"], _DIGEST_RE, "instance key digest") + if record["previous_record_digest"] is not None: + _string( + record["previous_record_digest"], + _DIGEST_RE, + "previous instance key record digest", + ) + _string(record["record_digest"], _DIGEST_RE, "instance key record digest") + if record["record_digest"] != _instance_key_record_digest(record): + raise RouteControllerError("instance key record digest changed") + return record + + +def _instance_key_record( + plan: RoutePlan, + resource: ResourcePlan, + instance_id: str, + creation_timestamp: str, + *, + key: bytes, + key_epoch: int, + issued_at_unix: int, + previous_record_digest: str | None, +) -> dict[str, Any]: + generation = instance_generation_digest(resource.name, instance_id, creation_timestamp) + value: dict[str, Any] = { + "schema_version": INSTANCE_KEY_SCHEMA_VERSION, + "scope": INSTANCE_KEY_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": _action_id(plan, "start_route"), + "resource_name": resource.name, + "resource_kind": resource.kind, + "instance_id": instance_id, + "creation_timestamp": creation_timestamp, + "instance_generation_digest": generation, + "key_epoch": key_epoch, + "issued_at_unix": issued_at_unix, + "expires_at_unix": plan.deadline_unix, + "key_sha256": "sha256:" + hashlib.sha256(key).hexdigest(), + "key_bytes": INSTANCE_KEY_BYTES, + "previous_record_digest": previous_record_digest, + "record_digest": "", + } + value["record_digest"] = _instance_key_record_digest(value) + return validate_instance_generation_key_record( + value, + plan, + expected_resource_name=resource.name, + expected_generation_digest=generation, + ) + + +def _instance_key_active( + plan: RoutePlan, + record: Mapping[str, Any], +) -> dict[str, Any]: + value: dict[str, Any] = { + "schema_version": INSTANCE_KEY_SCHEMA_VERSION, + "scope": INSTANCE_KEY_ACTIVE_SCOPE, + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "resource_name": record["resource_name"], + "instance_generation_digest": record["instance_generation_digest"], + "key_epoch": record["key_epoch"], + "record_digest": record["record_digest"], + "active_digest": "", + } + value["active_digest"] = _instance_key_active_digest(value) + return _validate_instance_key_active(value, plan) + + +def _validate_instance_key_active( + value: Any, + plan: RoutePlan, + *, + expected_resource_name: str | None = None, + expected_generation_digest: str | None = None, +) -> dict[str, Any]: + active = dict(_mapping(value, _INSTANCE_KEY_ACTIVE_FIELDS, "active instance key")) + resource = _instance_key_resource(plan, active.get("resource_name")) + fixed = { + "schema_version": INSTANCE_KEY_SCHEMA_VERSION, + "scope": INSTANCE_KEY_ACTIVE_SCOPE, + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "resource_name": resource.name, + } + if any(active[field] != expected for field, expected in fixed.items()): + raise RouteControllerError("active instance key binding is invalid") + _string(active["instance_generation_digest"], _DIGEST_RE, "active instance key generation") + epoch = _integer(active["key_epoch"], "active instance key epoch", 1) + if epoch > 99_999_999: + raise RouteControllerError("active instance key epoch is invalid") + _string(active["record_digest"], _DIGEST_RE, "active instance key record digest") + _string(active["active_digest"], _DIGEST_RE, "active instance key digest") + if active["active_digest"] != _instance_key_active_digest(active): + raise RouteControllerError("active instance key digest changed") + if expected_resource_name is not None and resource.name != expected_resource_name: + raise RouteControllerError("active instance key resource changed") + if expected_generation_digest is not None and active["instance_generation_digest"] != expected_generation_digest: + raise RouteControllerError("active instance key provider generation changed") + return active + + +def _instance_key_tombstone( + plan: RoutePlan, + resource: ResourcePlan, + generation: str, + *, + revoked_at_unix: int, + last_key_epoch: int, + last_record_digest: str | None, +) -> dict[str, Any]: + value: dict[str, Any] = { + "schema_version": INSTANCE_KEY_SCHEMA_VERSION, + "scope": INSTANCE_KEY_TOMBSTONE_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": _action_id(plan, "start_route"), + "resource_name": resource.name, + "instance_generation_digest": generation, + "revoked_at_unix": revoked_at_unix, + "last_key_epoch": last_key_epoch, + "last_record_digest": last_record_digest, + "tombstone_digest": "", + } + value["tombstone_digest"] = _instance_key_tombstone_digest(value) + return _validate_instance_key_tombstone( + value, + plan, + expected_resource_name=resource.name, + expected_generation_digest=generation, + ) + + +def _validate_instance_key_tombstone( + value: Any, + plan: RoutePlan, + *, + expected_resource_name: str, + expected_generation_digest: str, +) -> dict[str, Any]: + tombstone = dict(_mapping(value, _INSTANCE_KEY_TOMBSTONE_FIELDS, "instance key tombstone")) + resource = _instance_key_resource(plan, tombstone.get("resource_name")) + fixed = { + "schema_version": INSTANCE_KEY_SCHEMA_VERSION, + "scope": INSTANCE_KEY_TOMBSTONE_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "execution_inventory_digest": plan.execution_inventory_digest, + "start_action_id": _action_id(plan, "start_route"), + "resource_name": resource.name, + "instance_generation_digest": expected_generation_digest, + } + if resource.name != expected_resource_name or any( + tombstone[field] != expected for field, expected in fixed.items() + ): + raise RouteControllerError("instance key tombstone binding is invalid") + _integer(tombstone["revoked_at_unix"], "instance key revocation time", 1) + _integer(tombstone["last_key_epoch"], "instance key tombstone epoch") + if tombstone["last_record_digest"] is not None: + _string( + tombstone["last_record_digest"], + _DIGEST_RE, + "instance key tombstone record digest", + ) + _string(tombstone["tombstone_digest"], _DIGEST_RE, "instance key tombstone digest") + if tombstone["tombstone_digest"] != _instance_key_tombstone_digest(tombstone): + raise RouteControllerError("instance key tombstone digest changed") + return tombstone + + +def _instance_key_reparse(path: Path, metadata: os.stat_result) -> bool: + return ( + bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + or path.is_symlink() + ) + + +def _assert_windows_instance_key_acl(path: Path, *, directory: bool) -> None: + import ctypes + from ctypes import wintypes + + class SidAndAttributes(ctypes.Structure): + _fields_ = [("sid", ctypes.c_void_p), ("attributes", wintypes.DWORD)] + + class TokenUser(ctypes.Structure): + _fields_ = [("user", SidAndAttributes)] + + class AclSizeInformation(ctypes.Structure): + _fields_ = [ + ("ace_count", wintypes.DWORD), + ("acl_bytes_in_use", wintypes.DWORD), + ("acl_bytes_free", wintypes.DWORD), + ] + + class AceHeader(ctypes.Structure): + _fields_ = [ + ("ace_type", wintypes.BYTE), + ("ace_flags", wintypes.BYTE), + ("ace_size", wintypes.WORD), + ] + + class AccessAllowedAce(ctypes.Structure): + _fields_ = [ + ("header", AceHeader), + ("mask", wintypes.DWORD), + ("sid_start", wintypes.DWORD), + ] + + advapi32 = ctypes.WinDLL("advapi32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + close_handle = kernel32.CloseHandle + close_handle.argtypes = (wintypes.HANDLE,) + close_handle.restype = wintypes.BOOL + token = wintypes.HANDLE() + open_process_token = advapi32.OpenProcessToken + open_process_token.argtypes = ( + wintypes.HANDLE, + wintypes.DWORD, + ctypes.POINTER(wintypes.HANDLE), + ) + open_process_token.restype = wintypes.BOOL + kernel32.GetCurrentProcess.restype = wintypes.HANDLE + if not open_process_token(kernel32.GetCurrentProcess(), 0x0008, ctypes.byref(token)): + raise RouteControllerError("instance key vault owner is unavailable") + try: + get_token_information = advapi32.GetTokenInformation + get_token_information.argtypes = ( + wintypes.HANDLE, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ) + get_token_information.restype = wintypes.BOOL + required = wintypes.DWORD() + get_token_information(token, 1, None, 0, ctypes.byref(required)) + if required.value == 0: + raise RouteControllerError("instance key vault owner is unavailable") + token_buffer = ctypes.create_string_buffer(required.value) + if not get_token_information( + token, + 1, + token_buffer, + required, + ctypes.byref(required), + ): + raise RouteControllerError("instance key vault owner is unavailable") + current_sid = ctypes.cast( + token_buffer, + ctypes.POINTER(TokenUser), + ).contents.user.sid + + owner = ctypes.c_void_p() + dacl = ctypes.c_void_p() + descriptor = ctypes.c_void_p() + get_security = advapi32.GetNamedSecurityInfoW + get_security.argtypes = ( + wintypes.LPWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ) + get_security.restype = wintypes.DWORD + if ( + get_security( + os.fspath(path), + 1, + 0x1 | 0x4, + ctypes.byref(owner), + None, + ctypes.byref(dacl), + None, + ctypes.byref(descriptor), + ) + != 0 + ): + raise RouteControllerError("instance key vault ACL is unavailable") + try: + if not owner.value or not dacl.value: + raise RouteControllerError("instance key vault ACL is unsafe") + equal_sid = advapi32.EqualSid + equal_sid.argtypes = (ctypes.c_void_p, ctypes.c_void_p) + equal_sid.restype = wintypes.BOOL + if not equal_sid(owner, current_sid): + raise RouteControllerError("instance key vault owner is unsafe") + + control = wintypes.WORD() + revision = wintypes.DWORD() + get_control = advapi32.GetSecurityDescriptorControl + get_control.argtypes = ( + ctypes.c_void_p, + ctypes.POINTER(wintypes.WORD), + ctypes.POINTER(wintypes.DWORD), + ) + get_control.restype = wintypes.BOOL + if ( + not get_control( + descriptor, + ctypes.byref(control), + ctypes.byref(revision), + ) + or not control.value & 0x1000 + ): + raise RouteControllerError("instance key vault DACL is not protected") + + information = AclSizeInformation() + get_acl_information = advapi32.GetAclInformation + get_acl_information.argtypes = ( + ctypes.c_void_p, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + ) + get_acl_information.restype = wintypes.BOOL + if ( + not get_acl_information( + dacl, + ctypes.byref(information), + ctypes.sizeof(information), + 2, + ) + or information.ace_count != 1 + ): + raise RouteControllerError("instance key vault DACL is not private") + + ace_pointer = ctypes.c_void_p() + get_ace = advapi32.GetAce + get_ace.argtypes = ( + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_void_p), + ) + get_ace.restype = wintypes.BOOL + if not get_ace(dacl, 0, ctypes.byref(ace_pointer)): + raise RouteControllerError("instance key vault DACL is unavailable") + ace = ctypes.cast( + ace_pointer, + ctypes.POINTER(AccessAllowedAce), + ).contents + expected_flags = 0x1 | 0x2 if directory else 0 + if ( + ace.header.ace_type != 0 + or ace.header.ace_flags != expected_flags + or ace.header.ace_size < ctypes.sizeof(AccessAllowedAce) + or ace.mask != 0x001F01FF + ): + raise RouteControllerError("instance key vault DACL is not private") + ace_sid = ctypes.c_void_p(ace_pointer.value + AccessAllowedAce.sid_start.offset) + is_valid_sid = advapi32.IsValidSid + is_valid_sid.argtypes = (ctypes.c_void_p,) + is_valid_sid.restype = wintypes.BOOL + if not is_valid_sid(ace_sid) or not equal_sid(ace_sid, current_sid): + raise RouteControllerError("instance key vault DACL is not private") + finally: + local_free = kernel32.LocalFree + local_free.argtypes = (ctypes.c_void_p,) + local_free.restype = ctypes.c_void_p + local_free(descriptor) + finally: + close_handle(token) + + +def _windows_instance_key_acl( + path: Path, + *, + directory: bool, + apply: bool, +) -> None: + if os.name != "nt": + return + if apply: + executable = shutil.which("powershell.exe") or shutil.which("powershell") + if executable is None: + raise RouteControllerError("instance key vault ACL setter is unavailable") + script = _WINDOWS_INSTANCE_KEY_DIRECTORY_ACL if directory else _WINDOWS_INSTANCE_KEY_FILE_ACL + try: + completed = subprocess.run( + [ + executable, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-Command", + f"& {{\n{script}\n}}", + os.fspath(path), + "$true", + ], + check=False, + shell=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise RouteControllerError("instance key vault ACL update failed") from exc + if completed.returncode != 0: + raise RouteControllerError("instance key vault ACL is not private") + _assert_windows_instance_key_acl(path, directory=directory) + + +def _assert_instance_key_directory(path: Path) -> None: + try: + metadata = path.lstat() + except OSError as exc: + raise RouteControllerError("instance key vault directory is unavailable") from exc + if _instance_key_reparse(path, metadata) or not stat.S_ISDIR(metadata.st_mode): + raise RouteControllerError("instance key vault directory is unsafe") + if os.name == "posix" and (metadata.st_uid != os.geteuid() or stat.S_IMODE(metadata.st_mode) != 0o700): + raise RouteControllerError("instance key vault directory is not private") + _windows_instance_key_acl(path, directory=True, apply=False) + + +def _ensure_instance_key_directory(path: Path) -> None: + if path.exists() or path.is_symlink(): + _assert_instance_key_directory(path) + return + try: + path.mkdir(mode=0o700) + except OSError as exc: + raise RouteControllerError("instance key vault directory could not be created") from exc + if os.name == "posix": + os.chmod(path, 0o700) + _windows_instance_key_acl(path, directory=True, apply=True) + _assert_instance_key_directory(path) + + +def _instance_key_directory( + plan: RoutePlan, + resource: ResourcePlan, + vault_root: Path, +) -> Path: + root = Path(vault_root) + if not root.is_absolute() or root == Path(root.anchor) or root == Path.home() or root.name in {"", ".", ".."}: + raise RouteControllerError("instance key vault root is invalid") + if not root.exists() and not root.is_symlink(): + try: + root.mkdir(mode=0o700, parents=True) + except OSError as exc: + raise RouteControllerError("instance key vault root could not be created") from exc + if os.name == "posix": + os.chmod(root, 0o700) + _windows_instance_key_acl(root, directory=True, apply=True) + _assert_instance_key_directory(root) + run_root = root / plan.run_id + _ensure_instance_key_directory(run_root) + resource_root = run_root / resource.name + _ensure_instance_key_directory(resource_root) + if resource_root.parent != run_root or run_root.parent != root: + raise RouteControllerError("instance key vault layout is invalid") + return resource_root + + +def _assert_instance_key_file(path: Path, *, expected_size: int | None = None) -> None: + try: + metadata = path.lstat() + except OSError as exc: + raise RouteControllerError("instance key vault file is unavailable") from exc + if ( + _instance_key_reparse(path, metadata) + or not stat.S_ISREG(metadata.st_mode) + or expected_size is not None + and metadata.st_size != expected_size + ): + raise RouteControllerError("instance key vault file is unsafe") + if os.name == "posix" and (metadata.st_uid != os.geteuid() or stat.S_IMODE(metadata.st_mode) != 0o600): + raise RouteControllerError("instance key vault file is not private") + _windows_instance_key_acl(path, directory=False, apply=False) + + +def _read_instance_key_file( + path: Path, + *, + maximum: int, + expected_size: int | None = None, +) -> bytes: + _assert_instance_key_file(path, expected_size=expected_size) + payload = _regular_bytes(path, maximum=maximum) + _assert_instance_key_file(path, expected_size=expected_size) + return payload + + +def _instance_key_json(value: Mapping[str, Any]) -> bytes: + try: + payload = ( + json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("ascii") + except (TypeError, ValueError) as exc: + raise RouteControllerError("instance key metadata is invalid") from exc + if not 1 <= len(payload) <= MAX_INSTANCE_KEY_RECORD_BYTES: + raise RouteControllerError("instance key metadata exceeded its bound") + return payload + + +def _write_instance_key_exclusive(path: Path, payload: bytes) -> None: + _assert_instance_key_directory(path.parent) + descriptor: int | None = None + created = False + try: + descriptor = os.open( + path, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_BINARY", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + created = True + if os.name == "posix": + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb", closefd=False) as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + _windows_instance_key_acl(path, directory=False, apply=True) + _assert_instance_key_file(path, expected_size=len(payload)) + except RouteControllerError: + raise + except OSError as exc: + raise RouteControllerError("instance key vault commit failed") from exc + finally: + if descriptor is not None: + os.close(descriptor) + if created and (not path.exists() or path.is_symlink()): + raise RouteControllerError("instance key vault commit was lost") + + +def _replace_instance_key_json(path: Path, value: Mapping[str, Any]) -> None: + payload = _instance_key_json(value) + _assert_instance_key_directory(path.parent) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + if os.name == "posix": + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle: + descriptor = -1 + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + _windows_instance_key_acl(temporary, directory=False, apply=True) + os.replace(temporary, path) + _assert_instance_key_file(path, expected_size=len(payload)) + _fsync_instance_key_directory(path.parent) + except RouteControllerError: + raise + except OSError as exc: + raise RouteControllerError("instance key metadata commit failed") from exc + finally: + if descriptor not in (None, -1): + os.close(descriptor) + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + +def _fsync_instance_key_directory(path: Path) -> None: + if os.name != "posix": + return + descriptor: int | None = None + try: + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + os.fsync(descriptor) + except OSError as exc: + raise RouteControllerError("instance key vault could not be synchronized") from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +@contextmanager +def _instance_key_lock(directory: Path) -> Iterator[None]: + _assert_instance_key_directory(directory) + path = directory / ".instance-key.lock" + descriptor: int | None = None + try: + descriptor = os.open( + path, + os.O_RDWR + | os.O_CREAT + | getattr(os, "O_BINARY", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + if os.name == "posix": + os.fchmod(descriptor, 0o600) + opened = os.fstat(descriptor) + _windows_instance_key_acl(path, directory=False, apply=True) + _assert_instance_key_file(path) + if not stat.S_ISREG(opened.st_mode): + raise RouteControllerError("instance key vault lock is unsafe") + if os.name == "nt": + import msvcrt + + if opened.st_size == 0: + os.write(descriptor, b"\0") + os.fsync(descriptor) + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + except RouteControllerError: + raise + except OSError as exc: + raise RouteControllerError("instance key vault is locked or unsafe") from exc + finally: + if descriptor is not None: + try: + if os.name == "nt": + import msvcrt + + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_UN) + except OSError: + pass + os.close(descriptor) + + +def _instance_key_record_path(directory: Path, generation: str, epoch: int) -> Path: + return directory / f"record-v1-{generation.removeprefix('sha256:')}-{epoch:08d}.json" + + +def _instance_key_bytes_path(directory: Path, generation: str, epoch: int) -> Path: + return directory / f"key-v1-{generation.removeprefix('sha256:')}-{epoch:08d}.key" + + +def _instance_key_tombstone_path(directory: Path, generation: str) -> Path: + return directory / f"revoked-v1-{generation.removeprefix('sha256:')}.json" + + +def _instance_key_epoch_from_name( + name: str, + *, + prefix: str, + suffix: str, +) -> int | None: + if not name.startswith(prefix): + return None + if not name.endswith(suffix): + raise RouteControllerError("instance key vault contains an unsafe epoch entry") + encoded = name[len(prefix) : -len(suffix)] + if len(encoded) != 8 or not encoded.isascii() or not encoded.isdigit(): + raise RouteControllerError("instance key vault contains an unsafe epoch entry") + epoch = int(encoded) + if not 1 <= epoch <= 99_999_999: + raise RouteControllerError("instance key vault contains an invalid epoch") + return epoch + + +def _reconcile_instance_key_epochs( + directory: Path, + generation: str, + active_epoch: int, +) -> None: + """Remove unreferenced key bytes and incomplete future rotation records.""" + + generation_value = generation.removeprefix("sha256:") + key_prefix = f"key-v1-{generation_value}-" + record_prefix = f"record-v1-{generation_value}-" + changed = False + try: + candidates = tuple(directory.iterdir()) + except OSError as exc: + raise RouteControllerError("instance key vault inventory failed") from exc + for candidate in candidates: + key_epoch = _instance_key_epoch_from_name( + candidate.name, + prefix=key_prefix, + suffix=".key", + ) + if key_epoch is not None: + if key_epoch != active_epoch: + _unlink_instance_key_file(candidate) + changed = True + continue + record_epoch = _instance_key_epoch_from_name( + candidate.name, + prefix=record_prefix, + suffix=".json", + ) + if record_epoch is not None and record_epoch > active_epoch: + _unlink_instance_key_file(candidate) + changed = True + if changed: + _fsync_instance_key_directory(directory) + + +def _load_instance_key_json(path: Path) -> dict[str, Any]: + return dict( + _strict_json( + _read_instance_key_file( + path, + maximum=MAX_INSTANCE_KEY_RECORD_BYTES, + ) + ) + ) + + +def _load_instance_key_material( + plan: RoutePlan, + resource: ResourcePlan, + directory: Path, + active: Mapping[str, Any], + *, + now_unix: int, + expected_generation_digest: str, +) -> InstanceGenerationKey: + validated_active = _validate_instance_key_active( + active, + plan, + expected_resource_name=resource.name, + expected_generation_digest=expected_generation_digest, + ) + epoch = validated_active["key_epoch"] + record = validate_instance_generation_key_record( + _load_instance_key_json(_instance_key_record_path(directory, expected_generation_digest, epoch)), + plan, + expected_resource_name=resource.name, + expected_generation_digest=expected_generation_digest, + ) + if record["record_digest"] != validated_active["record_digest"]: + raise RouteControllerError("active instance key record changed") + if now_unix >= record["expires_at_unix"]: + raise RouteControllerError("active instance key expired") + key = _read_instance_key_file( + _instance_key_bytes_path(directory, expected_generation_digest, epoch), + maximum=INSTANCE_KEY_BYTES, + expected_size=INSTANCE_KEY_BYTES, + ) + if "sha256:" + hashlib.sha256(key).hexdigest() != record["key_sha256"]: + raise RouteControllerError("active instance key digest changed") + return InstanceGenerationKey(MappingProxyType(record), key) + + +def _new_instance_key( + plan: RoutePlan, + resource: ResourcePlan, + directory: Path, + instance_id: str, + creation_timestamp: str, + *, + key_epoch: int, + now_unix: int, + previous_record_digest: str | None, + key_factory: Any, +) -> InstanceGenerationKey: + try: + generated = key_factory(INSTANCE_KEY_BYTES) + except Exception as exc: + raise RouteControllerError("instance key generation failed") from exc + if not isinstance(generated, (bytes, bytearray)) or len(generated) != INSTANCE_KEY_BYTES: + raise RouteControllerError("instance key generator returned invalid material") + key = bytes(generated) + record = _instance_key_record( + plan, + resource, + instance_id, + creation_timestamp, + key=key, + key_epoch=key_epoch, + issued_at_unix=now_unix, + previous_record_digest=previous_record_digest, + ) + generation = record["instance_generation_digest"] + key_path = _instance_key_bytes_path(directory, generation, key_epoch) + record_path = _instance_key_record_path(directory, generation, key_epoch) + if key_path.exists() or key_path.is_symlink() or record_path.exists() or record_path.is_symlink(): + raise RouteControllerError("instance key epoch already exists") + _write_instance_key_exclusive(key_path, key) + try: + _write_instance_key_exclusive(record_path, _instance_key_json(record)) + except BaseException: + try: + _unlink_instance_key_file(key_path) + except RouteControllerError: + pass + raise + _fsync_instance_key_directory(directory) + return InstanceGenerationKey(MappingProxyType(record), key) + + +def _unlink_instance_key_file(path: Path) -> None: + if not path.exists() and not path.is_symlink(): + return + _assert_instance_key_file(path) + try: + path.unlink() + except OSError as exc: + raise RouteControllerError("instance key vault cleanup failed") from exc + + +def ensure_instance_generation_key( + plan: RoutePlan, + resource_name: str, + instance_id: str, + creation_timestamp: str, + vault_root: Path, + *, + now_unix: int, + key_factory: Any = secrets.token_bytes, +) -> InstanceGenerationKey: + resource = _instance_key_resource(plan, resource_name) + generation = instance_generation_digest(resource.name, instance_id, creation_timestamp) + now = _integer(now_unix, "instance key generation time", 1) + if now >= plan.deadline_unix: + raise RouteControllerError("instance key generation is outside the route deadline") + directory = _instance_key_directory(plan, resource, vault_root) + active_path = directory / "active.json" + tombstone_path = _instance_key_tombstone_path(directory, generation) + with _instance_key_lock(directory): + if tombstone_path.exists() or tombstone_path.is_symlink(): + _validate_instance_key_tombstone( + _load_instance_key_json(tombstone_path), + plan, + expected_resource_name=resource.name, + expected_generation_digest=generation, + ) + raise RouteControllerError("instance key generation was revoked") + if active_path.exists() or active_path.is_symlink(): + material = _load_instance_key_material( + plan, + resource, + directory, + _load_instance_key_json(active_path), + now_unix=now, + expected_generation_digest=generation, + ) + _reconcile_instance_key_epochs( + directory, + generation, + material.record["key_epoch"], + ) + return material + _reconcile_instance_key_epochs(directory, generation, 0) + material = _new_instance_key( + plan, + resource, + directory, + instance_id, + creation_timestamp, + key_epoch=1, + now_unix=now, + previous_record_digest=None, + key_factory=key_factory, + ) + _replace_instance_key_json(active_path, _instance_key_active(plan, material.record)) + return material + + +def load_instance_generation_key( + plan: RoutePlan, + resource_name: str, + instance_id: str, + creation_timestamp: str, + vault_root: Path, + *, + now_unix: int, +) -> InstanceGenerationKey: + resource = _instance_key_resource(plan, resource_name) + generation = instance_generation_digest(resource.name, instance_id, creation_timestamp) + now = _integer(now_unix, "instance key load time", 1) + directory = _instance_key_directory(plan, resource, vault_root) + tombstone_path = _instance_key_tombstone_path(directory, generation) + active_path = directory / "active.json" + with _instance_key_lock(directory): + if tombstone_path.exists() or tombstone_path.is_symlink(): + _validate_instance_key_tombstone( + _load_instance_key_json(tombstone_path), + plan, + expected_resource_name=resource.name, + expected_generation_digest=generation, + ) + raise RouteControllerError("instance key generation was revoked") + if not active_path.exists() and not active_path.is_symlink(): + raise RouteControllerError("active instance key is unavailable") + material = _load_instance_key_material( + plan, + resource, + directory, + _load_instance_key_json(active_path), + now_unix=now, + expected_generation_digest=generation, + ) + _reconcile_instance_key_epochs( + directory, + generation, + material.record["key_epoch"], + ) + return material + + +def rotate_instance_generation_key( + plan: RoutePlan, + resource_name: str, + instance_id: str, + creation_timestamp: str, + vault_root: Path, + *, + now_unix: int, + expected_record_digest: str, + key_factory: Any = secrets.token_bytes, +) -> InstanceGenerationKey: + resource = _instance_key_resource(plan, resource_name) + generation = instance_generation_digest(resource.name, instance_id, creation_timestamp) + now = _integer(now_unix, "instance key rotation time", 1) + _string( + expected_record_digest, + _DIGEST_RE, + "expected instance key record digest", + ) + if now >= plan.deadline_unix: + raise RouteControllerError("instance key rotation is outside the route deadline") + directory = _instance_key_directory(plan, resource, vault_root) + active_path = directory / "active.json" + tombstone_path = _instance_key_tombstone_path(directory, generation) + with _instance_key_lock(directory): + if tombstone_path.exists() or tombstone_path.is_symlink(): + _validate_instance_key_tombstone( + _load_instance_key_json(tombstone_path), + plan, + expected_resource_name=resource.name, + expected_generation_digest=generation, + ) + raise RouteControllerError("instance key generation was revoked") + if not active_path.exists() and not active_path.is_symlink(): + raise RouteControllerError("active instance key is unavailable") + current = _load_instance_key_material( + plan, + resource, + directory, + _load_instance_key_json(active_path), + now_unix=now, + expected_generation_digest=generation, + ) + _reconcile_instance_key_epochs( + directory, + generation, + current.record["key_epoch"], + ) + if current.record["record_digest"] != expected_record_digest: + if current.record["previous_record_digest"] == expected_record_digest: + return current + raise RouteControllerError("active instance key changed before rotation") + epoch = current.record["key_epoch"] + 1 + if epoch > 99_999_999: + raise RouteControllerError("instance key epoch is exhausted") + replacement = _new_instance_key( + plan, + resource, + directory, + instance_id, + creation_timestamp, + key_epoch=epoch, + now_unix=now, + previous_record_digest=current.record["record_digest"], + key_factory=key_factory, + ) + _replace_instance_key_json( + active_path, + _instance_key_active(plan, replacement.record), + ) + _unlink_instance_key_file( + _instance_key_bytes_path( + directory, + generation, + current.record["key_epoch"], + ) + ) + _fsync_instance_key_directory(directory) + return replacement + + +def revoke_instance_generation_key( + plan: RoutePlan, + resource_name: str, + instance_id: str, + creation_timestamp: str, + vault_root: Path, + *, + now_unix: int, +) -> dict[str, Any]: + resource = _instance_key_resource(plan, resource_name) + generation = instance_generation_digest(resource.name, instance_id, creation_timestamp) + now = _integer(now_unix, "instance key revocation time", 1) + directory = _instance_key_directory(plan, resource, vault_root) + active_path = directory / "active.json" + tombstone_path = _instance_key_tombstone_path(directory, generation) + prefix = f"key-v1-{generation.removeprefix('sha256:')}-" + with _instance_key_lock(directory): + if tombstone_path.exists() or tombstone_path.is_symlink(): + tombstone = _validate_instance_key_tombstone( + _load_instance_key_json(tombstone_path), + plan, + expected_resource_name=resource.name, + expected_generation_digest=generation, + ) + else: + epoch = 0 + record_digest: str | None = None + if active_path.exists() or active_path.is_symlink(): + active = _validate_instance_key_active( + _load_instance_key_json(active_path), + plan, + expected_resource_name=resource.name, + ) + if active["instance_generation_digest"] != generation: + raise RouteControllerError("cannot revoke a different active instance generation") + epoch = active["key_epoch"] + record_digest = active["record_digest"] + tombstone = _instance_key_tombstone( + plan, + resource, + generation, + revoked_at_unix=now, + last_key_epoch=epoch, + last_record_digest=record_digest, + ) + _write_instance_key_exclusive( + tombstone_path, + _instance_key_json(tombstone), + ) + _fsync_instance_key_directory(directory) + for candidate in directory.iterdir(): + if candidate.name.startswith(prefix) and candidate.name.endswith(".key"): + _unlink_instance_key_file(candidate) + if active_path.exists() or active_path.is_symlink(): + active = _validate_instance_key_active( + _load_instance_key_json(active_path), + plan, + expected_resource_name=resource.name, + ) + if active["instance_generation_digest"] == generation: + _unlink_instance_key_file(active_path) + _fsync_instance_key_directory(directory) + return tombstone + + +def cleanup_instance_generation_key( + plan: RoutePlan, + resource_name: str, + instance_id: str, + creation_timestamp: str, + vault_root: Path, + *, + now_unix: int, +) -> dict[str, Any]: + """Revoke key bytes idempotently after provider absence is independently proved.""" + + return revoke_instance_generation_key( + plan, + resource_name, + instance_id, + creation_timestamp, + vault_root, + now_unix=now_unix, + ) + + +def observation_instance_generations_digest( + resources: Mapping[str, Any], + plan: RoutePlan, +) -> str | None: + generations: list[dict[str, str]] = [] + for resource in plan.resources: + if not resource.kind.endswith("instance"): + continue + observed = resources.get(resource.name) + if not isinstance(observed, Mapping) or observed.get("present") is not True: + return None + digest = observed.get("instance_generation_digest") + _string(digest, _DIGEST_RE, "instance generation digest") + generations.append( + { + "resource_name": resource.name, + "instance_generation_digest": digest, + } + ) + return _canonical_digest({"instances": generations}) + + +def initial_state(plan: RoutePlan) -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "revision": 0, + "phase": "ABSENT", + "failure_code": None, + "evidence_digest": None, + "instance_generations_digest": None, + "cleanup_verified": False, + "next_action": "none", + } + + +def validate_state(value: Mapping[str, Any], plan: RoutePlan) -> dict[str, Any]: + state = dict(_mapping(value, _STATE_FIELDS, "state")) + if ( + state["schema_version"] != STATE_SCHEMA_VERSION + or state["run_id"] != plan.run_id + or state["plan_digest"] != plan.plan_digest + or state["phase"] not in PHASES + or state["next_action"] not in ACTIONS + ): + raise RouteControllerError("state binding is invalid") + _integer(state["revision"], "state revision") + if state["failure_code"] is not None: + _string(state["failure_code"], _LABEL_RE, "failure_code") + if state["evidence_digest"] is not None: + _string(state["evidence_digest"], _DIGEST_RE, "evidence_digest") + if state["instance_generations_digest"] is not None: + _string( + state["instance_generations_digest"], + _DIGEST_RE, + "instance_generations_digest", + ) + if state["phase"] in {"READY", "COLLECTING"} and state["instance_generations_digest"] is None: + raise RouteControllerError("active state lacks an instance generation latch") + _boolean(state["cleanup_verified"], "cleanup_verified") + if state["cleanup_verified"] is not (state["phase"] in TERMINAL_PHASES): + raise RouteControllerError("cleanup state is inconsistent") + if state["phase"] == "CLEANED_PASS" and state["evidence_digest"] is None: + raise RouteControllerError("passing terminal state lacks evidence") + if state["phase"] == "CLEANED_FAILURE" and state["failure_code"] is None: + raise RouteControllerError("failed terminal state lacks a failure code") + if state["evidence_digest"] is not None and state["phase"] not in { + "CLEANING", + "CLEANED_PASS", + "CLEANED_FAILURE", + }: + raise RouteControllerError("evidence state is inconsistent") + if state["phase"] not in {"CLEANING", "CLEANED_FAILURE"} and state["failure_code"] is not None: + raise RouteControllerError("failure code is inconsistent") + allowed_actions = { + "ABSENT": {"none"}, + "STARTING": {"none", "start_route"}, + "READY": {"none"}, + "COLLECTING": {"none", "collect_route"}, + "CLEANING": {"cleanup_route"}, + "CLEANED_PASS": {"none"}, + "CLEANED_FAILURE": {"none"}, + } + if state["next_action"] not in allowed_actions[state["phase"]]: + raise RouteControllerError("state action is inconsistent") + return state + + +def _validate_route_record( + value: Any, + plan: RoutePlan, + workers: Mapping[str, Any], +) -> Mapping[str, Any]: + record = _mapping(value, _ROUTE_RECORD_FIELDS, "route record") + if ( + record["schema_version"] != SCHEMA_VERSION + or record["result"] != "passed" + or record["run_id"] != plan.run_id + or record["job_id"] != plan.route_job_id + or record["collect_action_id"] != _action_id(plan, "collect_route") + or record["plan_digest"] != plan.plan_digest + or record["source_commit"] != plan.source_commit + or record["manifest_digest"] != plan.manifest_digest + or record["worker_plan_digest"] != plan.worker_plan_digest + or record["route_span"] != "0:64" + or record["cleanup_ready"] is not True + ): + raise RouteControllerError("route record binding is invalid") + _string(record["session_id"], _LABEL_RE, "route record session_id") + _string( + record["route_rpc_evidence_digest"], + _DIGEST_RE, + "route record RPC evidence digest", + ) + results = record["worker_results"] + if not isinstance(results, list) or len(results) != len(plan.workers): + raise RouteControllerError("route record worker inventory is invalid") + for index, (result_value, worker_plan) in enumerate(zip(results, plan.workers)): + result = _mapping( + result_value, + _ROUTE_WORKER_RESULT_FIELDS, + f"route record worker_results[{index}]", + ) + observed_worker = workers[worker_plan.worker_id] + if ( + result["worker_id"] != worker_plan.worker_id + or result["machine_id"] != worker_plan.machine_id + or result["peer_id"] != observed_worker["peer_id"] + or result["span"] != worker_plan.span + or result["source_commit"] != plan.source_commit + or result["manifest_digest"] != plan.manifest_digest + or result["artifact_bytes"] != worker_plan.artifact_bytes + or result["artifact_set_digest"] != worker_plan.artifact_set_digest + or result["cache_root"] != worker_plan.cache_root + ): + raise RouteControllerError("route record worker binding is invalid") + _string( + result["worker_evidence_digest"], + _DIGEST_RE, + "route record worker evidence digest", + ) + return record + + +def _protected_named_bytes( + plan: RoutePlan, + source_root: Path, + root: Path, + name: str, +) -> bytes: + if not _LABEL_RE.fullmatch(name): + raise RouteControllerError("protected evidence filename is invalid") + _assert_protected_path(root, plan, source_root, directory=True) + candidate = root.resolve() / name + _assert_protected_path(candidate, plan, source_root, directory=False) + return _regular_bytes(candidate) + + +def revalidate_route_evidence( + plan: RoutePlan, + observation: Mapping[str, Any], + evidence_root: Path, + source_root: Path, +) -> str: + """Validate protected host-job, RPC, and per-worker evidence records.""" + + workers = observation["workers"] + route_job = observation["route_job"] + if route_job["state"] != "passed": + raise RouteControllerError("route evidence is only valid for a passed job") + record = _validate_route_record(route_job["route_record"], plan, workers) + expected_names = { + "route-terminal.json", + "route-rpc.json", + *(f"{worker.worker_id}-evidence.json" for worker in plan.workers), + } + _assert_protected_path(evidence_root, plan, source_root, directory=True) + try: + observed_names = {entry.name for entry in evidence_root.iterdir()} + except OSError as exc: + raise RouteControllerError("protected route evidence inventory is unavailable") from exc + if observed_names != expected_names: + raise RouteControllerError("protected route evidence inventory is not exact") + + terminal_payload = _protected_named_bytes( + plan, + source_root, + evidence_root, + "route-terminal.json", + ) + terminal_record = _mapping( + _strict_json(terminal_payload), + _ROUTE_RECORD_FIELDS, + "protected route terminal record", + ) + if dict(terminal_record) != dict(record): + raise RouteControllerError("protected route terminal record does not match the observation") + + rpc_payload = _protected_named_bytes( + plan, + source_root, + evidence_root, + "route-rpc.json", + ) + if "sha256:" + hashlib.sha256(rpc_payload).hexdigest() != record["route_rpc_evidence_digest"]: + raise RouteControllerError("protected route RPC evidence digest changed") + rpc = _mapping( + _strict_json(rpc_payload), + _RPC_EVIDENCE_FIELDS, + "protected route RPC evidence", + ) + if ( + rpc["schema_version"] != SCHEMA_VERSION + or rpc["result"] != "passed" + or rpc["run_id"] != plan.run_id + or rpc["job_id"] != plan.route_job_id + or rpc["collect_action_id"] != _action_id(plan, "collect_route") + or rpc["plan_digest"] != plan.plan_digest + or rpc["source_commit"] != plan.source_commit + or rpc["manifest_digest"] != plan.manifest_digest + or rpc["worker_plan_digest"] != plan.worker_plan_digest + or rpc["route_span"] != "0:64" + or rpc["session_id"] != record["session_id"] + ): + raise RouteControllerError("protected route RPC evidence binding is invalid") + + for worker_plan, result in zip(plan.workers, record["worker_results"]): + payload = _protected_named_bytes( + plan, + source_root, + evidence_root, + f"{worker_plan.worker_id}-evidence.json", + ) + if "sha256:" + hashlib.sha256(payload).hexdigest() != result["worker_evidence_digest"]: + raise RouteControllerError("protected worker evidence digest changed") + evidence = _mapping( + _strict_json(payload), + _WORKER_EVIDENCE_FIELDS, + "protected worker evidence", + ) + if ( + evidence["schema_version"] != SCHEMA_VERSION + or evidence["result"] != "passed" + or evidence["run_id"] != plan.run_id + or evidence["job_id"] != plan.route_job_id + or evidence["collect_action_id"] != _action_id(plan, "collect_route") + or evidence["plan_digest"] != plan.plan_digest + or evidence["source_commit"] != plan.source_commit + or evidence["manifest_digest"] != plan.manifest_digest + or evidence["worker_plan_digest"] != plan.worker_plan_digest + or evidence["start_action_id"] != _action_id(plan, "start_route") + or evidence["worker_id"] != worker_plan.worker_id + or evidence["machine_id"] != worker_plan.machine_id + or evidence["peer_id"] != result["peer_id"] + or evidence["span"] != worker_plan.span + or evidence["artifact_bytes"] != worker_plan.artifact_bytes + or evidence["artifact_set_digest"] != worker_plan.artifact_set_digest + or evidence["cache_root"] != worker_plan.cache_root + ): + raise RouteControllerError("protected worker evidence binding is invalid") + return route_job["evidence_digest"] + + +def validate_observation( + value: Mapping[str, Any], + plan: RoutePlan, + *, + cleanup_only: bool = False, +) -> dict[str, Any]: + observation = dict(_mapping(value, _OBSERVATION_FIELDS, "observation")) + if observation["schema_version"] != SCHEMA_VERSION or observation["run_id"] != plan.run_id: + raise RouteControllerError("observation identity is invalid") + _boolean( + observation["protected_bootstrap_running"], + "protected_bootstrap_running", + ) + observed_at = _integer(observation["observed_at_unix"], "observed_at_unix", 1) + + if cleanup_only: + revalidation = observation["artifact_plan_revalidation"] + else: + revalidation = _mapping( + observation["artifact_plan_revalidation"], + _REVALIDATION_FIELDS, + "artifact_plan_revalidation", + ) + verified_at = _integer( + revalidation["verified_at_unix"], + "artifact plan verified_at_unix", + 1, + ) + verifier_binding = next( + (binding for binding in plan.source_bindings if binding["relative_path"] == VERIFIER_SOURCE_PATH), + None, + ) + if ( + verified_at > observed_at + or revalidation["source_commit"] != plan.source_commit + or revalidation["manifest_digest"] != plan.manifest_digest + or revalidation["model_revision"] != plan.model_revision + or revalidation["index_digest"] != EXPECTED_INDEX_DIGEST + or revalidation["block_prefix"] != EXPECTED_BLOCK_PREFIX + or revalidation["worker_plan_digest"] != plan.worker_plan_digest + or verifier_binding is None + or revalidation["verifier_source_sha256"] != verifier_binding["sha256"] + ): + raise RouteControllerError("production artifact plan revalidation is invalid") + + resources = observation["resources"] + if not isinstance(resources, dict) or set(resources) != set(plan.resource_by_name): + raise RouteControllerError("provider resource inventory is not exact") + for name, resource_plan in plan.resource_by_name.items(): + resource = _mapping(resources[name], _OBS_RESOURCE_FIELDS, f"resources[{name!r}]") + present = _boolean(resource["present"], "resource present") + if ( + resource["kind"] != resource_plan.kind + or resource["provider"] != resource_plan.provider + or resource["region"] != resource_plan.region + ): + raise RouteControllerError("provider resource identity is invalid") + if present: + if ( + resource["run_id"] != plan.run_id + or resource["source_commit"] != plan.source_commit + or _integer(resource["deadline_unix"], "resource deadline", 1) != plan.deadline_unix + or resource["plan_digest"] != plan.plan_digest + or resource["start_action_id"] != _action_id(plan, "start_route") + or resource["worker_id"] != resource_plan.worker_id + ): + raise RouteControllerError("provider resource binding is invalid") + if resource_plan.kind.endswith("instance"): + instance_id = _string( + resource["instance_id"], + _INSTANCE_ID_RE, + "provider instance id", + ) + creation_timestamp = _string( + resource["creation_timestamp"], + _CREATION_TIMESTAMP_RE, + "provider instance creation timestamp", + ) + if resource["instance_generation_digest"] != instance_generation_digest( + resource_plan.name, + instance_id, + creation_timestamp, + ): + raise RouteControllerError("provider instance generation binding is invalid") + elif any( + resource[field] is not None + for field in ( + "instance_id", + "creation_timestamp", + "instance_generation_digest", + ) + ): + raise RouteControllerError("non-instance resource exposed generation metadata") + elif any( + resource[field] is not None + for field in ( + "run_id", + "source_commit", + "deadline_unix", + "plan_digest", + "start_action_id", + "worker_id", + "instance_id", + "creation_timestamp", + "instance_generation_digest", + ) + ): + raise RouteControllerError("absent resource metadata is invalid") + + expected_generations_digest = observation_instance_generations_digest(resources, plan) + if observation["instance_generations_digest"] != expected_generations_digest: + raise RouteControllerError("provider instance generation inventory is invalid") + + workers = observation["workers"] + if not isinstance(workers, dict) or set(workers) != set(plan.worker_by_id): + raise RouteControllerError("worker inventory is not exact") + if cleanup_only: + for worker in workers.values(): + if not isinstance(worker, dict) or worker.get("state") not in WORKER_STATES: + raise RouteControllerError("cleanup worker inventory is invalid") + if worker["state"] == "absent" and ( + set(worker) != _OBS_WORKER_FIELDS + or any(value is not None for field, value in worker.items() if field != "state") + ): + raise RouteControllerError("absent worker metadata is invalid") + route_job = observation["route_job"] + if not isinstance(route_job, dict) or route_job.get("state") not in JOB_STATES: + raise RouteControllerError("cleanup route job inventory is invalid") + if route_job["state"] == "absent" and ( + set(route_job) != _ROUTE_JOB_FIELDS + or any(value is not None for field, value in route_job.items() if field != "state") + ): + raise RouteControllerError("absent route job metadata is invalid") + return observation + ready_peer_ids: list[str] = [] + for worker_id, worker_plan in plan.worker_by_id.items(): + worker = _mapping(workers[worker_id], _OBS_WORKER_FIELDS, f"workers[{worker_id!r}]") + state = worker["state"] + if state not in WORKER_STATES: + raise RouteControllerError("worker state is invalid") + if state == "absent": + if any(value is not None for field, value in worker.items() if field != "state"): + raise RouteControllerError("absent worker metadata is invalid") + continue + if ( + worker["machine_id"] != worker_plan.machine_id + or worker["source_commit"] != plan.source_commit + or worker["plan_digest"] != plan.plan_digest + or worker["worker_plan_digest"] != plan.worker_plan_digest + or worker["start_action_id"] != _action_id(plan, "start_route") + or worker["span"] != worker_plan.span + or worker["manifest_digest"] != plan.manifest_digest + or worker["artifact_bytes"] != worker_plan.artifact_bytes + or worker["artifact_set_digest"] != worker_plan.artifact_set_digest + or worker["cache_root"] != worker_plan.cache_root + ): + raise RouteControllerError("worker binding is invalid") + peer_id = worker["peer_id"] + if state == "ready": + ready_peer_ids.append(_string(peer_id, _PEER_RE, "worker peer_id")) + elif peer_id is not None: + raise RouteControllerError("unfinished worker exposed a peer identity") + if len(set(ready_peer_ids)) != len(ready_peer_ids): + raise RouteControllerError("ready workers must have unique peer identities") + + route_job = _mapping(observation["route_job"], _ROUTE_JOB_FIELDS, "route_job") + if route_job["state"] not in JOB_STATES: + raise RouteControllerError("route job state is invalid") + evidence_digest = route_job["evidence_digest"] + job_bindings = ( + "job_id", + "collect_action_id", + "run_id", + "plan_digest", + "source_commit", + "manifest_digest", + "worker_plan_digest", + ) + if route_job["state"] == "absent": + if any(route_job[field] is not None for field in (*job_bindings, "evidence_digest", "route_record")): + raise RouteControllerError("absent route job metadata is invalid") + elif ( + route_job["job_id"] != plan.route_job_id + or route_job["collect_action_id"] != _action_id(plan, "collect_route") + or route_job["run_id"] != plan.run_id + or route_job["plan_digest"] != plan.plan_digest + or route_job["source_commit"] != plan.source_commit + or route_job["manifest_digest"] != plan.manifest_digest + or route_job["worker_plan_digest"] != plan.worker_plan_digest + ): + raise RouteControllerError("route job binding is invalid") + if route_job["state"] == "passed": + _string(evidence_digest, _DIGEST_RE, "route evidence digest") + record = _validate_route_record(route_job["route_record"], plan, workers) + if evidence_digest != _canonical_digest(record): + raise RouteControllerError("route evidence digest does not bind the route record") + elif evidence_digest is not None or route_job["route_record"] is not None: + raise RouteControllerError("unfinished route job exposed evidence") + return observation + + +def _all_absent(observation: Mapping[str, Any]) -> bool: + return ( + all(not resource["present"] for resource in observation["resources"].values()) + and all(worker["state"] == "absent" for worker in observation["workers"].values()) + and observation["route_job"]["state"] == "absent" + ) + + +def _all_resources_present(observation: Mapping[str, Any]) -> bool: + return all(resource["present"] for resource in observation["resources"].values()) + + +def _all_workers_ready(observation: Mapping[str, Any]) -> bool: + return all(worker["state"] == "ready" for worker in observation["workers"].values()) + + +def _authorized(plan: RoutePlan) -> bool: + return all( + plan.authorization[field] is True + for field in ( + "reservation_recorded", + "native_auth_revalidated", + "inventory_revalidated", + "pricing_revalidated", + "provisioning_authorized", + ) + ) + + +def _next(state: Mapping[str, Any], **changes: Any) -> dict[str, Any]: + result = dict(state) + result.update(changes) + if all(result[key] == state[key] for key in state): + return result + result["revision"] = int(state["revision"]) + 1 + return result + + +def _cleanup_state(state: Mapping[str, Any], failure_code: str | None = None) -> dict[str, Any]: + changes: dict[str, Any] = { + "phase": "CLEANING", + "next_action": "cleanup_route", + } + if failure_code is not None: + changes["failure_code"] = failure_code + return _next(state, **changes) + + +def reconcile( + operation: str, + state_value: Mapping[str, Any], + observation_value: Mapping[str, Any], + plan: RoutePlan, + *, + now_unix: int | None = None, + route_evidence_validated: bool = False, + start_was_issued: bool = False, +) -> dict[str, Any]: + if operation not in {"start", "status", "collect", "cleanup"}: + raise RouteControllerError("operation is invalid") + state = validate_state(state_value, plan) + phase = state["phase"] + raw_observed_at = observation_value.get("observed_at_unix") + effective_now = ( + _integer(now_unix, "trusted current time", 1) + if now_unix is not None + else _integer(raw_observed_at, "observed_at_unix", 1) + ) + cleanup_only = ( + operation == "cleanup" + or phase == "CLEANING" + or observation_value.get("protected_bootstrap_running") is False + or effective_now >= plan.deadline_unix + ) + observation = validate_observation( + observation_value, + plan, + cleanup_only=cleanup_only, + ) + all_absent = _all_absent(observation) + resources_present = _all_resources_present(observation) + workers_ready = _all_workers_ready(observation) + worker_states = {worker["state"] for worker in observation["workers"].values()} + job_state = observation["route_job"]["state"] + observed_generations_digest = observation["instance_generations_digest"] + latched_generations_digest = state["instance_generations_digest"] + if ( + phase not in TERMINAL_PHASES + and not cleanup_only + and latched_generations_digest is not None + and observed_generations_digest != latched_generations_digest + ): + return _cleanup_state(state, "instance-generation-changed") + + def advance_with_generations(**changes: Any) -> dict[str, Any]: + if not cleanup_only and resources_present and latched_generations_digest is None: + if observed_generations_digest is None: + raise RouteControllerError("complete resources lack instance generations") + changes.setdefault( + "instance_generations_digest", + observed_generations_digest, + ) + return _next(state, **changes) + + if not cleanup_only and job_state == "passed" and not route_evidence_validated: + raise RouteControllerError("passed route evidence was not revalidated from protected records") + + if phase in TERMINAL_PHASES: + if not all_absent: + raise RouteControllerError("resources returned after terminal cleanup") + if observation["protected_bootstrap_running"] is not True: + raise RouteControllerError("protected bootstrap was lost after terminal cleanup") + return state + + if observation["protected_bootstrap_running"] is not True: + if all_absent: + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="protected-bootstrap-lost", + cleanup_verified=True, + next_action="none", + ) + return _cleanup_state(state, "protected-bootstrap-lost") + + if effective_now >= plan.deadline_unix: + if all_absent: + return _next( + state, + phase="CLEANED_FAILURE", + failure_code=state["failure_code"] or "run-expired", + cleanup_verified=True, + next_action="none", + ) + return _cleanup_state(state, state["failure_code"] or "run-expired") + + if operation == "cleanup": + if all_absent: + if state["evidence_digest"] is not None and state["failure_code"] is None: + return _next( + state, + phase="CLEANED_PASS", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANED_FAILURE", + failure_code=state["failure_code"] or "operator-cleanup", + cleanup_verified=True, + next_action="none", + ) + return _cleanup_state(state) + + if phase == "CLEANING": + if all_absent: + if state["evidence_digest"] is not None and state["failure_code"] is None: + return _next( + state, + phase="CLEANED_PASS", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANED_FAILURE", + failure_code=state["failure_code"] or "route-failed", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="cleanup_route") + + plan_verified_at = observation["artifact_plan_revalidation"]["verified_at_unix"] + if ( + observation["observed_at_unix"] > effective_now + or effective_now - observation["observed_at_unix"] > MAX_PLAN_REVALIDATION_AGE_SECONDS + or ( + (operation in {"start", "collect"} or job_state == "passed") + and effective_now - plan_verified_at > MAX_PLAN_REVALIDATION_AGE_SECONDS + ) + ): + raise RouteControllerError("controller observation or artifact plan is stale") + + if phase == "ABSENT" and not all_absent and not start_was_issued: + return _cleanup_state(state, "unrecorded-resources") + + if phase == "ABSENT" and operation == "start" and all_absent and start_was_issued: + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="state-lost-after-start", + cleanup_verified=True, + next_action="none", + ) + + if phase == "ABSENT" and resources_present and workers_ready and job_state == "passed": + return advance_with_generations( + phase="CLEANING", + evidence_digest=observation["route_job"]["evidence_digest"], + next_action="cleanup_route", + ) + + if operation == "start": + if phase == "ABSENT": + if not _authorized(plan): + raise RouteControllerError("paid provisioning is not authorized") + if all_absent: + return _next(state, phase="STARTING", next_action="start_route") + if resources_present and job_state == "failed": + return _cleanup_state(state, "qualification-failed") + if resources_present and workers_ready and job_state in {"absent", "running"}: + return advance_with_generations(phase="READY", next_action="none") + if resources_present and worker_states <= {"starting", "ready"} and job_state == "absent": + return advance_with_generations(phase="STARTING", next_action="none") + return _cleanup_state(state, "partial-reattach") + + if phase == "ABSENT": + if all_absent: + return _next(state, next_action="none") + return _cleanup_state(state, "unexpected-resources") + + if phase == "STARTING" and all_absent: + if state["next_action"] == "start_route": + return state + return _cleanup_state(state, "route-inventory-lost") + + if not resources_present or "failed" in worker_states or "absent" in worker_states: + return _cleanup_state(state, "route-inventory-lost") + + if phase == "STARTING": + if workers_ready: + return advance_with_generations(phase="READY", next_action="none") + if worker_states <= {"starting", "ready"} and job_state == "absent": + return advance_with_generations(next_action="none") + return _cleanup_state(state, "route-start-failed") + + if phase == "READY": + if not workers_ready: + return _cleanup_state(state, "route-readiness-lost") + if job_state == "passed": + return _next( + state, + phase="CLEANING", + evidence_digest=observation["route_job"]["evidence_digest"], + next_action="cleanup_route", + ) + if job_state == "failed": + return _cleanup_state(state, "qualification-failed") + if operation == "collect" and job_state == "absent": + return _next(state, phase="COLLECTING", next_action="collect_route") + if operation == "collect" and job_state == "running": + return _next(state, phase="COLLECTING", next_action="none") + return _next(state, next_action="none") + + if phase == "COLLECTING": + if not workers_ready: + return _cleanup_state(state, "route-readiness-lost") + if job_state == "passed": + return _next( + state, + phase="CLEANING", + evidence_digest=observation["route_job"]["evidence_digest"], + next_action="cleanup_route", + ) + if job_state == "failed": + return _cleanup_state(state, "qualification-failed") + if job_state == "absent" and state["next_action"] == "collect_route": + return state + return _next(state, next_action="none") + + raise RouteControllerError("state transition is invalid") + + +def action_record(state: Mapping[str, Any], plan: RoutePlan) -> dict[str, Any]: + workers = [ + { + "worker_id": worker.worker_id, + "machine_id": worker.machine_id, + "instance": worker.instance, + "disk": worker.disk, + "span": worker.span, + "artifact_bytes": worker.artifact_bytes, + "artifact_set_digest": worker.artifact_set_digest, + "cache_root": worker.cache_root, + } + for worker in plan.workers + ] + resources = [ + { + "name": resource.name, + "kind": resource.kind, + "provider": resource.provider, + "region": resource.region, + "worker_id": resource.worker_id, + } + for resource in plan.resources + ] + action = state["next_action"] + action_id = None + if action != "none": + action_id = _action_id(plan, action) + instance_generations_digest = state.get("instance_generations_digest") + if instance_generations_digest is not None: + _string( + instance_generations_digest, + _DIGEST_RE, + "instance_generations_digest", + ) + return { + "schema_version": SCHEMA_VERSION, + "gate": GATE, + "run_id": plan.run_id, + "route_job_id": plan.route_job_id, + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "model_revision": plan.model_revision, + "worker_plan_digest": plan.worker_plan_digest, + "deadline_unix": plan.deadline_unix, + "revision": state["revision"], + "instance_generations_digest": instance_generations_digest, + "action": action, + "action_id": action_id, + "authorization": dict(plan.authorization), + "source_bindings": [dict(binding) for binding in plan.source_bindings], + "runtime_package": dict(plan.runtime_package), + "resources": resources, + "resource_specs": [_expected_resource_spec(resource) for resource in plan.resources], + "workers": workers, + } + + +def _prepare_output_path(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + parent_metadata = path.parent.lstat() + parent_reparse = bool( + getattr(parent_metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + if parent_reparse or path.parent.is_symlink() or not stat.S_ISDIR(parent_metadata.st_mode): + raise RouteControllerError("output parent is unsafe") + if path.exists() or path.is_symlink(): + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise RouteControllerError("output target is unsafe") + + +def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: + payload = (json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8") + if len(payload) > MAX_JSON_BYTES: + raise RouteControllerError("output is too large") + _prepare_output_path(path) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Advance one durable Qwen3.8 complete-route controller operation") + parser.add_argument("operation", choices=("start", "status", "collect", "cleanup")) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--observation", type=Path, required=True) + parser.add_argument("--manifest", type=Path) + parser.add_argument("--artifact-root", type=Path) + parser.add_argument("--authorization-root", type=Path) + parser.add_argument("--evidence-root", type=Path) + parser.add_argument("--state", type=Path, required=True) + parser.add_argument("--decision", type=Path, required=True) + return parser + + +def _assert_output_isolation( + outputs: Sequence[Path], + input_files: Sequence[Path | None], + input_roots: Sequence[Path | None], +) -> None: + output_paths = [path.resolve(strict=False) for path in outputs] + file_paths = [path.resolve(strict=False) for path in input_files if path is not None] + if len(set(map(os.path.normcase, map(os.fspath, output_paths)))) != len(output_paths): + raise RouteControllerError("controller output paths must be distinct") + if any( + os.path.normcase(os.fspath(output)) == os.path.normcase(os.fspath(input_path)) + for output in output_paths + for input_path in file_paths + ): + raise RouteControllerError("controller input and output paths must be distinct") + for root in (path.resolve(strict=False) for path in input_roots if path is not None): + for output in output_paths: + try: + output.relative_to(root) + except ValueError: + continue + raise RouteControllerError("controller output overlaps a protected input root") + + +@contextmanager +def _controller_lock(path: Path) -> Iterator[None]: + _prepare_output_path(path) + with path.open("a+b") as handle: + if os.name == "nt": + import msvcrt + + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write(b"\\0") + handle.flush() + handle.seek(0) + try: + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + except OSError as exc: + raise RouteControllerError("another controller invocation holds the state lock") from exc + else: + import fcntl + + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + raise RouteControllerError("another controller invocation holds the state lock") from exc + try: + yield + finally: + try: + if os.name == "nt": + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: + pass + + +def _validate_journal( + value: Mapping[str, Any], + plan: RoutePlan, + *, + now_unix: int, +) -> dict[str, Any]: + journal = dict(_mapping(value, _JOURNAL_FIELDS, "issuance journal")) + issued_at = _integer( + journal["issued_at_unix"], + "issuance journal issued_at_unix", + 1, + ) + if ( + journal["schema_version"] != SCHEMA_VERSION + or journal["run_id"] != plan.run_id + or journal["plan_digest"] != plan.plan_digest + or journal["start_action_id"] != _action_id(plan, "start_route") + or issued_at > now_unix + or journal["status"] not in {"issued", "completed"} + ): + raise RouteControllerError("issuance journal binding is invalid") + if journal["status"] == "issued": + if ( + any( + journal[field] is not None + for field in ( + "completed_at_unix", + "terminal_phase", + "terminal_revision", + "failure_code", + "evidence_digest", + ) + ) + or journal["cleanup_verified"] is not False + ): + raise RouteControllerError("open issuance journal is invalid") + return journal + + completed_at = _integer( + journal["completed_at_unix"], + "issuance journal completed_at_unix", + issued_at, + ) + terminal_revision = _integer( + journal["terminal_revision"], + "issuance journal terminal_revision", + 1, + ) + phase = journal["terminal_phase"] + if completed_at > now_unix or phase not in TERMINAL_PHASES or journal["cleanup_verified"] is not True: + raise RouteControllerError("completed issuance journal is invalid") + if phase == "CLEANED_PASS": + _string(journal["evidence_digest"], _DIGEST_RE, "issuance journal evidence_digest") + if journal["failure_code"] is not None: + raise RouteControllerError("passing issuance journal is invalid") + elif journal["failure_code"] is None or journal["evidence_digest"] is not None: + raise RouteControllerError("failing issuance journal is invalid") + return journal + + +def _issued_journal(plan: RoutePlan, now_unix: int) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "start_action_id": _action_id(plan, "start_route"), + "status": "issued", + "issued_at_unix": now_unix, + "completed_at_unix": None, + "terminal_phase": None, + "terminal_revision": None, + "failure_code": None, + "evidence_digest": None, + "cleanup_verified": False, + } + + +def _completed_journal( + journal: Mapping[str, Any], + state: Mapping[str, Any], + now_unix: int, +) -> dict[str, Any]: + result = dict(journal) + result.update( + status="completed", + completed_at_unix=now_unix, + terminal_phase=state["phase"], + terminal_revision=state["revision"], + failure_code=state["failure_code"], + evidence_digest=state["evidence_digest"], + cleanup_verified=state["cleanup_verified"], + ) + return result + + +def _state_from_completed_journal( + journal: Mapping[str, Any], + plan: RoutePlan, +) -> dict[str, Any]: + state = initial_state(plan) + state.update( + revision=journal["terminal_revision"], + phase=journal["terminal_phase"], + failure_code=journal["failure_code"], + evidence_digest=journal["evidence_digest"], + cleanup_verified=True, + next_action="none", + ) + return validate_state(state, plan) + + +def _execute_controller( + args: argparse.Namespace, + *, + now_unix: int, + journal_path: Path, +) -> dict[str, Any]: + plan = load_plan(args.plan, args.source_root) + observation = _strict_json(_regular_bytes(args.observation)) + journal = None + if journal_path.exists(): + journal = _validate_journal( + _strict_json(_regular_bytes(journal_path)), + plan, + now_unix=now_unix, + ) + if args.state.exists(): + state = _strict_json(_regular_bytes(args.state)) + validated_state = validate_state(state, plan) + if ( + journal is not None + and journal["status"] == "completed" + and ( + validated_state["phase"] != journal["terminal_phase"] + or validated_state["revision"] != journal["terminal_revision"] + or validated_state["failure_code"] != journal["failure_code"] + or validated_state["evidence_digest"] != journal["evidence_digest"] + or validated_state["cleanup_verified"] is not True + ) + ): + raise RouteControllerError("state conflicts with the completed issuance journal") + elif journal is not None and journal["status"] == "completed": + validated_state = _state_from_completed_journal(journal, plan) + else: + validated_state = initial_state(plan) + raw_observed_at = observation.get("observed_at_unix") + cleanup_first = ( + args.operation == "cleanup" + or validated_state["phase"] == "CLEANING" + or observation.get("protected_bootstrap_running") is False + or now_unix >= plan.deadline_unix + ) + validated_observation = validate_observation( + observation, + plan, + cleanup_only=cleanup_first, + ) + if not cleanup_first and ( + type(raw_observed_at) is not int + or raw_observed_at > now_unix + or now_unix - raw_observed_at > MAX_PLAN_REVALIDATION_AGE_SECONDS + ): + raise RouteControllerError("controller observation is stale") + job_state = validated_observation["route_job"]["state"] + requires_production_revalidation = not cleanup_first and ( + args.operation in {"start", "collect"} or job_state == "passed" + ) + requires_start_authorization = ( + not cleanup_first + and args.operation == "start" + and _authorized(plan) + and journal is None + and validated_state["phase"] == "ABSENT" + and _all_absent(validated_observation) + ) + if requires_start_authorization: + if args.authorization_root is None: + raise RouteControllerError("paid start requires controller-owned authorization records") + revalidate_authorization_evidence( + plan, + args.authorization_root, + args.source_root, + now_unix=now_unix, + ) + if requires_production_revalidation: + if args.manifest is None or args.artifact_root is None: + raise RouteControllerError("start and collection require production artifact plan inputs") + revalidation = validated_observation["artifact_plan_revalidation"] + expected_revalidation = revalidate_production_artifact_plan( + plan, + args.manifest, + args.artifact_root, + args.source_root, + verified_at_unix=revalidation["verified_at_unix"], + ) + if dict(revalidation) != expected_revalidation: + raise RouteControllerError("observation was not produced by the revalidated production artifact plan") + route_evidence_validated = False + if not cleanup_first and job_state == "passed": + if args.evidence_root is None: + raise RouteControllerError("passed route requires protected evidence records") + revalidate_route_evidence( + plan, + validated_observation, + args.evidence_root, + args.source_root, + ) + route_evidence_validated = True + next_state = reconcile( + args.operation, + validated_state, + validated_observation, + plan, + now_unix=now_unix, + route_evidence_validated=route_evidence_validated, + start_was_issued=journal is not None, + ) + decision = action_record(next_state, plan) + if decision["action"] == "start_route" and journal is None: + journal = _issued_journal(plan, now_unix) + _atomic_json(journal_path, journal) + if next_state["phase"] in TERMINAL_PHASES and journal is not None and journal["status"] != "completed": + journal = _completed_journal(journal, next_state, now_unix) + _atomic_json(journal_path, journal) + neutral_state = dict(validated_state) + neutral_state["next_action"] = "none" + _atomic_json(args.decision, action_record(neutral_state, plan)) + _atomic_json(args.state, next_state) + _atomic_json(args.decision, decision) + return decision + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + lock_path = args.state.with_name(f".{args.state.name}.lock") + journal_path = args.state.with_name(f".{args.state.name}.issuance.json") + try: + _assert_output_isolation( + (args.state, args.decision, lock_path, journal_path), + (args.plan, args.observation, args.manifest), + (args.source_root, args.artifact_root, args.authorization_root, args.evidence_root), + ) + with _controller_lock(lock_path): + decision = _execute_controller( + args, + now_unix=int(time.time()), + journal_path=journal_path, + ) + except (OSError, RouteControllerError) as exc: + print(f"Gate Q3.8 route controller failed: {exc}") + return 2 + print(json.dumps(decision, allow_nan=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gateq38_stage_package.py b/scripts/gateq38_stage_package.py new file mode 100644 index 000000000..7c4401336 --- /dev/null +++ b/scripts/gateq38_stage_package.py @@ -0,0 +1,585 @@ +"""Validate and bind the exact Linux packaged runtime for the Qwen3.8 route. + +This module is controller-side only. It consumes an already extracted production +desktop bundle and its release attestations, revalidates the complete install +archive and onedir inventory, and emits one canonical record for the later +privileged host-stage operation. It never downloads model data or invokes a +provider. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import tempfile +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from desktop import build_desktop +from drift.model_manifest import ManifestError, ModelManifest +from scripts import gateq38_route_controller as controller + +SCHEMA_VERSION = controller.RUNTIME_PACKAGE_SCHEMA_VERSION +SCOPE = controller.RUNTIME_PACKAGE_SCOPE +SOURCE_CONTEXT_SCOPE = "qwen3.8-runtime-package-source-context" +MAX_RECORD_BYTES = 262_144 +MAX_PROVENANCE_BYTES = controller.MAX_RELEASE_PROVENANCE_BYTES +MAX_CHECKSUMS_BYTES = controller.MAX_RELEASE_CHECKSUMS_BYTES +MAX_METRICS_BYTES = controller.MAX_RELEASE_METRICS_BYTES +HASH_CHUNK_BYTES = 1_048_576 +NODE_ROOT = controller.RUNTIME_PACKAGE_NODE_ROOT +NODE_EXECUTABLE = controller.RUNTIME_PACKAGE_NODE_EXECUTABLE +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_WEIGHT_ROLES = {"converted_weight", "quantized_weight", "weight"} +ProtectionVerifier = Callable[[Path, bool], None] +_SOURCE_CONTEXT_FIELDS = { + "schema_version", + "scope", + "source_commit", + "source_tree", + "source_bindings", +} + + +class Q38StagePackageError(RuntimeError): + """The packaged runtime or its source binding failed closed.""" + + +def _regular_bytes(path: Path, maximum: int | None = None) -> bytes: + try: + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise Q38StagePackageError(f"required regular file is unsafe: {path.name}") + if maximum is not None and not 1 <= metadata.st_size <= maximum: + raise Q38StagePackageError(f"required file size is invalid: {path.name}") + payload = path.read_bytes() + except Q38StagePackageError: + raise + except OSError as exc: + raise Q38StagePackageError(f"required file could not be read: {path.name}") from exc + if len(payload) != metadata.st_size: + raise Q38StagePackageError(f"required file changed while read: {path.name}") + return payload + + +def _sha256(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _canonical_bytes(value: Any) -> bytes: + try: + return ( + json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise Q38StagePackageError("runtime package record is not canonical JSON") from exc + + +def _package_digest(value: Mapping[str, Any]) -> str: + try: + return controller._runtime_package_digest(value) + except controller.RouteControllerError as exc: + raise Q38StagePackageError(str(exc)) from exc + + +def _file_identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + stat.S_IFMT(metadata.st_mode), + metadata.st_size, + metadata.st_mtime_ns, + ) + + +def _stream_binding(path: Path, expected_size: int) -> tuple[str, int]: + descriptor: int | None = None + try: + before = path.lstat() + reparse = bool(getattr(before, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or path.is_symlink() + or not stat.S_ISREG(before.st_mode) + or type(expected_size) is not int + or expected_size <= 0 + or before.st_size != expected_size + ): + raise Q38StagePackageError("release archive identity is invalid") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _file_identity(opened) != _file_identity(before): + raise Q38StagePackageError("release archive identity changed while opened") + + digest = hashlib.sha256() + total = 0 + while True: + chunk = os.read(descriptor, HASH_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > expected_size: + raise Q38StagePackageError("release archive size changed while hashed") + digest.update(chunk) + + after = os.fstat(descriptor) + final = path.lstat() + final_reparse = bool(getattr(final, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + total != expected_size + or final_reparse + or path.is_symlink() + or not stat.S_ISREG(final.st_mode) + or _file_identity(after) != _file_identity(opened) + or _file_identity(final) != _file_identity(opened) + ): + raise Q38StagePackageError("release archive identity changed while hashed") + return "sha256:" + digest.hexdigest(), total + except Q38StagePackageError: + raise + except OSError as exc: + raise Q38StagePackageError("release archive could not be hashed safely") from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +def _assert_verifier_sources( + source_root: Path, + source_bindings: Sequence[Mapping[str, Any]], +) -> None: + bindings = {item.get("relative_path"): item for item in source_bindings if isinstance(item, Mapping)} + imported = { + controller.DESKTOP_RELEASE_VERIFIER_SOURCE_PATH: Path(build_desktop.__file__).resolve(), + controller.STAGE_PACKAGE_SOURCE_PATH: Path(__file__).resolve(), + } + root = source_root.resolve() + for relative, module_path in imported.items(): + expected_path = (root / Path(*relative.split("/"))).resolve() + binding = bindings.get(relative) + if module_path != expected_path or not isinstance(binding, Mapping): + raise Q38StagePackageError("runtime package verifier sources are not plan-bound") + payload = _regular_bytes(expected_path) + if binding.get("byte_size") != len(payload) or binding.get("sha256") != _sha256(payload): + raise Q38StagePackageError("runtime package verifier source binding changed") + + +def _protected_tree_snapshot( + root: Path, + protection_verifier: ProtectionVerifier, +) -> tuple[tuple[Any, ...], ...]: + try: + supplied = root.lstat() + supplied_reparse = bool( + getattr(supplied, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + if supplied_reparse or root.is_symlink() or not stat.S_ISDIR(supplied.st_mode): + raise Q38StagePackageError("runtime package root is unsafe") + root = root.resolve(strict=True) + protection_verifier(root, True) + entries: list[tuple[Any, ...]] = [] + for child in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()): + metadata = child.lstat() + relative = child.relative_to(root).as_posix() + reparse = bool( + getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + if stat.S_ISDIR(metadata.st_mode) and not reparse and not child.is_symlink(): + protection_verifier(child, True) + kind = "directory" + elif stat.S_ISREG(metadata.st_mode) and not reparse and not child.is_symlink(): + protection_verifier(child, False) + kind = "file" + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + else: + raise Q38StagePackageError("runtime package tree contains an unsafe entry") + entries.append( + ( + relative, + kind, + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + ) + return tuple(entries) + except Q38StagePackageError: + raise + except OSError as exc: + raise Q38StagePackageError("runtime package tree could not be snapshotted") from exc + + +def _is_model_weight(path: str, manifested_weight_names: set[str]) -> bool: + normalized = path.casefold() + name = normalized.rsplit("/", 1)[-1] + return ( + name in manifested_weight_names + or name in {"model.safetensors", "pytorch_model.bin"} + or name.endswith((".safetensors", ".gguf", ".ckpt", ".pt", ".pth", ".onnx")) + or (name.startswith("pytorch_model-") and name.endswith(".bin")) + or "/model-cache/" in normalized + or "/model_cache/" in normalized + or "/models--" in normalized + ) + + +def validate_record( + value: Any, + *, + expected_source_commit: str | None = None, + expected_source_tree: str | None = None, + expected_manifest_digest: str | None = None, + expected_source_bindings: Sequence[Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + try: + record = dict( + controller.validate_runtime_package_record( + value, + expected_source_commit=expected_source_commit, + expected_manifest_digest=expected_manifest_digest, + expected_source_bindings=expected_source_bindings, + ) + ) + except controller.RouteControllerError as exc: + raise Q38StagePackageError(str(exc)) from exc + if expected_source_commit is not None and record["source_commit"] != expected_source_commit: + raise Q38StagePackageError("runtime package source commit changed") + if expected_source_tree is not None and record["source_tree"] != expected_source_tree: + raise Q38StagePackageError("runtime package source tree changed") + if expected_manifest_digest is not None and record["manifest_digest"] != expected_manifest_digest: + raise Q38StagePackageError("runtime package manifest binding changed") + return record + + +def validate_release_root( + release_root: Path, + manifest_path: Path, + *, + expected_source_commit: str, + expected_source_tree: str, + source_root: Path, + source_bindings: Sequence[Mapping[str, Any]], + protection_verifier: ProtectionVerifier, +) -> dict[str, Any]: + _assert_verifier_sources(source_root, source_bindings) + if _COMMIT_RE.fullmatch(expected_source_commit) is None or _COMMIT_RE.fullmatch(expected_source_tree) is None: + raise Q38StagePackageError("expected source identity is invalid") + + protection_verifier(manifest_path, False) + manifest_payload = _regular_bytes(manifest_path, controller.MAX_JSON_BYTES) + manifest_sha256 = _sha256(manifest_payload) + manifest_bytes = len(manifest_payload) + try: + manifest = ModelManifest.from_dict(json.loads(manifest_payload.decode("utf-8"))) + except (UnicodeDecodeError, json.JSONDecodeError, ManifestError) as exc: + raise Q38StagePackageError("Qwen3.8 manifest is invalid") from exc + if manifest.digest_id != controller.EXPECTED_MANIFEST_DIGEST: + raise Q38StagePackageError("Qwen3.8 manifest identity is invalid") + manifested_weight_names = { + artifact.path.rsplit("/", 1)[-1].casefold() for artifact in manifest.artifacts if artifact.role in _WEIGHT_ROLES + } + + before = _protected_tree_snapshot(release_root, protection_verifier) + provenance_path = release_root / build_desktop.PROVENANCE_NAME + checksums_path = release_root / build_desktop.CHECKSUMS_NAME + metrics_path = release_root / build_desktop.DESKTOP_METRICS_NAME + provenance_payload = _regular_bytes(provenance_path, MAX_PROVENANCE_BYTES) + checksums_payload = _regular_bytes(checksums_path, MAX_CHECKSUMS_BYTES) + metrics_payload = _regular_bytes(metrics_path, MAX_METRICS_BYTES) + try: + provenance = json.loads(provenance_payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Q38StagePackageError("release provenance is invalid") from exc + if not isinstance(provenance, dict): + raise Q38StagePackageError("release provenance is invalid") + + try: + summary = build_desktop._verify_release_attestations( + release_root, + expected_source_commit=expected_source_commit, + expected_source_tree=expected_source_tree, + require_metrics=True, + ) + except RuntimeError as exc: + raise Q38StagePackageError("production release attestations are invalid") from exc + if ( + _regular_bytes(provenance_path, MAX_PROVENANCE_BYTES) != provenance_payload + or _regular_bytes(checksums_path, MAX_CHECKSUMS_BYTES) != checksums_payload + or _regular_bytes(metrics_path, MAX_METRICS_BYTES) != metrics_payload + ): + raise Q38StagePackageError("release attestations changed while validated") + + build_platform = provenance.get("build_platform") + archive = summary.get("install_archive") + if ( + not isinstance(build_platform, str) + or not build_platform.casefold().startswith("linux") + or not isinstance(archive, dict) + or archive.get("platform") != "Linux" + or archive.get("format") != "tar.gz" + or archive.get("path") != "communityai-desktop-linux.tar.gz" + or provenance.get("source_commit") != expected_source_commit + or provenance.get("source_tree") != expected_source_tree + ): + raise Q38StagePackageError("release is not the exact Linux production package") + + artifacts = provenance.get("artifacts") + if not isinstance(artifacts, list): + raise Q38StagePackageError("release artifact inventory is invalid") + node_entries: list[dict[str, Any]] = [] + seen: set[str] = set() + executable: dict[str, Any] | None = None + for raw in artifacts: + if not isinstance(raw, dict): + raise Q38StagePackageError("release artifact inventory is invalid") + path_value = raw.get("path") + if not isinstance(path_value, str): + raise Q38StagePackageError("release artifact path is invalid") + if path_value == NODE_EXECUTABLE or path_value.startswith(NODE_ROOT + "/"): + folded = path_value.casefold() + if folded in seen: + raise Q38StagePackageError("node runtime inventory has a case collision") + seen.add(folded) + if _is_model_weight(path_value, manifested_weight_names): + raise Q38StagePackageError("node runtime contains model weights") + entry = dict(raw) + if entry.get("kind") == "symlink": + target = entry.get("link_target") + if not isinstance(target, str) or not target.startswith(NODE_ROOT + "/"): + raise Q38StagePackageError("node runtime symlink escapes its runtime root") + node_entries.append(entry) + if path_value == NODE_EXECUTABLE: + executable = entry + if executable is None or executable.get("kind") != "file": + raise Q38StagePackageError("packaged node executable is missing") + mode = executable.get("mode") + if type(mode) is not int or mode != 0o755: + raise Q38StagePackageError("packaged node executable mode is not 0755") + if any(type(entry.get("size_bytes")) is not int or entry["size_bytes"] <= 0 for entry in node_entries): + raise Q38StagePackageError("node runtime entry size is invalid") + + archive_path = release_root / str(archive["path"]) + archive_digest, archive_bytes = _stream_binding(archive_path, archive.get("size_bytes")) + expected_archive_digest = "sha256:" + str(archive["sha256"]) + if archive_digest != expected_archive_digest or archive_bytes != archive["size_bytes"]: + raise Q38StagePackageError("release archive binding changed") + if _protected_tree_snapshot(release_root, protection_verifier) != before: + raise Q38StagePackageError("runtime package tree changed while validated") + if ( + _regular_bytes(provenance_path, MAX_PROVENANCE_BYTES) != provenance_payload + or _regular_bytes(checksums_path, MAX_CHECKSUMS_BYTES) != checksums_payload + or _regular_bytes(metrics_path, MAX_METRICS_BYTES) != metrics_payload + or _regular_bytes(manifest_path, controller.MAX_JSON_BYTES) != manifest_payload + ): + raise Q38StagePackageError("runtime package inputs changed while validated") + + inventory = sorted(node_entries, key=lambda item: str(item["path"])) + record: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "platform": "linux", + "source_commit": expected_source_commit, + "source_tree": expected_source_tree, + "source_bindings_digest": controller._source_bindings_digest(source_bindings), + "release_archive_name": archive["path"], + "release_archive_sha256": archive_digest, + "release_archive_bytes": archive_bytes, + "checksums_sha256": _sha256(checksums_payload), + "checksums_bytes": len(checksums_payload), + "provenance_sha256": _sha256(provenance_payload), + "provenance_bytes": len(provenance_payload), + "desktop_metrics_sha256": _sha256(metrics_payload), + "desktop_metrics_bytes": len(metrics_payload), + "manifest_digest": manifest.digest_id, + "manifest_sha256": manifest_sha256, + "manifest_bytes": manifest_bytes, + "node_root": NODE_ROOT, + "node_executable": NODE_EXECUTABLE, + "node_executable_sha256": "sha256:" + str(executable["sha256"]), + "node_executable_bytes": executable["size_bytes"], + "node_runtime_entry_count": len(inventory), + "node_runtime_bytes": sum(int(entry["size_bytes"]) for entry in inventory), + "node_runtime_inventory_digest": _sha256(_canonical_bytes(inventory)), + "runtime_package_digest": "", + } + record["runtime_package_digest"] = _package_digest(record) + return validate_record( + record, + expected_source_commit=expected_source_commit, + expected_source_tree=expected_source_tree, + expected_manifest_digest=controller.EXPECTED_MANIFEST_DIGEST, + expected_source_bindings=source_bindings, + ) + + +def _assert_output_isolation( + output: Path, + *, + release_root: Path, + manifest_path: Path, + source_context_path: Path, + source_root: Path, +) -> None: + try: + candidate = output.resolve(strict=False) + protected_files = { + manifest_path.resolve(strict=True), + source_context_path.resolve(strict=True), + } + protected_roots = { + release_root.resolve(strict=True), + source_root.resolve(strict=True), + } + except (OSError, RuntimeError) as exc: + raise Q38StagePackageError("runtime package output isolation could not be resolved") from exc + if candidate in protected_files: + raise Q38StagePackageError("runtime package output aliases a protected input") + for root in protected_roots: + try: + candidate.relative_to(root) + except ValueError: + continue + raise Q38StagePackageError("runtime package output is inside a protected input root") + + +def _atomic_record(path: Path, record: Mapping[str, Any]) -> None: + payload = _canonical_bytes(validate_record(record)) + if len(payload) > MAX_RECORD_BYTES: + raise Q38StagePackageError("runtime package record is too large") + path.parent.mkdir(parents=True, exist_ok=True) + parent = path.parent.lstat() + reparse = bool(getattr(parent, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.parent.is_symlink() or not stat.S_ISDIR(parent.st_mode): + raise Q38StagePackageError("runtime package output parent is unsafe") + if path.exists() or path.is_symlink(): + target = path.lstat() + target_reparse = bool( + getattr(target, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + if target_reparse or path.is_symlink() or not stat.S_ISREG(target.st_mode): + raise Q38StagePackageError("runtime package output target is unsafe") + handle, raw_temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + temporary = Path(raw_temporary) + try: + with os.fdopen(handle, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except OSError as exc: + raise Q38StagePackageError("runtime package record could not be committed") from exc + finally: + temporary.unlink(missing_ok=True) + + +def load_source_context(path: Path, source_root: Path) -> dict[str, Any]: + payload = _regular_bytes(path, MAX_RECORD_BYTES) + try: + raw = controller._mapping( + controller._strict_json(payload), + _SOURCE_CONTEXT_FIELDS, + "runtime package source context", + ) + if ( + type(raw["schema_version"]) is not int + or raw["schema_version"] != SCHEMA_VERSION + or raw["scope"] != SOURCE_CONTEXT_SCOPE + or not isinstance(raw["source_commit"], str) + or _COMMIT_RE.fullmatch(raw["source_commit"]) is None + or not isinstance(raw["source_tree"], str) + or _COMMIT_RE.fullmatch(raw["source_tree"]) is None + ): + raise Q38StagePackageError("runtime package source context identity is invalid") + source_bindings = controller._validate_source_bindings(raw["source_bindings"], source_root) + except controller.RouteControllerError as exc: + raise Q38StagePackageError(str(exc)) from exc + if {item["relative_path"] for item in source_bindings} != controller.REQUIRED_SOURCE_PATHS: + raise Q38StagePackageError("runtime package source context is incomplete") + try: + controller._assert_protected_path_from_bindings( + path.parent, + source_bindings, + source_root, + directory=True, + ) + controller._assert_protected_path_from_bindings( + path, + source_bindings, + source_root, + directory=False, + ) + except controller.RouteControllerError as exc: + raise Q38StagePackageError(str(exc)) from exc + if _regular_bytes(path, MAX_RECORD_BYTES) != payload: + raise Q38StagePackageError("runtime package source context changed while validated") + _assert_verifier_sources(source_root, source_bindings) + return { + "schema_version": SCHEMA_VERSION, + "scope": SOURCE_CONTEXT_SCOPE, + "source_commit": raw["source_commit"], + "source_tree": raw["source_tree"], + "source_bindings": source_bindings, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Bind an exact Qwen3.8 Linux packaged runtime") + parser.add_argument("--release-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--source-context", type=Path, required=True) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + _assert_output_isolation( + args.output, + release_root=args.release_root, + manifest_path=args.manifest, + source_context_path=args.source_context, + source_root=args.source_root, + ) + context = load_source_context(args.source_context, args.source_root) + source_bindings = context["source_bindings"] + record = validate_release_root( + args.release_root, + args.manifest, + expected_source_commit=context["source_commit"], + expected_source_tree=context["source_tree"], + source_root=args.source_root, + source_bindings=source_bindings, + protection_verifier=lambda path, directory: controller._assert_protected_path_from_bindings( + path, + source_bindings, + args.source_root, + directory=directory, + ), + ) + _atomic_record(args.output, record) + except (Q38StagePackageError, controller.RouteControllerError) as exc: + raise SystemExit(f"Qwen3.8 runtime package validation failed: {exc}") from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_qwen_catalog_candidate.py b/scripts/prepare_qwen_catalog_candidate.py new file mode 100644 index 000000000..43b09a53c --- /dev/null +++ b/scripts/prepare_qwen_catalog_candidate.py @@ -0,0 +1,103 @@ +"""Prepare an unsigned, reviewable Qwen alpha catalog for the existing trust root.""" + +import argparse +import json +import time +from pathlib import Path + +from drift.catalog_release import verify_catalog_publication_bundle +from drift.model_catalog import CatalogSigningKey, CatalogTrustRoot, ModelCatalog, SignedModelCatalog +from drift.model_manifest import ModelManifest +from drift.node.catalog_bootstrap import CatalogBootstrapConfig + + +def prepare(root, output): + bootstrap = CatalogBootstrapConfig.load(root / "public-alpha/catalog-v1/catalog-bootstrap.json") + previous = SignedModelCatalog.load(root / "public-alpha/catalog-v1/catalog.signed.json").verify( + bootstrap.trust_root + ) + manifests = [ + ModelManifest.load(root / "manifests/candidates" / name) + for name in ("qwen3.5-0.8b-local-bfloat16-eager.json", "qwen3.8-27b-fp8-dequant-eager.json") + ] + base = previous.models[0].manifest_urls[0].rsplit("/", 1)[0] + models, rungs = [], [] + for i, manifest in enumerate(manifests): + rung = "local-qwen" if i == 0 else "community-qwen" + rungs.append( + { + "id": rung, + "order": i + 1, + "minimum_replicas": 1, + "minimum_independent_routes": 1, + "minimum_surviving_replicas": 0, + "minimum_soak_seconds": 60, + "maximum_observation_age_seconds": 30, + "maximum_p95_first_token_ms": 60000, + "minimum_tokens_per_minute": 1, + } + ) + models.append( + { + "manifest_digest": manifest.digest_id, + "manifest_urls": [f"{base}/{manifest.digest}.json"], + "rung": rung, + "role": "primary", + "execution": "local" if i == 0 else "distributed", + "total_parameters": 800000000 if i == 0 else 27000000000, + "active_parameters": 800000000 if i == 0 else 27000000000, + "weight_bytes": sum(a.size for a in manifest.artifacts if a.role == "weight"), + } + ) + payload = previous.to_dict() + payload.update(sequence=previous.sequence + 1, issued_at_ms=int(time.time() * 1000), rungs=rungs, models=models) + catalog = ModelCatalog.from_dict(payload) + envelope = SignedModelCatalog(1, catalog, ()) + # Validate transport, shape and manifest consistency using a throwaway in-memory + # key. Its signature and private key are never published or saved as release trust. + key = CatalogSigningKey.generate() + temporary_root = CatalogTrustRoot.from_dict( + {"schema_version": 1, "catalog_id": catalog.catalog_id, "threshold": 1, "keys": [key.trusted_key.to_dict()]} + ) + from dataclasses import replace + + verify_catalog_publication_bundle( + replace(bootstrap, trust_root=temporary_root), envelope.add_signature(key), manifests + ) + output.mkdir(parents=True, exist_ok=False) + (output / "manifests").mkdir() + for manifest in manifests: + (output / "manifests" / f"{manifest.digest}.json").write_text( + manifest.canonical_json() + "\n", encoding="utf-8" + ) + (output / "catalog.unsigned.json").write_text( + json.dumps(envelope.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + # Keep both local fallback and CPU client-side community weights resident when + # available. Admission and the per-model envelope remain independently bounded. + bootstrap_data = bootstrap.to_dict() + bootstrap_data["max_loaded_models"] = 2 + (output / "catalog-bootstrap.json").write_text( + json.dumps(bootstrap_data, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (output / "review.json").write_text( + json.dumps( + { + "status": "unsigned-candidate", + "catalog_digest": catalog.digest, + "requires_existing_trusted_key_ids": [k.key_id for k in bootstrap.trust_root.keys], + "publication_performed": False, + "release_qualification_complete": False, + }, + indent=2, + ) + + "\n" + ) + print(json.dumps({"candidate": str(output.resolve()), "catalog_digest": catalog.digest, "signed": False})) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + prepare(Path(__file__).resolve().parents[1], args.output) diff --git a/scripts/provision_catalog_signer.ps1 b/scripts/provision_catalog_signer.ps1 new file mode 100644 index 000000000..74e1e8631 --- /dev/null +++ b/scripts/provision_catalog_signer.ps1 @@ -0,0 +1,93 @@ +param( + [Parameter(Mandatory = $true)][string]$Python, + [Parameter(Mandatory = $true)][string]$BackupDirectory, + [Parameter(Mandatory = $true)][string]$SecretName, + [string]$GoogleProject = 'community-ai-506321', + [string]$KeyDirectory = (Join-Path $env:LOCALAPPDATA 'CommunityAI\publisher-keys\catalog-20260906') +) +$ErrorActionPreference = 'Stop' +$Python = (Get-Command $Python -ErrorAction Stop).Source + +# Working keys and the second-volume copy stay outside the repo. The owner also +# requested a third copy under the repo's explicitly ignored secret directory. +function New-ProtectedDirectory([string]$Target, [bool]$AllowIgnoredBackup = $false) { + $resolved = [IO.Path]::GetFullPath($Target) + $repository = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) + if ($AllowIgnoredBackup) { + $allowed = Join-Path $repository '.publisher-secrets' + if (-not $resolved.StartsWith($allowed + '\', [StringComparison]::OrdinalIgnoreCase)) { + throw 'Repository backup must remain inside .publisher-secrets.' + } + & git -C $repository check-ignore --quiet -- (Join-Path $resolved 'catalog-private.pem') + if ($LASTEXITCODE -ne 0) { throw 'Repository backup is not ignored; refusing to create private material.' } + } elseif ($resolved.StartsWith($repository + '\', [StringComparison]::OrdinalIgnoreCase)) { + throw 'Publisher keys must be outside the repository.' + } + if (Test-Path -LiteralPath $resolved) { throw "Refusing to replace existing directory: $resolved" } + New-Item -ItemType Directory -Path $resolved | Out-Null + $sid = [Security.Principal.WindowsIdentity]::GetCurrent().User + $acl = [Security.AccessControl.DirectorySecurity]::new() + $acl.SetOwner($sid) + $acl.SetAccessRuleProtection($true, $false) + $rule = [Security.AccessControl.FileSystemAccessRule]::new( + $sid, 'FullControl', 'ContainerInherit, ObjectInherit', 'None', 'Allow' + ) + $acl.AddAccessRule($rule) + Set-Acl -LiteralPath $resolved -AclObject $acl + $checked = Get-Acl -LiteralPath $resolved + if (-not $checked.AreAccessRulesProtected -or $checked.Access.Count -ne 1) { + throw "Could not establish exclusive publisher-directory permissions: $resolved" + } + return $resolved +} + +$recordPath = Join-Path (Split-Path -Parent ([IO.Path]::GetFullPath($KeyDirectory))) 'active-catalog.json' +if (Test-Path -LiteralPath $recordPath) { throw 'An active signer registry already exists; review rotation explicitly.' } +if ($SecretName -notmatch '^[a-zA-Z0-9_-]{1,255}$') { throw 'Invalid Google secret name.' } +Get-Command gcloud -ErrorAction Stop | Out-Null +$primary = New-ProtectedDirectory $KeyDirectory +$backup = New-ProtectedDirectory $BackupDirectory +$repositoryBackup = New-ProtectedDirectory (Join-Path (Join-Path $PSScriptRoot '..\.publisher-secrets') (Split-Path -Leaf $primary)) $true +$keyPath = Join-Path $primary 'catalog-private.pem' +$publicPath = Join-Path $primary 'catalog-public.json' +& $Python -m drift.cli.run_catalog keygen $keyPath --public-output $publicPath +if ($LASTEXITCODE -ne 0) { throw 'Catalog key generation failed.' } +foreach ($destination in @($backup, $repositoryBackup)) { + Copy-Item -LiteralPath $keyPath -Destination (Join-Path $destination 'catalog-private.pem') + Copy-Item -LiteralPath $publicPath -Destination (Join-Path $destination 'catalog-public.json') + if ((Get-FileHash -LiteralPath $keyPath).Hash -ne (Get-FileHash -LiteralPath (Join-Path $destination 'catalog-private.pem')).Hash) { + throw 'Publisher backup verification failed.' + } +} +$publicKey = Get-Content -LiteralPath $publicPath -Raw | ConvertFrom-Json +& gcloud secrets create $SecretName --project=$GoogleProject --replication-policy=automatic --quiet +if ($LASTEXITCODE -ne 0) { throw 'Could not create a new emergency secret; local copies were retained.' } +$onlineVersion = & gcloud secrets versions add $SecretName --project=$GoogleProject --data-file=$keyPath '--format=value(name)' --quiet +if ($LASTEXITCODE -ne 0) { throw 'Emergency upload failed; local copies were retained.' } +$version = ($onlineVersion.Trim() -split '/')[-1] +$verification = @' +import sys +from catalog_key_backup import load_online_backup +key = load_online_backup(*sys.argv[1:]) +challenge = b'CommunityAI publisher provisioning backup verification' +key.trusted_key.public_key_object.verify(key.sign(challenge), challenge) +'@ +Push-Location $PSScriptRoot +try { + & $Python -c $verification $GoogleProject $SecretName $version $publicKey.key_id + if ($LASTEXITCODE -ne 0) { throw 'Emergency recovery verification failed; all created copies were retained.' } +} finally { Pop-Location } +$record = [ordered]@{ + schema_version = 1 + key_id = $publicKey.key_id + private_key_path = $keyPath + backup_private_key_path = (Join-Path $backup 'catalog-private.pem') + repository_backup_private_key_path = (Join-Path $repositoryBackup 'catalog-private.pem') + emergency_backup = @{ project = $GoogleProject; secret = $SecretName; version = $version } + created_at = [DateTime]::UtcNow.ToString('o') + backup_kind = 'Google Secret Manager, second local volume, gitignored repository copy' +} +$record | ConvertTo-Json | Set-Content -LiteralPath $recordPath -Encoding utf8 +$record | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $backup 'signer-registry.json') -Encoding utf8 +$record | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $repositoryBackup 'signer-registry.json') -Encoding utf8 +Write-Output "Created and backed up publisher key $($publicKey.key_id). Registry: $recordPath" diff --git a/scripts/qualify_catalog_desktop.py b/scripts/qualify_catalog_desktop.py new file mode 100644 index 000000000..a9afc417d --- /dev/null +++ b/scripts/qualify_catalog_desktop.py @@ -0,0 +1,416 @@ +"""Replay an ordinary-user frozen Windows/Linux desktop's signed catalog migration. + +Uses a new private state directory and native credential account. Sharing remains +off; no inference request, catalog publication, or installer action is performed. +Qt runs offscreen unless --visible-ui explicitly requests native-window acceptance. +""" + +import argparse +import ctypes +import hashlib +import json +import os +import platform +import subprocess +import time +from pathlib import Path + +import psutil +from communityai_desktop.client import NodeClient, NodeClientError +from communityai_desktop.credentials import CredentialMissingError, NativeCredentialStore + +ROOT = Path(__file__).resolve().parents[1] + + +def build_parser(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--desktop", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--node-url", default="http://127.0.0.1:18104") + parser.add_argument("--visible-ui", action="store_true", help="Explicitly open native qualification windows") + return parser + + +def desktop_environment(visible_ui, *, system=None): + environment = os.environ.copy() + system = platform.system() if system is None else system + if visible_ui and system != "Windows": + raise ValueError("Native-window observation currently supports Windows; use offscreen on Linux") + environment["QT_QPA_PLATFORM"] = "windows" if visible_ui else "offscreen" + return environment + + +def qualification_platform(): + system = platform.system() + if system == "Windows": + if ctypes.windll.shell32.IsUserAnAdmin(): + raise RuntimeError("Run this acceptance from an ordinary, non-elevated Windows session") + elif system == "Linux": + if os.geteuid() == 0: + raise RuntimeError("Run this acceptance as an ordinary Linux user with a native credential store") + else: + raise RuntimeError("This acceptance supports Windows and Linux") + return system + + +def node_executable(desktop, system): + return desktop.parent / "node" / ("CommunityAI-Node.exe" if system == "Windows" else "CommunityAI-Node") + + +def identity_is_live(process, created): + # A Linux zombie cannot execute or retain an open model runtime. Its parent + # or container init still owns reaping the remaining process-table entry. + return process.create_time() == created and process.is_running() and process.status() != psutil.STATUS_ZOMBIE + + +def sha256(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def process_tree(pid): + process = psutil.Process(pid) + identities = [] + for child in (process, *process.children(recursive=True)): + try: + identities.append((child.pid, child.create_time())) + except psutil.NoSuchProcess: + pass + return identities + + +def wait_tree_gone(identities): + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + remaining = [] + for pid, created in identities: + try: + process = psutil.Process(pid) + if identity_is_live(process, created): + remaining.append(pid) + except psutil.NoSuchProcess: + pass + if not remaining: + return + time.sleep(0.2) + raise AssertionError(f"Recorded desktop tree remains alive: {remaining}") + + +def force_stop_owned_tree(identities, *, timeout=15): + """Bound emergency cleanup to recorded identities and their descendants. + + A PID alone is insufficient: a reused PID must never be signaled. psutil's + terminate/kill methods also guard identity reuse immediately before signaling. + Keep discovering children while an owned ancestor remains alive so a helper + created during shutdown cannot escape a fixed initial snapshot. + """ + known = set(identities) + signaled = set() + deadline = time.monotonic() + timeout + while True: + live = {} + for pid, created in tuple(known): + try: + process = psutil.Process(pid) + if not identity_is_live(process, created): + continue + live[(pid, created)] = process + for child in process.children(recursive=True): + identity = (child.pid, child.create_time()) + known.add(identity) + if identity_is_live(child, identity[1]): + live[identity] = child + except psutil.NoSuchProcess: + continue + identities[:] = sorted(known) + if not live: + return + if time.monotonic() >= deadline: + raise TimeoutError("Owned desktop cleanup exceeded its deadline") + for identity, process in reversed(tuple(live.items())): + try: + if identity not in signaled: + process.terminate() + signaled.add(identity) + else: + process.kill() + except psutil.NoSuchProcess: + pass + time.sleep(0.2) + + +def visible_window(pid): + if platform.system() != "Windows": + return False + found = [] + callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p) + + @callback_type + def observe(window, unused): + owner = ctypes.c_ulong() + ctypes.windll.user32.GetWindowThreadProcessId(window, ctypes.byref(owner)) + if owner.value == pid and ctypes.windll.user32.IsWindowVisible(window): + title = ctypes.create_unicode_buffer(512) + ctypes.windll.user32.GetWindowTextW(window, title, len(title)) + if title.value == "CommunityAI": + found.append(True) + return True + + ctypes.windll.user32.EnumWindows(observe, 0) + return bool(found) + + +def require_desktop_stopped(): + for process in psutil.process_iter(("name",)): + if (process.info["name"] or "").casefold() in ("communityai.exe", "communityai"): + try: + if process.status() != psutil.STATUS_ZOMBIE: + raise RuntimeError("Close the existing CommunityAI desktop before this isolated replay") + except psutil.NoSuchProcess: + continue + + +def run(args): + system = qualification_platform() + environment = desktop_environment(args.visible_ui, system=system) + require_desktop_stopped() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + desktop = args.desktop.resolve() + node = node_executable(desktop, system) + bootstrap = desktop.parent / "_internal/bootstrap/catalog-bootstrap.json" + assert bootstrap.read_bytes() == (ROOT / "public-alpha/catalog-qwen-v2/catalog-bootstrap.json").read_bytes() + state = output / "state" + config_path = state / "node-config.json" + store = NativeCredentialStore("org.communityai.catalog-acceptance." + output.name, "control") + try: + store.get() + except CredentialMissingError: + pass + else: + raise RuntimeError("The private qualification credential already exists") + process_options = {"creationflags": subprocess.CREATE_NO_WINDOW} if system == "Windows" else {} + result = { + "result": "failed", + "scope": f"ordinary-user-frozen-{system}-desktop-signed-catalog-startup-migration", + "platform": system, + "non_elevated": True, + "zombies_treated_as_non_executing": system == "Linux", + "ui_mode": "visible-native" if args.visible_ui else "offscreen", + "desktop_sha256": sha256(desktop), + "node_sha256": sha256(node), + "bootstrap_sha256": sha256(bootstrap), + "replay_script_sha256": sha256(Path(__file__)), + "launches": [], + } + gui = None + identities = [] + + def stop(): + nonlocal gui + if gui is None: + return + try: + if gui.poll() is None: + identities.extend(process_tree(gui.pid)) + subprocess.run( + [str(desktop), "--prepare-update"], check=True, timeout=60, env=environment, **process_options + ) + assert gui.wait(timeout=60) == 0 + wait_tree_gone(identities) + except BaseException as exc: + result["result"] = "failed" + result["shutdown_failure_type"] = type(exc).__name__ + try: + force_stop_owned_tree(identities) + gui.wait(timeout=5) + except BaseException as cleanup_error: + result["owned_shutdown_fallback"] = "failed" + result["cleanup_failure_type"] = type(cleanup_error).__name__ + raise + else: + result["owned_shutdown_fallback"] = "passed" + gui = None + raise + gui = None + + def launch(label): + nonlocal gui + started = time.monotonic() + with (output / f"{label}.log").open("wb") as log: + gui = subprocess.Popen( + [ + str(desktop), + "--node-url", + args.node_url, + "--node-config", + str(config_path), + "--node-data-dir", + str(state), + "--credential-service", + store.service, + "--credential-account", + store.account, + ], + stdout=log, + stderr=subprocess.STDOUT, + env=environment, + **process_options, + ) + # Record ownership immediately, including failures before authenticated + # readiness. Later snapshots expand this set with owned descendants. + identities.append((gui.pid, psutil.Process(gui.pid).create_time())) + deadline = started + 240 + while time.monotonic() < deadline: + if gui.poll() is not None: + raise RuntimeError("Frozen desktop exited before authenticated readiness") + try: + client = NodeClient(args.node_url, store.get(), timeout=5) + status = client.status() + if not args.visible_ui or visible_window(gui.pid): + break + except (CredentialMissingError, NodeClientError): + pass + time.sleep(0.5) + else: + raise TimeoutError("Frozen desktop did not open and expose its authenticated node") + identities.extend(process_tree(gui.pid)) + workers = client.list_workers() + assert all(worker["state"] == "paused" for worker in workers) + assert client.get_contribution_policy()["policy"]["sharing_enabled"] is False + record = { + "phase": label, + "seconds_to_authenticated_status": round(time.monotonic() - started, 3), + "visible_native_window": visible_window(gui.pid), + "model_ids": [model["id"] for model in status["models"]], + "worker_states": [worker["state"] for worker in workers], + "owned_process_count": len(set(identities)), + } + result["launches"].append(record) + print(json.dumps(record), flush=True) + + try: + proc = subprocess.run( + [ + str(node), + "bootstrap", + str(ROOT / "public-alpha/catalog-v1/catalog-bootstrap.json"), + "--data_dir", + str(state), + ], + capture_output=True, + timeout=300, + **process_options, + ) + (output / "legacy-install.log").write_bytes(proc.stdout + proc.stderr) + assert proc.returncode == 0, "Legacy signed catalog bootstrap failed; see retained log" + result["legacy_install"] = json.loads(proc.stdout.decode().splitlines()[-1]) + assert result["legacy_install"]["catalog_sequence"] == 1 + legacy = json.loads(config_path.read_text()) + legacy["inference_mode"] = "local_only" + legacy["contribution_policy"] = { + "sharing_enabled": False, + "max_vram": "37%", + "max_processing_percent": 43, + "max_disk_space": "8GiB", + } + config_path.write_text(json.dumps(legacy, indent=2)) + sentinel = state / "model-cache/retained-cache-sentinel" + sentinel.parent.mkdir(exist_ok=True) + sentinel.write_bytes(b"Ordinary user retained cache choice\n") + sentinel_digest = sha256(sentinel) + launch("automatic-migration") + migrated = json.loads(config_path.read_text()) + assert migrated["inference_mode"] == legacy["inference_mode"] + assert migrated["contribution_policy"] == legacy["contribution_policy"] + assert migrated["workers"] == legacy["workers"] + assert sha256(sentinel) == sentinel_digest + assert ( + Path(migrated["catalog_path"]).read_bytes() + == (ROOT / "public-alpha/catalog-qwen-v2/catalog.signed.json").read_bytes() + ) + assert ( + json.loads(Path(migrated["catalog_bootstrap_path"]).read_text())["trust_root"] + == json.loads(bootstrap.read_text())["trust_root"] + ) + assert set(migrated["auto_model_priority"]) == { + "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + } + result["activated_exact_published_sequence_2"] = True + result["preferences_workers_and_cache_preserved"] = True + native_credential = store.get() + before = config_path.read_bytes() + stop() + launch("repeat-start") + assert config_path.read_bytes() == before + assert store.get() == native_credential + result["repeat_start_config_and_native_credential_preserved"] = True + stop() + proc = subprocess.run( + [ + str(node), + "bootstrap", + str(ROOT / "public-alpha/catalog-v1/catalog-bootstrap.json"), + "--data_dir", + str(state), + "--refresh", + ], + capture_output=True, + timeout=300, + **process_options, + ) + (output / "old-root-rejected.log").write_bytes(proc.stdout + proc.stderr) + assert proc.returncode != 0 and config_path.read_bytes() == before + result["old_trust_root_rejected_without_config_change"] = True + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + shutdown_complete = False + try: + stop() + wait_tree_gone(identities) + result["recorded_runtime_trees_gone"] = True + shutdown_complete = True + except BaseException as cleanup_error: + result["result"] = "failed" + result["recorded_runtime_trees_gone"] = False + result["cleanup_failure_type"] = type(cleanup_error).__name__ + raise + finally: + if shutdown_complete: + try: + store.delete() + store.get() + except CredentialMissingError: + result["native_qualification_credential_removed"] = True + except BaseException as credential_error: + result["result"] = "failed" + result["native_qualification_credential_removed"] = False + result["credential_cleanup_failure_type"] = type(credential_error).__name__ + else: + result["result"] = "failed" + result["native_qualification_credential_removed"] = False + result["credential_cleanup_failure_type"] = "CredentialStillPresent" + else: + # Retain access until a remaining owned runtime can be shut down; + # never claim cleanup because its credential was merely erased. + result["native_qualification_credential_removed"] = False + result["credential_preserved_for_cleanup"] = True + result["limitations"] = [ + f"{system} startup migration and unchanged-catalog restart only; other platforms require separate replay.", + "The legacy state was installed from the actual signed online catalog, with private test preferences/cache marker.", + "No inference, worker execution, periodic newer-sequence activation, or active-generation drain was exercised.", + "The existing frozen bundle was launched directly; its installer lifecycle has separate evidence.", + ] + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({"result": result["result"], "output": str(output)}), flush=True) + + if result["result"] != "passed": + raise RuntimeError("Catalog replay did not complete verified cleanup") + + +if __name__ == "__main__": + run(build_parser().parse_args()) diff --git a/scripts/qualify_catalog_online.py b/scripts/qualify_catalog_online.py new file mode 100644 index 000000000..2c334d9d9 --- /dev/null +++ b/scripts/qualify_catalog_online.py @@ -0,0 +1,116 @@ +"""Qualify real packaged HTTPS catalog installation and explicit trust-root migration.""" + +import argparse +import hashlib +import json +import subprocess +import time +from pathlib import Path + +import requests + +ROOT = Path(__file__).resolve().parents[1] + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + bundle = ROOT / "public-alpha/catalog-qwen-v2" + bootstrap = args.node.resolve().parent.parent / "_internal/bootstrap/catalog-bootstrap.json" + assert bootstrap.read_bytes() == (bundle / "catalog-bootstrap.json").read_bytes() + config = json.loads(bootstrap.read_text()) + base = config["catalog_mirrors"][0].rsplit("/", 1)[0] + result = {"result": "failed", "scope": "packaged-HTTPS-catalog-install-and-old-root-migration", "files": []} + try: + for path in sorted(bundle.rglob("*.json")): + relative = path.relative_to(bundle).as_posix() + response = requests.get(base + "/" + relative, timeout=(10, 30), allow_redirects=False) + assert response.status_code == 200, (relative, response.status_code) + assert response.content == path.read_bytes(), relative + result["files"].append({"path": relative, "sha256": hashlib.sha256(response.content).hexdigest()}) + + def install(name, trusted, *, refresh=False): + data = output / name + command = [str(args.node.resolve()), "bootstrap", str(trusted), "--data_dir", str(data)] + if refresh: + command.append("--refresh_if_needed") + started = time.monotonic() + proc = subprocess.run( + command, capture_output=True, timeout=300, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0) + ) + label = name + ("-refresh" if refresh else "-install") + (output / (label + ".log")).write_bytes(proc.stdout + proc.stderr) + assert proc.returncode == 0, label + ": inspect retained log" + receipt = json.loads(proc.stdout.decode().splitlines()[-1]) + receipt["seconds"] = time.monotonic() - started + return receipt + + result["clean_install"] = install("clean", bootstrap) + assert result["clean_install"]["catalog_sequence"] == 2 + clean = json.loads((output / "clean/node-config.json").read_text()) + assert [m.get("execution") for m in clean["models"]] == ["local", "distributed"] + assert clean["max_loaded_models"] == 2 + + result["legacy_install"] = install("legacy", ROOT / "public-alpha/catalog-v1/catalog-bootstrap.json") + assert result["legacy_install"]["catalog_sequence"] == 1 + legacy_path = output / "legacy/node-config.json" + legacy = json.loads(legacy_path.read_text()) + legacy["inference_mode"] = "local_only" + legacy["contribution_policy"] = {"sharing_enabled": False, "max_vram": "2GiB", "max_disk_space": "8GiB"} + legacy_path.write_text(json.dumps(legacy, indent=2)) + marker = output / "legacy/model-cache/user-retained-cache.txt" + marker.parent.mkdir(exist_ok=True) + marker.write_text("retained-cache-choice\n") + marker_hash = hashlib.sha256(marker.read_bytes()).hexdigest() + result["migration"] = install("legacy", bootstrap, refresh=True) + assert result["migration"]["catalog_sequence"] == 2 + migrated = json.loads(legacy_path.read_text()) + assert migrated["inference_mode"] == legacy["inference_mode"] + assert migrated["contribution_policy"] == legacy["contribution_policy"] + assert migrated["workers"] == legacy["workers"] + assert hashlib.sha256(marker.read_bytes()).hexdigest() == marker_hash + installed_bootstrap = json.loads(Path(migrated["catalog_bootstrap_path"]).read_text()) + assert installed_bootstrap["trust_root"] == config["trust_root"] + result["preferences_workers_and_cache_marker_preserved"] = True + before = legacy_path.read_bytes() + # A supplied old application root has no authority to reverse migration. + proc = subprocess.run( + [ + str(args.node.resolve()), + "bootstrap", + str(ROOT / "public-alpha/catalog-v1/catalog-bootstrap.json"), + "--data_dir", + str(output / "legacy"), + "--refresh", + ], + capture_output=True, + timeout=300, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + (output / "legacy-old-root-rejected.log").write_bytes(proc.stdout + proc.stderr) + assert proc.returncode != 0 + assert legacy_path.read_bytes() == before + result["old_root_rejected"] = True + result["repeat_start"] = install("legacy", bootstrap, refresh=True) + assert legacy_path.read_bytes() == before + result["repeat_start_config_unchanged"] = True + result["node_sha256"] = hashlib.sha256(args.node.read_bytes()).hexdigest() + result["complete_gate15"] = False + result["limitations"] = [ + "Catalog/bootstrap CLI only; no ordinary-user installer lifecycle or full UI acceptance.", + "Migration fixture installed the actual online sequence 1, then added test preferences and a cache marker.", + ] + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({"result": result["result"], "output": str(output)}), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--node", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + run(parser.parse_args()) diff --git a/scripts/qualify_login_startup_linux.py b/scripts/qualify_login_startup_linux.py new file mode 100644 index 000000000..5d8779b25 --- /dev/null +++ b/scripts/qualify_login_startup_linux.py @@ -0,0 +1,403 @@ +"""Exercise the unmodified frozen Linux checkbox through isolated Xvfb/AT-SPI. + +Run the host with the desktop Python environment inside a private dbus-run-session +and Xvfb. Accessibility subprocesses use system Python's python3-pyatspi. No test +telemetry, inference, sharing work, or Windows UI automation is used. +""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +CHECKBOX_NAME = "Start CommunityAI when I sign in" +MAX_ACCESSIBLE_NODES = 4096 + + +def private_display(display): + if not isinstance(display, str) or re.fullmatch(r":[1-9][0-9]*", display) is None: + raise RuntimeError("Use a private numbered Xvfb display, not an inherited host display") + return display + + +def children(node): + count = node.childCount + if not 0 <= count <= MAX_ACCESSIBLE_NODES: + raise RuntimeError("Accessible child count exceeds the qualification bound") + return [node.getChildAtIndex(index) for index in range(count)] + + +def find_named(root, name, roles): + pending = [root] + found = [] + visited = 0 + while pending: + node = pending.pop() + visited += 1 + if visited > MAX_ACCESSIBLE_NODES: + raise RuntimeError("Accessible tree exceeds the qualification bound") + if node.name == name and node.getRoleName() in roles: + found.append(node) + pending.extend(children(node)) + if len(found) != 1: + raise RuntimeError(f"Expected exactly one {name!r} control; found {len(found)}") + return found[0] + + +def select_application(desktop, pid): + apps = [node for node in children(desktop) if node.get_process_id() == pid] + if len(apps) != 1: + raise RuntimeError("Expected exactly one accessibility application for the owned GUI PID") + return apps[0] + + +def invoke_action(control, requested): + action = control.queryAction() + supported = [action.getName(index) for index in range(action.nActions)] + acceptable = [index for index, name in enumerate(supported) if name.casefold() == requested.casefold()] + if len(acceptable) != 1: + raise RuntimeError(f"Expected one {requested!r} accessible action; received {supported!r}") + selected = acceptable[0] + if not action.doAction(selected): + raise RuntimeError("Accessible control rejected its action") + return supported[selected] + + +def accessibility_actor(pid, operation): + import pyatspi + + deadline = time.monotonic() + 25 + app = None + while time.monotonic() < deadline: + try: + app = select_application(pyatspi.Registry.getDesktop(0), pid) + break + except RuntimeError: + time.sleep(0.2) + if app is None: + raise TimeoutError("Owned frozen GUI did not expose an AT-SPI application") + if operation == "application": + return {"owned_application_exposed": True} + navigation = find_named(app, "Sharing", {"push button", "check box", "toggle button", "radio button"}) + invoke_action(navigation, "Press") + time.sleep(0.1) + checkbox = find_named(app, CHECKBOX_NAME, {"check box"}) + state = checkbox.getState() + before = bool(state.contains(pyatspi.STATE_CHECKED)) + if not state.contains(pyatspi.STATE_ENABLED): + raise RuntimeError("The actual frozen sign-in checkbox is disabled") + selected_action = None + if operation == "toggle": + selected_action = invoke_action(checkbox, "Toggle") + time.sleep(0.1) + after = bool(checkbox.getState().contains(pyatspi.STATE_CHECKED)) + if operation == "toggle" and before == after: + raise RuntimeError("The actual frozen checkbox did not change state") + return {"checked_before": before, "checked_after": after, "action": selected_action} + + +def run_host(args): + import psutil + from communityai_desktop.client import NodeClient, NodeClientError + from communityai_desktop.credentials import CredentialMissingError, NativeCredentialStore + from communityai_desktop.startup import _linux_autostart_bytes + + if os.name != "posix" or os.geteuid() == 0: + raise RuntimeError("Run as the ordinary Linux desktop user") + if os.environ.get("QT_QPA_PLATFORM") != "xcb" or not os.environ.get("DISPLAY"): + raise RuntimeError("This frozen acceptance requires the private Xvfb xcb display") + display = private_display(os.environ["DISPLAY"]) + authority = Path(os.environ.get("XAUTHORITY", "")).resolve(strict=True) + if not authority.is_file() or authority.stat().st_uid != os.geteuid(): + raise RuntimeError("The private Xvfb authority must belong to this ordinary user") + owned_xvfb = [] + for process in psutil.process_iter(): + try: + if process.name() != "Xvfb" or process.uids().real != os.geteuid(): + continue + command = process.cmdline() + if display in command and "-auth" in command and command[command.index("-auth") + 1] == str(authority): + owned_xvfb.append(process.pid) + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if len(owned_xvfb) != 1: + raise RuntimeError("Could not bind the display to one owned Xvfb process and its private authority") + if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): + raise RuntimeError("A private accessibility/credential D-Bus session is required") + desktop = args.desktop.resolve(strict=True) + node = desktop.parent / "node/CommunityAI-Node" + bootstrap = desktop.parent / "_internal/bootstrap/catalog-bootstrap.json" + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + config_home = Path(os.environ["XDG_CONFIG_HOME"]).resolve(strict=True) + if not config_home.is_relative_to(output.parent) or config_home == output.parent: + raise RuntimeError("XDG_CONFIG_HOME must be a private directory beside the evidence") + entry = config_home / "autostart/communityai.desktop" + if entry.exists() or entry.is_symlink(): + raise RuntimeError("Private qualification autostart entry must initially be absent") + store = NativeCredentialStore("org.communityai.gate15.login." + output.parent.name, "control") + try: + store.get() + except CredentialMissingError: + pass + else: + raise RuntimeError("Qualification credential already exists") + state = output / "state" + config = state / "node-config.json" + result = { + "result": "failed", + "scope": "unmodified-frozen-Linux-Qt-sign-in-checkbox-via-AT-SPI", + "desktop_sha256": hashlib.sha256(desktop.read_bytes()).hexdigest(), + "node_sha256": hashlib.sha256(node.read_bytes()).hexdigest(), + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "ordinary_user": True, + "isolated_xvfb_verified": True, + "mock_telemetry": False, + "phases": [], + } + gui = None + identities = set() + deadline = time.monotonic() + 540 + + def remaining_timeout(maximum): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("Qualification deadline reached") + return min(maximum, remaining) + + def capture(): + for pid, created in tuple(identities): + try: + parent = psutil.Process(pid) + if parent.create_time() != created or not parent.is_running(): + continue + for process in (parent, *parent.children(recursive=True)): + identities.add((process.pid, process.create_time())) + except psutil.NoSuchProcess: + pass + + def live_owned(): + remaining = [] + for pid, created in identities: + try: + process = psutil.Process(pid) + if ( + process.create_time() == created + and process.is_running() + and process.status() != psutil.STATUS_ZOMBIE + ): + remaining.append(process) + except psutil.NoSuchProcess: + pass + return remaining + + def stop(*, cleanup_only=False): + nonlocal gui + capture() + if gui is None and not live_owned(): + return + record = {"normal_shutdown": False, "forced_cleanup": False} + if gui is not None and gui.poll() is None: + gui.terminate() # The product's POSIX bridge requests Qt/node cleanup. + normal_end = min(deadline, time.monotonic() + 35) + while live_owned() and time.monotonic() < normal_end: + capture() + if gui is not None: + gui.poll() + time.sleep(0.2) + returncode = None if gui is None else gui.poll() + record["gui_returncode"] = returncode + record["normal_shutdown"] = returncode == 0 and not live_owned() + if not record["normal_shutdown"]: + result["result"] = "failed" + record["forced_cleanup"] = bool(live_owned()) + # Emergency cleanup has its own finite budget even after acceptance time + # expires. A forced cleanup can never turn a failed shutdown into a pass. + end = time.monotonic() + 10 + while live_owned() and time.monotonic() < end: + capture() + for process in reversed(live_owned()): + try: + process.kill() + except psutil.NoSuchProcess: + pass + time.sleep(0.2) + if gui is not None: + gui.wait(timeout=5) + record["owned_identities_stopped"] = not live_owned() + result.setdefault("shutdowns", []).append(record) + if live_owned(): + raise RuntimeError("An exact owned runtime identity remains alive") + gui = None + if not record["normal_shutdown"] and not cleanup_only: + raise RuntimeError("Normal frozen desktop shutdown failed; emergency cleanup recorded") + + def actor(operation): + completed = subprocess.run( + ["/usr/bin/python3", str(Path(__file__).resolve()), "--actor", operation, "--pid", str(gui.pid)], + capture_output=True, + text=True, + timeout=remaining_timeout(35), + ) + if completed.returncode: + (output / f"actor-{len(result['phases'])}-failure.log").write_text(completed.stderr, encoding="utf-8") + raise RuntimeError("AT-SPI actor failed; retained private diagnostic") + return json.loads(completed.stdout) + + def cache_observation(): + files = [path for path in (state / "model-cache").rglob("*") if path.is_file()] + facts = {"model_cache_file_count": len(files), "model_cache_bytes": sum(path.stat().st_size for path in files)} + if files: + raise RuntimeError("Unexpected model cache acquisition during sign-in qualification") + return facts + + def launch(label, *, login=False): + nonlocal gui + if time.monotonic() >= deadline: + raise TimeoutError("Qualification deadline reached") + command = [ + str(desktop), + "--node-url", + args.node_url, + "--node-config", + str(config), + "--node-data-dir", + str(state), + "--credential-service", + store.service, + "--credential-account", + store.account, + ] + if login: + command.append("--started-at-login") + with (output / f"{label}.log").open("wb") as log: + gui = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT) + process = psutil.Process(gui.pid) + identities.add((process.pid, process.create_time())) + end = min(deadline, time.monotonic() + 150) + while time.monotonic() < end: + if gui.poll() is not None: + raise RuntimeError("Frozen desktop exited before readiness") + capture() + try: + client = NodeClient(args.node_url, store.get(), timeout=remaining_timeout(3)) + status = client.status() + assert status["runtime_budget"]["resident_models"] == 0 + remaining_timeout(3) + assert client.get_contribution_policy()["policy"]["sharing_enabled"] is False + remaining_timeout(3) + assert all(worker["state"] == "paused" for worker in client.list_workers()) + break + except (CredentialMissingError, NodeClientError): + time.sleep(0.5) + else: + raise TimeoutError("Frozen desktop/node readiness timed out") + result["phases"].append( + { + "phase": label, + "authenticated_node_ready": True, + "sharing_paused": True, + "resident_models": 0, + **cache_observation(), + } + ) + + try: + with (output / "bootstrap.log").open("wb") as log: + subprocess.run( + [str(node), "bootstrap", str(bootstrap), "--data_dir", str(state)], + stdout=log, + stderr=subprocess.STDOUT, + timeout=remaining_timeout(150), + check=True, + ) + document = json.loads(config.read_text()) + document.setdefault("contribution_policy", {})["sharing_enabled"] = False + document["inference_mode"] = "local_only" + config.write_text(json.dumps(document, indent=2), encoding="utf-8") + launch("enable") + observed = actor("toggle") + assert observed["checked_before"] is False and observed["checked_after"] is True + assert entry.read_bytes() == _linux_autostart_bytes((str(desktop), "--started-at-login")) + result["phases"][-1].update(observed, exact_frozen_autostart_file=True) + native = store.get() + stop() + launch("restart-and-disable") + observed = actor("toggle") + assert observed["checked_before"] is True and observed["checked_after"] is False + assert not entry.exists() and store.get() == native + result["phases"][-1].update(observed, autostart_file_removed=True, native_credential_preserved=True) + stop() + launch("started-at-login", login=True) + result["phases"][-1].update(actor("application"), started_at_login_argument=True) + stop() + result["final_model_cache"] = cache_observation() + result["result"] = "passed" + except BaseException as exc: + result["error_type"] = type(exc).__name__ + result["error"] = str(exc) + raise + finally: + cleanup_errors = [] + try: + stop(cleanup_only=True) + result["recorded_runtime_identities_stopped"] = not live_owned() + except BaseException as exc: + cleanup_errors.append(f"runtime: {type(exc).__name__}") + result["recorded_runtime_identities_stopped"] = False + result["recorded_runtime_identity_count"] = len(identities) + if result["recorded_runtime_identities_stopped"]: + try: + if entry.is_file() and not entry.is_symlink(): + expected = _linux_autostart_bytes((str(desktop), "--started-at-login")) + if entry.read_bytes() == expected: + entry.unlink() + result["private_autostart_entry_absent"] = not entry.exists() and not entry.is_symlink() + if not result["private_autostart_entry_absent"]: + raise RuntimeError("Unexpected private autostart entry remains") + store.delete() + try: + store.get() + except CredentialMissingError: + result["native_qualification_credential_absent"] = True + else: + raise RuntimeError("Native qualification credential remains after deletion") + except BaseException as exc: + cleanup_errors.append(f"credential-or-entry: {type(exc).__name__}") + else: + result["credential_preserved_for_running_owned_runtime"] = True + if cleanup_errors: + result["result"] = "failed" + result["cleanup_errors"] = cleanup_errors + result["limitations"] = [ + "Linux Xvfb/AT-SPI, not Windows frozen-checkbox acceptance or physical desktop sign-in.", + "Minimized/tray presentation without a window manager is not qualified; executed launch modes are in phases.", + "No model acquisition, inference, or sharing work was requested.", + ] + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + if result["result"] != "passed": + raise RuntimeError("Qualification or final cleanup failed; see retained evidence") + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--actor", choices=("application", "inspect", "toggle")) + parser.add_argument("--pid", type=int) + parser.add_argument("--desktop", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--node-url", default="http://127.0.0.1:18108") + args = parser.parse_args() + if args.actor: + if args.pid is None or args.pid <= 0: + parser.error("--actor requires a positive --pid") + print(json.dumps(accessibility_actor(args.pid, args.actor))) + else: + if args.desktop is None or args.output is None: + parser.error("--desktop and --output are required") + print(json.dumps({"result": run_host(args)["result"]})) diff --git a/scripts/qualify_login_startup_source.py b/scripts/qualify_login_startup_source.py new file mode 100644 index 000000000..79dd3c675 --- /dev/null +++ b/scripts/qualify_login_startup_source.py @@ -0,0 +1,243 @@ +"""Offscreen source-Qt sign-in checkbox regression; never frozen acceptance. + +The native Windows replay redirects registration to a new, non-autostart HKCU +qualification key. It never changes the user's actual CommunityAI Run entry, +creates a credential, starts a node, or presents a visible window. +""" + +import argparse +import hashlib +import json +import os +import sys +import types +import uuid +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "desktop" / "src")) + + +def code_objects(code): + yield code + for value in code.co_consts: + if isinstance(value, types.CodeType): + yield from code_objects(value) + + +def inspect_frozen_hooks(executable): + """Read embedded code constants without loading or modifying the application.""" + from PyInstaller.archive.readers import CArchiveReader + + archive = CArchiveReader(executable).open_embedded_archive("PYZ.pyz") + resource = archive.extract("communityai_desktop.resource_playthrough") + initializer = next(code for code in code_objects(resource) if code.co_name == "__init__") + actions = next(value for value in initializer.co_consts if value == ("observe", "limits", "start", "pause")) + parser = next( + code for code in code_objects(archive.extract("communityai_desktop.app")) if code.co_name == "build_parser" + ) + return { + "frozen_executable_sha256": hashlib.sha256(executable.read_bytes()).hexdigest(), + "embedded_resource_actions": list(actions), + "embedded_cli_options": [ + value for value in parser.co_consts if isinstance(value, str) and value.startswith("--") + ], + "read_only_bytecode_inspection": True, + "frozen_checkbox_exercised": False, + } + + +def checkbox_session(read_enabled, write_enabled, *, click=False): + # An inherited native Qt setting must not open a test window. If another + # caller already constructed a native QApplication, refuse before run(). + os.environ["QT_QPA_PLATFORM"] = "offscreen" + from communityai_desktop.pyside_shell import run + from PySide6.QtCore import Qt + from PySide6.QtTest import QTest + from PySide6.QtWidgets import QApplication, QMessageBox, QPushButton, QStyle, QStyleOptionButton + + application = QApplication.instance() or QApplication([]) + if application.platformName() != "offscreen": + raise RuntimeError("Source checkbox regression requires an offscreen QApplication") + observed = {} + errors = [] + session_timers = [] + + class Automation: + def install(self, window, app, qt): + watchdog = qt["QTimer"](window) + exercise_timer = qt["QTimer"](window) + session_timers.extend((watchdog, exercise_timer)) + watchdog.setSingleShot(True) + exercise_timer.setSingleShot(True) + + def finish(code): + for timer in session_timers: + timer.stop() + window.close() + app.exit(code) + + def expired(): + errors.append(TimeoutError("Source checkbox callback exceeded its deadline")) + finish(1) + + def exercise(): + try: + checkbox = window.login_startup_toggle + more = [ + button + for button in window.findChildren(QPushButton) + if button.accessibleName() == "More sharing settings" + ] + if len(more) != 1 or more[0].isChecked(): + raise AssertionError("Sharing settings must start collapsed") + observed["settings_initially_collapsed"] = not checkbox.isVisible() + window.pages.currentWidget().ensureWidgetVisible(more[0]) + app.processEvents() + QTest.mouseClick(more[0], Qt.LeftButton) + if not more[0].isChecked(): + raise AssertionError("Sharing settings did not expand after clicking their control") + window.pages.currentWidget().ensureWidgetVisible(checkbox) + app.processEvents() + if not checkbox.isVisible(): + raise AssertionError("Sign-in checkbox is not visible on the offscreen Sharing page") + observed.update( + initial_checked=checkbox.isChecked(), + initial_enabled=checkbox.isEnabled(), + initial_detail=window.login_startup_detail.text(), + checkbox_visible=checkbox.isVisible(), + ) + if click: + if not checkbox.isEnabled(): + raise AssertionError("Sign-in checkbox is disabled") + # Checkbox hit regions differ by style; a stretched + # widget's center can lie outside its clickable label. + option = QStyleOptionButton() + checkbox.initStyleOption(option) + indicator = checkbox.style().subElementRect(QStyle.SE_CheckBoxIndicator, option, checkbox) + hit_region = checkbox.style().subElementRect(QStyle.SE_CheckBoxClickRect, option, checkbox) + position = indicator.center() + if ( + not indicator.isValid() + or not checkbox.rect().contains(position) + or not hit_region.contains(position) + ): + raise AssertionError("The checkbox style did not expose a valid indicator click target") + QTest.mouseClick(checkbox, Qt.LeftButton, pos=position) + observed.update( + final_checked=checkbox.isChecked(), + final_detail=window.login_startup_detail.text(), + final_detail_visible=window.login_startup_detail.isVisible(), + qt_platform=app.platformName(), + ) + except BaseException as exc: + errors.append(exc) + finally: + finish(0) + + # Cancel both timers when this session ends. Static singleShot quit + # timers survive a fast run and can interrupt a later QApplication + # event loop in the same unittest process. + watchdog.timeout.connect(expired) + exercise_timer.timeout.connect(exercise) + watchdog.start(3_000) + exercise_timer.start(50) + + def offline(): + raise RuntimeError("Isolated offline UI regression; no node or fixture telemetry") + + with ( + patch("communityai_desktop.pyside_shell.login_startup_enabled", read_enabled), + patch("communityai_desktop.pyside_shell.set_login_startup", write_enabled), + patch.object(QMessageBox, "warning") as warning, + ): + try: + exit_code = run( + connect=offline, + single_instance=False, + screenshot_page=2, + qualification_automation=Automation(), + ) + finally: + for timer in session_timers: + timer.stop() + timer.timeout.disconnect() + observed["warning_count"] = warning.call_count + if errors: + raise errors[0] + if exit_code != 0 or "final_checked" not in observed: + raise RuntimeError("Source checkbox callback did not complete") + return observed + + +def run_native_windows(executable, output): + if os.name != "nt": + raise RuntimeError("Native registry replay requires Windows") + import ctypes + import winreg + + from communityai_desktop import startup + + if ctypes.windll.shell32.IsUserAnAdmin(): + raise RuntimeError("Run the checkbox regression as a non-elevated ordinary user") + output.mkdir(parents=True, exist_ok=False) + real_key = startup.WINDOWS_RUN_KEY + value_name = startup.WINDOWS_VALUE_NAME + private_key = r"Software\CommunityAI\Qualification" + "\\" + uuid.uuid4().hex + + def read_value(key_path): + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path) as key: + return winreg.QueryValueEx(key, value_name) + except FileNotFoundError: + return None + + original = read_value(real_key) + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, private_key): + raise RuntimeError("The unique qualification key is unexpectedly occupied") + except FileNotFoundError: + pass + result = { + "result": "failed", + "scope": "source-Qt-checkbox-with-native-isolated-registry", + "frozen_gate_passed": False, + "visible_windows": False, + "native_credentials_created": False, + "nodes_workers_or_inference_started": False, + "frozen_hook_inspection": inspect_frozen_hooks(executable), + } + try: + with patch.object(startup, "WINDOWS_RUN_KEY", private_key): + enabled = checkbox_session(startup.login_startup_enabled, startup.set_login_startup, click=True) + assert enabled["initial_checked"] is False and enabled["final_checked"] is True + assert enabled["final_detail"] == "CommunityAI will open when you sign in." + assert startup.login_startup_enabled() + reopened = checkbox_session(startup.login_startup_enabled, startup.set_login_startup, click=True) + assert reopened["initial_checked"] is True and reopened["final_checked"] is False + assert reopened["initial_detail"] == "" and reopened["final_detail"] == "Automatic opening is off." + assert not startup.login_startup_enabled() and read_value(private_key) is None + result["enable"] = enabled + result["reopen_and_disable"] = reopened + result["result"] = "passed-source-only" + finally: + try: + winreg.DeleteKey(winreg.HKEY_CURRENT_USER, private_key) + except FileNotFoundError: + pass + result["qualification_registry_key_removed"] = True + result["real_login_entry_unchanged"] = read_value(real_key) == original + if not result["real_login_entry_unchanged"]: + result["result"] = "failed-concurrent-real-login-entry-change" + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--desktop", type=Path, required=True, help="Read-only frozen-hook inspection target") + parser.add_argument("--output", type=Path, required=True, help="New evidence directory") + arguments = parser.parse_args() + facts = run_native_windows(arguments.desktop.resolve(strict=True), arguments.output.resolve()) + print(json.dumps({"result": facts["result"], "frozen_gate_passed": False})) diff --git a/scripts/qualify_login_startup_windows.cs b/scripts/qualify_login_startup_windows.cs new file mode 100644 index 000000000..d43670234 --- /dev/null +++ b/scripts/qualify_login_startup_windows.cs @@ -0,0 +1,148 @@ +// Windows-only UIA companion. It never switches the input desktop, sends input, +// accesses the registry. Default read mode never toggles the login checkbox; +// explicit enable/disable modes wait for the Python Run-state guard handshake. +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Windows.Automation; + +class PrivateQtRead { + [DllImport("user32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern IntPtr CreateDesktop(string n,IntPtr d,IntPtr m,uint f,uint a,IntPtr s); + [DllImport("user32.dll",SetLastError=true)] static extern bool SetThreadDesktop(IntPtr d); + [DllImport("user32.dll")] static extern bool CloseDesktop(IntPtr d); + [DllImport("user32.dll",SetLastError=true)] static extern IntPtr OpenInputDesktop(uint f,bool i,uint a); + [DllImport("user32.dll",CharSet=CharSet.Unicode)] static extern bool GetUserObjectInformation(IntPtr h,int i,StringBuilder b,uint n,out uint needed); + [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr h,out uint pid); + [DllImport("user32.dll",CharSet=CharSet.Unicode)] static extern int GetWindowText(IntPtr h,StringBuilder text,int count); + [DllImport("user32.dll")] static extern IntPtr GetThreadDesktop(uint thread); + [DllImport("kernel32.dll")] static extern uint GetCurrentThreadId(); + [DllImport("kernel32.dll",SetLastError=true)] static extern IntPtr OpenProcess(uint access,bool inherit,uint pid); + delegate bool EnumWindow(IntPtr window,IntPtr unused); + [DllImport("user32.dll",ExactSpelling=true,SetLastError=true)] static extern bool EnumDesktopWindows(IntPtr desktop,EnumWindow callback,IntPtr unused); + [DllImport("kernel32.dll",ExactSpelling=true)] static extern void SetLastError(uint code); + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern bool CreateProcess(string app,StringBuilder cmd,IntPtr pa,IntPtr ta,bool inherit,uint flags,IntPtr env,string cwd,ref STARTUPINFO s,out PROCESS_INFORMATION p); + [DllImport("kernel32.dll")] static extern bool GetProcessTimes(IntPtr p,out long created,out long exited,out long kernel,out long user); + [DllImport("kernel32.dll")] static extern uint WaitForSingleObject(IntPtr h,uint ms); + [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr h); + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern IntPtr CreateJobObject(IntPtr attributes,string name); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool SetInformationJobObject(IntPtr job,int info,ref EXTENDED_LIMIT limit,uint length); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job,int info,ref JOB_ACCOUNTING accounting,uint length,IntPtr returned); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job,IntPtr process); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool TerminateJobObject(IntPtr job,uint code); + [DllImport("kernel32.dll",SetLastError=true)] static extern uint ResumeThread(IntPtr thread); + [DllImport("kernel32.dll")] static extern bool TerminateProcess(IntPtr process,uint code); + [StructLayout(LayoutKind.Sequential)] struct BASIC_LIMIT {public long processTime,jobTime;public uint flags;public UIntPtr minWorkingSet,maxWorkingSet;public uint activeProcesses;public UIntPtr affinity;public uint priority,scheduling;} + [StructLayout(LayoutKind.Sequential)] struct IO_COUNTERS {public ulong readOps,writeOps,otherOps,readBytes,writeBytes,otherBytes;} + [StructLayout(LayoutKind.Sequential)] struct EXTENDED_LIMIT {public BASIC_LIMIT basic;public IO_COUNTERS io;public UIntPtr processMemory,jobMemory,peakProcessMemory,peakJobMemory;} + [StructLayout(LayoutKind.Sequential)] struct JOB_ACCOUNTING {public long totalUser,totalKernel,periodUser,periodKernel;public uint pageFaults,totalProcesses,activeProcesses,terminatedProcesses;} + [StructLayout(LayoutKind.Sequential,CharSet=CharSet.Unicode)] struct STARTUPINFO {public int cb; public string reserved,desktop,title; public uint x,y,xSize,ySize,xCount,yCount,fill,flags;public short show,reserved2;public IntPtr reservedPtr,input,output,error;} + [StructLayout(LayoutKind.Sequential)] struct PROCESS_INFORMATION {public IntPtr process,thread;public uint pid,tid;} + static string InputName(){IntPtr h=OpenInputDesktop(0,false,1);if(h==IntPtr.Zero)throw new Exception("input_desktop_unavailable");try{var s=new StringBuilder(512);uint n;if(!GetUserObjectInformation(h,2,s,1024,out n))throw new Exception("input_desktop_name");return s.ToString();}finally{CloseDesktop(h);}} + static void Write(string root,string file,string text){string temporary=Path.Combine(root,file+".tmp");File.WriteAllText(temporary,text);File.Move(temporary,Path.Combine(root,file));} + static void Stage(string root,string value){File.AppendAllText(Path.Combine(root,"uia-stages.txt"),DateTime.UtcNow.ToString("o")+" "+value+"\n");} + [MTAThread] static int Main(string[] args) { + try{return args[0]=="--inspect"?Inspect(args):Run(args);}catch(Exception error){try{string output=args[0]=="--inspect"?args[1]:args[2];if(Directory.Exists(output))File.WriteAllText(Path.Combine(output,args[0]=="--inspect"?"actor-fatal.txt":"helper-fatal.txt"),"error_type="+error.GetType().Name+"\n");}catch{}return 1;} + } + static int Inspect(string[] args) { + string output=args[1],desktopName=args[4],action=args[5];uint expectedPid=UInt32.Parse(args[2]);long expectedCreated=Int64.Parse(args[3]); + if(action!="read"&&action!="enable"&&action!="disable")throw new Exception("unknown_action"); + Stage(output,"actor_started");var name=new StringBuilder(512);uint needed; + if(!GetUserObjectInformation(GetThreadDesktop(GetCurrentThreadId()),2,name,1024,out needed)||name.ToString()!=desktopName||InputName()==desktopName)throw new Exception("actor_desktop_mismatch"); + Stage(output,"private_desktop_verified");IntPtr gui=OpenProcess(0x101000,false,expectedPid);if(gui==IntPtr.Zero)throw new Exception("owned_gui_unavailable"); + int retries=0,lastError=0;string navigationRole="",navigationPatterns="";bool navigationInvoked=false; + try { + long created,exited,kernel,user;if(!GetProcessTimes(gui,out created,out exited,out kernel,out user)||created!=expectedCreated)throw new Exception("owned_gui_identity_mismatch"); + var windows=new List();EnumWindow collect=(window,unused)=>{uint pid;GetWindowThreadProcessId(window,out pid);if(pid==expectedPid)windows.Add(window);return true;}; + // .NET Framework's first callback-marshalling/native-binding call was observed + // to leave error127 on an empty desktop. Warm that exact callback once, then + // reset last-error and evaluate every measured enumeration normally. + bool warm=EnumDesktopWindows(IntPtr.Zero,collect,IntPtr.Zero);int warmError=Marshal.GetLastWin32Error();Stage(output,"enumeration_binding_warmup_"+warm+"_"+warmError); + DateTime deadline=DateTime.UtcNow.AddSeconds(80); + while(DateTime.UtcNow();foreach(var available in sharing.GetSupportedPatterns())patterns.Add(available.ProgrammaticName);navigationPatterns=String.Join(",",patterns.ToArray());Stage(output,"sharing_role_"+navigationRole);Stage(output,"sharing_patterns_"+navigationPatterns); + object invoke;if(!sharing.TryGetCurrentPattern(InvokePattern.Pattern,out invoke))throw new Exception("sharing_invoke_unavailable");Stage(output,"sharing_invoke");((InvokePattern)invoke).Invoke();navigationInvoked=true;} + Stage(output,"checkbox_query");var boxes=top.FindAll(TreeScope.Descendants,new AndCondition(new PropertyCondition(AutomationElement.NameProperty,"Start CommunityAI when I sign in"),new PropertyCondition(AutomationElement.ControlTypeProperty,ControlType.CheckBox)));if(boxes.Count>1)throw new Exception("ambiguous_login_checkbox");var box=boxes.Count==1?boxes[0]:null; + if(box==null)continue;Stage(output,"checkbox_pattern");object pattern;if(!box.TryGetCurrentPattern(TogglePattern.Pattern,out pattern))throw new Exception("checkbox_toggle_pattern_unavailable"); + Stage(output,"checkbox_state");string state=((TogglePattern)pattern).Current.ToggleState.ToString(),initial=state,type=box.Current.ControlType.ProgrammaticName;bool enabled=box.Current.IsEnabled; + if(type!="ControlType.CheckBox")throw new Exception("unexpected_login_control_type"); + if(action!="read") { + string before=action=="enable"?"Off":"On",after=action=="enable"?"On":"Off"; + if(!enabled||state!=before)throw new Exception("unexpected_initial_checkbox_state"); + Write(output,"action-ready.txt",action);Stage(output,"waiting_for_Run_guard");DateTime guardDeadline=DateTime.UtcNow.AddSeconds(15); + while(!File.Exists(Path.Combine(output,"allow-action"))&&DateTime.UtcNow3?args[3]:"read"; + if(action!="read"&&action!="enable"&&action!="disable")throw new Exception("unknown_action"); + string desktopName="CommunityAIReadOnly-"+Guid.NewGuid().ToString("N"),before=InputName(); + // Deliberately omit DESKTOP_SWITCHDESKTOP permission. + IntPtr desktop=CreateDesktop(desktopName,IntPtr.Zero,IntPtr.Zero,0,0x00CB,IntPtr.Zero); + if(desktop==IntPtr.Zero)throw new Exception("CreateDesktop_"+Marshal.GetLastWin32Error()); + PROCESS_INFORMATION process=new PROCESS_INFORMATION(),actor=new PROCESS_INFORMATION();bool noSwitch=true,closed=false,contained=false,actorContained=false; + IntPtr job=IntPtr.Zero;uint activeAtClose=UInt32.MaxValue;bool jobClosed=false,jobCleanupVerified=false; + string result="failed",error=""; + try { + job=CreateJobObject(IntPtr.Zero,null);if(job==IntPtr.Zero)throw new Exception("CreateJobObject_"+Marshal.GetLastWin32Error()); + var limits=new EXTENDED_LIMIT();limits.basic.flags=0x2000; + if(!SetInformationJobObject(job,9,ref limits,(uint)Marshal.SizeOf(limits)))throw new Exception("job_kill_on_close_"+Marshal.GetLastWin32Error()); + var startup=new STARTUPINFO();startup.cb=Marshal.SizeOf(startup);startup.desktop="WinSta0\\"+desktopName;startup.flags=0x80; + if(!CreateProcess(executable,new StringBuilder(command),IntPtr.Zero,IntPtr.Zero,false,0x08000004,IntPtr.Zero,output,ref startup,out process))throw new Exception("CreateProcess_"+Marshal.GetLastWin32Error()); + if(!AssignProcessToJobObject(job,process.process)){TerminateProcess(process.process,91);WaitForSingleObject(process.process,3000);throw new Exception("job_assignment_"+Marshal.GetLastWin32Error());} + contained=true; + long created,exited,kernel,user;if(!GetProcessTimes(process.process,out created,out exited,out kernel,out user))throw new Exception("process_creation_time"); + Write(output,"launch.txt",process.pid+"\n"+created+"\n"+desktopName+"\n"+before); + if(ResumeThread(process.thread)==UInt32.MaxValue)throw new Exception("resume_thread_"+Marshal.GetLastWin32Error()); + string self=System.Reflection.Assembly.GetExecutingAssembly().Location; + string actorCommand="\""+self+"\" --inspect \""+output+"\" "+process.pid+" "+created+" "+desktopName+" "+action; + if(!CreateProcess(self,new StringBuilder(actorCommand),IntPtr.Zero,IntPtr.Zero,false,0x08000004,IntPtr.Zero,output,ref startup,out actor))throw new Exception("actor_create_process_"+Marshal.GetLastWin32Error()); + if(!AssignProcessToJobObject(job,actor.process)){TerminateProcess(actor.process,94);WaitForSingleObject(actor.process,3000);throw new Exception("actor_job_assignment");}actorContained=true; + long actorCreated;if(!GetProcessTimes(actor.process,out actorCreated,out exited,out kernel,out user))throw new Exception("actor_creation_time"); + Write(output,"actor-launch.txt",actor.pid+"\n"+actorCreated+"\n"+desktopName); + if(ResumeThread(actor.thread)==UInt32.MaxValue)throw new Exception("actor_resume_thread"); + DateTime until=DateTime.UtcNow.AddSeconds(115); + while(DateTime.UtcNow= deadline: + raise TimeoutError("Owned process shutdown exceeded the shared probe deadline") + time.sleep(0.1) + + +def run(args): + assert qualification_platform() == "Windows" + require_desktop_stopped() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + desktop = args.desktop.resolve() + node = desktop.parent / "node/CommunityAI-Node.exe" + bootstrap = desktop.parent / "_internal/bootstrap/catalog-bootstrap.json" + assert bootstrap.read_bytes() == (ROOT / "public-alpha/catalog-qwen-v2/catalog-bootstrap.json").read_bytes() + original = run_value() + action = getattr(args, "action", "read") + shared_state = getattr(args, "shared_state", None) + store = getattr(args, "shared_store", None) or NativeCredentialStore( + "org.communityai.private-desktop." + output.name, "control" + ) + if shared_state is None: + try: + store.get() + except CredentialMissingError: + pass + else: + raise RuntimeError("The private qualification credential already exists") + else: + store.get() + if action != "read": + assert shared_state is not None and action in ("enable", "disable") + assert callable(args.before_action) and callable(args.after_action) + result = { + "result": "failed", + "scope": "frozen-Windows-private-desktop-login-checkbox-" + action, + "desktop_sha256": sha256(desktop), + "node_sha256": sha256(node), + "bootstrap_sha256": sha256(bootstrap), + "replay_sha256": sha256(Path(__file__)), + "uia_source_sha256": sha256(Path(__file__).with_suffix(".cs")), + "non_elevated": True, + "run_entry_original_present": original is not None, + "registry_mutation": action != "read", + "visible_input_desktop_acceptance": False, + "gui_probe_deadline_seconds": 120, + } + state = Path(shared_state) if shared_state is not None else output / "state" + config_path = state / "node-config.json" + identities = [] + helper = None + gui_identity = None + gui_deadline = None + credential_created = False + environment = os.environ.copy() + environment.update(QT_QPA_PLATFORM="windows", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1") + flags = {"creationflags": subprocess.CREATE_NO_WINDOW} + try: + compiler_root = Path(os.environ["WINDIR"]) / "Microsoft.NET/Framework64/v4.0.30319" + compiled = output / "private-desktop-reader.exe" + subprocess.run( + [ + str(compiler_root / "csc.exe"), + "/nologo", + "/target:winexe", + "/out:" + str(compiled), + *[ + "/reference:" + str(compiler_root / "WPF" / name) + for name in ("UIAutomationClient.dll", "UIAutomationTypes.dll", "WindowsBase.dll") + ], + str(Path(__file__).with_suffix(".cs").resolve()), + ], + check=True, + capture_output=True, + timeout=30, + **flags, + ) + if shared_state is None: + proc = subprocess.run( + [str(node), "bootstrap", str(bootstrap), "--data_dir", str(state)], + capture_output=True, + timeout=90, + env=environment, + **flags, + ) + (output / "bootstrap.private.log").write_bytes(proc.stdout + proc.stderr) + assert proc.returncode == 0, "Signed bootstrap failed; private log retained" + installed = json.loads(proc.stdout.decode().splitlines()[-1]) + assert installed["catalog_sequence"] == 2 + config = json.loads(config_path.read_text()) + config["inference_mode"] = "local_only" + config["contribution_policy"]["sharing_enabled"] = False + config_path.write_text(json.dumps(config, indent=2)) + credential_created = True + store.provision(state / "unused-private-legacy-key", allow_create=True) + command = [ + str(desktop), + "--node-url", + args.node_url, + "--node-config", + str(config_path), + "--node-data-dir", + str(state), + "--credential-service", + store.service, + "--credential-account", + store.account, + ] + command_file = output / "command.private.txt" + command_file.write_text(subprocess.list2cmdline(command)) + started = time.monotonic() + gui_deadline = started + 120 + helper = subprocess.Popen( + [str(compiled), str(desktop), str(command_file), str(output), action], + env=environment, + **flags, + ) + result["helper_containment_required"] = True + identities.append((helper.pid, psutil.Process(helper.pid).create_time())) + status = None + gui = None + while time.monotonic() - started < 75: + if helper.poll() is not None: + raise RuntimeError("Private desktop helper exited before acceptance") + identities.extend(process_tree(helper.pid)) + if action != "read" and (output / "action-ready.txt").exists() and not (output / "allow-action").exists(): + assert (output / "action-ready.txt").read_text() == action + args.before_action() + (output / "allow-action").touch() + launch = output / "launch.txt" + if gui is None and launch.exists(): + pid, filetime, private_name, input_name = launch.read_text().splitlines() + gui = psutil.Process(int(pid)) + assert abs(gui.create_time() - (int(filetime) / 10000000 - 11644473600)) < 0.001 + identities.append((gui.pid, gui.create_time())) + gui_identity = (gui.pid, gui.create_time()) + result.update(private_desktop_name=private_name, input_desktop_before=input_name) + try: + client = NodeClient(args.node_url, store.get(), timeout=2) + status = client.status() + result["last_authenticated_status"] = { + "status": status["status"], + "resident_models": status["runtime_budget"]["resident_models"], + "worker_states": [worker["state"] for worker in status["workers"]], + } + if (output / "uia.txt").exists(): + break + except NodeClientError: + pass + time.sleep(0.2) + else: + raise TimeoutError("Frozen Qt checkbox/authenticated node exceeded observation deadline") + assert status["status"] == "running" + assert status["runtime_budget"]["resident_models"] == 0 + assert status["inference_mode"] == "local_only" + assert not status["contribution"]["policy"]["policy"]["sharing_enabled"] + assert all(worker["state"] == "paused" for worker in status["workers"]) + assert not list(state.rglob("*.safetensors")) + result.update( + seconds_to_uia_and_authenticated_status=round(time.monotonic() - started, 3), + uia=fields(output / "uia.txt"), + resident_models=0, + private_model_weight_files=0, + worker_states=[worker["state"] for worker in status["workers"]], + catalog_sequence=2, + ) + assert result["uia"]["result"] == "passed" + assert result["uia"]["control_type"] == "ControlType.CheckBox" + if action == "read": + assert run_value() == original + else: + args.after_action() + result["result"] = "passed" + except BaseException as exc: + result["error_type"] = type(exc).__name__ + raise + finally: + try: + if helper is not None: + refresh_owned_identities(identities) + # Existing desktop guard ensures this maintenance message targets + # only the GUI launched above. It creates no normal app window. + if owned_gui_is_live(gui_identity): + maintenance = subprocess.Popen([str(desktop), "--prepare-update"], env=environment, **flags) + identities.append((maintenance.pid, psutil.Process(maintenance.pid).create_time())) + assert maintenance.wait(timeout=min(30, max(0.1, gui_deadline - time.monotonic() - 10))) == 0 + refresh_owned_identities(identities) + app_identities = [item for item in set(identities) if item[0] != helper.pid] + wait_owned_gone(app_identities, gui_deadline - 5) + (output / "stop").touch() + assert helper.wait(timeout=min(5, max(0.1, gui_deadline - time.monotonic()))) == 0 + wait_owned_gone(identities, gui_deadline) + result["private_desktop_helper"] = fields(output / "helper-result.txt") + assert result["private_desktop_helper"]["input_desktop_unchanged"] == "True" + assert result["private_desktop_helper"]["desktop_handle_closed"] == "True" + assert result["private_desktop_helper"]["job_assigned_before_resume"] == "True" + assert result["private_desktop_helper"]["actor_job_assigned_before_resume"] == "True" + assert result["private_desktop_helper"]["job_active_processes_at_close"] == "0" + assert result["private_desktop_helper"]["job_handle_closed"] == "True" + result["owned_processes_stopped"] = True + except BaseException as exc: + result["result"] = "failed" + result["cleanup_error_type"] = type(exc).__name__ + force_stop_owned_tree(identities, timeout=max(1, min(10, gui_deadline - time.monotonic()))) + wait_owned_gone(identities, gui_deadline) + result["owned_processes_stopped"] = True + raise + finally: + finalize_evidence( + result, + output, + store, + credential_created, + original if action == "read" else args.expected_run_after, + identities, + comparison_field="run_entry_unchanged" if action == "read" else "expected_run_state_verified", + ) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--desktop", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--node-url", default="http://127.0.0.1:18118") + parser.add_argument("--exercise-login-startup", action="store_true") + args = parser.parse_args() + if args.exercise_login_startup: + from qualify_login_startup_windows_cycle import run_cycle + + answer = run_cycle(args) + else: + answer = run(args) + print(json.dumps({"result": answer["result"]})) + raise SystemExit(0 if answer["result"] == "passed" else 1) diff --git a/scripts/qualify_login_startup_windows_cycle.py b/scripts/qualify_login_startup_windows_cycle.py new file mode 100644 index 000000000..e10a374fe --- /dev/null +++ b/scripts/qualify_login_startup_windows_cycle.py @@ -0,0 +1,170 @@ +"""Opt-in frozen Windows login-startup cycle on unswitched private desktops.""" + +import json +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import qualify_login_startup_windows as probe +from communityai_desktop.credentials import CredentialMissingError, NativeCredentialStore +from communityai_desktop.startup import WINDOWS_RUN_KEY, WINDOWS_VALUE_NAME, login_startup_command +from qualify_login_startup_windows_state import RunValueGuard + + +def write_run(value): + import winreg + + with winreg.CreateKeyEx(winreg.HKEY_CURRENT_USER, WINDOWS_RUN_KEY, 0, winreg.KEY_SET_VALUE) as key: + winreg.SetValueEx(key, WINDOWS_VALUE_NAME, 0, value[1], value[0]) + + +def delete_run(): + import winreg + + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, WINDOWS_RUN_KEY, 0, winreg.KEY_SET_VALUE) as key: + winreg.DeleteValue(key, WINDOWS_VALUE_NAME) + except FileNotFoundError: + pass + + +def finalize_cycle(result, output, guard, store, credential_created): + """Restore after proven process shutdown, retaining unconditional evidence.""" + try: + stopped = all(phase.get("owned_processes_stopped") is True for phase in result["phases"]) + result["owned_processes_stopped"] = stopped + result["original_run_state_restored"] = False + if stopped: + try: + result["restoration_required_registry_write"] = guard.restore() + result["original_run_state_restored"] = True + except BaseException as exc: + result.update(result="failed", restoration_error_type=type(exc).__name__) + else: + result.update(result="failed", restoration_deferred_for_unverified_processes=True) + if credential_created: + result["private_credential_removed"] = False + if stopped: + try: + store.delete() + try: + store.get() + except CredentialMissingError: + result["private_credential_removed"] = True + else: + result.update(result="failed", credential_cleanup_error_type="CredentialStillPresent") + except BaseException as exc: + result.update(result="failed", credential_cleanup_error_type=type(exc).__name__) + else: + result.update(result="failed", private_credential_retained_for_live_processes=True) + finally: + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + + +def run_cycle(args): + if not args.exercise_login_startup: + raise ValueError("The real login-startup cycle requires explicit opt-in") + if probe.qualification_platform() != "Windows": + raise RuntimeError("The private-desktop login cycle requires Windows") + probe.require_desktop_stopped() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + desktop = args.desktop.resolve() + node = desktop.parent / "node/CommunityAI-Node.exe" + bootstrap = desktop.parent / "_internal/bootstrap/catalog-bootstrap.json" + assert bootstrap.read_bytes() == (probe.ROOT / "public-alpha/catalog-qwen-v2/catalog-bootstrap.json").read_bytes() + guard = RunValueGuard(probe.run_value, write_run, delete_run) + # Private recovery record; binary registry data is encoded without evaluating it. + (output / "original-run.private.json").write_text( + json.dumps(guard.original, default=lambda value: {"bytes_hex": value.hex()}) + "\n" + ) + expected = (subprocess.list2cmdline(login_startup_command(desktop, frozen=True)), 1) # REG_SZ + guard.expect(expected) + guard.expect(None) + store = NativeCredentialStore("org.communityai.private-desktop." + output.name, "control") + try: + store.get() + except CredentialMissingError: + pass + else: + raise RuntimeError("The private qualification credential already exists") + state = output / "state" + result = { + "result": "failed", + "scope": "frozen-Windows-private-desktop-login-enable-restart-disable", + "explicit_mutation_opt_in": True, + "run_entry_original_present": guard.original is not None, + "expected_startup_argv": list(login_startup_command(desktop, frozen=True)), + "expected_run_type": "REG_SZ", + "credential_service": store.service, + "credential_account": store.account, + "replay_sha256": probe.sha256(Path(__file__)), + "run_state_guard_sha256": probe.sha256(Path(__file__).with_name("qualify_login_startup_windows_state.py")), + "phases": [], + "actual_os_sign_in_exercised": False, + } + credential_created = False + try: + environment = os.environ.copy() + environment.update(OMP_NUM_THREADS="1", MKL_NUM_THREADS="1") + initialized = subprocess.run( + [str(node), "bootstrap", str(bootstrap), "--data_dir", str(state)], + capture_output=True, + timeout=90, + env=environment, + creationflags=subprocess.CREATE_NO_WINDOW, + ) + (output / "bootstrap.private.log").write_bytes(initialized.stdout + initialized.stderr) + assert initialized.returncode == 0, "Signed bootstrap failed; private log retained" + assert json.loads(initialized.stdout.decode().splitlines()[-1])["catalog_sequence"] == 2 + config_path = state / "node-config.json" + config = json.loads(config_path.read_text()) + config["inference_mode"] = "local_only" + config["contribution_policy"]["sharing_enabled"] = False + config_path.write_text(json.dumps(config, indent=2)) + config_before = config_path.read_bytes() + credential_created = True + store.provision(state / "unused-private-legacy-key", allow_create=True) + original_credential = store.get() + for action, before, after, initial, final in ( + ("enable", guard.original, expected, "Off", "On"), + ("disable", expected, None, "On", "Off"), + ): + phase_output = output / action + phase_record = {"action": action, "result": "failed", "owned_processes_stopped": True} + result["phases"].append(phase_record) + guard.verify(before) + assert store.get() == original_credential + phase_args = SimpleNamespace( + desktop=desktop, + output=phase_output, + node_url=args.node_url, + action=action, + shared_state=state, + shared_store=store, + before_action=lambda before=before: guard.verify(before), + after_action=lambda after=after: guard.verify(after), + expected_run_after=after, + ) + phase_record["owned_processes_stopped"] = False + try: + phase = probe.run(phase_args) + finally: + if (phase_output / "result.json").exists(): + phase_record.update(json.loads((phase_output / "result.json").read_text())) + assert phase["result"] == "passed" + assert phase["uia"]["initial_state"] == initial and phase["uia"]["state"] == final + assert phase["uia"]["login_action"] == action and phase["uia"]["registry_mutation"] == "True" + assert phase["owned_processes_stopped"] is True + assert store.get() == original_credential + assert config_path.read_bytes() == config_before + phase_record["native_credential_unchanged_after_shutdown"] = True + phase_record["private_config_unchanged_after_shutdown"] = True + result.update(result="passed", restart_preserved_enabled_checkbox=True, private_credential_continuity=True) + except BaseException as exc: + result.update(result="failed", error_type=type(exc).__name__) + raise + finally: + finalize_cycle(result, output, guard, store, credential_created) + return result diff --git a/scripts/qualify_login_startup_windows_state.py b/scripts/qualify_login_startup_windows_state.py new file mode 100644 index 000000000..d4f7e0e2f --- /dev/null +++ b/scripts/qualify_login_startup_windows_state.py @@ -0,0 +1,58 @@ +"""Preserve a qualification run's original Run value without importing winreg. + +Values use QueryValueEx's ``(data, registry_type)`` pair, or None for absence. +The injected write callback accepts that pair; delete removes only that value. +This guard detects changes observed before restoration and never overwrites an +unrecognized value. Registry reads and writes are not an atomic compare-and-swap. +""" + +from copy import deepcopy + + +class RunStateConflict(RuntimeError): + """An unrelated value was observed; restoring it would overwrite another edit.""" + + +class RunStateVerificationError(RuntimeError): + """The observed registry state does not match the required exact value/type.""" + + +class RunValueGuard: + def __init__(self, read, write, delete): + self._read, self._write, self._delete = read, write, delete + self._original = deepcopy(read()) + self._expected = [] + + @property + def original(self): + return deepcopy(self._original) + + def expect(self, value): + """Register an exact value this run may leave after a product action.""" + self._expected.append(deepcopy(value)) + + def verify(self, value): + """Read without mutation and require the supplied exact value/type.""" + if self._read() != value: + raise RunStateVerificationError("The Run value does not match the expected state") + + def restore(self): + """Restore only a recognized state, then independently verify the original. + + Return True if restoration required a write/delete. The verification + still runs when that operation raises; an operation error is never + converted into successful acceptance, even if the original was restored. + """ + current = self._read() + changed = current != self._original + if changed and current not in self._expected: + raise RunStateConflict("The Run value changed outside this qualification; it was left untouched") + try: + if changed: + if self._original is None: + self._delete() + else: + self._write(deepcopy(self._original)) + finally: + self.verify(self._original) + return changed diff --git a/scripts/qualify_qwen_formation_local.py b/scripts/qualify_qwen_formation_local.py new file mode 100644 index 000000000..d2f6076c3 --- /dev/null +++ b/scripts/qualify_qwen_formation_local.py @@ -0,0 +1,55 @@ +"""Real packaged-node/Qt preflight for the formation harness; no cloud resources. + +This checks only local inference and desktop controls on an empty isolated DHT. +It is never accepted as the distributed formation result. +""" + +import hashlib +import json +import secrets +import time + +from hivemind import DHT +from run_qwen_formation import ROOT, FormationRun, LauncherLock, _write_json + + +def main(): + runs = ROOT / ".gate13-runs/qwen-formation-local" + with LauncherLock(runs / "launcher.lock"): + path = runs / (time.strftime("q38local-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(3)) + config = json.loads((ROOT / "config/qwen_formation.json").read_text()) + run = FormationRun(path, config) + run.deadline = time.time() + 900 + run.packaged = json.loads((ROOT / "config/qwen_product_test.json").read_text()) + if hashlib.sha256((ROOT / run.packaged["node"]).read_bytes()).hexdigest() != run.packaged["node_sha256"]: + raise RuntimeError("Retained node hash mismatch") + result = {"result": "failed", "scope": "local-formation-harness-preflight", "distributed_formation": False} + dht = None + try: + run.bundle() + result["source_inventory"] = "source-inventory.json" + result["packaged_node_sha256"] = run.packaged["node_sha256"] + dht = DHT(initial_peers=[], host_maddrs=["/ip4/127.0.0.1/tcp/0"], client_mode=False, start=True, tls=True) + run.start_desktop([str(value) for value in dht.get_visible_maddrs()]) + result["desktop_local"] = run.desktop("local") + result["local_inference"] = run.command("desktop", "infer", source="local") + result["local_only_clicked"] = run.desktop("local", toggle=True) + result["auto_clicked"] = run.desktop("local", toggle=True, inference_mode="auto") + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + finally: + try: + run.stop_desktop() + result["local_cleanup_verified"] = True + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + if dht is not None: + dht.shutdown() + _write_json(path / "result.json", result) + print(json.dumps({"result": result["result"], "evidence": str(path / "result.json")}), flush=True) + return 0 if result["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qualify_qwen_local_product.py b/scripts/qualify_qwen_local_product.py new file mode 100644 index 000000000..e4179db47 --- /dev/null +++ b/scripts/qualify_qwen_local_product.py @@ -0,0 +1,177 @@ +"""Exercise the real authenticated node/desktop contract with pinned local Qwen weights. + +Accepts either the current Python node or a packaged CommunityAI-Node executable. +Uses an isolated data directory and never prints control or inference credentials. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import httpx +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController + + +def run(args): + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False) + config_path = root / "node-config.json" + configuration = { + "schema_version": 1, + "models": [ + { + "manifest": str(args.manifest.resolve()), + "initial_peers": [], + "execution": "local", + "cache_dir": str(args.cache.resolve()), + "local_device": args.device, + "local_max_context": 1024, + "local_max_new_tokens": 64, + } + ], + "max_loaded_models": 1, + "auto_model_priority": ["local-qwen"], + "contribution_policy": {"sharing_enabled": False}, + } + config_path.write_text(json.dumps(configuration), encoding="utf-8") + command = [str(args.node.resolve())] if args.node else [sys.executable, "-m", "drift.cli", "node"] + command += ["--config", str(config_path), "--data_dir", str(root), "--port", str(args.port)] + environment = dict(os.environ, HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HUB_DISABLE_XET="1") + evidence = { + "result": "failed", + "packaged": args.node is not None, + "offline": True, + "manifest": str(args.manifest.resolve()), + "device_requested": args.device, + } + process = None + try: + with (root / "node.log").open("wb") as log: + process = subprocess.Popen( + command, + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + url = f"http://127.0.0.1:{args.port}" + deadline = time.monotonic() + 120 + with httpx.Client(base_url=url, timeout=180) as api: + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"node stopped before readiness: {process.returncode}; inspect node.log") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(1) + else: + raise TimeoutError("local node startup deadline") + control = (root / "control-api.key").read_text().strip() + key = (root / "local-api.key").read_text().strip() + client = NodeClient(url, control) + controller = DesktopController(client) + evidence["before"] = controller.snapshot() + assert evidence["before"]["auto_selection"]["source"] == "local" + assert api.post("/v1/completions", json={"model": "auto", "prompt": "Hello"}).status_code == 401 + api.headers["Authorization"] = "Bearer " + key + payload = {"model": "auto", "prompt": "The capital of France is", "max_tokens": 8, "temperature": 0} + started = time.monotonic() + reply = api.post("/v1/completions", json=payload) + reply.raise_for_status() + evidence["completion_seconds"] = time.monotonic() - started + evidence["completion"] = reply.json() + assert evidence["completion"]["usage"]["completion_tokens"] > 0 + assert "paris" in evidence["completion"]["choices"][0]["text"].casefold() + budget = api.post("/v1/completions", json={**payload, "max_tokens": 65}) + assert budget.status_code == 400 + evidence["token_budget_rejected"] = True + client.set_inference_mode("local_only") + assert controller.snapshot()["inference_mode"] == "local_only" + assert json.loads(config_path.read_text())["inference_mode"] == "local_only" + evidence["local_only_persisted"] = True + short_chat = api.post( + "/v1/chat/completions", + json={ + "model": "auto", + "max_tokens": 16, + "temperature": 0, + "enable_thinking": False, + "messages": [ + {"role": "system", "content": "Reply with only the city name."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + }, + ) + short_chat.raise_for_status() + evidence["short_chat"] = short_chat.json() + assert "paris" in evidence["short_chat"]["choices"][0]["message"]["content"].casefold() + chunks = [] + with api.stream( + "POST", + "/v1/chat/completions", + json={ + "model": "auto", + "max_tokens": 64, + "temperature": 0, + "stream": True, + "messages": [{"role": "user", "content": "Count from one to twenty."}], + }, + ) as response: + response.raise_for_status() + for line in response.iter_lines(): + if line.startswith("data: {"): + chunk = json.loads(line[6:]) + if chunk.get("choices", [{}])[0].get("delta", {}).get("content"): + chunks.append(chunk) + break # A real client disconnect must stop the generation and release its lease. + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + status = client.status() + if not any(model["active_requests"] for model in status["models"]): + break + time.sleep(0.25) + else: + raise TimeoutError("stream cancellation did not release the model lease") + assert chunks + evidence["stream_cancel_released_lease"] = True + evidence["after"] = status + memory = status["models"][0]["route"].get("memory") + if args.device.startswith("cuda"): + assert memory is not None + assert memory["peak_reserved_bytes"] <= memory["budget_bytes"] + client.set_inference_mode("auto") + evidence["result"] = "passed" + except BaseException as exc: + evidence["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + evidence["node_stopped"] = process is None or process.poll() is not None + (root / "result.json").write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"result": evidence["result"], "evidence": str(root / "result.json")})) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--node", type=Path) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--port", type=int, default=18087) + run(parser.parse_args()) diff --git a/scripts/qualify_qwen_modal_transport.py b/scripts/qualify_qwen_modal_transport.py new file mode 100644 index 000000000..bb28760a7 --- /dev/null +++ b/scripts/qualify_qwen_modal_transport.py @@ -0,0 +1,120 @@ +"""Bounded Modal raw-TCP reachability check before launching model containers.""" + +import json +import os +import secrets +import socket +import subprocess +import sys +import time +from pathlib import Path + +import modal + + +def confirm_app_stopped(app_id, timeout=60): + """Modal's app list is eventually consistent after a successful stop.""" + subprocess.run( + [sys.executable, "-m", "modal", "app", "stop", "--yes", app_id], + capture_output=True, + text=True, + encoding="utf-8", + timeout=60, + ) + observations = [] + until = time.time() + timeout + while time.time() < until: + listed = subprocess.run( + [sys.executable, "-m", "modal", "app", "list", "--json"], + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + check=True, + ) + record = next((v for v in json.loads(listed.stdout) if v["app_id"] == app_id), None) + observations.append(record) + if record and record["state"].casefold() == "stopped" and int(record["tasks"]) == 0: + return {"verified": True, "observations": observations} + time.sleep(2) + return {"verified": False, "observations": observations} + + +def main(): + os.environ["PYTHONIOENCODING"] = "utf-8" + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8") + root = Path(__file__).resolve().parents[1] + run_id = time.strftime("q38mt-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(3) + path = root / ".gate13-runs/qwen-modal-transport" / run_id + path.mkdir(parents=True, exist_ok=False) + result = {"result": "failed", "run_id": run_id, "scope": "raw-tcp-nonce-probe", "model_inference": False} + sandbox = None + app = modal.App(run_id) + program = ( + "import socket\n" + "s=socket.socket(); s.bind(('0.0.0.0',31330)); s.listen()\n" + "while True:\n" + " c,a=s.accept(); c.settimeout(5)\n" + " try: c.sendall(b'communityai:'+c.recv(64))\n" + " except OSError: pass\n" + " finally: c.close()\n" + ) + try: + with modal.enable_output(), app.run(): + try: + sandbox = modal.Sandbox.create( + "python", + "-u", + "-c", + program, + app=app, + image=modal.Image.debian_slim(python_version="3.12"), + cpu=(0.125, 0.25), + memory=(128, 256), + timeout=180, + unencrypted_ports=[31330], + tags={"communityai-run": run_id}, + ) + result["sandbox_id"] = sandbox.object_id + (path / "resource.json").write_text(json.dumps(result), encoding="utf-8") + address = sandbox.tunnels()[31330].tcp_socket + nonce = secrets.token_hex(12).encode() + until = time.time() + 60 + while time.time() < until: + try: + with socket.create_connection(address, timeout=10) as connection: + connection.sendall(nonce) + reply = connection.recv(128) + if reply != b"communityai:" + nonce: + raise RuntimeError("Modal tunnel nonce response mismatch") + break + except OSError: + time.sleep(2) + else: + raise TimeoutError("Modal TCP tunnel did not become reachable") + result.update(result="passed", tcp_nonce_verified=True) + finally: + if sandbox is not None: + sandbox.terminate(wait=True) + result["cleanup_verified"] = sandbox.poll() is not None + except BaseException as exc: + result.update(result="failed", error=f"{type(exc).__name__}: {exc}") + finally: + if app.app_id: + result["app_id"] = app.app_id + result["app_cleanup"] = confirm_app_stopped(app.app_id) + result["app_stop_confirmed"] = result["app_cleanup"]["verified"] + result["cleanup_verified"] = result["app_stop_confirmed"] and ( + sandbox is None or sandbox.poll() is not None + ) + if not result.get("cleanup_verified"): + result["result"] = "failed" + (path / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"result": result["result"], "evidence": str(path / "result.json")}), flush=True) + return 0 if result["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qualify_qwen_remote_product.py b/scripts/qualify_qwen_remote_product.py new file mode 100644 index 000000000..7f894b55e --- /dev/null +++ b/scripts/qualify_qwen_remote_product.py @@ -0,0 +1,321 @@ +"""Real packaged Windows/Linux client against an owned, already qualified mixed route.""" + +import argparse +import hashlib +import json +import os +import subprocess +import threading +import time +from pathlib import Path + +import httpx +import psutil +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController +from qwen_offline_http import HttpDownloadBlocker + +ROOT = Path(__file__).resolve().parents[1] +LOCAL = "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0" +REMOTE = "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + + +def run(args): + cloud = args.cloud_run.resolve() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + ready = json.loads((cloud / "packaged-client-ready.json").read_text()) + assert ready["run_id"] == cloud.name + deadline = min(time.time() + 3300, ready["deadline_unix"] - 30) + if deadline <= time.time(): + raise RuntimeError("Owned cloud client window has expired") + bundle = ROOT / "public-alpha/catalog-qwen-v2" + cache = args.remote_cache.resolve() if args.remote_cache else output / "model-cache" + seeded_cache = args.remote_cache is not None + if seeded_cache and (args.cache_provenance is None or not args.cache_provenance.is_file()): + raise ValueError("A reused remote cache requires its acquisition provenance receipt") + config = { + "schema_version": 1, + "max_loaded_models": 2, + "inference_mode": "auto", + "auto_model_priority": [REMOTE, LOCAL], + "models": [ + { + "manifest": str(ROOT / "manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json"), + "execution": "local", + "initial_peers": [], + "local_device": args.device, + "cache_dir": str(args.local_cache.resolve()), + "local_max_new_tokens": 64, + }, + { + "manifest": str(ROOT / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json"), + "initial_peers": json.loads((cloud / "bootstrap.json").read_text())["peers"], + "cache_dir": str(cache), + "request_timeout": 180, + "max_retries": 2, + }, + ], + "catalog_path": str(bundle / "catalog.signed.json"), + "catalog_bootstrap_path": str(bundle / "catalog-bootstrap.json"), + "catalog_refresh_seconds": 86400, + "discovery_update_period": 5, + "contribution_policy": {"sharing_enabled": False}, + } + config_path = output / "node-config.json" + config_path.write_text(json.dumps(config, indent=2)) + evidence = { + "result": "failed", + "run_id": cloud.name, + "packaged": True, + "scope": "packaged-client-community-chat-local-preference-and-offline-cache-restart", + "catalog_scope": "staged signed public sequence 2, explicit test configuration", + "node_sha256": hashlib.sha256(args.node.read_bytes()).hexdigest(), + "initial_remote_cache_empty": not cache.exists(), + "remote_cache_source": "explicitly seeded cache" if seeded_cache else "direct Hub cold acquisition", + "cache_provenance": json.loads(args.cache_provenance.read_text()) if seeded_cache else None, + "direct_hub_cold_acquisition_claimed": not seeded_cache, + "phases": [], + } + process = None + http_blocker = None + sampler = None + stop_sampling = threading.Event() + + def checkpoint(stage): + temporary = output / "progress.tmp" + temporary.write_text(json.dumps(dict(evidence, stage=stage, observed_at_unix=time.time()), indent=2)) + temporary.replace(output / "progress.json") + print(json.dumps({"stage": stage}), flush=True) + + try: + for offline in (False, True): + phase = {"hub_offline": offline, "initial_cache_seeded": seeded_cache} + phase["peak_observed_process_tree_rss_bytes"] = 0 + evidence["phases"].append(phase) + node = args.warm_node if offline and args.warm_node else args.node + if offline and args.warm_node: + if args.warm_ready_file is None: + raise ValueError("A warm replacement package requires its successful-build receipt") + while not args.warm_ready_file.is_file(): + if time.time() >= deadline: + raise TimeoutError("Replacement package was not verified within the cloud window") + time.sleep(5) + phase["node_sha256"] = hashlib.sha256(node.read_bytes()).hexdigest() + phase["application_replaced"] = offline and args.warm_node is not None + env = dict(os.environ, HF_HUB_DISABLE_XET="1", HF_HUB_DISABLE_IMPLICIT_TOKEN="1") + for key in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"): + env.pop(key, None) + if offline: + env[key] = "1" + if offline: + http_blocker = HttpDownloadBlocker() + env = http_blocker.environment(env) + phase["http_downloads_blocked"] = True + with (output / ("warm-node.log" if offline else "cold-node.log")).open("wb") as log: + process = subprocess.Popen( + [ + str(node.resolve()), + "--config", + str(config_path), + "--data_dir", + str(output / "data"), + "--port", + str(args.port), + ], + env=env, + stdout=log, + stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + + def sample_memory(): + while not stop_sampling.wait(1): + try: + root = psutil.Process(process.pid) + total = 0 + for child in [root, *root.children(recursive=True)]: + try: + total += child.memory_info().rss + except psutil.NoSuchProcess: + pass + phase["peak_observed_process_tree_rss_bytes"] = max( + phase["peak_observed_process_tree_rss_bytes"], total + ) + except psutil.NoSuchProcess: + return + + stop_sampling.clear() + sampler = threading.Thread(target=sample_memory, daemon=True) + sampler.start() + url = f"http://127.0.0.1:{args.port}" + with httpx.Client(base_url=url, timeout=600) as api: + startup_deadline = min(deadline, time.time() + 120) + while time.time() < startup_deadline: + if process.poll() is not None: + raise RuntimeError("Packaged node exited during startup") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(2) + else: + raise TimeoutError("Packaged node startup deadline") + client = NodeClient(url, (output / "data/control-api.key").read_text().strip()) + api.headers["Authorization"] = "Bearer " + (output / "data/local-api.key").read_text().strip() + + def wait_remote(until=deadline): + while time.time() < min(deadline, until): + status = client.status() + (output / "latest-status.json").write_text(json.dumps(status, indent=2)) + if status["auto_selection"].get("manifest_digest") == REMOTE: + return status + if process.poll() is not None: + raise RuntimeError("Packaged node exited during route qualification") + time.sleep(5) + raise TimeoutError("Public catalog route thresholds were not satisfied") + + status = wait_remote() + phase["promoted_status"] = status + phase["desktop_snapshot"] = DesktopController(client).snapshot() + + def infer(chat=False): + started = time.monotonic() + payload = {"model": "auto", "max_tokens": 3, "temperature": 0} + payload.update( + { + "messages": [ + {"role": "system", "content": "Reply with only the city name."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + "enable_thinking": False, + "max_tokens": 6, + } + if chat + else {"prompt": "The capital of France is"} + ) + remaining = deadline - time.time() + if remaining <= 0: + raise TimeoutError("Owned packaged-client window expired before inference") + response = api.post( + "/v1/chat/completions" if chat else "/v1/completions", + json=payload, + timeout=min(600, remaining), + ) + response.raise_for_status() + value = response.json() + assert value["usage"]["completion_tokens"] > 0 + if not chat: + assert "paris" in value["choices"][0]["text"].lower() + else: + answer = value["choices"][0]["message"]["content"].lower() + assert "paris" in answer and "" not in answer + return {"response": value, "seconds": time.monotonic() - started} + + phase["intervening_local_fallbacks"] = [] + + def community_infer(chat=False): + until = min(deadline, time.time() + 600) + for attempt in range(3): + wait_remote(until) + reply = infer(chat) + if reply["response"]["model"] == "Qwen3.8 27B FP8 Dequant": + return reply + assert reply["response"]["model"] == "Qwen3.5-0.8B-Local" + phase["intervening_local_fallbacks"].append(dict(reply, chat=chat)) + raise AssertionError("Three freshly selected auto requests fell back before acquiring Qwen3.8") + + phase["community_completion"] = community_infer() + checkpoint("offline-completion" if offline else "online-completion") + phase["community_chat"] = community_infer(chat=True) + checkpoint("offline-chat" if offline else "online-chat") + if args.worker_recovery and not offline: + from qwen_packaged_worker_action import worker_action + + loss = phase["worker_outage"] = {"scope": "CPU worker service stop and same-identity restart"} + # A slow chat can temporarily make the route ineligible. + # Re-establish community selection before injecting loss, + # so an already-local client cannot count as a downgrade. + loss["before_stop_status"] = wait_remote(min(deadline, time.time() + 600)) + checkpoint("community-selected-before-worker-loss") + stopped = True + try: + loss["stop"] = worker_action(cloud, "stop") + checkpoint("worker-stopped") + until = min(deadline, time.time() + 240) + while time.time() < until: + status = client.status() + if status["auto_selection"].get("source") == "local": + loss["fallback_status"] = status + break + time.sleep(2) + else: + raise TimeoutError("Packaged auto did not fall back after worker outage") + loss["local_completion"] = infer() + assert loss["local_completion"]["response"]["model"] == "Qwen3.5-0.8B-Local" + checkpoint("local-after-worker-loss") + finally: + if stopped: + loss["restart"] = worker_action(cloud, "start") + checkpoint("worker-restarted") + loss["recovered_status"] = wait_remote(min(deadline, time.time() + 600)) + loss["community_after_rejoin"] = community_infer() + checkpoint("community-after-rejoin") + client.set_inference_mode("local_only") + phase["local_only_completion"] = infer() + assert phase["local_only_completion"]["response"]["model"] == "Qwen3.5-0.8B-Local" + checkpoint("offline-local-only" if offline else "online-local-only") + client.set_inference_mode("auto") + process.terminate() + process.wait(timeout=30) + stop_sampling.set() + sampler.join(timeout=5) + phase["node_stopped"] = True + if http_blocker is not None: + phase["denied_http_requests"] = http_blocker.denied_requests + http_blocker.close() + http_blocker = None + print( + json.dumps({"phase": "offline-cache" if offline else "online", "result": "passed"}), + flush=True, + ) + evidence["result"] = "passed" + except BaseException as exc: + evidence["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + stop_sampling.set() + if sampler is not None: + sampler.join(timeout=5) + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + evidence["node_stopped"] = process is None or process.poll() is not None + if http_blocker is not None: + phase["denied_http_requests"] = http_blocker.denied_requests + http_blocker.close() + (output / "result.json").write_text(json.dumps(evidence, indent=2)) + temporary = cloud / "packaged-client-result.tmp" + temporary.write_text(json.dumps(evidence, indent=2)) + temporary.replace(cloud / "packaged-client-result.json") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--node", type=Path, required=True) + parser.add_argument("--warm-node", type=Path) + parser.add_argument("--warm-ready-file", type=Path) + parser.add_argument("--cloud-run", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--local-cache", type=Path, required=True) + parser.add_argument("--remote-cache", type=Path) + parser.add_argument("--cache-provenance", type=Path) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--worker-recovery", action="store_true") + parser.add_argument("--port", type=int, default=18089) + run(parser.parse_args()) diff --git a/scripts/qualify_qwen_resource_controls.py b/scripts/qualify_qwen_resource_controls.py new file mode 100644 index 000000000..5e301c220 --- /dev/null +++ b/scripts/qualify_qwen_resource_controls.py @@ -0,0 +1,191 @@ +"""Bounded packaged resource admission using real host telemetry and control API.""" + +import argparse +import datetime +import hashlib +import json +import os +import subprocess +import time +from pathlib import Path + +import httpx +from communityai_desktop.client import NodeClient, NodeClientError + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + config = json.loads(args.config.read_text()) + config["contribution_policy"]["sharing_enabled"] = False + config_path = output / "node-config.json" + config_path.write_text(json.dumps(config, indent=2)) + result = {"result": "failed", "packaged": True, "scope": "resource-admission-guards", "checks": {}} + process, client = None, None + try: + with (output / "node.log").open("wb") as log: + process = subprocess.Popen( + [ + str(args.node.resolve()), + "--config", + str(config_path), + "--data_dir", + str(output / "data"), + "--port", + str(args.port), + ], + stdout=log, + stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + url = f"http://127.0.0.1:{args.port}" + with httpx.Client(base_url=url, timeout=180) as api: + until = time.monotonic() + 120 + while time.monotonic() < until: + if process.poll() is not None: + raise RuntimeError("Packaged node stopped during startup") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(2) + else: + raise TimeoutError("Node startup") + client = NodeClient(url, (output / "data/control-api.key").read_text().strip()) + api.headers["Authorization"] = "Bearer " + (output / "data/local-api.key").read_text().strip() + baseline = client.get_contribution_policy()["policy"] + tomorrow = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1) + day = ("mon", "tue", "wed", "thu", "fri", "sat", "sun")[tomorrow.weekday()] + cases = [ + ( + "schedule", + { + "schedule": { + "timezone": "UTC", + "windows": [{"days": [day], "start": "09:00", "end": "10:00"}], + } + }, + lambda w: not w["schedule_admitted"] and "schedule" in (w["schedule_reason"] or ""), + ), + ( + "power", + {"max_power_watts": 1}, + lambda w: not w["resource_admitted"] and "power" in (w["resource_reason"] or ""), + ), + ( + "bandwidth", + {"max_bandwidth_mbps": 0.000001}, + lambda w: not w["resource_admitted"] and "bandwidth" in (w["resource_reason"] or ""), + ), + ( + "storage", + {"max_disk_space": "1MiB"}, + lambda w: not w["policy_admitted"] + and any(s in (w["policy_reason"] or "").lower() for s in ("disk", "artifact", "storage")), + ), + ] + for name, change, predicate in cases: + policy = client.get_contribution_policy() + client.update_contribution_policy( + dict(baseline, sharing_enabled=True, **change), expected_revision=policy["config_revision"] + ) + until = time.monotonic() + 180 + while time.monotonic() < until: + worker = client.list_workers()[0] + (output / (name + "-latest-worker.json")).write_text(json.dumps(worker, indent=2)) + other_guards_open = ( + (name == "storage" or worker["policy_admitted"]) + and (name == "schedule" or worker["schedule_admitted"]) + and (name in ("power", "bandwidth") or worker["resource_admitted"]) + ) + if ( + worker["state"] == "paused" + and predicate(worker) + and worker["pid"] is None + and other_guards_open + ): + break + time.sleep(2) + else: + raise TimeoutError(name + " guard did not pause the worker") + try: + client.worker_action(worker["id"], "start") + except NodeClientError: + pass + else: + raise AssertionError(name + " guard admitted a forbidden start") + assert client.list_workers()[0]["pid"] is None + result["checks"][name] = { + k: worker.get(k) + for k in ( + "state", + "pid", + "policy_admitted", + "policy_reason", + "resource_admitted", + "resource_reason", + "schedule_admitted", + "schedule_reason", + "current_bandwidth_mbps", + "current_power_watts", + "max_disk_space_bytes", + "max_vram_bytes", + ) + } + result["checks"][name]["start_rejected"] = True + response = api.post( + "/v1/completions", + json={"model": "auto", "prompt": "The capital of France is", "max_tokens": 3, "temperature": 0}, + ) + response.raise_for_status() + reply = response.json() + assert reply["model"] == "Qwen3.5-0.8B-Local" and "paris" in reply["choices"][0]["text"].lower() + result["checks"][name]["local_tokens"] = reply["usage"]["completion_tokens"] + # A resource-suspended worker still has Start intent. The + # real editor requires explicit Pause before changing limits. + for configured_worker in client.list_workers(): + client.worker_action(configured_worker["id"], "pause") + policy = client.get_contribution_policy() + client.update_contribution_policy( + dict(baseline, sharing_enabled=False), expected_revision=policy["config_revision"] + ) + print(json.dumps({"check": name, "result": "passed"}), flush=True) + result["node_sha256"] = hashlib.sha256(args.node.read_bytes()).hexdigest() + result["complete_gate14"] = False + result["limitations"] = [ + "Power and bandwidth are sampled host telemetry pause guards, not OS hard caps or traffic shapers.", + "Tiny thresholds prove blocked admission; sustained load, overshoot and automatic resumption are separate checks.", + "Storage checks declared manifested artifact admission, not total disk-cache quota.", + "Control API admission test; literal desktop slider acceptance is recorded separately.", + ] + result["platform"] = os.name + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + if client is not None: + try: + for worker in client.list_workers(): + client.worker_action(worker["id"], "pause") + except Exception: + pass + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + result["node_stopped"] = process is None or process.poll() is not None + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--node", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--port", type=int, default=18088) + run(parser.parse_args()) diff --git a/scripts/qualify_qwen_resource_desktop.py b/scripts/qualify_qwen_resource_desktop.py new file mode 100644 index 000000000..41a1ba160 --- /dev/null +++ b/scripts/qualify_qwen_resource_desktop.py @@ -0,0 +1,447 @@ +"""Run real Qwen sharing through the packaged resource sliders on one native host.""" + +import argparse +import hashlib +import json +import os +import statistics +import subprocess +import threading +import time +from pathlib import Path + +import httpx +import psutil +import torch +from communityai_desktop.client import NodeClient +from communityai_desktop.credentials import CredentialMissingError, NativeCredentialStore +from hivemind import DHT +from qualify_qwen_sharing_product import process_tree, wait_tree_gone + +from drift import AutoDistributedConfig +from drift.client.remote_sequential import RemoteSequential +from drift.model_manifest import ManifestArtifactVerifier, ModelManifest +from drift.protocol_identity import NodeIdentity + +ROOT = Path(__file__).resolve().parents[1] +COMMUNITY = ROOT / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json" +LOCAL = ROOT / "manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json" + + +def write_json(path, value): + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def rpc_load(manifest, cache, dht, peer, block_range, *, samples=6, tokens=128, seconds=20): + verifier = ManifestArtifactVerifier( + manifest, manifest.source.repository, manifest.source.revision, token=False, cache_dir=cache + ) + config = AutoDistributedConfig.from_pretrained(verifier.ensure_startup_metadata(), local_files_only=True) + config.dht_prefix = manifest.dht_prefix + config.manifest_digest = manifest.digest + config.manifest_execution_profile = manifest.runtime.to_dict() + config.allowed_servers = [peer] + config.request_timeout = 90 + config.max_retries = 1 + config.update_period = 2 + config.show_route = False + start, end = map(int, block_range.split(":")) + remote = RemoteSequential(config, dht=dht, start_block=start, end_block=end) + torch.manual_seed(14) + inputs = torch.randn(1, tokens, config.hidden_size, dtype=torch.bfloat16) + durations = [] + output_hashes = [] + gpu_samples = [] + stop = threading.Event() + + def sample_gpu(): + import pynvml + + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + try: + while not stop.is_set(): + utilization = pynvml.nvmlDeviceGetUtilizationRates(handle) + gpu_samples.append({"at": time.monotonic(), "gpu_percent": utilization.gpu}) + stop.wait(0.2) + finally: + pynvml.nvmlShutdown() + + sampler = threading.Thread(target=sample_gpu, daemon=True) + try: + remote.sequence_manager.make_sequence(mode="min_latency", cache_tokens_needed=tokens) + sampler.start() + time.sleep(3) + baseline_until = time.monotonic() + index = 0 + measured_until = float("inf") + with torch.inference_mode(), remote.inference_session(max_length=tokens) as session: + while index < samples + 2 or time.monotonic() < measured_until: + session.position = 0 + started = time.monotonic() + output = remote(inputs) + duration = time.monotonic() - started + assert output.shape == inputs.shape and torch.isfinite(output).all().item() + if index >= 2: + if index == 2: + measured_until = started + seconds + measured_from = started + durations.append(duration) + if len(output_hashes) < samples: + output_hashes.append( + hashlib.sha256(output.contiguous().view(torch.uint8).numpy().tobytes()).hexdigest() + ) + index += 1 + stop.set() + sampler.join(timeout=5) + baseline = [s["gpu_percent"] for s in gpu_samples if s["at"] < baseline_until] + active = [s["gpu_percent"] for s in gpu_samples if s["at"] >= measured_from + 1] + assert len(active) >= 10, "GPU utilization sampling did not cover the workload" + return { + "block_indices": block_range, + "tokens_per_request": tokens, + "pattern": "repeated prefill in one admitted session, rewound before each request", + "request_seconds": durations, + "median_seconds": statistics.median(durations), + "finite_outputs": True, + "output_hashes": output_hashes, + "gpu_utilization": { + "scope": "whole GPU; other applications remain untouched", + "sample_interval_seconds": 0.2, + "baseline_percent": baseline, + "workload_percent": active, + "workload_mean_percent": statistics.mean(active), + }, + } + finally: + stop.set() + if sampler.is_alive(): + sampler.join(timeout=5) + remote.sequence_manager.shutdown() + + +def run(args): + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False) + desktop = args.desktop.resolve() + store = NativeCredentialStore("org.communityai.gate14." + root.name, "control") + try: + store.get() + except CredentialMissingError: + pass + else: + raise RuntimeError("The qualification credential already exists") + dht = DHT( + start=True, num_workers=2, host_maddrs=["/ip4/127.0.0.1/tcp/0"], use_relay=False, use_auto_relay=False, tls=True + ) + peers = [str(address) for address in dht.get_visible_maddrs()] + identity_path = (args.identity_path or root / "worker-identity.key").resolve() + identity = NodeIdentity.load(identity_path) if args.identity_path else NodeIdentity.create(identity_path) + manifest = ModelManifest.load(COMMUNITY) + config = { + "schema_version": 1, + "max_loaded_models": 2, + "inference_mode": "local_only", + "auto_model_priority": [manifest.digest_id, ModelManifest.load(LOCAL).digest_id], + "models": [ + { + "manifest": str(LOCAL), + "execution": "local", + "initial_peers": [], + "cache_dir": str(args.local_cache.resolve()), + "local_device": args.device, + "local_max_new_tokens": 64, + "local_max_context": 1024, + }, + {"manifest": str(COMMUNITY), "initial_peers": peers, "cache_dir": str(args.worker_cache.resolve())}, + ], + "discovery_update_period": 2, + "workers": [ + { + "id": "automatic", + "model": "auto", + "num_blocks": 1, + "enabled": False, + "identity_path": str(identity_path), + "device": args.device, + "cache_dir": str(args.worker_cache.resolve()), + "throughput": 0.01, + } + ], + "contribution_policy": {"sharing_enabled": False, "max_disk_space": "8GiB", "max_vram": "100%"}, + } + config_path = root / "node-config.json" + write_json(config_path, config) + steps = [ + {"action": "observe", "vram_percent": 100, "processing_percent": 100}, + {"action": "limits", "vram_percent": 25, "processing_percent": 100}, + {"action": "start"}, + {"action": "limits", "vram_percent": 25, "processing_percent": 50}, + {"action": "limits", "vram_percent": 20, "processing_percent": 25}, + {"action": "limits", "vram_percent": 1, "processing_percent": 25}, + {"action": "limits", "vram_percent": 25, "processing_percent": 100}, + {"action": "pause"}, + {"action": "start"}, + {"action": "pause"}, + ] + result = { + "scope": "native-packaged-Qwen-resource-controls", + "result": "failed", + "platform": os.name, + "device": args.device, + "manifest_digest": manifest.digest_id, + "desktop_sha256": hashlib.sha256(desktop.read_bytes()).hexdigest(), + "node_sha256": hashlib.sha256( + (desktop.parent / "node" / ("CommunityAI-Node.exe" if os.name == "nt" else "CommunityAI-Node")).read_bytes() + ).hexdigest(), + "observations": [], + } + gui = None + client = None + owned = [] + try: + for stage, stage_steps in ( + ("initial", steps), + ("restart", [{"action": "observe", "vram_percent": 25, "processing_percent": 100}]), + ): + plan_path, ui_path, ack = (root / f"{stage}-{name}.json" for name in ("plan", "ui", "ack")) + write_json(plan_path, {"steps": stage_steps, "timeout_seconds": 2700, "acknowledgement": str(ack)}) + command = [ + str(desktop), + "--node-url", + f"http://127.0.0.1:{args.port}", + "--node-config", + str(config_path), + "--node-data-dir", + str(root / "data"), + "--credential-service", + store.service, + "--credential-account", + store.account, + "--resource-ui-playthrough", + str(plan_path), + "--resource-ui-evidence", + str(ui_path), + ] + with (root / f"{stage}-gui.log").open("wb") as log: + gui = subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + deadline = time.monotonic() + 2700 + for index, step in enumerate(stage_steps): + while time.monotonic() < deadline: + if gui.poll() is not None: + raise RuntimeError(f"Packaged GUI exited during {stage} step {index}") + if ui_path.exists(): + ui = json.loads(ui_path.read_text()) + if ui.get("result") == "failed": + raise RuntimeError(f"Packaged UI failed: {ui.get('error')}") + if len(ui["steps"]) > index: + break + time.sleep(0.2) + else: + raise TimeoutError(f"Packaged UI step {index}") + if client is None: + client = NodeClient(f"http://127.0.0.1:{args.port}", store.get(), timeout=30) + owned = process_tree(gui.pid) + observation = {"stage": stage, "step": index, **step} + previous_tree = result.get("_worker_tree", []) + if step["action"] in ("limits", "pause") and previous_tree: + wait_tree_gone(previous_tree) + observation["old_worker_tree_gone"] = True + result["_worker_tree"] = [] + should_run = stage == "initial" and index in (2, 3, 4, 6, 8) + until = min(deadline, time.monotonic() + (900 if index == 2 else 240)) + restart_baseline = None + while time.monotonic() < until: + worker = client.list_workers()[0] + write_json(root / "latest-worker.json", worker) + if should_run: + if restart_baseline is None: + restart_baseline = worker["restart_count"] + if worker["restart_count"] >= restart_baseline + 3: + raise RuntimeError("Worker repeatedly exited before readiness; see latest-worker.json") + if ( + worker["state"] == "running" + and worker["download_progress"] + and worker["download_progress"]["state"] == "ready" + ): + break + elif index == 5 and stage == "initial": + if ( + worker["pid"] is None + and worker["state"] == "paused" + and any( + word + in ( + (worker.get("placement_reason") or "") + + (worker.get("resource_reason") or "") + + (worker.get("policy_reason") or "") + ).lower() + for word in ("memory", "vram", "budget", "capacity") + ) + ): + break + elif worker["pid"] is None: + break + time.sleep(1) + else: + raise TimeoutError(f"Worker state for {stage} step {index}") + observation["worker_state"] = worker["state"] + saved = json.loads(config_path.read_text())["contribution_policy"] + if step["action"] in ("limits", "observe"): + assert saved.get("max_processing_percent", 100) == step["processing_percent"] + assert saved["max_vram"] == f"{step['vram_percent']}%" + observation["policy_persisted"] = True + if should_run: + result["_worker_tree"] = process_tree(worker["pid"]) + observation["block_indices"] = worker["block_indices"] + observation["max_vram_bytes"] = worker["max_vram_bytes"] + if args.device.startswith("cuda"): + total = torch.cuda.get_device_properties(args.device).total_memory + assert worker["max_vram_bytes"] <= int(total * float(saved["max_vram"].rstrip("%")) / 100) + progress = worker["download_progress"] + assert progress["verified_bytes"] == progress["selected_bytes"] > 0 + observation["verified_download_bytes"] = progress["verified_bytes"] + if index in (2, 3, 4) and not args.skip_rpc: + observation["rpc_load"] = rpc_load( + manifest, + args.worker_cache, + dht, + identity.peer_id, + worker["block_indices"], + tokens=args.tokens, + ) + if index == 5 and stage == "initial": + observation["low_memory_blocked_without_worker"] = True + observation["resource_reason"] = worker["resource_reason"] + assert worker["desired_running"] and worker["resource_suspended"] + time.sleep(5) + stable = client.list_workers()[0] + assert stable["pid"] is None and stable["restart_count"] == worker["restart_count"] + observation["no_restart_loop"] = True + inference_key = (root / "data/local-api.key").read_text().strip() + with httpx.Client(timeout=180) as api: + response = api.post( + f"http://127.0.0.1:{args.port}/v1/completions", + headers={"Authorization": "Bearer " + inference_key}, + json={ + "model": "auto", + "prompt": "The capital of France is", + "max_tokens": 3, + "temperature": 0, + }, + ) + response.raise_for_status() + reply = response.json() + assert reply["model"] == "Qwen3.5-0.8B-Local" and "paris" in reply["choices"][0]["text"].lower() + observation["local_inference_tokens"] = reply["usage"]["completion_tokens"] + result["observations"].append(observation) + write_json(root / "result.json", {k: v for k, v in result.items() if not k.startswith("_")}) + print( + json.dumps( + {"stage": stage, "step": index, "result": "passed", "worker_state": worker["state"]} + ), + flush=True, + ) + write_json(ack, index) + assert gui.wait(timeout=60) == 0 + wait_tree_gone(owned) + assert json.loads(ui_path.read_text())["result"] == "passed" + result[f"{stage}_owned_tree_gone"] = True + client = None + if not args.skip_rpc: + loads = [item["rpc_load"] for item in result["observations"] if "rpc_load" in item] + assert len(loads) == 3 and len({item["block_indices"] for item in loads}) == 1 + assert len({digest for item in loads for digest in item["output_hashes"]}) == 1 + result["same_Qwen_outputs_at_all_processing_limits"] = True + if args.with_admission_guards: + from qualify_qwen_resource_controls import run as run_admission_guards + + guard_output = root / "admission-guards" + run_admission_guards( + argparse.Namespace( + node=desktop.parent / "node" / ("CommunityAI-Node.exe" if os.name == "nt" else "CommunityAI-Node"), + config=config_path, + output=guard_output, + port=args.port + 1, + ) + ) + result["admission_guards"] = json.loads((guard_output / "result.json").read_text()) + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + cleanup_errors = [] + try: + if gui is not None and gui.poll() is None: + owned = process_tree(gui.pid) + subprocess.run( + [str(desktop), "--prepare-update"], + timeout=60, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + check=True, + ) + gui.wait(timeout=30) + if owned: + wait_tree_gone(owned) + except Exception as exc: + cleanup_errors.append(f"GUI cleanup: {type(exc).__name__}") + for pid, created in reversed(owned): + try: + process = psutil.Process(pid) + if process.create_time() == created: + process.kill() + except psutil.NoSuchProcess: + pass + # Reap our direct child before checking PID disappearance on Linux. + # A killed but unreaped GUI is still visible as a zombie process. + if gui is not None: + try: + gui.wait(timeout=30) + except Exception as exc: + cleanup_errors.append(f"GUI reap: {type(exc).__name__}") + try: + wait_tree_gone(owned) + except Exception as exc: + cleanup_errors.append(f"Owned tree cleanup: {type(exc).__name__}") + finally: + try: + store.delete() + result["test_credential_removed"] = True + except Exception as exc: + cleanup_errors.append(f"Credential cleanup: {type(exc).__name__}") + try: + dht.shutdown() + dht.join(timeout=20) + except Exception as exc: + cleanup_errors.append(f"DHT cleanup: {type(exc).__name__}") + result.pop("_worker_tree", None) + result["gui_stopped"] = gui is None or gui.poll() is not None + if cleanup_errors: + result["cleanup_errors"] = cleanup_errors + result["result"] = "failed" + write_json(root / "result.json", result) + if cleanup_errors: + raise RuntimeError("Qualification cleanup required intervention") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--desktop", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--local-cache", type=Path, required=True) + parser.add_argument("--worker-cache", type=Path, required=True) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--port", type=int, default=18096) + parser.add_argument("--tokens", type=int, default=128) + parser.add_argument("--identity-path", type=Path, help="An owned, stopped qualification worker's reusable identity") + parser.add_argument("--skip-rpc", action="store_true") + parser.add_argument("--with-admission-guards", action="store_true") + run(parser.parse_args()) diff --git a/scripts/qualify_qwen_sharing_product.py b/scripts/qualify_qwen_sharing_product.py new file mode 100644 index 000000000..443c1a5ac --- /dev/null +++ b/scripts/qualify_qwen_sharing_product.py @@ -0,0 +1,341 @@ +"""Bounded real Windows/Linux node check: local Qwen plus one automatic sharing worker.""" + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import httpx +import psutil +from communityai_desktop.client import NodeClient + +ROOT = Path(__file__).resolve().parents[1] + + +def worker_is_ready(status, worker, prior_ready_lines=()): + if worker["state"] != "running" or not worker["remote_acknowledged"]: + return False + if not any( + "Connection handlers are ready" in line and line not in prior_ready_lines for line in worker["recent_logs"] + ): + return False + start, end = map(int, worker["block_indices"].split(":")) + for model in status["models"]: + if model["id"] != worker["model"]: + continue + route = model["route"] + counts = route.get("replica_counts") + return ( + route["status"] in ("complete", "incomplete") + and isinstance(counts, list) + and len(counts) >= end + and all(counts[index] > 0 for index in range(start, end)) + ) + return False + + +def process_tree(pid): + root = psutil.Process(pid) + result = [] + for process in [root, *root.children(recursive=True)]: + try: + result.append((process.pid, process.create_time())) + except psutil.NoSuchProcess: + pass + return result + + +def wait_tree_gone(identities, timeout=30): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + remaining = [] + for pid, created in identities: + try: + process = psutil.Process(pid) + if process.create_time() == created and process.is_running(): + remaining.append(pid) + except psutil.NoSuchProcess: + pass + if not remaining: + return + time.sleep(0.2) + raise AssertionError(f"Pause left owned worker processes alive: {remaining}") + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + bundle = ROOT / "public-alpha/catalog-qwen-v2" + bootstrap = json.loads((bundle / "catalog-bootstrap.json").read_text()) + manifests = [ + ROOT / "manifests/candidates" / name + for name in ("qwen3.5-0.8b-local-bfloat16-eager.json", "qwen3.8-27b-fp8-dequant-eager.json") + ] + config = { + "schema_version": 1, + "max_loaded_models": 2, + "inference_mode": "local_only", + "auto_model_priority": [ + "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4", + "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0", + ], + "models": [ + { + "manifest": str(manifests[0]), + "execution": "local", + "initial_peers": [], + "cache_dir": str(args.local_cache.resolve()), + "local_device": args.device, + "local_max_new_tokens": 64, + "local_max_context": 1024, + }, + { + "manifest": str(manifests[1]), + "initial_peers": bootstrap["initial_peers"], + "cache_dir": str(args.worker_cache.resolve()), + }, + ], + "catalog_path": str(bundle / "catalog.signed.json"), + "catalog_bootstrap_path": str(bundle / "catalog-bootstrap.json"), + "catalog_refresh_seconds": 86400, + "discovery_update_period": 5, + "workers": [ + { + "id": "automatic", + "model": "auto", + "num_blocks": 1, + "enabled": True, + "identity_path": str( + args.identity_path.resolve() if args.identity_path else output / "worker-identity.key" + ), + "device": args.device, + "cache_dir": str(args.worker_cache.resolve()), + "throughput": 0.01, + } + ], + "contribution_policy": {"sharing_enabled": False, "max_disk_space": "8GiB", "max_vram": "2GiB"}, + } + config_path = output / "node-config.json" + if args.power_recovery_watts is not None: + if not 1 <= args.power_recovery_watts <= 300: + raise ValueError("Power recovery test requires a finite 1..300 W threshold") + config["contribution_policy"]["max_power_watts"] = args.power_recovery_watts + config_path.write_text(json.dumps(config, indent=2)) + command = [str(args.node.resolve())] if args.node else [sys.executable, "-m", "drift.cli", "node"] + command += ["--config", str(config_path), "--data_dir", str(output / "data"), "--port", str(args.port)] + evidence = { + "result": "failed", + "scope": "local-inference-and-one-automatic-contribution-worker", + "packaged": args.node is not None, + "complete_gate14": False, + "worker_budget_bytes": 2 * 1024**3, + "local_budget_bytes": 3 * 1024**3, + } + process, client = None, None + try: + with (output / "node.log").open("wb") as log: + env = dict(os.environ, HF_HUB_DISABLE_XET="1", HF_HUB_DISABLE_IMPLICIT_TOKEN="1") + env.pop("HF_HUB_OFFLINE", None) + env.pop("TRANSFORMERS_OFFLINE", None) + process = subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + env=env, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + url = f"http://127.0.0.1:{args.port}" + deadline = time.monotonic() + 1800 + with httpx.Client(base_url=url, timeout=180) as api: + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Node exited before readiness; inspect node.log") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(2) + control = (output / "data/control-api.key").read_text().strip() + inference = (output / "data/local-api.key").read_text().strip() + client = NodeClient(url, control) + + def infer(): + started = time.monotonic() + response = api.post( + "/v1/completions", + headers={"Authorization": "Bearer " + inference}, + json={"model": "auto", "prompt": "The capital of France is", "max_tokens": 3, "temperature": 0}, + ) + response.raise_for_status() + value = response.json() + assert "paris" in value["choices"][0]["text"].lower() + assert value["model"] == "Qwen3.5-0.8B-Local" + return {"seconds": time.monotonic() - started, "response": value} + + def wait_worker(predicate, *, timeout): + until = min(deadline, time.monotonic() + timeout) + previous = None + while time.monotonic() < until: + snapshot = client.status() + snapshot["workers"] = client.list_workers() + worker = snapshot["workers"][0] + compact = { + key: worker.get(key) + for key in ("state", "block_indices", "policy_reason", "resource_reason", "last_error") + } + if compact != previous: + print(json.dumps(compact), flush=True) + previous = compact + (output / "latest-status.json").write_text(json.dumps(snapshot, indent=2)) + if predicate(snapshot, worker): + return snapshot + if process.poll() is not None: + raise RuntimeError("Node stopped during sharing qualification") + time.sleep(5) + raise TimeoutError("Automatic sharing did not reach the required state") + + evidence["before_sharing"] = infer() + policy = client.get_contribution_policy() + client.update_contribution_policy( + dict(policy["policy"], sharing_enabled=True), expected_revision=policy["config_revision"] + ) + ready = wait_worker(worker_is_ready, timeout=1500) + worker = ready["workers"][0] + assert worker["automatic"] and worker["max_vram_bytes"] <= evidence["worker_budget_bytes"] + evidence["automatic_ready"] = ready + evidence["while_sharing"] = infer() + if args.power_recovery_watts is not None: + policy_before = client.get_contribution_policy() + tree_before = process_tree(worker["pid"]) + prior_ready = {line for line in worker["recent_logs"] if "Connection handlers are ready" in line} + samples = [] + load = None + try: + with (output / "bounded-gpu-load.log").open("wb") as load_log: + load = subprocess.Popen( + [ + sys.executable, + str(ROOT / "scripts/qwen_bounded_gpu_load.py"), + "--seconds", + "25", + "--device", + args.device, + ], + stdout=load_log, + stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + started_power = time.monotonic() + while time.monotonic() - started_power < 45: + observed = client.list_workers()[0] + samples.append( + { + "seconds": time.monotonic() - started_power, + "watts": observed["current_power_watts"], + "state": observed["state"], + "pid": observed["pid"], + "resource_admitted": observed["resource_admitted"], + "resource_reason": observed["resource_reason"], + } + ) + (output / "power-recovery-samples.json").write_text(json.dumps(samples, indent=2)) + if ( + observed["state"] == "paused" + and observed["pid"] is None + and not observed["resource_admitted"] + and "power" in (observed["resource_reason"] or "") + ): + break + time.sleep(0.2) + else: + raise TimeoutError("Real GPU load did not trigger a measured power pause") + wait_tree_gone(tree_before) + load.wait(timeout=45) + assert load.returncode == 0 + finally: + if load is not None and load.poll() is None: + load.terminate() + load.wait(timeout=10) + recovered_power = wait_worker( + lambda status, candidate: candidate["pid"] != worker["pid"] + and candidate["resource_admitted"] + and worker_is_ready(status, candidate, prior_ready), + timeout=240, + ) + assert client.get_contribution_policy() == policy_before + evidence["power_recovery"] = { + "threshold_watts": args.power_recovery_watts, + "sampled_pause_seconds": samples[-1]["seconds"], + "peak_observed_watts": max(s["watts"] for s in samples if s["watts"] is not None), + "paused_process_tree_gone": True, + "resumed_without_policy_change_or_start_command": True, + "recovered": recovered_power, + "local_after_recovery": infer(), + } + ready = recovered_power + worker = ready["workers"][0] + owned = process_tree(worker["pid"]) + started = time.monotonic() + client.worker_action("automatic", "pause") + paused = wait_worker( + lambda status, worker: worker["state"] == "paused" and worker["pid"] is None, timeout=45 + ) + wait_tree_gone(owned) + evidence["paused_worker_tree_gone"] = True + evidence["pause_seconds"] = time.monotonic() - started + evidence["paused"] = paused + evidence["after_pause"] = infer() + prior_ready_lines = { + line for line in paused["workers"][0]["recent_logs"] if "Connection handlers are ready" in line + } + client.worker_action("automatic", "start") + evidence["restarted"] = wait_worker( + lambda status, worker: worker["pid"] != ready["workers"][0]["pid"] + and worker_is_ready(status, worker, prior_ready_lines), + timeout=180, + ) + evidence["after_restart"] = infer() + restarted_tree = process_tree(evidence["restarted"]["workers"][0]["pid"]) + client.worker_action("automatic", "pause") + wait_worker(lambda status, worker: worker["state"] == "paused" and worker["pid"] is None, timeout=45) + wait_tree_gone(restarted_tree) + evidence["restarted_worker_tree_gone"] = True + evidence["result"] = "passed" + except BaseException as exc: + evidence["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + if client is not None: + try: + client.worker_action("automatic", "pause") + except Exception: + pass + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + evidence["node_stopped"] = process is None or process.poll() is not None + (output / "result.json").write_text(json.dumps(evidence, indent=2) + "\n") + print(json.dumps({"result": evidence["result"]}), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--local-cache", type=Path, required=True) + parser.add_argument("--worker-cache", type=Path, required=True) + parser.add_argument("--node", type=Path) + parser.add_argument( + "--identity-path", type=Path, help="Reuse an owned stopped test worker's identity across retries" + ) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--power-recovery-watts", type=float) + parser.add_argument("--port", type=int, default=18088) + run(parser.parse_args()) diff --git a/scripts/qualify_resource_processing.py b/scripts/qualify_resource_processing.py new file mode 100644 index 000000000..22f5d7437 --- /dev/null +++ b/scripts/qualify_resource_processing.py @@ -0,0 +1,109 @@ +"""Bounded, offline CPU/CUDA probes of the production contribution compute limiter. + +This measures synthetic tensor work through RuntimeWithDeduplicatedPools, not +Qwen inference or full Gate 14 acceptance. It downloads no models or software. +""" + +import argparse +import json +import time +from pathlib import Path +from types import SimpleNamespace + +import torch + +from drift.server.server import RuntimeWithDeduplicatedPools +from drift.utils.hardware import get_device_total_memory, set_device_memory_limit, synchronize_device + + +def qualify(device_name, output): + device = torch.device(device_name) + torch.set_num_threads(2) + output.mkdir(parents=True, exist_ok=False) + result = { + "scope": "synthetic-production-runtime-processing-budget", + "device": device_name, + "complete_gate14": False, + "runs": [], + } + if device.type == "cuda": + result["hardware"] = torch.cuda.get_device_name(device) + cap = 256 * 1024**2 + set_device_memory_limit(device, cap) + under = torch.empty(32 * 1024**2, device=device, dtype=torch.uint8) + try: + over = torch.empty(cap + 1024**2, device=device, dtype=torch.uint8) + except torch.OutOfMemoryError: + result["memory_allocator"] = { + "cap_bytes": cap, + "under_limit_succeeded": True, + "over_limit_rejected": True, + "device_total_bytes": get_device_total_memory(device), + } + else: + del over + raise AssertionError("GPU allocator accepted an allocation above its configured ceiling") + del under + torch.cuda.empty_cache() + set_device_memory_limit(device, get_device_total_memory(device)) + matrix = torch.randn(768, 768, device=device) + out = torch.empty_like(matrix) + for _ in range(5): + torch.mm(matrix, matrix, out=out) + synchronize_device(device) + for percent in (100, 50, 25): + backend = SimpleNamespace(get_pools=lambda: (), module=SimpleNamespace(devices=[device])) + runtime = RuntimeWithDeduplicatedPools( + {"probe": backend}, max_processing_percent=percent, processing_budget_path=output / "budget" + ) + work = [0.0] + steps = 0 + + def compute(): + started = time.monotonic() + while time.monotonic() - started < 0.025: + torch.mm(matrix, matrix, out=out) + synchronize_device(device) + work[0] += time.monotonic() - started + return (out,) + + pool = SimpleNamespace(process_func=compute) + started = time.monotonic() + try: + while time.monotonic() - started < 4: + values, _ = runtime.process_batch(pool, steps) + assert torch.isfinite(values[0]).all().item() + steps += 1 + elapsed = time.monotonic() - started + duty = work[0] / elapsed * 100 + assert duty <= percent + 1 + assert duty >= percent * 0.85 + result["runs"].append( + { + "requested_percent": percent, + "measured_compute_duty_percent": duty, + "wall_seconds": elapsed, + "compute_seconds": work[0], + "steps": steps, + } + ) + print(json.dumps(result["runs"][-1]), flush=True) + finally: + runtime.shutdown_trigger.set() + runtime.shutdown_recv.close() + runtime.shutdown_send.close() + result["result"] = "passed" + result["limitations"] = [ + "Compute duty cycle, not an instantaneous whole-device utilization guarantee", + "No Qwen model, download, cold startup or final-package lifecycle exercised", + "Local inference, downloads and other applications are outside the sharing compute budget", + ] + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device", choices=("cpu", "cuda:0"), required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + qualify(args.device, args.output) diff --git a/scripts/qwen_bounded_gpu_load.py b/scripts/qwen_bounded_gpu_load.py new file mode 100644 index 000000000..72549dd06 --- /dev/null +++ b/scripts/qwen_bounded_gpu_load.py @@ -0,0 +1,27 @@ +"""Generate at most 40 seconds of GPU work for a real contribution-pause check.""" + +import argparse +import json +import time + +import torch + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seconds", type=int, default=25, choices=range(1, 41)) + parser.add_argument("--device", default="cuda:0") + args = parser.parse_args() + if not args.device.startswith("cuda"): + parser.error("This bounded load generator requires a CUDA device") + # Three 32 MiB matrices; no model downloads or persistent storage. + left = torch.ones((4096, 4096), dtype=torch.float16, device=args.device) + right = torch.ones_like(left) + output = torch.empty_like(left) + started = time.monotonic() + count = 0 + with torch.inference_mode(): + while time.monotonic() - started < args.seconds: + torch.mm(left, right, out=output) + torch.cuda.synchronize(args.device) + count += 1 + print(json.dumps({"seconds": time.monotonic() - started, "matrix_products": count})) diff --git a/scripts/qwen_formation_desktop.py b/scripts/qwen_formation_desktop.py new file mode 100644 index 000000000..d8ee96d69 --- /dev/null +++ b/scripts/qwen_formation_desktop.py @@ -0,0 +1,160 @@ +"""Real Qt window attached to the formation test's isolated packaged node. + +Uses the same Qt automation hook as Gate 13. It observes live selections and +clicks the production local-only control; no fake node or mocked controller. +""" + +import argparse +import json +import time +from pathlib import Path + +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController +from communityai_desktop.pyside_shell import run +from qwen_formation_node import selected, write + + +class FormationDesktop: + def __init__(self, root): + self.root = root + self.completed = set() + self.clicked = set() + self.policy_opened = set() + self.baseline_paused = set() + + def install(self, window, application, qt): + self.window, self.application = window, application + self.qt = qt + original_failure = window._sharing_action_failed + + def sharing_failed(message): + write(self.root / "desktop-error.json", {"error": "Production desktop rejected sharing: " + str(message)}) + original_failure(message) + + window._sharing_action_failed = sharing_failed + self.timer = qt["QTimer"](window) + self.timer.setInterval(500) + self.timer.timeout.connect(self.tick) + self.timer.start() + + def tick(self): + try: + if (self.root / "desktop-stop").exists(): + self.application.quit() + return + window = self.window + if window._controller is None or window._busy: + return + path = self.root / "desktop-command.json" + if not path.exists(): + return + action = json.loads(path.read_text()) + identity = action["id"] + if identity in self.completed: + return + if action["action"] == "start-sharing": + window._page_buttons[2].click() + if not window._snapshot.get("contribution", {}).get("policy", {}).get("sharing_enabled"): + if identity not in self.policy_opened and window.edit_policy_button.isEnabled(): + self.policy_opened.add(identity) + self.qt["QTimer"].singleShot(100, self.enable_policy) + window.edit_policy_button.click() + return + if identity not in self.clicked: + # Gate 13 saves policy first, then normalizes an automatic + # start through the real per-model Pause control. Keep the + # persistent startup setting intact for restart recovery. + if window._snapshot.get("contribution", {}).get("intent_enabled"): + if identity in self.baseline_paused: + return + desired = {w["model"] for w in window._snapshot.get("workers", []) if w.get("desired_running")} + matches = [ + checkbox + for checkbox in window.findChildren(self.qt["QCheckBox"]) + if checkbox.accessibleName() in {"Share compute with " + name for name in desired} + and checkbox.isChecked() + ] + if len(matches) != 1 or not matches[0].isEnabled(): + return + self.baseline_paused.add(identity) + matches[0].click() + return + if not window.master_share_button.isEnabled(): + return + if window.master_share_button.text() != "Start sharing": + return + window._page_buttons[2].click() + window.master_share_button.click() + self.clicked.add(identity) + return + if not window._snapshot.get("contribution", {}).get("intent_enabled"): + return + if action["action"] == "toggle" and identity not in self.clicked: + if not window.inference_mode_button.isEnabled(): + return + window.inference_mode_button.click() + self.clicked.add(identity) + return + if not selected(window._snapshot, action["source"]): + return + if action.get("inference_mode") and window._snapshot.get("inference_mode") != action["inference_mode"]: + return + window._show_page(1) + screenshot = self.root / ("desktop-" + identity + ".png") + if not window.grab().save(str(screenshot)): + raise RuntimeError("Could not capture the real desktop window") + write( + self.root / ("desktop-response-" + identity + ".json"), + { + "result": "passed", + "id": identity, + "source": action["source"], + "real_window_visible": window.isVisible(), + "button_clicked": identity in self.clicked, + "baseline_pause_clicked": identity in self.baseline_paused, + "selection": window._snapshot["auto_selection"], + "inference_mode": window._snapshot["inference_mode"], + "title": window.auto_selection_title.text(), + "detail": window.auto_selection_detail.text(), + "screenshot": screenshot.name, + "ui_runtime": "production Qt source", + "fake_node": False, + }, + ) + self.completed.add(identity) + except BaseException as exc: + write(self.root / "desktop-error.json", {"error": f"{type(exc).__name__}: {exc}"}) + self.application.exit(1) + + def enable_policy(self): + try: + dialog = self.window.findChild(self.qt["QDialog"], "sharingPolicyDialog") + if dialog is None: + raise RuntimeError("Production sharing policy dialog did not open") + dialog.findChild(self.qt["QCheckBox"], "policy_sharing_enabled").setChecked(True) + buttons = dialog.findChild(self.qt["QDialogButtonBox"], "sharingPolicyButtons") + buttons.button(self.qt["QDialogButtonBox"].StandardButton.Save).click() + except BaseException as exc: + write(self.root / "desktop-error.json", {"error": f"{type(exc).__name__}: {exc}"}) + self.application.exit(1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + config = json.loads((args.root / "config.json").read_text()) + key_path = args.root / "node/control-api.key" + while not key_path.exists() and time.time() < config["expires_at_unix"]: + time.sleep(1) + client = NodeClient(f"http://127.0.0.1:{config.get('api_port', 8080)}", key_path.read_text().strip()) + try: + code = ( + run(DesktopController(client), single_instance=False, qualification_automation=FormationDesktop(args.root)) + or 0 + ) + except BaseException as exc: + write(args.root / "desktop-error.json", {"error": f"{type(exc).__name__}: {exc}"}) + raise + raise SystemExit(code) diff --git a/scripts/qwen_formation_node.py b/scripts/qwen_formation_node.py new file mode 100644 index 000000000..8b8b19a9d --- /dev/null +++ b/scripts/qwen_formation_node.py @@ -0,0 +1,296 @@ +"""Exercise an ordinary production node; cloud and local desktop share this agent. + +Only capacity is configured. All model/range decisions belong to run_node. +Commands carry a fresh ID and responses cannot be reused across checkpoints. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +import httpx +import psutil + +LOCAL = "sha256:e62b19ad7d0c6af3dabe730105aefd4cf067ddc50063ffa74c00bd94a29bd7d0" +REMOTE = "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + + +def write(path, value): + value = dict(value, observed_at_unix=time.time()) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def node_config(source, root, host): + if "span" in host or "block_indices" in host: + raise ValueError("Formation must not receive an assigned block range") + paths = [ + source / "manifests/candidates" / name + for name in ("qwen3.5-0.8b-local-bfloat16-eager.json", "qwen3.8-27b-fp8-dequant-eager.json") + ] + bundle = source / "public-alpha/catalog-qwen-v2" + capacity = host.get("capacity_blocks", 0) + return { + "schema_version": 1, + "max_loaded_models": 2, + "inference_mode": "auto", + "auto_model_priority": [REMOTE, LOCAL], + "models": [ + { + "manifest": str(paths[0]), + "execution": "local", + "initial_peers": [], + "cache_dir": host.get("local_cache", str(root / "cache")), + "local_device": host.get("local_device", "cpu"), + "local_max_new_tokens": 64, + }, + { + "manifest": str(paths[1]), + "initial_peers": host["peers"], + "cache_dir": host.get("remote_cache", str(root / "cache")), + "request_timeout": 180, + "max_retries": 2, + }, + ], + "catalog_path": str(bundle / "catalog.signed.json"), + "catalog_bootstrap_path": str(bundle / "catalog-bootstrap.json"), + "catalog_refresh_seconds": 86400, + "discovery_update_period": 5, + "contribution_policy": {"sharing_enabled": False, "max_disk_space": "32GiB"}, + "workers": [] + if not capacity + else [ + { + "id": "automatic", + "model": "auto", + "num_blocks": capacity, + # Match Gate 13: policy gates initial startup; the saved worker + # remains eligible to auto-start after a whole-node restart. + "enabled": True, + "identity_path": str(root / "worker-identity.key"), + "device": "cpu", + "cache_dir": str(root / "worker-cache"), + "throughput": "auto", + "port": 31330, + "public_ip": host["ip"], + **({"public_port": host["public_port"]} if host.get("public_port") else {}), + } + ], + } + + +def selected(snapshot, source): + choice = snapshot.get("auto_selection", {}) + return choice.get("status") == "selected" and choice.get("manifest_digest") == ( + LOCAL if source == "local" else REMOTE + ) + + +def ready_worker(snapshot): + workers = snapshot.get("workers", []) + if len(workers) != 1: + return False + worker = workers[0] + return bool( + worker.get("automatic") + and worker.get("state") == "running" + and worker.get("remote_acknowledged") + and worker.get("block_indices") + and any("Connection handlers are ready" in line for line in worker.get("recent_logs", [])) + ) + + +def coverage(snapshot): + model = next((m for m in snapshot.get("models", []) if m.get("manifest_digest") == REMOTE), {}) + return model.get("route", {}).get("covered_blocks") or 0 + + +def coverage_observed(snapshot): + model = next((m for m in snapshot.get("models", []) if m.get("manifest_digest") == REMOTE), {}) + route = model.get("route", {}) + age = route.get("last_updated_age") + return ( + route.get("status") in {"complete", "incomplete"} + and type(route.get("covered_blocks")) is int + and isinstance(age, (int, float)) + and 0 <= age <= 60 + ) + + +def serve(root, source): + root.mkdir(parents=True, exist_ok=True) + host = json.loads((root / "config.json").read_text()) + deadline = host["expires_at_unix"] + config_path = root / "node-config.json" + # A restart preserves the policy and identity of the participant being lost. + if not config_path.exists(): + config_path.write_text(json.dumps(node_config(source, root, host), indent=2), encoding="utf-8") + port = host.get("api_port", 8080) + command = [host["node_executable"]] if host.get("node_executable") else [sys.executable, "-m", "drift.cli", "node"] + command += [ + "--config", + str(config_path), + "--data_dir", + str(root / "node"), + "--port", + str(port), + "--default_max_tokens", + "8", + ] + process = None + try: + with (root / "formation-node.log").open("ab") as log: + env = dict( + os.environ, + HF_HUB_DISABLE_XET="1", + HF_HUB_DISABLE_IMPLICIT_TOKEN="1", + OMP_NUM_THREADS="4", + MKL_NUM_THREADS="4", + ) + env.pop("HF_TOKEN", None) + process = subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + env=env, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + write( + root / "formation-process.json", + {"pid": process.pid, "started": time.time(), "packaged_node": bool(host.get("node_executable"))}, + ) + url = f"http://127.0.0.1:{port}" + with httpx.Client(base_url=url, timeout=30) as api, concurrent.futures.ThreadPoolExecutor(1) as pool: + while time.time() < deadline: + if process.poll() is not None: + raise RuntimeError("Node stopped during startup") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(2) + else: + raise TimeoutError("Node API startup deadline") + control = (root / "node/control-api.key").read_text().strip() + inference = (root / "node/local-api.key").read_text().strip() + headers = {"Authorization": "Bearer " + control} + + def request(action): + if action["action"] == "sharing": + current = api.get("/control/v1/contribution-policy", headers=headers) + current.raise_for_status() + policy = current.json() + reply = api.put( + "/control/v1/contribution-policy", + headers=headers, + json={ + "schema_version": 1, + "policy": dict(policy["policy"], sharing_enabled=action["enabled"]), + "expected_config_revision": policy["config_revision"], + }, + ) + reply.raise_for_status() + return {"result": "passed", "enabled": action["enabled"]} + if action["action"] != "infer" or action["source"] not in {"local", "community"}: + raise ValueError("Unknown formation action") + started = time.time() + with httpx.Client(base_url=url, timeout=600) as inference_api: + response = inference_api.post( + "/v1/completions", + headers={"Authorization": "Bearer " + inference}, + json={ + "model": "auto", + "prompt": "The capital of France is", + "max_tokens": 3, + "temperature": 0, + }, + ) + response.raise_for_status() + reply = response.json() + # Resolve the display name from the exact pinned manifest. + from drift.model_manifest import ModelManifest + + config = json.loads(config_path.read_text()) + manifest_path = config["models"][0 if action["source"] == "local" else 1]["manifest"] + expected = ModelManifest.load(manifest_path).name + if reply["model"] != expected or "paris" not in reply["choices"][0]["text"].casefold(): + raise RuntimeError("Automatic inference did not answer on the required model: " + str(reply)) + if reply.get("usage", {}).get("completion_tokens") != 3: + raise RuntimeError("Expected three real generated tokens") + return {"result": "passed", "seconds": time.time() - started, "response": reply} + + active, action, completed = None, None, set() + while time.time() < deadline: + if process.poll() is not None: + raise RuntimeError("Production node exited") + response = api.get("/control/v1/status", headers=headers) + response.raise_for_status() + snapshot = response.json() + workers = api.get("/control/v1/workers", headers=headers) + workers.raise_for_status() + value = workers.json() + snapshot["workers"] = value["workers"] if isinstance(value, dict) else value + write(root / "formation-status.json", snapshot) + if active is not None and active.done(): + try: + value = active.result() + except Exception as exc: + value = {"result": "failed", "error": f"{type(exc).__name__}: {exc}"} + write(root / ("formation-response-" + action["id"] + ".json"), dict(value, id=action["id"])) + completed.add(action["id"]) + active = None + path = root / "formation-command.json" + if active is None and path.exists(): + action = json.loads(path.read_text()) + if not re.fullmatch(r"[a-f0-9]{24}", action["id"]): + raise ValueError("Invalid command ID") + if ( + action["id"] not in completed + and not (root / ("formation-response-" + action["id"] + ".json")).exists() + ): + active = pool.submit(request, action) + time.sleep(3) + raise TimeoutError("Formation participant reached its bounded lifetime") + except BaseException as exc: + write(root / "formation-error.json", {"error": f"{type(exc).__name__}: {exc}"}) + raise + finally: + if process is not None and process.poll() is None: + try: + tree = psutil.Process(process.pid).children(recursive=True) + except psutil.NoSuchProcess: + tree = [] + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + for child in tree: + try: + child.terminate() + except psutil.NoSuchProcess: + pass + _, live = psutil.wait_procs(tree, timeout=10) + for child in live: + try: + child.kill() + except psutil.NoSuchProcess: + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path("/srv/q38")) + args = parser.parse_args() + serve(args.root.resolve(), Path(__file__).resolve().parents[1]) diff --git a/scripts/qwen_formation_platform_probe.py b/scripts/qwen_formation_platform_probe.py new file mode 100644 index 000000000..24bd97a30 --- /dev/null +++ b/scripts/qwen_formation_platform_probe.py @@ -0,0 +1,28 @@ +"""Reject hosts that cannot persist ordinary desktop settings atomically.""" + +import tempfile +from pathlib import Path + +from drift.node.policy_store import _exchange_paths + + +def main(): + with tempfile.TemporaryDirectory(prefix="formation-platform-", dir="/srv/q38") as directory: + left, right = Path(directory) / "left", Path(directory) / "right" + left.write_text("before") + right.write_text("after") + _exchange_paths(left, right) + if (left.read_text(), right.read_text()) != ("after", "before"): + raise RuntimeError("Host does not preserve atomic settings exchange") + from PySide6.QtWidgets import QApplication, QWidget + + app = QApplication([]) + window = QWidget() + window.show() + app.processEvents() + if not window.isVisible(): + raise RuntimeError("Host cannot show the real desktop") + + +if __name__ == "__main__": + main() diff --git a/scripts/qwen_full_inference_host.py b/scripts/qwen_full_inference_host.py new file mode 100644 index 000000000..40d0cd956 --- /dev/null +++ b/scripts/qwen_full_inference_host.py @@ -0,0 +1,306 @@ +"""Linux host jobs for the isolated, manifested Qwen full-inference swarm.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import time +from pathlib import Path + +ROOT = Path("/srv/q38") +SOURCE = Path("/opt/q38/source") +MANIFEST = SOURCE / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json" + + +def write(name, value): + value = dict(value, observed_at_unix=time.time()) + path = ROOT / name + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n") + temporary.replace(path) + print(json.dumps(value), flush=True) + + +def bootstrap(config): + from hivemind import DHT + + dht = DHT( + initial_peers=[], + host_maddrs=["/ip4/0.0.0.0/tcp/31330"], + announce_maddrs=[f"/ip4/{config['ip']}/tcp/{config.get('public_port', 31330)}"], + client_mode=False, + start=True, + tls=True, + ) + try: + write("bootstrap.json", {"peers": [str(a) for a in dht.get_visible_maddrs()]}) + while dht.is_alive(): + time.sleep(5) + raise RuntimeError("bootstrap DHT exited") + finally: + dht.shutdown() + + +def worker(config): + import torch + + from drift.cli.run_server import build_parser, serve, server_from_args + + torch.set_num_threads(4) + argv = [ + "Qwen/Qwen3.8-27B-FP8", + "--model_manifest", + str(MANIFEST), + "--block_indices", + config["span"], + "--device", + config.get("device", "cpu"), + "--cache_dir", + str(ROOT / "cache"), + "--max_disk_space", + "24GiB", + "--torch_dtype", + "bfloat16", + "--quant_type", + "fp8_dequant", + "--attn_implementation", + "eager", + "--throughput", + "0.01", + "--num_handlers", + "1", + "--inference_max_length", + "64", + "--attn_cache_tokens", + "128", + "--max_batch_size", + "64", + "--update_period", + "10", + "--expiration", + "40", + "--request_timeout", + "1800", + "--session_timeout", + "14400", + "--step_timeout", + "3600", + "--ready_timeout", + "300", + "--balance_quality", + "0", + "--no_auto_relay", + "--host_maddrs", + "/ip4/0.0.0.0/tcp/31330", + "--announce_maddrs", + f"/ip4/{config['ip']}/tcp/31330", + "--health_state_path", + str(ROOT / "health.json"), + "--identity_path", + str(ROOT / "identity.key"), + "--initial_peers", + *config["peers"], + ] + parsed = vars(build_parser().parse_args(argv)) + parsed.pop("config", None) + server = server_from_args(parsed) + hardware = {"device": str(server.device), "dtype": str(server.torch_dtype)} + if server.device.type == "cuda": + hardware.update( + gpu_name=torch.cuda.get_device_name(server.device), + gpu_memory_bytes=torch.cuda.get_device_properties(server.device).total_memory, + compute_capability=list(torch.cuda.get_device_capability(server.device)), + ) + write( + "worker.json", + { + "span": config["span"], + "peer_id": str(server.dht.peer_id), + "hardware": hardware, + "peers": [str(a) for a in server.dht.get_visible_maddrs()], + }, + ) + serve(server, model="Qwen/Qwen3.8-27B-FP8") + + +def route(session): + return [ + {"start": s.span.start, "end": s.span.end, "peer_id": str(s.span.peer_id), "session_id": s.session_id} + for s in session._server_sessions + ] + + +def gpu_probe(config): + import torch + + if not torch.cuda.is_available(): + raise RuntimeError("the requested GPU is unavailable") + matrix = torch.ones(256, 256, device="cuda", dtype=torch.bfloat16) + product = matrix @ matrix + convolution = torch.nn.functional.conv1d( + torch.ones(1, 16, 8, device="cuda", dtype=torch.bfloat16), + torch.ones(16, 16, 4, device="cuda", dtype=torch.bfloat16), + ) + torch.cuda.synchronize() + if not torch.isfinite(product).all() or not torch.isfinite(convolution).all(): + raise RuntimeError("GPU BF16 kernels produced non-finite values") + write( + "gpu-probe.json", + { + "result": "passed", + "gpu_name": torch.cuda.get_device_name(), + "memory_bytes": torch.cuda.get_device_properties(0).total_memory, + "compute_capability": list(torch.cuda.get_device_capability()), + "torch": torch.__version__, + "cuda": torch.version.cuda, + "bf16_matmul": True, + "bf16_convolution": True, + }, + ) + + +def client(config): + import hivemind + import torch + import transformers + from transformers import AutoTokenizer + + from drift import AutoDistributedModelForCausalLM + from drift.model_manifest import ManifestArtifactVerifier, ModelManifest + from drift.node.loading import _runtime_closer + + torch.set_num_threads(4) + torch.manual_seed(0) + manifest = ModelManifest.load(MANIFEST) + verifier = ManifestArtifactVerifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=False, + cache_dir=str(ROOT / "cache"), + ) + write("client-status.json", {"phase": "loading-client"}) + verifier.ensure_startup_metadata(include_tokenizer=True) + tokenizer = AutoTokenizer.from_pretrained(verifier.snapshot_root, local_files_only=True) + model = AutoDistributedModelForCausalLM.from_pretrained( + manifest.source.repository, + revision=manifest.source.revision, + token=False, + initial_peers=config["peers"], + dht_prefix=manifest.dht_prefix, + manifest_digest=manifest.digest, + manifest_execution_profile=manifest.runtime.to_dict(), + torch_dtype=torch.bfloat16, + artifact_verifier=verifier, + request_timeout=config.get("request_timeout", 180), + connect_timeout=30, + max_retries=240, + min_backoff=2, + max_backoff=10, + update_period=5, + use_server_to_server=True, + ).eval() + try: + prompt = "The capital of France is" + inputs = tokenizer(prompt, return_tensors="pt")["input_ids"] + generation = dict(do_sample=False, min_new_tokens=3, max_new_tokens=3, pad_token_id=tokenizer.eos_token_id) + write("client-status.json", {"phase": "baseline", "prompt_tokens": inputs.shape[1]}) + started = time.monotonic() + with torch.inference_mode(), model.inference_session(max_length=64) as session: + output = model.generate(inputs, **generation) + baseline_route = route(session) + baseline = { + "token_ids": output[0, inputs.shape[1] :].tolist(), + "text": tokenizer.decode(output[0, inputs.shape[1] :]), + "route": baseline_route, + "seconds": time.monotonic() - started, + } + write("baseline.json", baseline) + if config.get("run_recovery", True) is False: + write( + "client-result.json", + { + "result": "passed", + "manifest_digest": manifest.digest_id, + "model_revision": manifest.source.revision, + "baseline": baseline, + "recovery": None, + "versions": { + "python": platform.python_version(), + "torch": torch.__version__, + "transformers": transformers.__version__, + "hivemind": hivemind.__version__, + }, + }, + ) + return + with torch.inference_mode(), model.inference_session(max_length=64) as session: + first = model.generate( + inputs, do_sample=False, min_new_tokens=1, max_new_tokens=1, pad_token_id=tokenizer.eos_token_id + ) + before = route(session) + position = session.position + write("recovery-ready.json", {"route": before, "position": position, "first_token_id": first[0, -1].item()}) + deadline = time.monotonic() + 5400 + while not (ROOT / "continue-recovery").exists(): + if time.monotonic() >= deadline: + raise TimeoutError("replacement did not become ready") + time.sleep(2) + started = time.monotonic() + # The exact same InferenceSession and history survive the worker loss. + output = model.generate( + do_sample=False, min_new_tokens=2, max_new_tokens=2, pad_token_id=tokenizer.eos_token_id + ) + output = session.output_ids + after = route(session) + recovery = { + "token_ids": output[0, inputs.shape[1] :].tolist(), + "text": tokenizer.decode(output[0, inputs.shape[1] :]), + "before_route": before, + "after_route": after, + "position_before": position, + "position_after": session.position, + "same_session": True, + "seconds": time.monotonic() - started, + } + recovery["matches_baseline"] = recovery["token_ids"] == baseline["token_ids"] + write("recovery.json", recovery) + if not recovery["matches_baseline"]: + raise RuntimeError("recovery tokens differ from uninterrupted generation") + write( + "client-result.json", + { + "result": "passed", + "manifest_digest": manifest.digest_id, + "model_revision": manifest.source.revision, + "baseline": baseline, + "recovery": recovery, + "versions": { + "python": platform.python_version(), + "torch": torch.__version__, + "transformers": transformers.__version__, + "hivemind": hivemind.__version__, + }, + }, + ) + finally: + _runtime_closer(model)() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("role", choices=["bootstrap", "worker", "client", "gpu_probe"]) + args = parser.parse_args() + ROOT.mkdir(parents=True, exist_ok=True) + config = json.loads((ROOT / "config.json").read_text()) + try: + globals()[args.role](config) + except BaseException as exc: + write(f"{args.role}-error.json", {"error": type(exc).__name__, "message": str(exc)}) + raise + + +if __name__ == "__main__": + main() diff --git a/scripts/qwen_modal_participant.py b/scripts/qwen_modal_participant.py new file mode 100644 index 000000000..1817a9923 --- /dev/null +++ b/scripts/qwen_modal_participant.py @@ -0,0 +1,64 @@ +"""Start/kill an owned node and real Xvfb desktop inside one Modal sandbox.""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import psutil + +ROOT = Path("/srv/q38") +SOURCE = Path("/opt/q38/source") + + +def main(action): + record = ROOT / "participant-processes.json" + if action == "start": + for name in ("formation-error.json", "desktop-error.json", "desktop-stop"): + (ROOT / name).unlink(missing_ok=True) + processes = [] + for script in ("qwen_formation_node.py", "qwen_formation_desktop.py"): + command = [sys.executable, str(SOURCE / "scripts" / script), "--root", str(ROOT)] + if "desktop" in script: + command = ["xvfb-run", "-a", "-s", "-screen 0 1440x1000x24", *command] + with (ROOT / (script + ".log")).open("ab") as log: + process = subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + env=dict( + os.environ, PYTHONPATH=str(SOURCE / "desktop/src"), OMP_NUM_THREADS="4", MKL_NUM_THREADS="4" + ), + ) + processes.append({"pid": process.pid, "created": psutil.Process(process.pid).create_time()}) + record.write_text(json.dumps(processes)) + return {"started": time.time(), "processes": processes} + if action != "stop": + raise ValueError("Unknown participant action") + targets = {} + for original in json.loads(record.read_text()): + try: + parent = psutil.Process(original["pid"]) + if parent.create_time() != original["created"]: + raise RuntimeError("Owned participant PID was reused") + for process in [*parent.children(recursive=True), parent]: + targets[process.pid] = process + except psutil.NoSuchProcess: + pass + for process in targets.values(): + try: + process.kill() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs(list(targets.values()), timeout=20) + live = [p.pid for p in alive if p.is_running() and p.status() != psutil.STATUS_ZOMBIE] + if live: + raise RuntimeError("Participant processes survived loss: " + str(live)) + return {"verified": True, "killed_pids": list(targets), "scope": "complete node and desktop process trees"} + + +if __name__ == "__main__": + print(json.dumps(main(sys.argv[1]))) diff --git a/scripts/qwen_offline_http.py b/scripts/qwen_offline_http.py new file mode 100644 index 000000000..0d6c65b39 --- /dev/null +++ b/scripts/qwen_offline_http.py @@ -0,0 +1,39 @@ +"""Deny HTTP downloads in a qualification child without blocking swarm RPC.""" + +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +class HttpDownloadBlocker: + def __init__(self): + self.denied_requests = 0 + self._lock = threading.Lock() + owner = self + + class Reject(BaseHTTPRequestHandler): + def deny(self): + with owner._lock: + owner.denied_requests += 1 + self.send_error(403, "HTTP downloads disabled for cache qualification") + + do_CONNECT = do_GET = do_HEAD = do_POST = deny + + def log_message(self, *args): + pass # Do not retain URLs, headers or credentials. + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Reject) + self.server.daemon_threads = True + self.url = "http://127.0.0.1:" + str(self.server.server_port) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def environment(self, original): + names = {"http_proxy", "https_proxy", "all_proxy", "no_proxy"} + env = {key: value for key, value in original.items() if key.lower() not in names} + env.update(HTTP_PROXY=self.url, HTTPS_PROXY=self.url, ALL_PROXY=self.url, NO_PROXY="127.0.0.1,localhost,::1") + return env + + def close(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) diff --git a/scripts/qwen_packaged_worker_action.py b/scripts/qwen_packaged_worker_action.py new file mode 100644 index 000000000..5abd78665 --- /dev/null +++ b/scripts/qwen_packaged_worker_action.py @@ -0,0 +1,33 @@ +"""Stop/start only one labeled CPU worker inside an active owned package-test window.""" + +import json +import time +from pathlib import Path + +from qwen_product_recovery import WORKER_STATE_COMMAND, worker_is_stopped +from run_qwen_product_mixed import RUNS, MixedProductRun + + +def worker_action(cloud_run: Path, action: str): + path = cloud_run.resolve() + if path.parent != RUNS.resolve() or action not in {"stop", "start"}: + raise ValueError("Worker action must name an owned mixed product run and stop/start") + ready = json.loads((path / "packaged-client-ready.json").read_text()) + if ready.get("run_id") != path.name or time.time() >= ready["deadline_unix"] or (path / "result.json").exists(): + raise ValueError("Owned packaged-client window is not active") + config = json.loads((path / "provider-config.json").read_text()) + run = MixedProductRun(path, config) + name = run.names[3] # The CPU span 32:48, never the persistent public bootstrap. + instance = run.cloud_json(["compute", "instances", "describe", name, "--zone", config["zone"]]) + if ( + instance.get("name") != path.name + "-w2" + or instance.get("labels", {}).get("q38-run") != path.name + or path.name not in instance.get("tags", {}).get("items", []) + ): + raise ValueError("CPU worker ownership does not match this live run") + before = run.ssh(name, WORKER_STATE_COMMAND).stdout + run.ssh(name, "sudo systemctl " + action + " q38-worker") + after = run.ssh(name, WORKER_STATE_COMMAND).stdout + if action == "stop" and not worker_is_stopped(after): + raise RuntimeError("Owned worker did not stop") + return {"instance": name, "action": action, "before": before, "after": after, "observed_at_unix": time.time()} diff --git a/scripts/qwen_product_inference_host.py b/scripts/qwen_product_inference_host.py new file mode 100644 index 000000000..2fe345b99 --- /dev/null +++ b/scripts/qwen_product_inference_host.py @@ -0,0 +1,234 @@ +"""Real local-fallback/64-block-promotion exercise on the isolated GCP coordinator.""" + +import concurrent.futures +import json +import os +import secrets +import subprocess +import sys +import time +from pathlib import Path + +import httpx +from qwen_product_recovery import wait_recovery_control + +from drift.model_manifest import ModelManifest + +ROOT = Path("/srv/q38") +SOURCE = Path(__file__).resolve().parents[1] + + +def write(name, data): + data = dict(data, observed_at_unix=time.time()) + temporary = ROOT / (name + ".tmp") + temporary.write_text(json.dumps(data, indent=2) + "\n") + temporary.replace(ROOT / name) + print(json.dumps({"phase": name}), flush=True) + + +def main(): + host = json.loads((ROOT / "config.json").read_text()) + paths = [ + SOURCE / "manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json", + SOURCE / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json", + ] + manifests = [ModelManifest.load(path) for path in paths] + bundle = SOURCE / "public-alpha/catalog-qwen-v2" + node_config = { + "schema_version": 1, + "max_loaded_models": 2, + "models": [ + { + "manifest": str(path), + "initial_peers": [] if i == 0 else host["peers"], + "execution": "local" if i == 0 else "distributed", + "cache_dir": str(ROOT / "cache"), + **( + {"local_device": "cpu", "local_max_new_tokens": 64} + if i == 0 + else {"request_timeout": 180, "max_retries": 2} + ), + } + for i, path in enumerate(paths) + ], + "auto_model_priority": [m.digest_id for m in reversed(manifests)], + "catalog_path": str(bundle / "catalog.signed.json"), + "catalog_bootstrap_path": str(bundle / "catalog-bootstrap.json"), + "catalog_refresh_seconds": 86400, + "discovery_update_period": 5, + "contribution_policy": {"sharing_enabled": False}, + } + (ROOT / "node-config.json").write_text(json.dumps(node_config)) + result = { + "result": "failed", + "catalog_scope": "staged signed public sequence 2; explicit owned test seed configuration", + "manifest_digests": [m.digest_id for m in manifests], + "all_blocks": 64, + "packaged": False, + } + process = None + try: + with (ROOT / "product-node.log").open("wb") as log: + process = subprocess.Popen( + [ + sys.executable, + "-m", + "drift.cli", + "node", + "--config", + str(ROOT / "node-config.json"), + "--data_dir", + str(ROOT / "node"), + "--port", + "8080", + "--default_max_tokens", + "8", + ], + stdout=log, + stderr=subprocess.STDOUT, + env=dict(os.environ, HF_HUB_DISABLE_XET="1"), + ) + deadline = time.monotonic() + 7200 + with httpx.Client(base_url="http://127.0.0.1:8080", timeout=600) as api: + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("product node stopped during startup") + try: + if api.get("/health").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(2) + control = (ROOT / "node/control-api.key").read_text().strip() + inference = (ROOT / "node/local-api.key").read_text().strip() + headers = {"Authorization": "Bearer " + control} + + def status(): + response = api.get("/control/v1/status", headers=headers) + response.raise_for_status() + return response.json() + + def wait_source(source, *, until=None): + until = min(deadline, until if until is not None else deadline) + while time.monotonic() < until: + current = status() + selection = current["auto_selection"] + if selection["status"] == "selected" and ( + (selection["source"] == "local") == (source == "local") + ): + return current + if process.poll() is not None: + raise RuntimeError("product node stopped while selecting a route") + time.sleep(5) + raise TimeoutError("product selection deadline: " + source) + + def infer(): + started = time.monotonic() + response = api.post( + "/v1/completions", + headers={"Authorization": "Bearer " + inference}, + json={"model": "auto", "prompt": "The capital of France is", "max_tokens": 3, "temperature": 0}, + ) + response.raise_for_status() + reply = response.json() + assert "paris" in reply["choices"][0]["text"].casefold() + return {"response": reply, "seconds": time.monotonic() - started} + + def mode(value): + policy = api.get("/control/v1/contribution-policy", headers=headers).json() + response = api.put( + "/control/v1/inference-mode", + headers=headers, + json={"inference_mode": value, "expected_config_revision": policy["config_revision"]}, + ) + response.raise_for_status() + + wait_source("local") + result["local_before_growth"] = infer() + assert result["local_before_growth"]["response"]["model"] == manifests[0].name + write("product-local-ready.json", result["local_before_growth"]) + promoted = wait_source("community") + assert promoted["auto_selection"]["covered_blocks"] == 64 + assert promoted["auto_selection"]["peer_count"] == 4 + result["promoted_status"] = promoted + result["qwen38_baseline"] = infer() + assert result["qwen38_baseline"]["response"]["model"] == manifests[1].name + # The HTTP response can arrive before its generation thread has + # released the prior lease. Do not mistake that old request for + # the next generation when changing inference mode. + while any(m["active_requests"] for m in status()["models"]): + if time.monotonic() >= deadline: + raise TimeoutError("baseline generation lease did not drain") + time.sleep(0.1) + # A long completion can outlive its last fresh route observation. + # Auto may correctly fall back between requests; wait for fresh + # readiness and record that race instead of assuming a lease. + transition_deadline = min(deadline, time.monotonic() + 600) + result["local_fallbacks_before_active_transition"] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + while time.monotonic() < transition_deadline: + wait_source("community", until=transition_deadline) + active = pool.submit(infer) + while not any( + m["active_requests"] and m["manifest_digest"] == manifests[1].digest_id + for m in status()["models"] + ): + if active.done(): + reply = active.result() + assert reply["response"]["model"] == manifests[0].name + result["local_fallbacks_before_active_transition"].append(reply) + break + if time.monotonic() >= transition_deadline: + raise TimeoutError("new community generation did not start") + time.sleep(0.2) + else: + mode("local_only") + result["active_answer_after_mode_change"] = active.result(timeout=600) + break + else: + raise TimeoutError("fresh community generation was not acquired within ten minutes") + assert result["active_answer_after_mode_change"]["response"]["model"] == manifests[1].name + result["local_only_next_request"] = infer() + assert result["local_only_next_request"]["response"]["model"] == manifests[0].name + mode("auto") + wait_source("community") + nonce = secrets.token_hex(16) + write("product-ready-for-loss.json", {"ready": True, "recovery_nonce": nonce}) + result["worker_stopped_acknowledgement"] = wait_recovery_control( + ROOT / "product-worker-stopped.json", nonce, deadline + ) + wait_source("local") + result["local_after_worker_loss"] = infer() + assert result["local_after_worker_loss"]["response"]["model"] == manifests[0].name + write("product-local-after-loss.json", dict(result["local_after_worker_loss"], recovery_nonce=nonce)) + result["worker_replaced_acknowledgement"] = wait_recovery_control( + ROOT / "product-worker-replaced.json", nonce, deadline + ) + result["recovered_status"] = wait_source("community") + result["qwen38_after_replacement"] = infer() + assert result["qwen38_after_replacement"]["response"]["model"] == manifests[1].name + assert ( + result["qwen38_baseline"]["response"]["choices"] + == result["qwen38_after_replacement"]["response"]["choices"] + ) + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=45) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + write("product-result.json", result) + + +if __name__ == "__main__": + try: + main() + except BaseException as exc: + write("product-error.json", {"error": f"{type(exc).__name__}: {exc}"}) + raise diff --git a/scripts/qwen_product_provenance.py b/scripts/qwen_product_provenance.py new file mode 100644 index 000000000..46116134b --- /dev/null +++ b/scripts/qwen_product_provenance.py @@ -0,0 +1,106 @@ +"""Capture the replay's actual source bytes and package inputs before cloud mutation.""" + +import hashlib +import io +import json +import subprocess +import tarfile +import time +from pathlib import Path + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(4 * 1024**2), b""): + digest.update(block) + return digest.hexdigest() + + +def source_files(root): + # Deliberate source-only roots: never include keys, caches, logs or build trees. + names = {"pyproject.toml", "README.md", "LICENSE", "desktop/pyproject.toml"} + names.update(p.name for p in root.glob("Run Qwen*.cmd")) + for directory, suffixes in ( + ("scripts", {".py", ".ps1"}), + ("src/drift", None), + ("desktop/src", None), + ("manifests/candidates", {".json"}), + ("public-alpha/catalog-qwen-v2", {".json"}), + ("config", {".json"}), + ): + names.update( + p.relative_to(root).as_posix() + for p in (root / directory).rglob("*") + if p.is_file() + and (suffixes is None or p.suffix in suffixes) + and p.suffix not in {".pyc", ".pyo"} + and "__pycache__" not in p.parts + ) + return sorted(names) + + +def snapshot(root, run, input_paths): + files = {} + archive_path = run / "launcher-source.tar.gz" + with tarfile.open(archive_path, "x:gz") as archive: + for name in source_files(root): + payload = (root / name).read_bytes() + files[name] = hashlib.sha256(payload).hexdigest() + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + git = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, check=True, timeout=30) + value = { + "schema_version": 1, + "run_id": run.name, + "recorded_at_unix": time.time(), + "checkout_head": git.stdout.strip(), + "source_identity": "archived working-tree bytes; checkout HEAD is not a package build claim", + "files": files, + "archive_sha256": sha256(archive_path), + "inputs": {label: {"path": str(p.resolve()), "sha256": sha256(p)} for label, p in input_paths.items()}, + } + (run / "launcher-source.json").write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return value + + +def verify_snapshot(root, run, value): + changed = sorted(set(source_files(root)) ^ set(value["files"])) + changed += [ + name for name, digest in value["files"].items() if not (root / name).is_file() or sha256(root / name) != digest + ] + changed += [ + "input:" + label + for label, binding in value["inputs"].items() + if not Path(binding["path"]).is_file() or sha256(binding["path"]) != binding["sha256"] + ] + if sha256(run / "launcher-source.tar.gz") != value["archive_sha256"]: + changed.append("launcher-source.tar.gz") + if changed: + raise ValueError("Replay inputs changed during execution: " + ", ".join(changed)) + + +def verify_package(node, provenance_path, expected_sha256): + """Verify the complete existing package, including DLLs, not only its small entrypoint.""" + if len(expected_sha256) != 64 or sha256(node) != expected_sha256: + raise ValueError("Packaged node SHA-256 does not match the configured build") + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + base = provenance_path.parent.resolve() + observed = set() + for item in provenance["artifacts"]: + path = (base / item["path"]).resolve() + if not path.is_relative_to(base) or item["kind"] != "file": + raise ValueError("This Windows replay requires regular package files within its build directory") + if not path.is_file() or path.stat().st_size != item["size_bytes"] or sha256(path) != item["sha256"]: + raise ValueError("Package provenance mismatch: " + item["path"]) + observed.add(path) + if node.resolve() not in observed: + raise ValueError("Package provenance does not bind the selected node") + artifact_root = (base / provenance.get("artifact_root", "CommunityAI")).resolve() + if not artifact_root.is_relative_to(base): + raise ValueError("Package root escapes its build directory") + actual = {p.resolve() for p in artifact_root.rglob("*") if p.is_file()} + if actual != observed: + raise ValueError("Package contains missing or unlisted runtime files") + return {"verified_files": len(observed), "node_sha256": expected_sha256} diff --git a/scripts/qwen_product_recovery.py b/scripts/qwen_product_recovery.py new file mode 100644 index 000000000..284b969d7 --- /dev/null +++ b/scripts/qwen_product_recovery.py @@ -0,0 +1,41 @@ +"""Bind source-client recovery observations to the orchestrator's actual outage.""" + +import json +import time +from pathlib import Path + +WORKER_STATE_COMMAND = "systemctl show q38-worker -p MainPID -p ActiveState -p KillMode" + + +def worker_is_stopped(state: str): + values = dict(line.split("=", 1) for line in state.splitlines() if "=" in line) + # A signal-terminated worker can remain a failed unit after stop. + return ( + values.get("MainPID") == "0" + and values.get("ActiveState") in {"inactive", "failed"} + and values.get("KillMode") == "control-group" + ) + + +def wait_recovery_control(path: Path, nonce: str, deadline: float): + """Ignore stale receipts and do not infer an injected fault from route state.""" + while time.monotonic() < deadline: + if path.exists(): + value = json.loads(path.read_text()) + if value.get("recovery_nonce") == nonce: + return value + time.sleep(1) + raise TimeoutError("Recovery control receipt deadline: " + path.name) + + +def require_recovery_acknowledgements(evidence, nonce, original_peer, replacement_peer): + stopped = evidence.get("worker_stopped_acknowledgement", {}) + replaced = evidence.get("worker_replaced_acknowledgement", {}) + if ( + stopped.get("recovery_nonce") != nonce + or replaced.get("recovery_nonce") != nonce + or stopped.get("peer_id") != original_peer + or replaced.get("peer_id") != replacement_peer + or original_peer == replacement_peer + ): + raise RuntimeError("Source result lacks matching recovery acknowledgements") diff --git a/scripts/qwen_qualification.py b/scripts/qwen_qualification.py new file mode 100644 index 000000000..07c2c4c1a --- /dev/null +++ b/scripts/qwen_qualification.py @@ -0,0 +1,191 @@ +"""Gate 13's durable phase recording around the established Qwen cloud/client test.""" + +import sys +import time + +from gate13_cloud_orchestrator import Gate13CloudError, RunRecorder +from gate13_gcp_provider import LoggedRunner +from qwen_product_provenance import sha256, snapshot, verify_package, verify_snapshot +from report_qwen_product import report +from run_gate13_gcp import _write_json +from run_qwen_product_mixed import MixedProductRun +from run_qwen_product_test import execute_packaged + + +class QwenRecorder(RunRecorder): + def _document(self): + value = super()._document() + value["scope"] = "qwen-one-click-source-and-packaged-qualification" + return value + + def phase(self, name, **details): + super().phase(name, **details) + print(f"[{time.strftime('%H:%M:%S')}] {name}", flush=True) + + +class QualificationRun(MixedProductRun): + """Add phase boundaries and an inline Windows client; reuse cloud lifecycle/cleanup.""" + + def __init__(self, path, cloud_config, *, root, packaged_config, inputs, inventory, recorder): + super().__init__(path, cloud_config) + self.root = root + self.packaged_config = packaged_config + self.inputs = inputs + self.inventory = inventory + self.recorder = recorder + self.cloud_creation_reached = False + self.packaged_exit_code = None + self.package_verification = None + self.runner = LoggedRunner(path / "command-journal.jsonl") + + def event(self, phase, **details): + if phase == "failed" and not any(e["phase"] == "FAILURE" for e in self.recorder.events): + self.recorder.phase( + "FAILURE", + failed_phase=self.recorder.current_phase, + failure_reason=details.get("error", "Cloud test failed"), + ) + super().event(phase, **details) + + def preflight(self): + self.recorder.phase("PREFLIGHT") + super().preflight() + self.recorder.phase("PACKAGES_VERIFYING") + self.runner.run( + [sys.executable, "-c", "import httpx, psutil, communityai_desktop.controller"], + action="Checking the Qwen desktop test runtime", + timeout=60, + ) + self.package_verification = verify_package( + self.inputs["node"], self.inputs["package_provenance"], self.packaged_config["node_sha256"] + ) + self.recorder.package_records["windows"] = { + **self.package_verification, + "provenance_sha256": sha256(self.inputs["package_provenance"]), + "scope": "explicit verified engineering package; no current-HEAD build claim", + } + verify_snapshot(self.root, self.path, self.inventory) + + def bundle(self): + self.recorder.phase("SOURCE_BUNDLING") + super().bundle() + verify_snapshot(self.root, self.path, self.inventory) + + def create_firewalls(self): + self.cloud_creation_reached = True # Set before the first resource mutation. + self.recorder.phase("ROUTE_CREATING") + super().create_firewalls() + + def stage(self, name, span=None, peers=()): + self.recorder.phase("ROUTE_PREPARING", instance=name) + super().stage(name, span, peers) + + def exercise_workers(self): + self.recorder.phase("SOURCE_RUNNING") + try: + return super().exercise_workers() + except BaseException as exc: + # A failed source run may still collect an independent packaged + # result. Preserve the original failed phase across that later step. + self.recorder.phase("FAILURE", failed_phase="SOURCE_RUNNING", failure_reason=str(exc)) + raise + + def run_packaged_client(self): + # Called after the owned ready/deadline receipt is written, inside run()'s + # try/finally. No background orchestrator or manual client launch is needed. + self.recorder.phase("CLIENT_RUNNING", platform="windows") + verify_snapshot(self.root, self.path, self.inventory) + self.packaged_exit_code = execute_packaged(self.path, self.path / "packaged", self.packaged_config, self.inputs) + + def cleanup(self): + self.recorder.phase("CLEANUP") + return super().cleanup() + + +class QwenQualification: + def __init__(self, *, root, output_root, cloud_config, packaged_config, inputs): + self.root = root + self.output_root = output_root + self.cloud_config = cloud_config + self.packaged_config = packaged_config + self.inputs = inputs + + def run(self): + path = self.output_root + # Keep the established raw cloud result.json intact. Gate 13's recorder + # writes the aggregate into a separate directory within this fresh run. + recorder = QwenRecorder(path.name, "gcp-and-azure", path / "qualification", time.time) + value = {"result": "failed", "run_id": path.name} + run = None + raw = {} + failure_code = None + failure_reason = None + try: + recorder.phase("SOURCE_RECORDING") + inventory = snapshot(self.root, path, self.inputs) + value["source_inventory_sha256"] = sha256(path / "launcher-source.json") + run = QualificationRun( + path, + self.cloud_config, + root=self.root, + packaged_config=self.packaged_config, + inputs=self.inputs, + inventory=inventory, + recorder=recorder, + ) + raw = run.run() # The proven cloud runner attempts cleanup in finally, on every failure. + value.update( + cloud_result=raw, + package_verification=run.package_verification, + packaged_exit_code=run.packaged_exit_code, + ) + if raw.get("result") != "passed": + raise Gate13CloudError(raw.get("error") or raw.get("cleanup_error") or "Qwen cloud/client test failed") + if run.packaged_exit_code != 0: + raise Gate13CloudError("The packaged client did not complete successfully") + recorder.phase("INPUTS_VERIFYING") + verify_snapshot(self.root, path, inventory) + verify_package(self.inputs["node"], self.inputs["package_provenance"], self.packaged_config["node_sha256"]) + value.update(result="passed", inputs_unchanged=True) + except BaseException as exc: + failure_code, failure_reason = type(exc).__name__, str(exc) or type(exc).__name__ + value["error"] = failure_reason + if not any(e["phase"] == "FAILURE" for e in recorder.events): + recorder.phase("FAILURE", failed_phase=recorder.current_phase, failure_reason=failure_reason) + finally: + _write_json(path / "launcher-result.json", value) + + recorder.phase("CLEANUP_VERIFYING") + cleanup = raw.get("cleanup", {}) + if cleanup.get("verified") is True: + recorder.cleanup = {"result": "passed", "evidence": cleanup} + elif run is None or not run.cloud_creation_reached: + recorder.cleanup = {"result": "not-needed", "reason": "This run did not reach cloud resource creation"} + else: + recorder.cleanup = {"result": "failed", "evidence": cleanup} + failure_code = failure_code or "CleanupError" + failure_reason = failure_reason or "Owned cloud cleanup was not verified" + + recorder.phase("REPORT_VALIDATING") + try: + summary = report(path, path / "packaged", path / "product-report.json", launcher=value) + if summary["result"] != "passed": + raise Gate13CloudError(summary.get("error", "Qwen evidence validation failed")) + recorder.client_evidence["windows"] = { + "result": "passed", + "sha256": "sha256:" + sha256(path / "packaged/result.json"), + } + except BaseException as exc: + failure_code = failure_code or type(exc).__name__ + failure_reason = failure_reason or str(exc) or type(exc).__name__ + if not any(e["phase"] == "FAILURE" for e in recorder.events): + recorder.phase("FAILURE", failed_phase="REPORT_VALIDATING", failure_reason=failure_reason) + passed = ( + failure_code is None + and value["result"] == "passed" + and recorder.cleanup.get("result") == "passed" + and "windows" in recorder.client_evidence + ) + return recorder.finish( + "passed" if passed else "failed", failure_code=failure_code, failure_reason=failure_reason + ) diff --git a/scripts/qwen_reference_inference_host.py b/scripts/qwen_reference_inference_host.py new file mode 100644 index 000000000..9aaf31dd6 --- /dev/null +++ b/scripts/qwen_reference_inference_host.py @@ -0,0 +1,281 @@ +"""Compare the full manifested model with stock Transformers, including cached decoding.""" + +import concurrent.futures +import gc +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path("/srv/q38") +SOURCE = Path("/opt/q38/source") +MANIFEST = SOURCE / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json" +PROMPTS = ("The capital of France is", "Two plus three equals", "In one sentence, explain why the sky looks blue.") +# Declared before running either implementation. Compare every vocabulary logit +# and require the same greedy token at all nine prefill/cached-decode positions. +ATOL, RTOL, STEPS = 0.5, 0.01, 3 + + +def write(name, value): + value = dict(value, time=time.time()) + path = ROOT / name + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n") + temporary.replace(path) + print(json.dumps({"phase": name, "detail": value.get("phase", value.get("result"))}), flush=True) + + +def main(): + import torch + import transformers + from hivemind import DHT + from transformers import AutoModelForImageTextToText, AutoTokenizer, FineGrainedFP8Config + + from drift import AutoDistributedModelForCausalLM + from drift.model_manifest import ManifestArtifactVerifier, ModelManifest + from drift.node.loading import _runtime_closer + + torch.set_num_threads(4) + manifest = ModelManifest.load(MANIFEST) + verifier = ManifestArtifactVerifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=False, + cache_dir=str(ROOT / "cache"), + ) + result = { + "result": "failed", + "scope": "stock-versus-four-RPC-worker numerical reference on one CPU host", + "manifest_digest": manifest.digest_id, + "revision": manifest.source.revision, + "torch": torch.__version__, + "transformers": transformers.__version__, + "atol": ATOL, + "rtol": RTOL, + "cached_decode_steps_per_prompt": STEPS - 1, + "prompts": list(PROMPTS), + "cross_host_qualification": False, + "checks": [], + } + workers, logs, dht, remote, stock = [], [], None, None, None + try: + write("reference-status.json", {"phase": "verify-all-artifacts"}) + # Bind the verified snapshot before parallel downloads. This verifier's + # first materialization establishes mutable root state and is serial. + verifier.ensure_startup_metadata(include_tokenizer=True) + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda a: verifier.ensure_path(a.path), manifest.artifacts)) + tokenizer = AutoTokenizer.from_pretrained(verifier.snapshot_root, local_files_only=True) + write("reference-status.json", {"phase": "load-stock-transformers"}) + stock = AutoModelForImageTextToText.from_pretrained( + verifier.snapshot_root, + local_files_only=True, + trust_remote_code=False, + dtype=torch.bfloat16, + device_map="cpu", + attn_implementation="eager", + quantization_config=FineGrainedFP8Config(dequantize=True), + ).eval() + result["stock_class"] = stock.__class__.__name__ + result["stock_dequantizer"] = "Transformers FineGrainedFP8Config(dequantize=True)" + references = [] + with torch.inference_mode(): + for index, prompt in enumerate(PROMPTS): + inputs = tokenizer(prompt, return_tensors="pt").input_ids + cache, logits, tokens = None, [], [] + current = inputs + for step in range(STEPS): + started = time.monotonic() + output = stock(input_ids=current, past_key_values=cache, use_cache=True, logits_to_keep=1) + cache = output.past_key_values + values = output.logits[0, -1].float().cpu().clone() + assert torch.isfinite(values).all() + token = int(values.argmax()) + logits.append(values) + tokens.append(token) + current = torch.tensor([[token]]) + write( + "reference-status.json", + { + "phase": "stock-forward", + "prompt_index": index, + "step": step, + "seconds": time.monotonic() - started, + }, + ) + references.append((inputs, logits, tokens)) + del cache, output + stock = None + gc.collect() + # Stock weights are released before allocating worker weights, so the + # numerical comparison fits inside this explicitly bounded 96 GiB host. + dht = DHT(initial_peers=[], host_maddrs=["/ip4/127.0.0.1/tcp/31330"], start=True, tls=True) + peers = [str(a) for a in dht.get_visible_maddrs()] + for index in range(4): + directory = ROOT / f"reference-worker-{index}" + directory.mkdir(exist_ok=False) + log = (directory / "worker.log").open("wb") + logs.append(log) + port = str(31331 + index) + command = [ + sys.executable, + "-m", + "drift.cli", + "server", + manifest.source.repository, + "--model_manifest", + str(MANIFEST), + "--block_indices", + f"{index*16}:{(index+1)*16}", + "--device", + "cpu", + "--cache_dir", + str(ROOT / "cache"), + "--max_disk_space", + "40GiB", + "--torch_dtype", + "bfloat16", + "--quant_type", + "fp8_dequant", + "--attn_implementation", + "eager", + "--throughput", + "0.01", + "--num_handlers", + "1", + "--inference_max_length", + "64", + "--attn_cache_tokens", + "128", + "--max_batch_size", + "64", + "--update_period", + "10", + "--expiration", + "40", + "--request_timeout", + "600", + "--session_timeout", + "3600", + "--step_timeout", + "600", + "--ready_timeout", + "300", + "--balance_quality", + "0", + "--no_auto_relay", + "--host_maddrs", + "/ip4/127.0.0.1/tcp/" + port, + "--announce_maddrs", + "/ip4/127.0.0.1/tcp/" + port, + "--health_state_path", + str(directory / "health.json"), + "--identity_path", + str(directory / "identity.key"), + "--initial_peers", + *peers, + ] + workers.append( + subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + env=dict(os.environ, OMP_NUM_THREADS="1", MKL_NUM_THREADS="1"), + ) + ) + write("reference-status.json", {"phase": "load-four-rpc-workers"}) + deadline = time.monotonic() + 1800 + while time.monotonic() < deadline: + if any(p.poll() is not None for p in workers): + raise RuntimeError("A reference RPC worker exited during loading; inspect worker logs") + health_paths = [ROOT / f"reference-worker-{i}/health.json" for i in range(4)] + if all(p.exists() and json.loads(p.read_text()).get("worker_healthy") for p in health_paths): + break + time.sleep(5) + else: + raise TimeoutError("RPC workers did not finish loading in 30 minutes") + remote = AutoDistributedModelForCausalLM.from_pretrained( + manifest.source.repository, + revision=manifest.source.revision, + token=False, + initial_peers=peers, + dht_prefix=manifest.dht_prefix, + manifest_digest=manifest.digest, + manifest_execution_profile=manifest.runtime.to_dict(), + torch_dtype=torch.bfloat16, + artifact_verifier=verifier, + request_timeout=300, + connect_timeout=30, + max_retries=2, + update_period=5, + use_server_to_server=True, + ).eval() + for index, (inputs, expected_steps, tokens) in enumerate(references): + current, cache = inputs, None + with torch.inference_mode(), remote.inference_session(max_length=64) as session: + for step, expected in enumerate(expected_steps): + started = time.monotonic() + output = remote(input_ids=current, past_key_values=cache, use_cache=True, logits_to_keep=1) + cache = output.past_key_values + actual = output.logits[0, -1].float().cpu() + delta = (actual - expected).abs() + finite = bool(torch.isfinite(actual).all()) + numeric = finite and bool(torch.allclose(actual, expected, atol=ATOL, rtol=RTOL)) + greedy = int(actual.argmax()) == tokens[step] + check = { + "prompt_index": index, + "step": step, + "finite": finite, + "all_vocabulary_logits_within_tolerance": numeric, + "greedy_token_equal": greedy, + "reference_token": tokens[step], + "actual_token": int(actual.argmax()), + "max_absolute_error": float(delta.max()), + "mean_absolute_error": float(delta.mean()), + "seconds": time.monotonic() - started, + } + result["checks"].append(check) + write("reference-status.json", dict(check, phase="distributed-forward")) + current = torch.tensor([[tokens[step]]]) + route = [(s.span.start, s.span.end, str(s.span.peer_id)) for s in session._server_sessions] + assert len(route) == 4 and route[0][0] == 0 and route[-1][1] == 64 + result["result"] = ( + "passed" + if all(c["all_vocabulary_logits_within_tolerance"] and c["greedy_token_equal"] for c in result["checks"]) + and len(result["checks"]) == len(PROMPTS) * STEPS + else "failed" + ) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + if remote is not None: + try: + _runtime_closer(remote)() + except Exception as exc: + result.update(result="failed", client_cleanup_error=str(exc)) + for process in workers: + if process.poll() is None: + process.terminate() + for process in workers: + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + for log in logs: + log.close() + if dht is not None: + dht.shutdown() + write("reference-result.json", result) + + +if __name__ == "__main__": + try: + main() + except BaseException as exc: + write("reference-error.json", {"error": f"{type(exc).__name__}: {exc}"}) + raise diff --git a/scripts/report_qwen_product.py b/scripts/report_qwen_product.py new file mode 100644 index 000000000..0d3b31d33 --- /dev/null +++ b/scripts/report_qwen_product.py @@ -0,0 +1,148 @@ +"""Validate and summarize a mixed source/packaged Qwen run without changing its raw receipts.""" + +import argparse +import json +from pathlib import Path + +from qwen_product_provenance import sha256 +from qwen_product_recovery import require_recovery_acknowledgements, worker_is_stopped + +REMOTE = "Qwen3.8 27B FP8 Dequant" +LOCAL = "Qwen3.5-0.8B-Local" + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def read(path): + return json.loads(path.read_text(encoding="utf-8-sig")) + + +def completion(value, model): + response = value["response"] + require(response["model"] == model, "Completion selected the wrong model") + require(response["usage"]["completion_tokens"] > 0 and bool(response["choices"]), "Missing generated tokens") + + +def summarize(run, output): + final, packaged = read(run / "result.json"), read(output / "result.json") + receipt = read(run / "packaged-client-result.json") + require(final["run_id"] == packaged["run_id"] == receipt["run_id"] == run.name, "Run ID mismatch") + require(receipt == packaged == final["packaged_client"], "Packaged receipts disagree") + require(final["result"] == packaged["result"] == final["evidence"]["result"] == "passed", "A test failed") + require(final["cleanup"].get("verified") is True, "Cloud cleanup is not verified") + require(packaged.get("node_stopped") is True, "Packaged client cleanup is not verified") + phases = packaged["phases"] + require([p["hub_offline"] for p in phases] == [False, True], "Both ordered cache phases are required") + for phase in phases: + require(phase.get("node_stopped") is True, "A phase left its node running") + require(phase["node_sha256"] == packaged["node_sha256"], "The replay changed packages between phases") + completion(phase["community_completion"], REMOTE) + completion(phase["community_chat"], REMOTE) + completion(phase["local_only_completion"], LOCAL) + require(phases[1].get("http_downloads_blocked") is True, "Offline flags alone do not prove download blocking") + loss = phases[0]["worker_outage"] + require(loss["before_stop_status"]["auto_selection"]["model"] == REMOTE, "Worker loss began while already local") + require(loss["fallback_status"]["auto_selection"]["source"] == "local", "Missing automatic local fallback") + require(loss["recovered_status"]["auto_selection"]["model"] == REMOTE, "Missing automatic community recovery") + completion(loss["local_completion"], LOCAL) + completion(loss["community_after_rejoin"], REMOTE) + require(loss["stop"]["action"] == "stop" and loss["restart"]["action"] == "start", "Missing worker actions") + require(loss["stop"]["instance"] == loss["restart"]["instance"] == run.name + "-w2", "Wrong worker was stopped") + require(worker_is_stopped(loss["stop"]["after"]), "Worker stop was not confirmed") + require(loss["stop"]["observed_at_unix"] < loss["restart"]["observed_at_unix"], "Worker actions are out of order") + source_evidence = final["evidence"] + stopped, replaced = ( + source_evidence["worker_stopped_acknowledgement"], + source_evidence["worker_replaced_acknowledgement"], + ) + require_recovery_acknowledgements( + source_evidence, stopped["recovery_nonce"], stopped["peer_id"], replaced["peer_id"] + ) + config = read(run / "provider-config.json") + instances = [] + # Azure's w1 metadata has a different schema; never glob it as a GCP instance. + paths = [run / f"{run.name}-{suffix}-instance.json" for suffix in ("c", "w0", "w2", "w3")] + for path, machine in zip( + paths, [config["client_machine_type"], config["gpu_machine_type"]] + [config["worker_machine_type"]] * 2 + ): + observed = read(path) + require(observed["name"] == path.name.removesuffix("-instance.json"), "Instance identity mismatch") + require(observed["machineType"].split("/")[-1] == machine, "GCP topology mismatch") + instances.append({"name": observed["name"], "machine_type": machine}) + source = read(run / "source-inventory.json") + require(sha256(run / "source.tar.gz") == source["bundle_sha256"], "Cloud source archive mismatch") + if (run / "launcher-source.json").exists(): + launch = read(run / "launcher-source.json") + launch_result = read(run / "launcher-result.json") + require(launch_result.get("run_id") == run.name, "Launcher result belongs to another run") + require( + launch_result.get("result") == "passed" and launch_result.get("inputs_unchanged") is True, + "Launcher or input verification failed", + ) + require( + launch_result.get("source_inventory_sha256") == sha256(run / "launcher-source.json"), + "Launcher result does not bind its source inventory", + ) + require(launch["run_id"] == run.name, "Launcher inventory belongs to another run") + require(sha256(run / "launcher-source.tar.gz") == launch["archive_sha256"], "Launcher archive mismatch") + require( + all(launch["files"].get(p) == digest for p, digest in source["files"].items()), + "Cloud source differs from launch snapshot", + ) + require( + launch["inputs"]["node"]["sha256"] == packaged["node_sha256"], "Packaged node differs from launch input" + ) + paths += [run / "launcher-source.json", run / "launcher-source.tar.gz", run / "launcher-result.json"] + paths += [ + run / "result.json", + output / "result.json", + run / "packaged-client-result.json", + run / "provider-config.json", + run / "source-inventory.json", + ] + return { + "result": "passed", + "run_id": run.name, + "scope": "assigned mixed route: source recovery and packaged completion/chat, worker stop/rejoin and cache restart", + "node_sha256": packaged["node_sha256"], + "catalog_scope": packaged["catalog_scope"], + "remote_cache_source": packaged["remote_cache_source"], + "gcp_instances": instances, + "phases": phases, + "cleanup": final["cleanup"], + "source_product": final["evidence"], + "source_bundle_sha256": source["bundle_sha256"], + "bindings": [{"path": str(p.resolve()), "sha256": sha256(p)} for p in paths], + "limits": [ + "Assigned spans and explicit bootstrap seeds; autonomous desktop formation remains open", + "Worker service stop/restart between requests; active-generation VM replacement is a separate test", + "HTTP-blocked cache restart keeps swarm RPC online", + "Frozen node/controller exercise; final package installation and ordinary-user UI remain open", + ], + } + + +def report(run, output, destination, *, launcher=None): + require(not destination.exists(), "Preserve the existing report; choose a fresh output") + try: + value = summarize(run, output) + if launcher is not None: + require(launcher["result"] == "passed", "Launcher or input verification failed") + require(launcher["run_id"] == run.name, "Launcher run ID mismatch") + value["launcher"] = launcher + except (OSError, KeyError, TypeError, ValueError, RuntimeError) as exc: + value = {"result": "failed", "run_id": run.name, "error": f"{type(exc).__name__}: {exc}", "launcher": launcher} + destination.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return value + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cloud-run", type=Path, required=True) + parser.add_argument("--packaged-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + raise SystemExit(0 if report(args.cloud_run, args.packaged_output, args.output)["result"] == "passed" else 1) diff --git a/scripts/run_gate13_gcp.py b/scripts/run_gate13_gcp.py new file mode 100644 index 000000000..ae8347ab7 --- /dev/null +++ b/scripts/run_gate13_gcp.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Zero-input, one-command Gate 13 qualification on GCP.""" + +from __future__ import annotations + +import json +import os +import secrets +import sys +import time +from dataclasses import asdict +from pathlib import Path +from typing import Any, Mapping, Sequence + +import gate13_packaged_lifecycle as lifecycle +from gate13_cloud_orchestrator import Gate13CloudError, Gate13CloudOrchestrator, PackageArtifact +from gate13_gcp_provider import GcpConfig, GcpProvider, GitHubPackageSource, LoggedRunner + +REPOSITORY = "flujo-app/CommunityAI" +WORKFLOW = "desktop.yaml" + + +def _write_json(path: Path, value: Mapping[str, Any]) -> None: + payload = json.dumps(value, allow_nan=False, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + path.parent.mkdir(parents=True, exist_ok=True) + try: + with temporary.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def _process_exists(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +class LauncherLock: + def __init__(self, path: Path) -> None: + self.path = path + + def __enter__(self) -> "LauncherLock": + self.path.parent.mkdir(parents=True, exist_ok=True) + try: + with self.path.open("x", encoding="ascii", newline="\n") as stream: + stream.write(str(os.getpid()) + "\n") + stream.flush() + os.fsync(stream.fileno()) + except FileExistsError: + try: + pid = int(self.path.read_text(encoding="ascii").strip()) + except (OSError, UnicodeError, ValueError): + pid = -1 + if _process_exists(pid): + raise Gate13CloudError("another Gate 13 launcher is already running") + self.path.unlink(missing_ok=True) + with self.path.open("x", encoding="ascii", newline="\n") as stream: + stream.write(str(os.getpid()) + "\n") + stream.flush() + os.fsync(stream.fileno()) + return self + + def __exit__(self, *_args: object) -> None: + self.path.unlink(missing_ok=True) + + +def _evidence_validator(run_id: str): + def validate(platform: str, payload: bytes, package: PackageArtifact) -> Mapping[str, Any]: + try: + raw = lifecycle.load_lifecycle_json(payload.decode("utf-8")) + value = lifecycle.validate_lifecycle_document(raw) + except Exception as exc: + raise Gate13CloudError(f"{platform} lifecycle evidence is invalid") from exc + expected = { + "windows": ( + "Qwen3.5 2B", + "3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + ), + "linux": ( + "Gemma 4 E2B IT", + "2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + ), + }[platform] + if ( + value.get("result") != "passed" + or value.get("run_id") != f"{run_id}-{platform}" + or value.get("platform") != platform + or value.get("source_commit") != package.source_commit + or value.get("package_sha256") != package.archive_sha256 + or value.get("package_bytes") != package.archive_bytes + or value.get("model_id") != expected[0] + or value.get("manifest_digest") != expected[1] + ): + raise Gate13CloudError(f"{platform} lifecycle evidence binding changed") + durations = raw.get("session_duration_seconds") + session_duration = ( + round(sum(float(item) for item in durations.values()), 6) if isinstance(durations, dict) else None + ) + return {"result": "passed", "session_duration_seconds": session_duration} + + return validate + + +def _provider( + *, + run_id: str, + repository_root: Path, + output_root: Path, + config: GcpConfig, + runner: LoggedRunner, + packages: GitHubPackageSource | None, +) -> GcpProvider: + def unavailable(_artifact: PackageArtifact) -> str: + raise Gate13CloudError("signed package URL is unavailable during recovery") + + return GcpProvider( + run_id=run_id, + repository_root=repository_root, + output_root=output_root, + config=config, + runner=runner, + signed_url=unavailable if packages is None else packages.signed_download_url, + ) + + +def _new_run_id() -> str: + return time.strftime("g13-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2) + + +def main(argv: Sequence[str] | None = None) -> int: + if list(sys.argv[1:] if argv is None else argv): + print("This launcher accepts no arguments.", file=sys.stderr) + return 2 + repository_root = Path(__file__).resolve().parent.parent + runs_root = repository_root / ".gate13-runs" / "gcp" + try: + config = GcpConfig.load(repository_root / "config" / "gate13_gcp.json") + with LauncherLock(runs_root / "launcher.lock"): + run_id = _new_run_id() + output_root = runs_root / run_id + output_root.mkdir(parents=True, exist_ok=False) + _write_json(output_root / "provider-config.json", asdict(config)) + runner = LoggedRunner(output_root / "command-journal.jsonl") + packages = GitHubPackageSource( + repository_root=repository_root, + output_root=output_root, + repository=REPOSITORY, + workflow=WORKFLOW, + runner=runner, + ) + provider = _provider( + run_id=run_id, + repository_root=repository_root, + output_root=output_root, + config=config, + runner=runner, + packages=packages, + ) + result = Gate13CloudOrchestrator( + run_id=run_id, + package_source=packages, + provider=provider, + output_root=output_root, + evidence_validator=_evidence_validator(run_id), + ).run() + duration = result.get("duration_seconds") + print() + print("=" * 68) + print(f"GATE 13 GCP: {str(result.get('result')).upper()}") + print(f"Run: {run_id}") + print(f"Duration: {duration} seconds") + if result.get("result") != "passed": + failure = next( + ( + event.get("details", {}).get("failed_phase") + for event in result.get("events", []) + if event.get("phase") == "FAILURE" + ), + None, + ) + print(f"Failed phase: {failure or 'cleanup verification'}") + print(f"Reason: {result.get('failure_reason') or result.get('failure_code') or 'unknown'}") + print(f"Result: {output_root / 'result.json'}") + print("=" * 68) + return 0 if result.get("result") == "passed" else 1 + except BaseException as exc: + print() + print("=" * 68) + print("GATE 13 GCP: FAILED BEFORE OR DURING ORCHESTRATION") + print(f"Failure: {type(exc).__name__}: {exc}") + print(f"Runs directory: {runs_root}") + print("=" * 68) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_formation.py b/scripts/run_qwen_formation.py new file mode 100644 index 000000000..2a0bc98d4 --- /dev/null +++ b/scripts/run_qwen_formation.py @@ -0,0 +1,555 @@ +"""One-click organic Qwen formation, live desktop transitions, loss and cleanup. + +Reuses Gate 13's journal and Qwen's proven provisioning/cleanup methods. Unlike +the assigned-route runners, no contributor is given a model or block range. +""" + +from __future__ import annotations + +import hashlib +import ipaddress +import json +import os +import secrets +import shlex +import subprocess +import sys +import tarfile +import time +import urllib.request +from pathlib import Path + +from gate13_cloud_orchestrator import RunRecorder +from qwen_formation_node import coverage, coverage_observed, ready_worker, selected +from run_qwen_full_inference_gcp import ROOT, LauncherLock, _write_json +from run_qwen_mixed_inference import MixedRun +from run_qwen_product_gcp import ProductRun + +RUNS = ROOT / ".gate13-runs/qwen-formation" + + +def validate_config(config): + if "spans" in config or "block_indices" in config: + raise ValueError("Formation rejects operator-assigned spans") + expected = { + "project": "community-ai-506321", + "region": "us-central1", + "client_machine_type": "e2-standard-4", + "capacity_blocks": 16, + "disk_gb": 80, + } + if any(config.get(key) != value for key, value in expected.items()): + raise ValueError("Formation configuration differs from the bounded five-VM CPU topology") + if config.get("zone") not in {"us-central1-b", "us-central1-c", "us-central1-f"}: + raise ValueError("Formation zone must remain in the approved us-central1 test zones") + if config.get("worker_machine_type") not in {"c3-highmem-4", "n2-highmem-4"}: + raise ValueError("Formation contributors must retain the four-vCPU, 32-GB CPU profile") + if not 1800 <= config["max_duration_seconds"] <= 21600: + raise ValueError("Formation lifetime must be bounded between 30 minutes and six hours") + + +class FormationRun(ProductRun): + remote_desktops = True + + def __init__(self, path, config): + super().__init__(path, config) + self.public_ips = {} + self.firewalls.append(self.run_id + "-public") + self.local_root = self.path / "desktop" + self.processes = [] + self.logs = [] + self.recorder = RunRecorder(self.run_id, "gcp-automatic-formation", self.path / "qualification", time.time) + + def phase(self, name, **details): + self.event(name, **details) + self.recorder.phase(name.upper(), **details) + + def preflight(self): + validate_config(self.config) + if os.name == "nt" and ROOT.drive.upper() != "C:": + raise ValueError("This test must run from C:") + packaged = json.loads((ROOT / "config/qwen_product_test.json").read_text()) + node = (ROOT / packaged["node"]).resolve() + if hashlib.sha256(node.read_bytes()).hexdigest() != packaged["node_sha256"]: + raise ValueError("Retained Windows node hash differs from its qualification input") + for field in ("local_cache", "remote_cache"): + if not (ROOT / packaged[field]).is_dir(): + raise ValueError("Verified reusable client cache is missing: " + field) + self.packaged = packaged + self.config["desktop_node_sha256"] = packaged["node_sha256"] + region = self.cloud_json(["compute", "regions", "describe", self.config["region"]]) + project = self.cloud_json(["compute", "project-info", "describe"]) + quotas = {q["metric"]: q for q in region["quotas"] + project["quotas"]} + required = { + "C3_CPUS" if self.config["worker_machine_type"] == "c3-highmem-4" else "N2_CPUS": 16, + "E2_CPUS": 4, + "CPUS_ALL_REGIONS": 20, + "INSTANCES": 5, + "IN_USE_ADDRESSES": 5, + "SSD_TOTAL_GB": 400, + } + for metric, amount in required.items(): + if quotas[metric]["limit"] - quotas[metric]["usage"] < amount: + raise RuntimeError("Insufficient existing quota: " + metric) + instances = self.cloud_json(["compute", "instances", "list"]) + if any(value["name"] in self.names for value in instances): + raise RuntimeError("An exact target VM already exists") + self.cloud(["compute", "images", "describe", self.config["image"]], project=self.config["image_project"]) + self.cloud( + ["compute", "networks", "subnets", "describe", self.config["subnet"], "--region", self.config["region"]] + ) + with urllib.request.urlopen("https://api.ipify.org", timeout=30) as response: + self.config["admin_ip"] = str(ipaddress.IPv4Address(response.read().decode().strip())) + _write_json( + self.path / "preflight.json", + { + "quotas": quotas, + "required": required, + "instances": instances, + "windows_node_sha256": packaged["node_sha256"], + "no_quota_request": True, + }, + ) + _write_json(self.path / "provider-config.json", self.config) + + def bundle(self): + super().bundle() + inventory = json.loads((self.path / "source-inventory.json").read_text()) + names = set(inventory["files"]) | { + "scripts/run_qwen_formation.py", + "scripts/qwen_formation_node.py", + "scripts/qwen_formation_desktop.py", + "scripts/qwen_formation_platform_probe.py", + "scripts/qualify_qwen_formation_local.py", + "config/qwen_formation.json", + } + with tarfile.open(self.path / "source.tar.gz", "w:gz") as archive: + for name in sorted(names): + archive.add(ROOT / name, arcname=name) + inventory["files"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in sorted(names)} + inventory["bundle_sha256"] = hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() + inventory[ + "scope" + ] = "Current source snapshot; source cloud nodes, retained frozen Windows node, production Qt source" + _write_json(self.path / "source-inventory.json", inventory) + + def setup_source(self, name, digest): + source = super().setup_source(name, digest) + if name != self.names[0]: + source = source.replace( + "touch /srv/q38/setup-ready", + "/opt/q38/venv/bin/pip install --no-cache-dir '/opt/q38/source[api]'\ntouch /srv/q38/setup-ready", + ) + source = source.replace( + "python3-venv python3-pip", + "python3-venv python3-pip xvfb xauth libgl1 libegl1 libglib2.0-0 libdbus-1-3 libxkbcommon-x11-0 libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0 libxcb-xinerama0 libxcb-randr0 libxcb-render-util0", + ) + return source.replace( + "touch /srv/q38/setup-ready", + "/opt/q38/venv/bin/pip install --no-cache-dir 'PySide6==6.11.2'\nxvfb-run -a /opt/q38/venv/bin/python /opt/q38/source/scripts/qwen_formation_platform_probe.py\ntouch /srv/q38/setup-ready", + ) + + def host_config(self, name, span, peers): + if span is not None: + raise ValueError("Formation staging rejects assigned ranges") + return { + "ip": self.public_ips[name], + "peers": list(peers), + "expires_at_unix": self.deadline, + "capacity_blocks": 0 if name == self.names[0] else self.config["capacity_blocks"], + "desktop_driven_sharing": True, + } + + def write_remote(self, name, filename, value): + self.ssh( + name, + "sudo /opt/q38/venv/bin/python -c " + + shlex.quote( + "from pathlib import Path; " + f"p=Path('/srv/q38/{filename}'); t=p.with_suffix('.tmp'); " + f"t.write_text({json.dumps(value)!r}); t.replace(p)" + ), + ) + + def start_participant(self, name): + self.ssh( + name, + "sudo systemd-run --unit=q38-formation --uid=q38 --property=KillMode=control-group " + "--property=TimeoutStopSec=45 --property=StandardOutput=append:/srv/q38/formation.log " + "--property=StandardError=append:/srv/q38/formation.log " + "--setenv=OMP_NUM_THREADS=4 --setenv=MKL_NUM_THREADS=4 " + "/opt/q38/venv/bin/python /opt/q38/source/scripts/qwen_formation_node.py", + ) + self.ssh( + name, + "sudo systemd-run --unit=q38-desktop --uid=q38 --property=KillMode=control-group " + "--property=TimeoutStopSec=45 --property=StandardOutput=append:/srv/q38/desktop.log " + "--property=StandardError=append:/srv/q38/desktop.log " + "--setenv=PYTHONPATH=/opt/q38/source/desktop/src " + "/usr/bin/xvfb-run -a /opt/q38/venv/bin/python /opt/q38/source/scripts/qwen_formation_desktop.py --root /srv/q38", + ) + + def read_participant(self, name, filename): + if name != "desktop": + return self.read(name, filename) + try: + return json.loads((self.local_root / filename).read_text()) + except (FileNotFoundError, ValueError): + return None + + def wait(self, name, filename, predicate=lambda _: True, timeout=1800): + until = min(self.deadline, time.time() + timeout) + while time.time() < until: + value = self.read_participant(name, filename) + fresh = value is not None and ( + filename != "formation-status.json" or time.time() - value["observed_at_unix"] <= 60 + ) + if fresh and predicate(value): + _write_json(self.path / (name + "-" + filename), value) + return value + for error_file in ("formation-error.json", "desktop-error.json"): + error = self.read_participant(name, error_file) + if error: + raise RuntimeError(name + ": " + str(error)) + if name == "desktop" and any(p.poll() is not None for p in self.processes): + raise RuntimeError("Owned desktop process exited") + if value and filename == "formation-status.json": + self.event( + "waiting-formation", + instance=name, + coverage=coverage(value), + selection=value.get("auto_selection", {}).get("reason"), + workers=[ + {k: w.get(k) for k in ("state", "block_indices", "policy_reason", "last_error")} + for w in value.get("workers", []) + ], + ) + time.sleep(10) + raise TimeoutError(f"{name}: {filename} did not satisfy its checkpoint") + + def command(self, name, action, **details): + identity = secrets.token_hex(12) + value = dict(id=identity, action=action, **details) + if name == "desktop": + _write_json(self.local_root / "formation-command.json", value) + else: + self.write_remote(name, "formation-command.json", value) + reply = self.wait(name, "formation-response-" + identity + ".json", timeout=660) + if reply.get("id") != identity or reply.get("result") != "passed": + raise RuntimeError("Participant command failed: " + str(reply)) + return reply + + def desktop(self, source, *, toggle=False, inference_mode=None, name="desktop", action=None): + identity = secrets.token_hex(12) + if toggle and inference_mode is None: + inference_mode = "local_only" if source == "local" else "auto" + request = { + "id": identity, + "action": action or ("toggle" if toggle else "observe"), + "source": source, + "inference_mode": inference_mode, + } + if name == "desktop": + _write_json(self.local_root / "desktop-command.json", request) + else: + self.write_remote(name, "desktop-command.json", request) + value = self.wait(name, "desktop-response-" + identity + ".json", timeout=1800) + if not value.get("real_window_visible") or ( + (toggle or action == "start-sharing") and not value.get("button_clicked") + ): + raise RuntimeError("Real desktop control evidence is missing") + return value + + def enable_contributor(self, name): + return self.desktop("local", name=name, action="start-sharing") + + def observe_remote_desktop(self, name, source): + if getattr(self, "remote_desktops", False) and name != "desktop": + return self.desktop(source, name=name) + return None + + def start_desktop(self, peers): + self.local_root.mkdir() + config = { + "peers": peers, + "expires_at_unix": self.deadline, + "capacity_blocks": 0, + "api_port": 18091, + "local_device": self.packaged["device"], + "node_executable": str((ROOT / self.packaged["node"]).resolve()), + "local_cache": str((ROOT / self.packaged["local_cache"]).resolve()), + "remote_cache": str((ROOT / self.packaged["remote_cache"]).resolve()), + } + _write_json(self.local_root / "config.json", config) + for script in ("qwen_formation_node.py", "qwen_formation_desktop.py"): + log = (self.local_root / (script + ".log")).open("wb") + self.logs.append(log) + self.processes.append( + subprocess.Popen( + [ + str(getattr(self, "desktop_python", sys.executable)), + str(ROOT / "scripts" / script), + "--root", + str(self.local_root), + ], + stdout=log, + stderr=subprocess.STDOUT, + env=dict(os.environ, PYTHONPATH=str(ROOT / "desktop/src")), + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + ) + + def exercise(self): + clients = [*self.names, "desktop"] + evidence = {"local_before_growth": {}, "joins": [], "promoted": {}, "after_loss": {}, "recovered": {}} + self.phase("prove-local-on-all-participants") + for name in clients: + self.wait( + name, + "formation-status.json", + lambda value: selected(value, "local") and coverage_observed(value), + timeout=180, + ) + evidence["local_before_growth"][name] = self.command(name, "infer", source="local") + self.observe_remote_desktop(name, "local") + evidence["desktop_initial"] = self.desktop("local") + for index, name in enumerate(self.names[1:]): + self.phase("join-automatic-contributor", instance=name, capacity_blocks=16) + self.wait(name, "formation-status.json", lambda value: coverage(value) >= index * 16) + self.enable_contributor(name) + ready = self.wait(name, "formation-status.json", ready_worker, timeout=2700) + complete = self.wait( + self.names[0], "formation-status.json", lambda value: coverage(value) == (index + 1) * 16 + ) + evidence["joins"].append( + {"instance": name, "automatic_worker": ready["workers"][0], "observed_coverage": coverage(complete)} + ) + _write_json(self.path / "formation-checkpoints.json", evidence) + self.phase("prove-automatic-promotion") + for name in clients: + status = self.wait( + name, "formation-status.json", lambda value: selected(value, "community") and coverage(value) == 64 + ) + evidence["promoted"][name] = { + "desktop": self.observe_remote_desktop(name, "community"), + "selection": status["auto_selection"], + "inference": self.command(name, "infer", source="community"), + } + _write_json(self.path / "formation-checkpoints.json", evidence) + evidence["desktop_promoted"] = self.desktop("community") + evidence["desktop_local_button"] = self.desktop("local", toggle=True) + evidence["desktop_local_request"] = self.command("desktop", "infer", source="local") + evidence["desktop_auto_button"] = self.desktop("community", toggle=True) + lost = self.names[2] + original = self.read(lost, "formation-process.json") + self.phase("kill-participant", instance=lost) + state = self.stop_participant(lost) + evidence["loss"] = {"instance": lost, "before": original, "stopped_service": state} + _write_json(self.path / "formation-checkpoints.json", evidence) + for name in [n for n in clients if n != lost]: + self.wait(name, "formation-status.json", lambda value: selected(value, "local") and coverage(value) < 64) + evidence["after_loss"][name] = self.command(name, "infer", source="local") + self.observe_remote_desktop(name, "local") + _write_json(self.path / "formation-checkpoints.json", evidence) + evidence["desktop_after_loss"] = self.desktop("local") + self.phase("restore-participant-without-assigning-blocks", instance=lost) + self.restart_participant(lost) + self.wait(lost, "formation-process.json", lambda value: value["started"] > original["started"]) + for name in clients: + status = self.wait( + name, "formation-status.json", lambda value: selected(value, "community") and coverage(value) == 64 + ) + evidence["recovered"][name] = { + "desktop": self.observe_remote_desktop(name, "community"), + "selection": status["auto_selection"], + "inference": self.command(name, "infer", source="community"), + } + _write_json(self.path / "formation-checkpoints.json", evidence) + evidence["desktop_recovered"] = self.desktop("community") + _write_json(self.path / "formation-checkpoints.json", evidence) + return evidence + + def stop_participant(self, name): + states = {} + for unit in ("q38-desktop", "q38-formation"): + self.ssh(name, "sudo systemctl kill --kill-whom=all --signal=SIGKILL " + unit) + self.ssh(name, "sudo systemctl stop " + unit) + state = self.ssh(name, "systemctl show " + unit + " --property=ActiveState --property=MainPID").stdout + if "MainPID=0" not in state or not any(s in state for s in ("ActiveState=failed", "ActiveState=inactive")): + raise RuntimeError("Participant loss was not confirmed: " + unit) + states[unit] = state + return states + + def restart_participant(self, name): + self.ssh(name, "sudo systemctl restart q38-formation") + self.ssh(name, "sudo systemctl restart q38-desktop") + + def capture(self): + super().capture() + for name in self.names: + archived = self.ssh( + name, + "sudo /opt/q38/venv/bin/python -c " + + shlex.quote( + "from pathlib import Path; import tarfile; root=Path('/srv/q38'); " + "t=tarfile.open('/tmp/formation-evidence.tar.gz','w:gz'); " + "[t.add(p,arcname=p.name) for p in root.iterdir() if p.is_file() and p.suffix in {'.json','.log','.png'}]; t.close()" + ), + check=False, + ) + if not archived.returncode: + self.cloud( + [ + "compute", + "scp", + name + ":/tmp/formation-evidence.tar.gz", + str(self.path / (name + "-evidence.tar.gz")), + "--zone", + self.config["zone"], + "--tunnel-through-iap", + ], + check=False, + ) + + def stop_desktop(self): + if self.local_root.exists(): + (self.local_root / "desktop-stop").touch() + import psutil + + trees = [] + for process in self.processes: + if process.poll() is None: + try: + trees.extend(psutil.Process(process.pid).children(recursive=True)) + except psutil.NoSuchProcess: + pass + process.terminate() + for process in self.processes: + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + for child in trees: + try: + child.terminate() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs(trees, timeout=15) + for child in alive: + try: + child.kill() + except psutil.NoSuchProcess: + pass + _, remaining = psutil.wait_procs(alive, timeout=10) + for log in self.logs: + log.close() + if remaining: + raise RuntimeError("Owned local process tree did not stop") + + def run(self): + mutated = False + result = { + "result": "failed", + "run_id": self.run_id, + "scope": "staggered-automatic-formation", + "catalog_policy_modified": False, + "assigned_spans": False, + "cloud_runtime": "production source node", + "desktop_node": "retained Windows package", + "desktop_ui": "production Qt source", + "remote_ui": "production Qt on Xvfb, actual Save and Start sharing clicks", + "isolated_test_seed": True, + "simultaneous_cold_join_proved": False, + "consumer_gpu_formation_proved": False, + } + try: + self.phase("preflight") + self.preflight() + self.bundle() + mutated = True + self.phase("create-owned-network-and-five-hosts") + self.create_firewalls() + for name in self.names: + MixedRun.create_gcp( + self, + name, + self.config["client_machine_type"] if name == self.names[0] else self.config["worker_machine_type"], + boot_disk_type="pd-balanced", + ) + self.cloud( + [ + "compute", + "firewall-rules", + "create", + self.run_id + "-public", + "--network", + self.config["network"], + "--allow=tcp:31330", + "--source-ranges", + ",".join(ip + "/32" for ip in [*self.public_ips.values(), self.config["admin_ip"]]), + "--target-tags", + self.run_id, + ] + ) + self.stage(self.names[0]) + self.wait_setup(self.names[:1]) + self.start_job(self.names[0], "bootstrap") + peers = self.wait_file(self.names[0], "bootstrap.json", role="bootstrap")["peers"] + self.write_remote(self.names[0], "config.json", self.host_config(self.names[0], None, peers)) + self.start_participant(self.names[0]) + self.start_desktop(peers) + for name in self.names[1:]: + self.stage(name, peers=peers) + self.wait_setup(self.names[1:]) + for name in self.names[1:]: + self.start_participant(name) + result["evidence"] = self.exercise() + result["result"] = "passed" + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.phase("failed", error=result["error"]) + finally: + try: + self.stop_desktop() + result["local_process_cleanup_verified"] = True + except Exception as exc: + result.update(result="failed", local_cleanup_error=str(exc)) + if mutated: + try: + self.capture() + except Exception as exc: + result["diagnostic_error"] = str(exc) + finally: + try: + self.phase("cleanup-owned-resources") + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + else: + result["cloud_resources_created"] = False + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.recorder.cleanup = result.get("cleanup") + self.recorder.finish(result["result"], failure_reason=result.get("error")) + self.event("finished", result=result["result"], result_path=str(self.path / "result.json")) + return result + + +def main(): + if sys.argv[1:]: + raise SystemExit("This one-click runner accepts no arguments") + with LauncherLock(RUNS / "launcher.lock"): + config = json.loads((ROOT / "config/qwen_formation.json").read_text()) + path = RUNS / (time.strftime("q38af-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(3)) + path.mkdir(parents=True, exist_ok=False) + _write_json(path / "provider-config.json", config) + result = FormationRun(path, config).run() + print(f"\nQWEN FORMATION: {result['result'].upper()}\nEvidence: {path / 'result.json'}", flush=True) + return 0 if result["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_formation_modal.py b/scripts/run_qwen_formation_modal.py new file mode 100644 index 000000000..c0e570009 --- /dev/null +++ b/scripts/run_qwen_formation_modal.py @@ -0,0 +1,301 @@ +"""Gate 13-style bounded Modal provider for the same automatic formation test.""" + +from __future__ import annotations + +import hashlib +import json +import os +import secrets +import socket +import sys +import tarfile +import time +from pathlib import Path + +import modal +from gate13_cloud_orchestrator import RunRecorder +from modal.exception import SandboxFilesystemNotFoundError +from qualify_qwen_modal_transport import confirm_app_stopped +from run_qwen_formation import ROOT, FormationRun, LauncherLock, _write_json + +RUNS = ROOT / ".gate13-runs/qwen-formation-modal" + + +class ModalFormationRun(FormationRun): + remote_desktops = True + + def __init__(self, path): + super().__init__(path, {"max_duration_seconds": 21600, "capacity_blocks": 16}) + self.sandboxes = {} + self.resources = {} + self.app = modal.App(self.run_id) + self.desktop_python = ROOT / ".gate13-runs/qwen-product-venv/Scripts/python.exe" + self.recorder = RunRecorder(self.run_id, "modal-automatic-formation", self.path / "qualification", time.time) + + def preflight(self): + if os.name != "nt" or ROOT.drive.upper() != "C:": + raise RuntimeError("This launcher requires the retained Windows desktop on C:") + if not self.desktop_python.is_file(): + raise RuntimeError("Qualified desktop Python is missing") + self.packaged = json.loads((ROOT / "config/qwen_product_test.json").read_text()) + if hashlib.sha256((ROOT / self.packaged["node"]).read_bytes()).hexdigest() != self.packaged["node_sha256"]: + raise RuntimeError("Retained Windows node hash mismatch") + for key in ("local_cache", "remote_cache"): + if not (ROOT / self.packaged[key]).is_dir(): + raise RuntimeError("Verified desktop cache is missing: " + key) + with socket.socket() as probe: + probe.bind(("127.0.0.1", 18091)) + _write_json( + self.path / "preflight.json", + { + "windows_node_sha256": self.packaged["node_sha256"], + "worker_count": 4, + "cpu_physical_cores_each": 2, + "worker_memory_mib": 32768, + "client_memory_mib": 16384, + "remote_ui": "production Qt on Xvfb", + "catalog_sequence": 2, + }, + ) + + def bundle(self): + super().bundle() + inventory = json.loads((self.path / "source-inventory.json").read_text()) + names = set(inventory["files"]) | { + "scripts/run_qwen_formation_modal.py", + "scripts/qwen_modal_participant.py", + "scripts/qualify_qwen_modal_transport.py", + } + with tarfile.open(self.path / "source.tar.gz", "w:gz") as archive: + for name in sorted(names): + archive.add(ROOT / name, arcname=name) + inventory["files"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in sorted(names)} + inventory["bundle_sha256"] = hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() + _write_json(self.path / "source-inventory.json", inventory) + + def image(self): + digest = hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() + return ( + modal.Image.debian_slim(python_version="3.12") + .apt_install( + "git", + "build-essential", + "xvfb", + "xauth", + "libgl1", + "libegl1", + "libdbus-1-3", + "libglib2.0-0", + "libxkbcommon-x11-0", + "libxcb-cursor0", + "libxcb-icccm4", + "libxcb-keysyms1", + "libxcb-shape0", + "libxcb-xinerama0", + "libxcb-randr0", + "libxcb-render-util0", + ) + .pip_install("torch==2.6.0", index_url="https://download.pytorch.org/whl/cpu") + .add_local_file(self.path / "source.tar.gz", "/tmp/source.tar.gz", copy=True) + .run_commands( + f"echo '{digest} /tmp/source.tar.gz' | sha256sum -c -", + "mkdir -p /opt/q38/source /srv/q38", + "tar -xzf /tmp/source.tar.gz -C /opt/q38/source", + "pip install --no-cache-dir '/opt/q38/source[api]' 'PySide6==6.11.2'", + "xvfb-run -a python /opt/q38/source/scripts/qwen_formation_platform_probe.py", + ) + .env( + { + "PYTHONPATH": "/opt/q38/source/desktop/src", + "OMP_NUM_THREADS": "4", + "MKL_NUM_THREADS": "4", + "HF_HUB_DISABLE_XET": "1", + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", + } + ) + ) + + def create_participant(self, name, image): + memory = 16384 if name == self.names[0] else 32768 + self.phase("create-modal-desktop", instance=name, memory_mib=memory) + sandbox = modal.Sandbox.create( + "sleep", + "21600", + app=self.app, + image=image, + cpu=(2, 2), + memory=(memory, memory), + timeout=max(600, int(self.deadline - time.time()) + 300), + unencrypted_ports=[31330], + tags={"communityai-run": self.run_id, "participant": name}, + ) + self.sandboxes[name] = sandbox + self.resources[name] = {"sandbox_id": sandbox.object_id} + _write_json(self.path / "resources.json", self.resources) + hostname, port = sandbox.tunnels()[31330].tcp_socket + self.resources[name].update(ip=socket.gethostbyname(hostname), public_port=port) + _write_json(self.path / "resources.json", self.resources) + + def execute(self, name, *args): + process = self.sandboxes[name].exec(*args, timeout=120) + output = process.stdout.read() + errors = process.stderr.read() + process.wait() + if process.returncode: + raise RuntimeError(f"{name}: command failed: {errors[-3000:]} {output[-1000:]}") + return output + + def read(self, name, filename): + try: + return json.loads(self.sandboxes[name].filesystem.read_text("/srv/q38/" + filename)) + except (SandboxFilesystemNotFoundError, json.JSONDecodeError): + return None + + def write_remote(self, name, filename, value): + # Atomic rename keeps the node from seeing a partially written command. + self.sandboxes[name].filesystem.write_text(json.dumps(value), "/srv/q38/" + filename + ".tmp") + self.execute( + name, + "python", + "-c", + f"from pathlib import Path; Path('/srv/q38/{filename}.tmp').replace('/srv/q38/{filename}')", + ) + + def config_for(self, name, peers): + endpoint = self.resources[name] + return dict( + ip=endpoint["ip"], + public_port=endpoint["public_port"], + peers=peers, + expires_at_unix=self.deadline, + capacity_blocks=0 if name == self.names[0] else 16, + desktop_driven_sharing=True, + api_port=8080, + ) + + def start_participant(self, name): + return json.loads(self.execute(name, "python", "/opt/q38/source/scripts/qwen_modal_participant.py", "start")) + + def stop_participant(self, name): + return json.loads(self.execute(name, "python", "/opt/q38/source/scripts/qwen_modal_participant.py", "stop")) + + def restart_participant(self, name): + return self.start_participant(name) + + def enable_contributor(self, name): + return self.desktop("local", name=name, action="start-sharing") + + def capture(self): + errors = {} + for name, sandbox in self.sandboxes.items(): + try: + # Only top-level evidence; never export API/identity keys or caches. + self.execute( + name, + "python", + "-c", + "from pathlib import Path; import tarfile; " + "root=Path('/srv/q38'); t=tarfile.open('/tmp/evidence.tar.gz','w:gz'); " + "[t.add(p,arcname=p.name) for p in root.iterdir() if p.is_file() and p.suffix in {'.json','.log','.png'}]; t.close()", + ) + sandbox.filesystem.copy_to_local("/tmp/evidence.tar.gz", self.path / (name + "-evidence.tar.gz")) + except Exception as exc: + errors[name] = str(exc) + if errors: + _write_json(self.path / "capture-errors.json", errors) + + def cleanup(self): + evidence = {} + for name, sandbox in self.sandboxes.items(): + try: + sandbox.terminate(wait=True) + evidence[name] = {"sandbox_id": sandbox.object_id, "exit_code": sandbox.poll()} + except Exception as exc: + evidence[name] = {"error": str(exc)} + result = {"verified": all(v.get("exit_code") is not None for v in evidence.values()), "sandboxes": evidence} + _write_json(self.path / "cleanup.json", result) + return result + + def run(self): + result = dict( + result="failed", + run_id=self.run_id, + scope="staggered-automatic-formation", + assigned_spans=False, + catalog_policy_modified=False, + isolated_test_seed=True, + remote_ui="production Qt on Xvfb", + local_ui="production Qt with retained frozen Windows node", + loss_scope="entire contributor node and desktop process trees", + simultaneous_cold_join_proved=False, + consumer_gpu_formation_proved=False, + ) + try: + self.phase("preflight") + self.preflight() + self.bundle() + with modal.enable_output(), self.app.run(): + try: + image = self.image() + for name in self.names: + self.create_participant(name, image) + coordinator = self.names[0] + self.write_remote(coordinator, "config.json", self.config_for(coordinator, [])) + self.execute( + coordinator, + "python", + "-c", + "import subprocess; " + "log=open('/srv/q38/bootstrap.log','ab'); " + "subprocess.Popen(['python','/opt/q38/source/scripts/qwen_full_inference_host.py','bootstrap']," + "stdout=log,stderr=log,start_new_session=True)", + ) + peers = self.wait(coordinator, "bootstrap.json", timeout=180)["peers"] + for name in self.names: + self.write_remote(name, "config.json", self.config_for(name, peers)) + self.start_participant(name) + self.start_desktop(peers) + result["evidence"] = self.exercise() + result["result"] = "passed" + finally: + self.capture() + result["cleanup"] = self.cleanup() + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.phase("failed", error=result["error"]) + finally: + try: + self.stop_desktop() + result["local_cleanup_verified"] = True + except Exception as exc: + result.update(result="failed", local_cleanup_error=str(exc)) + if self.app.app_id: + try: + result["app_cleanup"] = confirm_app_stopped(self.app.app_id) + except Exception as exc: + result["app_cleanup"] = {"verified": False, "error": str(exc)} + if not result.get("cleanup", {}).get("verified") or not result["app_cleanup"]["verified"]: + result["result"] = "failed" + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.recorder.cleanup = result.get("cleanup") + self.recorder.finish(result["result"], failure_reason=result.get("error")) + self.event("finished", result=result["result"], evidence=str(self.path / "result.json")) + return result + + +def main(): + if sys.argv[1:]: + raise SystemExit("This one-click runner accepts no arguments") + os.environ["PYTHONIOENCODING"] = "utf-8" + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8") + with LauncherLock(RUNS / "launcher.lock"): + path = RUNS / (time.strftime("q38mf-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(3)) + result = ModalFormationRun(path).run() + return 0 if result["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_full_inference_gcp.py b/scripts/run_qwen_full_inference_gcp.py new file mode 100644 index 000000000..318762b0a --- /dev/null +++ b/scripts/run_qwen_full_inference_gcp.py @@ -0,0 +1,712 @@ +#!/usr/bin/env python3 +"""Gate 13-derived one-click CPU swarm, inference, VM-loss recovery and cleanup. + +Uses Gate 13's argv-only GCP command runner, durable writes and launcher lock. +Unlike the desktop qualification runner, stages the current source snapshot and +the pinned Qwen manifest and runs a real source-runtime inference experiment. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import secrets +import shlex +import subprocess +import sys +import tarfile +import time +from pathlib import Path + +from gate13_gcp_provider import CommandError, LoggedRunner +from run_gate13_gcp import _write_json + +ROOT = Path(__file__).resolve().parent.parent +RUNS = ROOT / ".gate13-runs" / "qwen-full" +MANIFEST_DIGEST = "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" +REVISION = "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a" + + +def process_exists(pid): + if pid <= 0: + return False + if sys.platform == "win32": + # Windows os.kill(pid, 0) is not a safe process-existence probe. + import ctypes + from ctypes import wintypes + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel.OpenProcess.restype = wintypes.HANDLE + kernel.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + handle = kernel.OpenProcess(0x1000, False, pid) + if not handle: + return ctypes.get_last_error() != 87 + try: + code = wintypes.DWORD() + if not kernel.GetExitCodeProcess(handle, ctypes.byref(code)): + raise ctypes.WinError(ctypes.get_last_error()) + return code.value == 259 + finally: + kernel.CloseHandle(handle) + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +class LauncherLock: + """Gate 13 lock pattern with a read-only Windows process check.""" + + def __init__(self, path): + self.path = path + + def __enter__(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + if self.path.exists(): + try: + pid = int(self.path.read_text(encoding="ascii").strip()) + except (ValueError, UnicodeError): + pid = -1 + if process_exists(pid): + raise RuntimeError("another Qwen full-inference launcher is already running") + self.path.unlink() + with self.path.open("x", encoding="ascii") as stream: + stream.write(str(os.getpid()) + "\n") + stream.flush() + os.fsync(stream.fileno()) + return self + + def __exit__(self, *_args): + self.path.unlink(missing_ok=True) + + +def validate_route(route): + cursor = 0 + peers = set() + for span in route: + if span["start"] != cursor or span["end"] <= cursor or not span["peer_id"]: + raise ValueError("route has a gap, overlap or invalid peer") + cursor = span["end"] + peers.add(span["peer_id"]) + if cursor != 64 or len(peers) != 4: + raise ValueError("route must traverse all 64 blocks on four independent workers") + + +def validate_result(result, lost_peer): + if result.get("result") != "passed" or result.get("manifest_digest") != MANIFEST_DIGEST: + raise ValueError("client result/manifest binding failed") + if result.get("model_revision") != REVISION: + raise ValueError("client model revision changed") + baseline, recovery = result["baseline"], result["recovery"] + for route in (baseline["route"], recovery["before_route"], recovery["after_route"]): + validate_route(route) + if len(baseline["token_ids"]) != 3 or any(type(t) is not int or t < 0 for t in baseline["token_ids"]): + raise ValueError("baseline must contain three real token IDs") + if recovery["token_ids"] != baseline["token_ids"] or not recovery["matches_baseline"]: + raise ValueError("recovered token sequence differs from baseline") + before = {s["peer_id"] for s in recovery["before_route"]} + after = {s["peer_id"] for s in recovery["after_route"]} + if lost_peer not in before or lost_peer in after or len(after - before) != 1: + raise ValueError("route did not replace exactly the lost worker") + if not recovery["same_session"] or recovery["position_after"] <= recovery["position_before"]: + raise ValueError("client session did not advance after worker loss") + + +class SwarmRun: + def __init__(self, path, config): + self.path, self.config, self.run_id = path, config, path.name + path.mkdir(parents=True, exist_ok=True) + self.runner = LoggedRunner(path / "command-journal.jsonl", progress=lambda _: None) + self.names = [self.run_id + "-c"] + [self.run_id + f"-w{i}" for i in range(4)] + self.firewalls = [self.run_id + "-iap", self.run_id + "-swarm"] + self.started = time.time() + self.deadline = self.started + config["max_duration_seconds"] - 600 + self.ips = {} + + def event(self, phase, **details): + print(f"[{time.strftime('%H:%M:%S')}] {phase} " + json.dumps(details), flush=True) + value = {"phase": phase, "time": time.time(), **details} + _write_json(self.path / "status.json", value) + with (self.path / "events.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(value) + "\n") + + def cloud(self, args, *, timeout=300, check=True, project=None): + # Retry transport failures only for read-only control-plane requests. + # Mutations are reconciled by their callers, never blindly repeated. + attempts = 3 if len(args) > 2 and args[2] in {"list", "describe"} else 1 + for attempt in range(attempts): + try: + result = self.runner.run( + ["gcloud", *args, "--project", project or self.config["project"], "--quiet"], + action="gcloud:" + ":".join(args[:3]), + timeout=timeout, + check=False, + ) + except CommandError: + if attempt + 1 == attempts: + raise + else: + if not result.returncode or attempt + 1 == attempts: + break + self.event("retry-cloud-read", operation=args[:3], attempt=attempt + 1) + time.sleep(10) + if check and result.returncode: + raise RuntimeError(f"gcloud {' '.join(args[:3])}: {result.stderr[-2500:]}") + return result + + def cloud_json(self, args): + return json.loads(self.cloud([*args, "--format=json"]).stdout) + + def ssh(self, name, command, *, check=True): + try: + return self.cloud( + ["compute", "ssh", name, "--zone", self.config["zone"], "--tunnel-through-iap", "--command", command], + timeout=60, + check=check, + ) + except CommandError as exc: + if check: + raise + self.event("ssh-monitor-unavailable", instance=name, error=str(exc)) + return subprocess.CompletedProcess(["gcloud", "compute", "ssh"], 255, "", str(exc)) + + def scp(self, name, local, remote): + # Recopying the same staged file is idempotent; the host verifies its hash. + for attempt in range(5): + try: + self.cloud( + [ + "compute", + "scp", + str(local), + f"{name}:{remote}", + "--zone", + self.config["zone"], + "--tunnel-through-iap", + ], + timeout=180, + ) + return + except (CommandError, RuntimeError) as exc: + if attempt == 4 or time.time() >= self.deadline: + raise + self.event("retry-stage-copy", instance=name, attempt=attempt + 1, error=str(exc)[-500:]) + time.sleep(10) + + def read(self, name, filename): + response = self.ssh(name, "sudo cat " + shlex.quote("/srv/q38/" + filename), check=False) + if response.returncode: + return None + try: + return json.loads(response.stdout) + except ValueError: + return None + + def preflight(self): + c = self.config + manifest = json.loads((ROOT / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json").read_text()) + digest = hashlib.sha256( + json.dumps(manifest, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + if "sha256:" + digest != MANIFEST_DIGEST or manifest["source"]["revision"] != REVISION: + raise ValueError("model manifest differs from the pinned full-inference experiment") + if c["spans"] != ["0:16", "16:32", "32:48", "48:64"]: + raise ValueError("CPU topology must contain the four authorized 16-block spans") + if c["worker_machine_type"] != "e2-highmem-4" or c["client_machine_type"] != "e2-standard-4": + raise ValueError("machine types differ from the requested CPU test") + instances = self.cloud_json(["compute", "instances", "list"]) + if any(i["name"] in self.names for i in instances): + raise RuntimeError("exact target instance already exists") + region = self.cloud_json(["compute", "regions", "describe", c["region"]]) + project = self.cloud_json(["compute", "project-info", "describe"]) + quotas = {q["metric"]: q for q in region["quotas"] + project["quotas"]} + for metric, required in { + "E2_CPUS": 20, + "CPUS_ALL_REGIONS": 20, + "INSTANCES": 5, + "IN_USE_ADDRESSES": 5, + "DISKS_TOTAL_GB": c["disk_gb"] * 5, + }.items(): + q = quotas[metric] + if q["limit"] - q["usage"] < required: + raise RuntimeError(f"insufficient {metric} quota; no quota request will be made") + self.cloud(["compute", "images", "describe", c["image"]], project=c["image_project"]) + self.cloud(["compute", "networks", "subnets", "describe", c["subnet"], "--region", c["region"]]) + _write_json( + self.path / "preflight.json", + { + "instances": instances, + "quotas": quotas, + "authorization": "User requested five-VM CPU inference and worker replacement", + "max_duration_seconds": c["max_duration_seconds"], + }, + ) + + def bundle(self): + files = [ + p + for p in (ROOT / "src/drift").rglob("*") + if p.is_file() and "__pycache__" not in p.parts and p.suffix not in {".pyc", ".pyo"} + ] + files += [ + ROOT / "pyproject.toml", + ROOT / "README.md", + ROOT / "LICENSE", + ROOT / "scripts/qwen_full_inference_host.py", + ROOT / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json", + ] + files = [p for p in files if p.is_file()] + inventory = {p.relative_to(ROOT).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(files)} + with tarfile.open(self.path / "source.tar.gz", "w:gz") as tar: + for p in files: + tar.add(p, arcname=p.relative_to(ROOT).as_posix()) + _write_json( + self.path / "source-inventory.json", + { + "files": inventory, + "bundle_sha256": hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest(), + }, + ) + + def create_firewalls(self): + c = self.config + self.cloud( + [ + "compute", + "firewall-rules", + "create", + self.firewalls[0], + "--network", + c["network"], + "--allow=tcp:22", + "--source-ranges=35.235.240.0/20", + "--target-tags", + self.run_id, + ] + ) + self.cloud( + [ + "compute", + "firewall-rules", + "create", + self.firewalls[1], + "--network", + c["network"], + "--allow=tcp:31330", + "--source-tags", + self.run_id, + "--target-tags", + self.run_id, + ] + ) + + def create(self, names, machine): + c = self.config + self.cloud( + [ + "compute", + "instances", + "create", + *names, + "--zone", + c["zone"], + "--machine-type", + machine, + "--image", + c["image"], + "--image-project", + c["image_project"], + "--subnet", + c["subnet"], + "--boot-disk-size", + str(c["disk_gb"]), + "--boot-disk-type=pd-standard", + "--tags", + self.run_id, + "--labels", + f"q38-run={self.run_id}", + "--no-service-account", + "--no-scopes", + "--max-run-duration", + str(max(600, int(self.deadline - time.time() + 300))) + "s", + "--instance-termination-action=DELETE", + ], + timeout=600, + ) + for name in names: + instance = self.cloud_json(["compute", "instances", "describe", name, "--zone", c["zone"]]) + self.ips[name] = instance["networkInterfaces"][0]["networkIP"] + _write_json(self.path / f'{name}-instance-{instance["id"]}.json', instance) + + def stage(self, name, span=None, peers=()): + self.event("stage", instance=name, span=span) + deadline = min(self.deadline, time.time() + 600) + while time.time() < deadline: + if self.ssh(name, "true", check=False).returncode == 0: + break + time.sleep(10) + else: + raise TimeoutError("SSH did not become ready: " + name) + config = self.host_config(name, span, peers) + local_config = self.path / (name + "-config.json") + _write_json(local_config, config) + self.scp(name, self.path / "source.tar.gz", "/tmp/q38-source.tar.gz") + self.scp(name, local_config, "/tmp/q38-config.json") + digest = hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() + setup = self.path / (name + "-setup.sh") + setup.write_text(self.setup_source(name, digest), encoding="utf-8", newline="\n") + self.scp(name, setup, "/tmp/q38-setup.sh") + self.ssh(name, "sudo systemd-run --unit=q38-setup --collect /bin/bash /tmp/q38-setup.sh") + + def host_config(self, name, span, peers): + return {"ip": self.ips[name], "span": span, "peers": list(peers), "request_timeout": 180} + + def setup_source(self, name, digest): + return """#!/bin/bash +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +exec > /var/log/q38-setup.log 2>&1 +mkdir -p /opt/q38/source /srv/q38 +test "$(sha256sum /tmp/q38-source.tar.gz | cut -d' ' -f1)" = "DIGEST" +tar -xzf /tmp/q38-source.tar.gz -C /opt/q38/source +cp /tmp/q38-config.json /srv/q38/config.json +apt-get update -qq +apt-get install -y -qq python3-venv python3-pip +python3 -m venv /opt/q38/venv +/opt/q38/venv/bin/pip install --no-cache-dir 'torch==2.6.0' --index-url https://download.pytorch.org/whl/cpu +/opt/q38/venv/bin/pip install --no-cache-dir /opt/q38/source +id q38 >/dev/null 2>&1 || useradd --system --create-home --home-dir /srv/q38 --shell /usr/sbin/nologin q38 +chown -R q38:q38 /srv/q38 +/opt/q38/venv/bin/pip freeze > /srv/q38/pip-freeze.txt +touch /srv/q38/setup-ready +""".replace( + "DIGEST", digest + ) + + def wait_setup(self, names): + pending = set(names) + while pending: + if time.time() >= self.deadline: + raise TimeoutError("setup exceeded run deadline") + for name in list(pending): + result = self.ssh( + name, + "if test -f /srv/q38/setup-ready; then echo READY; " + "elif systemctl is-active --quiet q38-setup; then tail -n 1 /var/log/q38-setup.log; " + "else echo Q38_SETUP_FAILED; tail -n 25 /var/log/q38-setup.log; exit 1; fi", + check=False, + ) + if result.returncode: + if "Q38_SETUP_FAILED" in result.stdout: + raise RuntimeError("setup failed: " + name + "\n" + result.stdout) + self.event("setup-monitor-unavailable", instance=name, detail=result.stderr[-300:]) + continue + if "READY" in result.stdout: + pending.remove(name) + self.event("setup-progress", instance=name, detail=result.stdout.strip()[-250:]) + if pending: + time.sleep(15) + + def start_job(self, name, role): + command = " ".join( + shlex.quote(x) + for x in [ + "sudo", + "systemd-run", + "--unit=q38-" + role, + "--uid=q38", + "--property=KillMode=control-group", + "--property=TimeoutStopSec=30", + "--property=StandardOutput=append:/srv/q38/" + role + ".log", + "--property=StandardError=append:/srv/q38/" + role + ".log", + "--setenv=OMP_NUM_THREADS=4", + "--setenv=MKL_NUM_THREADS=4", + "--setenv=HF_HUB_DISABLE_IMPLICIT_TOKEN=1", + "--setenv=HF_TOKEN=", + "--setenv=PYTHONUNBUFFERED=1", + "--setenv=HF_HUB_DISABLE_XET=1", + "/opt/q38/venv/bin/python", + "/opt/q38/source/scripts/qwen_full_inference_host.py", + role, + ] + ) + self.ssh(name, command) + + def wait_file(self, name, filename, *, role, predicate=lambda v: True): + while time.time() < self.deadline: + value = self.read(name, filename) + if value is not None and predicate(value): + _write_json(self.path / filename, value) + _write_json(self.path / (name + "-" + filename), value) + return value + error = self.read(name, role + "-error.json") + if error is not None: + raise RuntimeError(f"{name} {role} failed: {error}") + log = self.ssh(name, "tail -n 2 /srv/q38/" + role + ".log", check=False) + self.event("waiting-" + filename, instance=name, detail=log.stdout.strip()[-400:]) + time.sleep(15) + raise TimeoutError(filename + " exceeded run deadline") + + def capture(self): + for name in self.names: + try: + result = self.ssh( + name, + "sudo sh -c 'cat /srv/q38/*.json; tail -n 120 /srv/q38/*.log; " + "cat /srv/q38/pip-freeze.txt; " + "sha256sum /opt/q38/source/scripts/qwen_full_inference_host.py " + "/opt/q38/source/src/drift/server/server.py; " + "tail -n 40 /var/log/q38-setup.log; free -m; " + "journalctl -k --no-pager -n 20'", + check=False, + ) + (self.path / (name + "-diagnostics.txt")).write_text( + result.stdout + "\n" + result.stderr, encoding="utf-8" + ) + except Exception as exc: + self.event("diagnostic-error", instance=name, error=str(exc)) + + def delete_instance(self, name): + rows = self.cloud_json(["compute", "instances", "list", "--filter", "name=" + name]) + if rows: + if len(rows) != 1 or rows[0].get("labels", {}).get("q38-run") != self.run_id: + raise RuntimeError("refusing cleanup of an unowned instance: " + name) + self.cloud( + ["compute", "instances", "delete", name, "--zone", self.config["zone"], "--delete-disks=all"], + timeout=300, + ) + + def cleanup(self): + self.event("cleanup") + errors = [] + for name in reversed(self.names): + try: + self.delete_instance(name) + except Exception as exc: + errors.append(str(exc)) + for name in reversed(self.firewalls): + try: + rows = self.cloud_json(["compute", "firewall-rules", "list", "--filter", "name=" + name]) + if rows: + if rows[0].get("targetTags") != [self.run_id]: + raise RuntimeError("firewall ownership mismatch") + self.cloud(["compute", "firewall-rules", "delete", name]) + except Exception as exc: + errors.append(str(exc)) + remaining = {} + for kind, names in [("instances", self.names), ("disks", self.names), ("firewall-rules", self.firewalls)]: + rows = self.cloud_json(["compute", kind, "list"]) + remaining[kind] = [r["name"] for r in rows if r["name"] in names] + result = {"verified": not errors and not any(remaining.values()), "remaining": remaining, "errors": errors} + _write_json(self.path / "cleanup.json", result) + return result + + def run(self): + result = {"result": "failed", "run_id": self.run_id, "topology": "gcp-cpu", "hardware_qualification": False} + mutation_started = False + try: + self.event("preflight") + self.preflight() + self.bundle() + mutation_started = True + self.create_firewalls() + self.event("create-coordinator") + self.create(self.names[:1], self.config["client_machine_type"]) + self.stage(self.names[0]) + self.event("create-four-workers") + self.create(self.names[1:], self.config["worker_machine_type"]) + # Coordinator installs while workers boot. + self.wait_setup(self.names[:1]) + self.start_job(self.names[0], "bootstrap") + peers = self.wait_file(self.names[0], "bootstrap.json", role="bootstrap")["peers"] + for name, span in zip(self.names[1:], self.config["spans"]): + self.stage(name, span, peers) + self.wait_setup(self.names[1:]) + for name in self.names[1:]: + self.start_job(name, "worker") + # Load the client while workers acquire their independent shard sets. + self.ssh( + self.names[0], + "sudo /opt/q38/venv/bin/python -c " + + shlex.quote( + "import json; p='/srv/q38/config.json'; d=json.load(open(p)); " + f"d['peers']={peers!r}; json.dump(d,open(p,'w'))" + ), + ) + for name in self.names[1:]: + self.wait_file(name, "health.json", role="worker", predicate=lambda v: v["worker_healthy"]) + self.event("all-64-blocks-ready") + self.start_job(self.names[0], "client") + baseline = self.wait_file(self.names[0], "baseline.json", role="client") + validate_route(baseline["route"]) + self.event("complete-short-inference", token_ids=baseline["token_ids"], text=baseline["text"]) + ready = self.wait_file(self.names[0], "recovery-ready.json", role="client") + client_pid = int( + self.ssh(self.names[0], "systemctl show q38-client --value --property=MainPID").stdout.strip() + ) + if client_pid <= 0: + raise RuntimeError("the warmed client process is no longer running") + _write_json(self.path / "client-before-recovery.json", {"client_pid": client_pid}) + lost_peer = next(s["peer_id"] for s in ready["route"] if s["start"] == 16) + worker_identity = self.wait_file(self.names[2], "worker.json", role="worker") + if worker_identity["peer_id"] != lost_peer: + raise ValueError("selected worker differs from the active client route") + self.event("delete-active-worker-vm", instance=self.names[2], peer_id=lost_peer) + self.delete_instance(self.names[2]) + self.create([self.names[2]], self.config["worker_machine_type"]) + self.stage(self.names[2], "16:32", peers) + self.wait_setup([self.names[2]]) + self.start_job(self.names[2], "worker") + self.wait_file(self.names[2], "health.json", role="worker", predicate=lambda v: v["worker_healthy"]) + replacement = self.wait_file(self.names[2], "worker.json", role="worker") + if replacement["peer_id"] == lost_peer: + raise ValueError("replacement did not acquire a new peer identity") + _write_json(self.path / "replacement.json", {"lost_peer": lost_peer, "replacement": replacement}) + self.ssh(self.names[0], "sudo touch /srv/q38/continue-recovery") + evidence = self.wait_file(self.names[0], "client-result.json", role="client") + validate_result(evidence, lost_peer) + result.update(result="passed", evidence=evidence) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.event("failure", error=result["error"]) + finally: + if mutation_started: + self.capture() + try: + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.event("finished", result=result["result"], result_path=str(self.path / "result.json")) + return result + + def resume_replacement(self): + """Resume a staged replacement after local transport failure, preserving the live session.""" + events = [json.loads(line) for line in (self.path / "events.jsonl").read_text().splitlines()] + self.started = events[0]["time"] + self.deadline = self.started + self.config["max_duration_seconds"] - 600 + if time.time() >= self.deadline: + raise TimeoutError("the original run deadline has expired") + inventory = json.loads((self.path / "source-inventory.json").read_text()) + if hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() != inventory["bundle_sha256"]: + raise ValueError("retained source bundle changed") + ready = json.loads((self.path / "recovery-ready.json").read_text()) + baseline = json.loads((self.path / "baseline.json").read_text()) + validate_route(ready["route"]) + validate_route(baseline["route"]) + lost_peer = next(span["peer_id"] for span in ready["route"] if span["start"] == 16) + for name in self.names: + instance = self.cloud_json(["compute", "instances", "describe", name, "--zone", self.config["zone"]]) + if instance.get("labels", {}).get("q38-run") != self.run_id: + raise ValueError("resume instance ownership mismatch") + originals = [json.loads(p.read_text()) for p in self.path.glob(name + "-instance-*.json")] + original = min(originals, key=lambda value: value["creationTimestamp"]) + if (instance["id"] == original["id"]) == (name == self.names[2]): + raise ValueError("resume requires exactly the original middle worker to have been replaced") + self.ips[name] = instance["networkInterfaces"][0]["networkIP"] + prior_client = json.loads((self.path / "client-before-recovery.json").read_text()) + pid = int(self.ssh(self.names[0], "systemctl show q38-client --value --property=MainPID").stdout.strip()) + if pid <= 0 or pid != prior_client["client_pid"]: + raise ValueError("original client process did not survive") + peers = json.loads((self.path / "bootstrap.json").read_text())["peers"] + result = { + "result": "failed", + "run_id": self.run_id, + "topology": "gcp-cpu", + "hardware_qualification": False, + "resumed_after_transport_failure": True, + "preserved_client_pid": pid, + } + self.event("resume-replacement", client_pid=pid, lost_peer=lost_peer) + try: + name = self.names[2] + state = self.ssh( + name, + "if test -f /srv/q38/setup-ready; then echo READY; " + "elif systemctl is-active --quiet q38-setup; then echo INSTALLING; " + "else echo NOT_STARTED; fi", + ).stdout.strip() + if state not in {"READY", "INSTALLING"}: + self.stage(name, "16:32", peers) + self.wait_setup([name]) + if self.read(name, "worker.json") is None: + self.start_job(name, "worker") + self.wait_file(name, "health.json", role="worker", predicate=lambda value: value["worker_healthy"]) + replacement = self.wait_file(name, "worker.json", role="worker") + if replacement["peer_id"] == lost_peer: + raise ValueError("replacement retained the lost peer identity") + _write_json(self.path / "replacement.json", {"lost_peer": lost_peer, "replacement": replacement}) + self.ssh(self.names[0], "sudo touch /srv/q38/continue-recovery") + evidence = self.wait_file(self.names[0], "client-result.json", role="client") + validate_result(evidence, lost_peer) + result.update(result="passed", evidence=evidence) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.event("resumed-failure", error=result["error"]) + finally: + self.capture() + try: + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.event("finished", result=result["result"]) + return result + + +def main(): + parser = argparse.ArgumentParser() + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--preflight-only", action="store_true") + mode.add_argument("--cleanup-run", type=Path) + mode.add_argument("--resume-replacement", type=Path) + args = parser.parse_args() + with LauncherLock(RUNS / "launcher.lock"): + if args.resume_replacement: + path = args.resume_replacement.resolve() + if path.parent != RUNS.resolve(): + raise ValueError("resume run must be inside the Qwen run directory") + config = json.loads((path / "provider-config.json").read_text()) + return 0 if SwarmRun(path, config).resume_replacement()["result"] == "passed" else 1 + if args.cleanup_run: + path = args.cleanup_run.resolve() + if path.parent != RUNS.resolve(): + raise ValueError("cleanup run must be inside the Qwen run directory") + config = json.loads((path / "provider-config.json").read_text()) + return 0 if SwarmRun(path, config).cleanup()["verified"] else 1 + # Recover exact owned resources from interrupted runs before starting another. + for path in [] if args.preflight_only else sorted(RUNS.glob("q38-*")): + cleanup = path / "cleanup.json" + if not (path / "provider-config.json").exists(): + continue + if cleanup.exists() and json.loads(cleanup.read_text()).get("verified"): + continue + if not SwarmRun(path, json.loads((path / "provider-config.json").read_text())).cleanup()["verified"]: + raise RuntimeError("prior run cleanup remains incomplete") + config = json.loads((ROOT / "config/qwen_full_inference_gcp.json").read_text()) + run_id = time.strftime("q38-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2) + run = SwarmRun(RUNS / run_id, config) + _write_json(run.path / "provider-config.json", config) + if args.preflight_only: + run.preflight() + run.bundle() + run.event("preflight-passed") + return 0 + return 0 if run.run()["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_mixed_inference.py b/scripts/run_qwen_mixed_inference.py new file mode 100644 index 000000000..3bd285ab2 --- /dev/null +++ b/scripts/run_qwen_mixed_inference.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python3 +"""Run the L4 + T4 + two-CPU route only after CPU inference/recovery pass.""" +from __future__ import annotations + +import argparse +import hashlib +import ipaddress +import json +import os +import secrets +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +from run_qwen_full_inference_gcp import ( + MANIFEST_DIGEST, + REVISION, + ROOT, + RUNS as CPU_RUNS, + CommandError, + LauncherLock, + SwarmRun, + _write_json, + validate_result, + validate_route, +) + +RUNS = ROOT / ".gate13-runs" / "qwen-mixed" + + +def require_cpu_proof(path): + result = json.loads((path / "result.json").read_text()) + replacement = json.loads((path / "replacement.json").read_text()) + if ( + result.get("result") != "passed" + or result.get("topology") != "gcp-cpu" + or result.get("cleanup", {}).get("verified") is not True + ): + raise ValueError("CPU inference, recovery and cleanup must pass before a mixed run") + validate_result(result["evidence"], replacement["lost_peer"]) + return {"run_id": path.name, "result_sha256": hashlib.sha256((path / "result.json").read_bytes()).hexdigest()} + + +def mixed_quota_requirements(config): + worker = config["worker_machine_type"] + if worker not in {"e2-highmem-4", "c3-highmem-4"}: + raise ValueError("Mixed CPU workers must have the selected four-vCPU, 32-GB profile") + if config["client_machine_type"] != "e2-standard-4" or config["gpu_machine_type"] != "g2-standard-8": + raise ValueError("Mixed coordinator and L4 machine types must retain the qualified topology") + if config["spans"] != ["0:16", "16:32", "32:48", "48:64"]: + raise ValueError("Mixed topology requires four complete 16-block spans") + c3 = worker == "c3-highmem-4" + requirements = { + "E2_CPUS": 4 if c3 else 12, + "CPUS_ALL_REGIONS": 20, + "NVIDIA_L4_GPUS": 1, + "INSTANCES": 4, + "IN_USE_ADDRESSES": 4, + "DISKS_TOTAL_GB": config["disk_gb"] * (1 if c3 else 3), + "SSD_TOTAL_GB": 100 + (2 * config["disk_gb"] if c3 else 0), + } + if c3: + requirements["C3_CPUS"] = 8 + return requirements + + +class MixedRun(SwarmRun): + def __init__(self, path, config): + super().__init__(path, config) + self.azure_name = self.names[2] + self.gcp_names = [name for name in self.names if name != self.azure_name] + self.group = self.run_id + self.public_ips = {} + self.key = path / "azure-key" + self.firewalls.append(self.run_id + "-public") + + def az(self, args, *, timeout=900, check=True): + command = ["az"] + if sys.platform == "win32": + entry = shutil.which("az.cmd") + if not entry: + raise RuntimeError("Azure CLI is unavailable") + python = Path(entry).resolve().parent.parent / "python.exe" + command = [str(python), "-IBm", "azure.cli"] + result = self.runner.run( + [*command, *args, "--subscription", self.config["azure_subscription"], "--only-show-errors"], + action="azure:" + ":".join(args[:3]), + check=False, + timeout=timeout, + ) + if check and result.returncode: + raise RuntimeError("Azure command failed: " + result.stderr[-3000:]) + return result + + def az_json(self, args, **kwargs): + return json.loads(self.az([*args, "-o", "json"], **kwargs).stdout) + + def preflight(self): + proof = require_cpu_proof(Path(self.config["cpu_proof_path"])) + c = self.config + region = self.cloud_json(["compute", "regions", "describe", c["region"]]) + project = self.cloud_json(["compute", "project-info", "describe"]) + quotas = {q["metric"]: q for q in region["quotas"] + project["quotas"]} + requirements = mixed_quota_requirements(c) + for metric, needed in requirements.items(): + if quotas[metric]["limit"] - quotas[metric]["usage"] < needed: + raise RuntimeError("insufficient existing GCP quota: " + metric) + if "GPUS_ALL_REGIONS" in quotas: + q = quotas["GPUS_ALL_REGIONS"] + if q["limit"] - q["usage"] < 1: + raise RuntimeError("insufficient existing global GPU quota") + instances = self.cloud_json(["compute", "instances", "list"]) + if any(i["name"] in self.names for i in instances): + raise RuntimeError("mixed-run target VM already exists") + for image, project_name in [(c["image"], c["image_project"]), (c["gpu_image"], c["gpu_image_project"])]: + self.cloud(["compute", "images", "describe", image], project=project_name) + account = self.az_json(["account", "show"]) + if account["id"] != c["azure_subscription"] or account["state"] != "Enabled": + raise RuntimeError("Azure account does not match the configured subscription") + if self.az_json(["group", "exists", "--name", self.group]): + raise RuntimeError("run-specific Azure resource group already exists") + usage = self.az_json(["vm", "list-usage", "--location", c["azure_location"]]) + for label, match in [("regional vCPU", lambda s: s == "cores"), ("T4 vCPU", lambda s: "t4" in s.lower())]: + rows = [q for q in usage if match(q["name"]["value"])] + if len(rows) != 1 or int(rows[0]["limit"]) - int(rows[0]["currentValue"]) < 4: + raise RuntimeError("insufficient existing Azure " + label + " quota") + self.az(["vm", "image", "show", "--urn", c["azure_image"], "--location", c["azure_location"]], timeout=300) + with urllib.request.urlopen("https://api.ipify.org", timeout=30) as response: + admin_ip = str(ipaddress.IPv4Address(response.read().decode().strip())) + self.config["admin_ip"] = admin_ip + _write_json(self.path / "provider-config.json", self.config) + _write_json( + self.path / "preflight.json", + { + "cpu_proof": proof, + "gcp_quotas": quotas, + "gcp_required_quota": requirements, + "azure_usage": usage, + "azure_subscription": account["id"], + "admin_ip": admin_ip, + }, + ) + + def create_gcp(self, name, machine, *, gpu=False, boot_disk_type=None): + c = self.config + c3 = machine == "c3-highmem-4" + disk_type = boot_disk_type or ("pd-balanced" if gpu or c3 else "pd-standard") + if disk_type not in {"pd-balanced", "pd-standard"} or (c3 and disk_type != "pd-balanced"): + raise ValueError("Unsupported boot disk type for the selected machine") + args = [ + "compute", + "instances", + "create", + name, + "--zone", + c["zone"], + "--machine-type", + machine, + "--image", + c["gpu_image"] if gpu else c["image"], + "--image-project", + c["gpu_image_project"] if gpu else c["image_project"], + *(["--network-interface", "nic-type=GVNIC,subnet=" + c["subnet"]] if c3 else ["--subnet", c["subnet"]]), + "--boot-disk-size", + "100" if gpu else str(c["disk_gb"]), + "--boot-disk-type=" + disk_type, + "--tags", + self.run_id, + "--labels", + f"q38-run={self.run_id}", + "--no-service-account", + "--no-scopes", + "--max-run-duration", + str(max(600, int(self.deadline - time.time() + 300))) + "s", + "--instance-termination-action=DELETE", + ] + if gpu: + args.append("--maintenance-policy=TERMINATE") + self.cloud(args, timeout=900) + instance = self.cloud_json(["compute", "instances", "describe", name, "--zone", c["zone"]]) + self.ips[name] = instance["networkInterfaces"][0]["networkIP"] + self.public_ips[name] = instance["networkInterfaces"][0]["accessConfigs"][0]["natIP"] + _write_json(self.path / (name + "-instance.json"), instance) + + def create_azure(self): + c = self.config + self.az( + [ + "group", + "create", + "--name", + self.group, + "--location", + c["azure_location"], + "--tags", + "q38-run=" + self.run_id, + ] + ) + self.runner.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(self.key)], + action="create-run-specific-azure-ssh-key", + ) + self.az( + [ + "vm", + "create", + "--resource-group", + self.group, + "--name", + self.azure_name, + "--location", + c["azure_location"], + "--size", + c["azure_size"], + "--image", + c["azure_image"], + "--admin-username", + "q38admin", + "--ssh-key-values", + str(self.key) + ".pub", + "--os-disk-size-gb", + str(c["disk_gb"]), + "--storage-sku", + "StandardSSD_LRS", + "--public-ip-sku", + "Standard", + "--public-ip-address", + self.run_id + "-pip", + "--nsg", + self.run_id + "-nsg", + "--nsg-rule", + "NONE", + "--vnet-name", + self.run_id + "-vnet", + "--subnet", + "swarm", + "--security-type", + "Standard", + "--tags", + "q38-run=" + self.run_id, + ], + timeout=1200, + ) + instance = self.az_json(["vm", "show", "-d", "--resource-group", self.group, "--name", self.azure_name]) + self.public_ips[self.azure_name] = str(ipaddress.IPv4Address(instance["publicIps"])) + self.ips[self.azure_name] = instance["privateIps"] + _write_json(self.path / (self.azure_name + "-instance.json"), instance) + shutdown_time = time.strftime("%H%M", time.gmtime(self.started + c["max_duration_seconds"])) + self.az( + ["vm", "auto-shutdown", "--resource-group", self.group, "--name", self.azure_name, "--time", shutdown_time] + ) + + def finish_network(self): + sources = [ip + "/32" for ip in self.public_ips.values()] + self.cloud( + [ + "compute", + "firewall-rules", + "create", + self.run_id + "-public", + "--network", + self.config["network"], + "--allow=tcp:31330", + "--source-ranges", + ",".join(sources), + "--target-tags", + self.run_id, + ] + ) + common = [ + "network", + "nsg", + "rule", + "create", + "--resource-group", + self.group, + "--nsg-name", + self.run_id + "-nsg", + "--direction", + "Inbound", + "--access", + "Allow", + "--protocol", + "Tcp", + "--destination-address-prefixes", + "*", + "--source-port-ranges", + "*", + ] + self.az( + [ + *common, + "--name", + "Swarm", + "--priority", + "110", + "--source-address-prefixes", + *sources, + "--destination-port-ranges", + "31330", + ] + ) + self.az( + [ + *common, + "--name", + "RunAdmin", + "--priority", + "100", + "--source-address-prefixes", + self.config["admin_ip"] + "/32", + "--destination-port-ranges", + "22", + ] + ) + _write_json(self.path / "endpoints.json", self.public_ips) + + def ssh(self, name, command, *, check=True): + if name != self.azure_name: + return super().ssh(name, command, check=check) + argv = [ + "ssh", + "-i", + str(self.key), + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=15", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "UserKnownHostsFile=" + str(self.path / "known_hosts"), + "q38admin@" + self.public_ips[name], + command, + ] + try: + return self.runner.run(argv, action="azure-ssh", check=check, timeout=60) + except CommandError as exc: + if check: + raise + self.event("ssh-monitor-unavailable", instance=name, error=str(exc)) + return subprocess.CompletedProcess(argv, 255, "", str(exc)) + + def scp(self, name, local, remote): + if name != self.azure_name: + return super().scp(name, local, remote) + argv = [ + "scp", + "-i", + str(self.key), + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=15", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "UserKnownHostsFile=" + str(self.path / "known_hosts"), + os.path.relpath(local, Path.cwd()), + "q38admin@" + self.public_ips[name] + ":" + remote, + ] + for attempt in range(5): + try: + return self.runner.run(argv, action="azure-scp", timeout=180) + except CommandError as exc: + if attempt == 4 or time.time() >= self.deadline: + raise + self.event("retry-stage-copy", instance=name, attempt=attempt + 1, error=str(exc)) + time.sleep(10) + + def host_config(self, name, span, peers): + return { + "ip": self.public_ips[name], + "span": span, + "peers": list(peers), + "device": "cuda" if name in (self.names[1], self.azure_name) else "cpu", + "request_timeout": 180, + "run_recovery": False, + } + + def setup_source(self, name, digest): + source = super().setup_source(name, digest) + if name in (self.names[1], self.azure_name): + version = self.config["ubuntu_driver_version"] + source = source.replace( + "python3 -m venv /opt/q38/venv", + 'apt-get install -y -qq "linux-headers-$(uname -r)" ' + "nvidia-driver-580-server=" + version + "\n" + "modprobe nvidia\n" + "nvidia-smi > /srv/q38/nvidia-smi.txt\n" + "python3 -m venv /opt/q38/venv", + ) + if name in (self.names[1], self.azure_name): + source = source.replace("https://download.pytorch.org/whl/cpu", "https://download.pytorch.org/whl/cu124") + source = source.replace( + "touch /srv/q38/setup-ready", + "/opt/q38/venv/bin/python /opt/q38/source/scripts/qwen_full_inference_host.py gpu_probe\n" + "touch /srv/q38/setup-ready", + ) + return source + + def cleanup(self): + self.event("cleanup-mixed") + errors = [] + try: + if self.az_json(["group", "exists", "--name", self.group]): + group = self.az_json(["group", "show", "--name", self.group]) + if group.get("tags", {}).get("q38-run") != self.run_id: + raise RuntimeError("refusing deletion of an unowned Azure resource group") + self.az(["group", "delete", "--name", self.group, "--yes", "--no-wait"]) + except Exception as exc: + errors.append(str(exc)) + saved = self.names + self.names = self.gcp_names + try: + gcp = super().cleanup() + finally: + self.names = saved + deadline = time.time() + 900 + while time.time() < deadline and self.az_json(["group", "exists", "--name", self.group]): + self.event("waiting-azure-group-deletion") + time.sleep(20) + azure_absent = not self.az_json(["group", "exists", "--name", self.group]) + result = { + "verified": gcp["verified"] and azure_absent and not errors, + "gcp": gcp, + "azure_group_absent": azure_absent, + "errors": errors, + } + _write_json(self.path / "cleanup.json", result) + return result + + def run(self): + result = { + "result": "failed", + "run_id": self.run_id, + "topology": "gcp-l4-azure-t4-cpu", + "hardware_qualification": False, + } + mutation_started = False + try: + self.event("mixed-preflight") + self.preflight() + self.bundle() + provider = self.az_json(["provider", "show", "--namespace", "Microsoft.DevTestLab"]) + if provider["registrationState"] != "Registered": + self.event("enable-azure-shutdown-provider") + self.az(["provider", "register", "--namespace", "Microsoft.DevTestLab", "--wait"], timeout=900) + _write_json( + self.path / "shutdown-provider.json", + { + "namespace": "Microsoft.DevTestLab", + "previous_state": provider["registrationState"], + "registered_for_vm_shutdown": True, + }, + ) + mutation_started = True + self.create_firewalls() + self.event("create-mixed-gcp-hosts") + self.create_gcp(self.names[0], self.config["client_machine_type"]) + self.create_gcp(self.names[1], self.config["gpu_machine_type"], gpu=True) + for name in self.names[3:]: + self.create_gcp(name, self.config["worker_machine_type"]) + self.event("create-azure-t4") + self.create_azure() + self.finish_network() + self.stage(self.names[0]) + self.wait_setup(self.names[:1]) + self.start_job(self.names[0], "bootstrap") + peers = self.wait_file(self.names[0], "bootstrap.json", role="bootstrap")["peers"] + for name, span in zip(self.names[1:], self.config["spans"]): + self.stage(name, span, peers) + self.wait_setup(self.names[1:]) + for name in (self.names[1], self.azure_name): + self.wait_file(name, "gpu-probe.json", role="gpu_probe") + for name in self.names[1:]: + self.start_job(name, "worker") + workers = [] + for name in self.names[1:]: + self.wait_file(name, "health.json", role="worker", predicate=lambda v: v["worker_healthy"]) + workers.append(self.wait_file(name, "worker.json", role="worker")) + if "L4" not in workers[0]["hardware"].get("gpu_name", ""): + raise ValueError("first worker is not the requested L4") + if "T4" not in workers[1]["hardware"].get("gpu_name", ""): + raise ValueError("second worker is not the requested T4") + if any(w["hardware"]["device"] != "cpu" for w in workers[2:]): + raise ValueError("CPU remainder is not running on CPU") + _write_json(self.path / "workers.json", {"workers": workers}) + config_path = self.path / "mixed-client-config.json" + _write_json(config_path, self.host_config(self.names[0], None, peers)) + self.scp(self.names[0], config_path, "/tmp/q38-client-config.json") + self.ssh(self.names[0], "sudo cp /tmp/q38-client-config.json /srv/q38/config.json") + self.start_job(self.names[0], "client") + evidence = self.wait_file(self.names[0], "client-result.json", role="client") + if ( + evidence.get("result") != "passed" + or evidence.get("manifest_digest") != MANIFEST_DIGEST + or evidence.get("model_revision") != REVISION + ): + raise ValueError("mixed client evidence binding failed") + validate_route(evidence["baseline"]["route"]) + if len(evidence["baseline"]["token_ids"]) != 3: + raise ValueError("mixed route did not produce three tokens") + if {r["peer_id"] for r in evidence["baseline"]["route"]} != {w["peer_id"] for w in workers}: + raise ValueError("mixed route did not use the inspected workers") + result.update(result="passed", evidence=evidence, workers=workers) + self.event("mixed-inference-passed", text=evidence["baseline"]["text"]) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.event("mixed-failure", error=result["error"]) + finally: + if mutation_started: + self.capture() + try: + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cleanup-run", type=Path) + args = parser.parse_args() + with LauncherLock(RUNS / "launcher.lock"): + if args.cleanup_run: + path = args.cleanup_run.resolve() + if path.parent != RUNS.resolve(): + raise ValueError("cleanup target is outside the mixed-run directory") + run = MixedRun(path, json.loads((path / "provider-config.json").read_text())) + return 0 if run.cleanup()["verified"] else 1 + for path in sorted(RUNS.glob("q38m-*")): + cleanup = path / "cleanup.json" + if cleanup.exists() and json.loads(cleanup.read_text()).get("verified"): + continue + if not (path / "provider-config.json").exists(): + continue + run = MixedRun(path, json.loads((path / "provider-config.json").read_text())) + if not run.cleanup()["verified"]: + raise RuntimeError("prior mixed-run cleanup is incomplete") + proofs = [] + for path in sorted(CPU_RUNS.glob("q38-*"), reverse=True): + try: + require_cpu_proof(path) + proofs.append(path) + break + except (OSError, ValueError, KeyError): + continue + if not proofs: + raise RuntimeError("no complete CPU inference + recovery + cleanup proof exists") + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + config["cpu_proof_path"] = str(proofs[0].resolve()) + run_id = time.strftime("q38m-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2) + run = MixedRun(RUNS / run_id, config) + _write_json(run.path / "provider-config.json", config) + return 0 if run.run()["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_product_gcp.py b/scripts/run_qwen_product_gcp.py new file mode 100644 index 000000000..d9874c9bc --- /dev/null +++ b/scripts/run_qwen_product_gcp.py @@ -0,0 +1,216 @@ +"""Bounded five-VM test of real node fallback, Qwen3.8 promotion and worker loss.""" + +import argparse +import hashlib +import json +import secrets +import shlex +import tarfile +import time +from pathlib import Path + +from qwen_product_recovery import WORKER_STATE_COMMAND, require_recovery_acknowledgements, worker_is_stopped +from run_qwen_full_inference_gcp import ROOT, LauncherLock, SwarmRun, _write_json + +RUNS = ROOT / ".gate13-runs/qwen-product-cloud" + + +class ProductRun(SwarmRun): + def bundle(self): + super().bundle() + inventory = json.loads((self.path / "source-inventory.json").read_text()) + names = set(inventory["files"]) + names.update( + path.relative_to(ROOT).as_posix() + for directory in ("desktop/src", "public-alpha/catalog-qwen-v2") + for path in (ROOT / directory).rglob("*") + if path.is_file() and "__pycache__" not in path.parts + ) + names.update( + [ + "scripts/qwen_product_inference_host.py", + "scripts/qwen_product_recovery.py", + "desktop/pyproject.toml", + "manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json", + ] + ) + with tarfile.open(self.path / "source.tar.gz", "w:gz") as archive: + for name in sorted(names): + archive.add(ROOT / name, arcname=name) + inventory["files"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in sorted(names)} + inventory["bundle_sha256"] = hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() + _write_json(self.path / "source-inventory.json", inventory) + + def setup_source(self, name, digest): + source = super().setup_source(name, digest) + if name == self.names[0]: + source = source.replace( + "touch /srv/q38/setup-ready", + "/opt/q38/venv/bin/pip install '/opt/q38/source[api]'\ntouch /srv/q38/setup-ready", + ) + return source + + def start_product(self, peers): + self.ssh( + self.names[0], + "sudo /opt/q38/venv/bin/python -c " + + shlex.quote( + "import json; p='/srv/q38/config.json'; d=json.load(open(p)); " + f"d['peers']={peers!r}; json.dump(d,open(p,'w'))" + ), + ) + self.ssh( + self.names[0], + "sudo systemd-run --unit=q38-product --uid=q38 --property=KillMode=control-group " + "--property=TimeoutStopSec=60 --setenv=OMP_NUM_THREADS=4 --setenv=MKL_NUM_THREADS=4 " + "/opt/q38/venv/bin/python /opt/q38/source/scripts/qwen_product_inference_host.py", + ) + + def signal_recovery(self, phase, nonce, peer_id): + if phase not in {"stopped", "replaced"}: + raise ValueError("Unknown recovery phase") + value = {"recovery_nonce": nonce, "peer_id": peer_id, "observed_at_unix": time.time()} + filename = "product-worker-" + phase + ".json" + self.ssh( + self.names[0], + "sudo /opt/q38/venv/bin/python -c " + + shlex.quote( + "from pathlib import Path; " + f"p=Path('/srv/q38/{filename}'); t=p.with_suffix('.tmp'); " + f"t.write_text({json.dumps(value)!r}); t.replace(p)" + ), + ) + _write_json(self.path / filename, value) + + def exercise_workers(self): + self.wait_file(self.names[0], "product-local-ready.json", role="product") + self.event("local-before-network-proved") + workers = [] + for name in self.names[1:]: + self.start_job(name, "worker") + for name in self.names[1:]: + self.wait_file(name, "health.json", role="worker", predicate=lambda value: value["worker_healthy"]) + workers.append(self.wait_file(name, "worker.json", role="worker")) + _write_json(self.path / "workers.json", {"workers": workers}) + ready = self.wait_file(self.names[0], "product-ready-for-loss.json", role="product") + nonce = ready.get("recovery_nonce") + if not isinstance(nonce, str) or not nonce: + raise RuntimeError("Source client did not establish a recovery challenge") + original = workers[1] + self.event("kill-worker", instance=self.names[2]) + self.ssh(self.names[2], "sudo systemctl kill --kill-whom=all --signal=SIGKILL q38-worker") + self.ssh(self.names[2], "sudo systemctl stop q38-worker") + stopped = self.ssh(self.names[2], WORKER_STATE_COMMAND).stdout + if not worker_is_stopped(stopped): + raise RuntimeError("Injected worker loss was not confirmed") + self.signal_recovery("stopped", nonce, original["peer_id"]) + self.wait_file( + self.names[0], + "product-local-after-loss.json", + role="product", + predicate=lambda value: value.get("recovery_nonce") == nonce, + ) + self.ssh( + self.names[2], + "sudo mv /srv/q38/identity.key /srv/q38/identity.before-replacement.key && " + "sudo systemctl restart q38-worker", + ) + replacement = self.wait_file( + self.names[2], + "worker.json", + role="worker", + predicate=lambda value: value["peer_id"] != original["peer_id"], + ) + self.signal_recovery("replaced", nonce, replacement["peer_id"]) + evidence = self.wait_file(self.names[0], "product-result.json", role="product") + if evidence["result"] != "passed": + raise RuntimeError("product exercise failed: " + evidence.get("error", "unknown")) + require_recovery_acknowledgements(evidence, nonce, original["peer_id"], replacement["peer_id"]) + return dict(result="passed", evidence=evidence, replacement={"before": original, "after": replacement}) + + def capture_product(self): + for filename in ("product-result.json", "product-local-ready.json", "product-local-after-loss.json"): + value = self.read(self.names[0], filename) + if value is not None: + _write_json(self.path / filename, value) + log = self.ssh(self.names[0], "sudo tail -n 300 /srv/q38/product-node.log", check=False) + (self.path / "product-node.log").write_text(log.stdout + "\n" + log.stderr, encoding="utf-8") + self.capture() + + def run(self): + result = { + "result": "failed", + "run_id": self.run_id, + "scope": "production-node-model-transitions", + "hardware_qualification": False, + "packaged_qualification": False, + } + mutated = False + try: + self.preflight() + self.bundle() + mutated = True + self.create_firewalls() + self.create(self.names[:1], self.config["client_machine_type"]) + self.stage(self.names[0]) + self.create(self.names[1:], self.config["worker_machine_type"]) + self.wait_setup(self.names[:1]) + self.start_job(self.names[0], "bootstrap") + peers = self.wait_file(self.names[0], "bootstrap.json", role="bootstrap")["peers"] + self.start_product(peers) + for name, span in zip(self.names[1:], self.config["spans"]): + self.stage(name, span, peers) + self.wait_setup(self.names[1:]) + result.update(self.exercise_workers()) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.event("failed", error=result["error"]) + finally: + if mutated: + try: + self.capture_product() + except Exception as exc: + result["diagnostic_error"] = f"{type(exc).__name__}: {exc}" + finally: + try: + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=f"{type(exc).__name__}: {exc}") + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.event("finished", result=result["result"]) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--preflight-only", action="store_true") + parser.add_argument("--cleanup-run", type=Path) + args = parser.parse_args() + with LauncherLock(RUNS / "launcher.lock"): + if args.cleanup_run: + path = args.cleanup_run.resolve() + if path.parent != RUNS.resolve(): + raise ValueError("cleanup must target an owned product run directory") + return ( + 0 + if ProductRun(path, json.loads((path / "provider-config.json").read_text())).cleanup()["verified"] + else 1 + ) + config = json.loads((ROOT / "config/qwen_full_inference_gcp.json").read_text()) + config["max_duration_seconds"] = 10800 + run_id = time.strftime("q38p-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2) + run = ProductRun(RUNS / run_id, config) + _write_json(run.path / "provider-config.json", config) + if args.preflight_only: + run.preflight() + run.bundle() + run.event("preflight-passed") + return 0 + return 0 if run.run()["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_product_mixed.py b/scripts/run_qwen_product_mixed.py new file mode 100644 index 000000000..997dfe21d --- /dev/null +++ b/scripts/run_qwen_product_mixed.py @@ -0,0 +1,197 @@ +"""Exercise measured node transitions on the previously proven L4/T4/CPU topology.""" + +import argparse +import ipaddress +import json +import secrets +import time +from pathlib import Path + +from run_qwen_full_inference_gcp import ROOT, LauncherLock, _write_json +from run_qwen_mixed_inference import CPU_RUNS, MixedRun, require_cpu_proof +from run_qwen_product_gcp import ProductRun + +RUNS = ROOT / ".gate13-runs/qwen-product-mixed" + + +class MixedProductRun(MixedRun, ProductRun): + def run_packaged_client(self): + """Optional synchronous client step; the original external handoff remains supported.""" + + def enable_packaged_client(self): + """Add only the preflight-resolved operator address to owned swarm rules.""" + address = str(ipaddress.IPv4Address(self.config["admin_ip"])) + "/32" + sources = sorted({ip + "/32" for ip in self.public_ips.values()} | {address}) + self.cloud( + [ + "compute", + "firewall-rules", + "update", + self.run_id + "-public", + "--source-ranges", + ",".join(sources), + ] + ) + self.az( + [ + "network", + "nsg", + "rule", + "update", + "--resource-group", + self.group, + "--nsg-name", + self.run_id + "-nsg", + "--name", + "Swarm", + "--source-address-prefixes", + *sources, + ] + ) + _write_json(self.path / "packaged-client-network.json", {"sources": sources, "tcp_port": 31330}) + + def wait_packaged_client(self): + seconds = self.config.get("packaged_client_wait_seconds", 0) + if not 1 <= seconds <= 3600: + raise ValueError("Packaged client wait must be bounded to 1..3600 seconds") + until = min(time.time() + seconds, self.started + self.config["max_duration_seconds"] - 600) + _write_json(self.path / "packaged-client-ready.json", {"run_id": self.run_id, "deadline_unix": until}) + self.event("waiting-for-packaged-client", deadline_unix=until) + self.run_packaged_client() + receipt = self.path / "packaged-client-result.json" + while time.time() < until: + if receipt.exists(): + value = json.loads(receipt.read_text()) + if value.get("run_id") != self.run_id or not value.get("node_stopped"): + raise ValueError("Packaged result must bind this run and confirm client cleanup") + if value.get("result") != "passed": + raise RuntimeError("Packaged client failed: " + value.get("error", "inspect receipt")) + return value + time.sleep(5) + raise TimeoutError("Packaged client deadline; proceeding to owned-resource cleanup") + + def run(self): + result = { + "result": "failed", + "run_id": self.run_id, + "scope": "production-node-model-transitions", + "topology": "gcp-l4-azure-t4-cpu", + "hardware_qualification": False, + "packaged_qualification": False, + } + mutated = False + try: + self.preflight() + self.bundle() + provider = self.az_json(["provider", "show", "--namespace", "Microsoft.DevTestLab"]) + if provider["registrationState"] != "Registered": + self.az(["provider", "register", "--namespace", "Microsoft.DevTestLab", "--wait"], timeout=900) + mutated = True + self.create_firewalls() + self.create_gcp(self.names[0], self.config["client_machine_type"]) + self.create_gcp(self.names[1], self.config["gpu_machine_type"], gpu=True) + for name in self.names[3:]: + self.create_gcp(name, self.config["worker_machine_type"]) + self.create_azure() + self.finish_network() + if self.config.get("packaged_client_wait_seconds"): + self.enable_packaged_client() + self.stage(self.names[0]) + self.wait_setup(self.names[:1]) + self.start_job(self.names[0], "bootstrap") + peers = self.wait_file(self.names[0], "bootstrap.json", role="bootstrap")["peers"] + self.start_product(peers) + for name, span in zip(self.names[1:], self.config["spans"]): + self.stage(name, span, peers) + self.wait_setup(self.names[1:]) + for name in (self.names[1], self.azure_name): + self.wait_file(name, "gpu-probe.json", role="gpu_probe") + try: + result.update(self.exercise_workers()) + except Exception as exc: + # Preserve independent packaged evidence after a completed + # source exercise fails. Only hold fully staged owned workers, + # and only after its client has written its final receipt. + source_receipt = self.read(self.names[0], "product-result.json") + if ( + self.config.get("packaged_client_wait_seconds") + and (self.path / "workers.json").exists() + and source_receipt is not None + ): + result["source_product_error"] = str(exc) + result["evidence"] = source_receipt + self.event("source-product-failed-packaged-check-remains-independent", error=str(exc)) + result["packaged_client"] = self.wait_packaged_client() + raise + workers = json.loads((self.path / "workers.json").read_text())["workers"] + if "L4" not in workers[0]["hardware"].get("gpu_name", "") or "T4" not in workers[1]["hardware"].get( + "gpu_name", "" + ): + raise ValueError("The inspected devices do not match the requested L4/T4 topology") + if any(w["hardware"]["device"] != "cpu" for w in workers[2:]): + raise ValueError("The remainder must execute on CPU") + if self.config.get("packaged_client_wait_seconds"): + result["packaged_client"] = self.wait_packaged_client() + except BaseException as exc: + result.update(result="failed", error=f"{type(exc).__name__}: {exc}") + self.event("failed", error=result["error"]) + finally: + if mutated: + try: + self.capture_product() + except Exception as exc: + result["diagnostic_error"] = str(exc) + finally: + try: + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.event("finished", result=result["result"]) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--preflight-only", action="store_true") + parser.add_argument("--cleanup-run", type=Path) + parser.add_argument("--packaged-client-wait-seconds", type=int, default=0) + parser.add_argument("--cpu-worker-machine-type", choices=("e2-highmem-4", "c3-highmem-4"), default="e2-highmem-4") + args = parser.parse_args() + with LauncherLock(RUNS / "launcher.lock"): + if args.cleanup_run: + path = args.cleanup_run.resolve() + if path.parent != RUNS.resolve(): + raise ValueError("Cleanup must name an owned mixed product run") + run = MixedProductRun(path, json.loads((path / "provider-config.json").read_text())) + raise SystemExit(0 if run.cleanup()["verified"] else 1) + proof = next( + ( + p + for p in sorted(CPU_RUNS.glob("q38-*"), reverse=True) + if (p / "result.json").exists() + and json.loads((p / "result.json").read_text()).get("result") == "passed" + ), + None, + ) + if proof is None: + raise RuntimeError("A complete CPU recovery proof is required") + require_cpu_proof(proof) + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + config.update(cpu_proof_path=str(proof.resolve()), max_duration_seconds=10800) + if not 0 <= args.packaged_client_wait_seconds <= 3600: + raise ValueError("Packaged client wait must be between zero and 3600 seconds") + config["packaged_client_wait_seconds"] = args.packaged_client_wait_seconds + config["worker_machine_type"] = args.cpu_worker_machine_type + path = RUNS / (time.strftime("q38pm-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2)) + run = MixedProductRun(path, config) + _write_json(path / "provider-config.json", config) + if args.preflight_only: + run.preflight() + run.bundle() + run.event("preflight-passed") + else: + raise SystemExit(0 if run.run()["result"] == "passed" else 1) diff --git a/scripts/run_qwen_product_test.py b/scripts/run_qwen_product_test.py new file mode 100644 index 000000000..e0afbab82 --- /dev/null +++ b/scripts/run_qwen_product_test.py @@ -0,0 +1,255 @@ +"""One command for the assigned L4/T4/C3 source and packaged Windows Qwen replay.""" + +import argparse +import concurrent.futures +import json +import os +import secrets +import subprocess +import sys +import time +from pathlib import Path + +from qwen_product_provenance import sha256, snapshot, verify_package, verify_snapshot +from report_qwen_product import report + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path): + return json.loads(path.read_text(encoding="utf-8-sig")) + + +def write(path, value): + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def load_inputs(config_path, root=ROOT): + config = read(config_path) + inputs = {"launcher_config": config_path.resolve()} + for label in ("node", "package_provenance", "cache_provenance", "cloud_config"): + path = (root / config[label]).resolve() + if not path.is_file(): + raise ValueError(f"Missing {label}: {path}; configure an existing verified package/cache") + inputs[label] = path + for label in ("local_cache", "remote_cache"): + path = (root / config[label]).resolve() + if not path.is_dir(): + raise ValueError(f"Missing {label}: {path}; acquire the cache before this bounded replay") + config[label] = str(path) + if os.name == "nt" and any(Path(config[k]).drive.upper() != "C:" for k in ("local_cache", "remote_cache")): + raise ValueError("This workspace's replay caches must stay on C:") + if not 1 <= config.get("port", 18089) <= 65535: + raise ValueError("Invalid localhost port") + if read(inputs["cache_provenance"]).get("result") != "passed": + raise ValueError("Reused remote cache requires a passed acquisition receipt") + return config, inputs + + +def stop_process_tree(process): + """Stop only this launched qualifier and its descendants on timeout/interruption.""" + import psutil + + try: + parent = psutil.Process(process.pid) + parent.suspend() # Prevent it from starting a new node while collecting descendants. + children = parent.children(recursive=True) + for child in children: + try: + child.kill() + except psutil.NoSuchProcess: + pass + parent.kill() + _, alive = psutil.wait_procs(children + [parent], timeout=15) + process.wait(timeout=15) + return not alive + except psutil.NoSuchProcess: + # A disappearing parent alone cannot certify that every child stopped. + return False + + +def execute_packaged(run, output, config, inputs, future=None): + ready = read(run / "packaged-client-ready.json") + if ready["run_id"] != run.name or ready["deadline_unix"] <= time.time() + 60: + raise ValueError("Packaged window is stale or belongs to another run") + command = [ + sys.executable, + str(ROOT / "scripts/qualify_qwen_remote_product.py"), + "--node", + str(inputs["node"]), + "--cloud-run", + str(run), + "--output", + str(output), + "--local-cache", + config["local_cache"], + "--remote-cache", + config["remote_cache"], + "--cache-provenance", + str(inputs["cache_provenance"]), + "--worker-recovery", + "--device", + config.get("device", "cuda:0"), + "--port", + str(config.get("port", 18089)), + ] + write(run / "packaged-command.json", {"argv": command, "run_id": run.name}) + process = None + try: + with (run / "packaged-client.log").open("xb") as log: + process = subprocess.Popen( + command, + cwd=ROOT, + stdout=log, + stderr=subprocess.STDOUT, + env=dict(os.environ, PYTHONUNBUFFERED="1"), + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + while process.poll() is None: + if time.time() >= ready["deadline_unix"] - 30 or (future is not None and future.done()): + raise TimeoutError("Packaged window ended; stopping the owned local process tree") + time.sleep(1) + if process.returncode: + raise RuntimeError(f"Packaged qualifier exited with {process.returncode}; inspect packaged-client.log") + receipt = read(run / "packaged-client-result.json") + if receipt.get("result") != "passed" or receipt.get("node_stopped") is not True: + raise ValueError("Packaged qualifier did not confirm success and local cleanup") + return process.returncode + except BaseException as exc: + stopped = process is None + if process is not None and process.poll() is None: + try: + stopped = stop_process_tree(process) + except Exception: + stopped = False + receipt_path = run / "packaged-client-result.json" + if receipt_path.exists(): + existing = read(receipt_path) + stopped = stopped or (existing.get("run_id") == run.name and existing.get("node_stopped") is True) + write(run / "packaged-client-original-result.json", existing) + write( + receipt_path, + {"result": "failed", "run_id": run.name, "node_stopped": stopped, "error": f"{type(exc).__name__}: {exc}"}, + ) + raise + + +def orchestrate(run, output, config, inputs, inventory, package_verification, *, execute=execute_packaged): + value = { + "result": "failed", + "run_id": run.run_id, + "package_verification": package_verification, + "source_inventory_sha256": sha256(run.path / "launcher-source.json"), + } + try: + verify_snapshot(ROOT, run.path, inventory) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(run.run) + try: + while not future.done() and not (run.path / "packaged-client-ready.json").exists(): + time.sleep(1) + if future.done(): + raise RuntimeError("Cloud run ended before its packaged handoff; inspect result.json") + verify_snapshot(ROOT, run.path, inventory) + value["packaged_exit_code"] = execute(run.path, output, config, inputs, future) + except BaseException as exc: + # Wake the bounded cloud waiter immediately, including on Ctrl+C. + if not (run.path / "packaged-client-result.json").exists(): + write( + run.path / "packaged-client-result.json", + { + "result": "failed", + "run_id": run.run_id, + "node_stopped": True, + "error": f"{type(exc).__name__}: {exc}", + }, + ) + raise + finally: + value["cloud_result"] = future.result() # The existing runner owns cleanup in finally. + verify_snapshot(ROOT, run.path, inventory) + # Also recheck installed runtime files, which the executable hash alone cannot cover. + verify_package(inputs["node"], inputs["package_provenance"], config["node_sha256"]) + value["inputs_unchanged"] = True + if value["cloud_result"]["result"] == "passed" and value["packaged_exit_code"] == 0: + value["result"] = "passed" + except BaseException as exc: + value["error"] = f"{type(exc).__name__}: {exc}" + finally: + write(run.path / "launcher-result.json", value) + summary = report(run.path, output, run.path / "product-report.json", launcher=value) + print( + json.dumps( + {"run_id": run.run_id, "result": summary["result"], "report": str(run.path / "product-report.json")} + ), + flush=True, + ) + return summary + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=ROOT / "config/qwen_product_test.json") + parser.add_argument("--cpu-proof", type=Path) + parser.add_argument("--validate-inputs", action="store_true", help="Local checks only; no provider calls") + parser.add_argument("--preflight-only", action="store_true", help="Local checks and read-only provider preflight") + args = parser.parse_args(argv) + if os.name == "nt" and ROOT.drive.upper() != "C:": + raise ValueError("Run this workspace's tests from C:; do not move outputs to a slower drive") + config, inputs = load_inputs(args.config) + # Missing desktop dependencies must fail before any cloud resources are created. + subprocess.run( + [sys.executable, "-c", "import httpx, psutil, communityai_desktop.controller"], check=True, timeout=60 + ) + package = verify_package(inputs["node"], inputs["package_provenance"], config["node_sha256"]) + if args.validate_inputs: + print(json.dumps({"result": "passed", "scope": "local-input-validation-only", **package})) + return 0 + from run_qwen_full_inference_gcp import LauncherLock + from run_qwen_mixed_inference import CPU_RUNS, require_cpu_proof + from run_qwen_product_mixed import RUNS, MixedProductRun + + proof = args.cpu_proof + if proof is None: + for candidate in sorted(CPU_RUNS.glob("q38-*"), reverse=True): + try: + require_cpu_proof(candidate) + except (OSError, KeyError, TypeError, ValueError): + continue + proof = candidate + break + if proof is None: + raise ValueError("A complete CPU recovery proof is required; pass --cpu-proof") + require_cpu_proof(proof) + inputs["cpu_proof"] = proof.resolve() / "result.json" + inputs["cpu_replacement"] = proof.resolve() / "replacement.json" + cloud_config = read(inputs["cloud_config"]) + cloud_config.update( + cpu_proof_path=str(proof.resolve()), + worker_machine_type="c3-highmem-4", + max_duration_seconds=10800, + packaged_client_wait_seconds=3600, + ) + with LauncherLock(RUNS / "launcher.lock"): + path = RUNS / (time.strftime("q38pm-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(3)) + path.mkdir(parents=True, exist_ok=False) + write(path / "provider-config.json", cloud_config) + # Preflight adds its observed admin IP to provider-config.json. Keep the + # requested configuration immutable, and bind the effective one in the report. + write(path / "requested-provider-config.json", cloud_config) + inputs["requested_provider_config"] = path / "requested-provider-config.json" + inventory = snapshot(ROOT, path, inputs) + run = MixedProductRun(path, cloud_config) + print("Qwen product run: " + str(path), flush=True) + if args.preflight_only: + run.preflight() + verify_snapshot(ROOT, path, inventory) + write(path / "preflight-result.json", {"result": "passed", "scope": "read-only-preflight"}) + return 0 + return 0 if orchestrate(run, path / "packaged", config, inputs, inventory, package)["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_qualification.py b/scripts/run_qwen_qualification.py new file mode 100644 index 000000000..7a18dcfba --- /dev/null +++ b/scripts/run_qwen_qualification.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Zero-input Qwen qualification launcher, following run_gate13_gcp.py's structure.""" + +import os +import secrets +import sys +import time +from pathlib import Path + +from gate13_cloud_orchestrator import Gate13CloudError +from qwen_qualification import QwenQualification +from run_gate13_gcp import _write_json +from run_qwen_full_inference_gcp import LauncherLock +from run_qwen_mixed_inference import require_cpu_proof +from run_qwen_product_test import load_inputs, read + + +def _new_run_id(): + return time.strftime("q38pm-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(3) + + +def _inputs(root): + packaged, inputs = load_inputs(root / "config/qwen_product_test.json", root) + proof = None + for candidate in sorted((root / ".gate13-runs/qwen-full").glob("q38-*"), reverse=True): + try: + require_cpu_proof(candidate) + except (OSError, KeyError, TypeError, ValueError): + continue + proof = candidate + break + if proof is None: + raise Gate13CloudError("A passed CPU inference/recovery/cleanup proof is required before this mixed test") + inputs.update(cpu_proof=proof / "result.json", cpu_replacement=proof / "replacement.json") + cloud = read(inputs["cloud_config"]) + cloud.update( + cpu_proof_path=str(proof.resolve()), + worker_machine_type="c3-highmem-4", + max_duration_seconds=10800, + packaged_client_wait_seconds=3600, + ) + return packaged, inputs, cloud + + +def main(argv=None): + if list(sys.argv[1:] if argv is None else argv): + print("This launcher accepts no arguments.", file=sys.stderr) + return 2 + repository_root = Path(__file__).resolve().parent.parent + runs_root = repository_root / ".gate13-runs/qwen-product-mixed" + try: + if os.name == "nt" and repository_root.drive.upper() != "C:": + raise Gate13CloudError("This workspace's Qwen qualification must run from C:") + with LauncherLock(runs_root / "launcher.lock"): + run_id = _new_run_id() + output_root = runs_root / run_id + output_root.mkdir(parents=True, exist_ok=False) + packaged, inputs, config = _inputs(repository_root) + _write_json(output_root / "provider-config.json", config) + _write_json(output_root / "requested-provider-config.json", config) + inputs["requested_provider_config"] = output_root / "requested-provider-config.json" + result = QwenQualification( + root=repository_root, + output_root=output_root, + cloud_config=config, + packaged_config=packaged, + inputs=inputs, + ).run() + print() + print("=" * 68) + print(f"QWEN QUALIFICATION: {str(result.get('result')).upper()}") + print(f"Run: {run_id}") + print(f"Duration: {result.get('duration_seconds')} seconds") + if result.get("result") != "passed": + failure = next( + ( + event.get("details", {}).get("failed_phase") + for event in result.get("events", []) + if event.get("phase") == "FAILURE" + ), + None, + ) + print(f"Failed phase: {failure or 'cleanup verification'}") + print(f"Reason: {result.get('failure_reason') or result.get('failure_code') or 'unknown'}") + print(f"Result: {output_root / 'qualification/result.json'}") + print(f"Evidence: {output_root / 'product-report.json'}") + print("=" * 68) + return 0 if result.get("result") == "passed" else 1 + except BaseException as exc: + print() + print("=" * 68) + print("QWEN QUALIFICATION: FAILED BEFORE OR DURING ORCHESTRATION") + print(f"Failure: {type(exc).__name__}: {exc}") + print(f"Runs directory: {runs_root}") + print("=" * 68) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen_reference_gcp.py b/scripts/run_qwen_reference_gcp.py new file mode 100644 index 000000000..580ab5d3d --- /dev/null +++ b/scripts/run_qwen_reference_gcp.py @@ -0,0 +1,139 @@ +"""Run bounded stock-versus-RPC numerical qualification on one 4-vCPU/96-GiB GCP VM.""" + +import argparse +import hashlib +import json +import secrets +import tarfile +import time +from pathlib import Path + +from run_qwen_full_inference_gcp import MANIFEST_DIGEST, REVISION, ROOT, LauncherLock, SwarmRun, _write_json + +RUNS = ROOT / ".gate13-runs/qwen-reference" +MACHINE = "n2-custom-4-98304-ext" + + +class ReferenceRun(SwarmRun): + def __init__(self, path, config): + super().__init__(path, config) + self.names = [self.run_id + "-r"] + + def preflight(self): + c = self.config + manifest = json.loads((ROOT / "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json").read_text()) + digest = ( + "sha256:" + + hashlib.sha256( + json.dumps(manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + ) + if manifest["source"]["revision"] != REVISION or digest != MANIFEST_DIGEST or c["disk_gb"] != 80: + raise ValueError("Reference run must use the pinned model and 80 GB disk") + instances = self.cloud_json(["compute", "instances", "list"]) + if any(i["name"] in self.names for i in instances): + raise ValueError("The exact reference VM name already exists") + region = self.cloud_json(["compute", "regions", "describe", c["region"]]) + project = self.cloud_json(["compute", "project-info", "describe"]) + quotas = {q["metric"]: q for q in region["quotas"] + project["quotas"]} + for metric, required in { + "N2_CPUS": 4, + "CPUS_ALL_REGIONS": 4, + "INSTANCES": 1, + "IN_USE_ADDRESSES": 1, + "DISKS_TOTAL_GB": 80, + }.items(): + q = quotas[metric] + if q["limit"] - q["usage"] < required: + raise RuntimeError("Insufficient " + metric + "; no quota request will be made") + machine = self.cloud_json(["compute", "machine-types", "describe", MACHINE, "--zone", c["zone"]]) + if machine["guestCpus"] != 4 or machine["memoryMb"] != 98304: + raise ValueError("Unexpected reference machine specification") + _write_json(self.path / "preflight.json", {"instances": instances, "quotas": quotas, "machine": machine}) + + def bundle(self): + super().bundle() + inventory = json.loads((self.path / "source-inventory.json").read_text()) + names = set(inventory["files"]) | {"scripts/qwen_reference_inference_host.py"} + with tarfile.open(self.path / "source.tar.gz", "w:gz") as archive: + for name in sorted(names): + archive.add(ROOT / name, arcname=name) + inventory["files"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in sorted(names)} + inventory["bundle_sha256"] = hashlib.sha256((self.path / "source.tar.gz").read_bytes()).hexdigest() + _write_json(self.path / "source-inventory.json", inventory) + + def run(self): + result = {"result": "failed", "run_id": self.run_id, "scope": "stock-reference-numerical-qualification"} + mutated = False + try: + self.preflight() + self.bundle() + mutated = True + self.create_firewalls() + self.create(self.names, MACHINE) + self.stage(self.names[0]) + self.wait_setup(self.names) + self.ssh( + self.names[0], + "sudo systemd-run --unit=q38-reference --uid=q38 " + "--property=KillMode=control-group --property=TimeoutStopSec=60 " + "--property=StandardOutput=append:/srv/q38/reference.log " + "--property=StandardError=append:/srv/q38/reference.log " + "--setenv=HF_HUB_DISABLE_XET=1 --setenv=HF_HUB_DISABLE_IMPLICIT_TOKEN=1 " + "--setenv=OMP_NUM_THREADS=4 --setenv=MKL_NUM_THREADS=4 " + "/opt/q38/venv/bin/python /opt/q38/source/scripts/qwen_reference_inference_host.py", + ) + evidence = self.wait_file(self.names[0], "reference-result.json", role="reference") + result.update(result=evidence["result"], evidence=evidence) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + self.event("failed", error=result["error"]) + finally: + if mutated: + try: + for name in ("reference-result.json", "reference-status.json", "reference-error.json"): + value = self.read(self.names[0], name) + if value is not None: + _write_json(self.path / name, value) + for i in range(-1, 4): + remote = "reference.log" if i == -1 else f"reference-worker-{i}/worker.log" + logs = self.ssh(self.names[0], "sudo tail -n 200 /srv/q38/" + remote, check=False) + (self.path / f"host-{i}.log").write_text(logs.stdout + "\n" + logs.stderr, encoding="utf-8") + except Exception as exc: + result["diagnostic_error"] = str(exc) + finally: + try: + result["cleanup"] = self.cleanup() + if not result["cleanup"]["verified"]: + result["result"] = "failed" + except Exception as exc: + result.update(result="failed", cleanup_error=str(exc)) + result["duration_seconds"] = time.time() - self.started + _write_json(self.path / "result.json", result) + self.event("finished", result=result["result"]) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--preflight-only", action="store_true") + parser.add_argument("--cleanup-run", type=Path) + args = parser.parse_args() + with LauncherLock(RUNS / "launcher.lock"): + if args.cleanup_run: + path = args.cleanup_run.resolve() + if path.parent != RUNS.resolve(): + raise ValueError("Cleanup must name an owned reference run") + run = ReferenceRun(path, json.loads((path / "provider-config.json").read_text())) + raise SystemExit(0 if run.cleanup()["verified"] else 1) + config = json.loads((ROOT / "config/qwen_full_inference_gcp.json").read_text()) + config["max_duration_seconds"] = 10800 + path = RUNS / (time.strftime("q38r-%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2)) + run = ReferenceRun(path, config) + _write_json(path / "provider-config.json", config) + if args.preflight_only: + run.preflight() + run.bundle() + run.event("preflight-passed") + else: + raise SystemExit(0 if run.run()["result"] == "passed" else 1) diff --git a/scripts/sign_catalog_candidate.py b/scripts/sign_catalog_candidate.py new file mode 100644 index 000000000..8b9a3cc5a --- /dev/null +++ b/scripts/sign_catalog_candidate.py @@ -0,0 +1,76 @@ +"""Seal a reviewed catalog candidate with the registered publisher key; never publish it.""" + +import argparse +import json +import os +from dataclasses import replace +from pathlib import Path + +from drift.catalog_release import write_catalog_publication_bundle +from drift.model_catalog import CatalogSigningKey, SignedModelCatalog +from drift.model_manifest import ModelManifest +from drift.node.catalog_bootstrap import CatalogBootstrapConfig + + +def seal(candidate, output, registry_path, publication_base, replace_root=None): + registry = json.loads(registry_path.read_text(encoding="utf-8-sig")) + key = CatalogSigningKey.load(registry["private_key_path"]) + if key.key_id != registry["key_id"]: + raise ValueError("Publisher key does not match its registered public identity") + from catalog_key_backup import load_online_backup + + remote = registry["emergency_backup"] + backup = load_online_backup(remote["project"], remote["secret"], remote["version"], key.key_id) + challenge = b"CommunityAI catalog publisher backup verification v1" + key.trusted_key.public_key_object.verify(backup.sign(challenge), challenge) + bootstrap = CatalogBootstrapConfig.load(candidate / "catalog-bootstrap.json") + envelope = SignedModelCatalog.load(candidate / "catalog.unsigned.json") + if envelope.signatures: + raise ValueError("Refusing to rewrite an already signed catalog candidate") + if key.key_id not in {k.key_id for k in bootstrap.trust_root.keys}: + if replace_root != bootstrap.trust_root_digest: + raise ValueError("A new publisher requires the exact --replace-trust-root digest") + bootstrap = replace( + bootstrap, + trust_root=replace(bootstrap.trust_root, keys=(key.trusted_key,), threshold=1), + replaces_trust_roots=(bootstrap.trust_root_digest,), + ) + base = publication_base.rstrip("/") + bootstrap = replace(bootstrap, catalog_mirrors=(base + "/catalog.signed.json",)) + # Re-parse transport fields so arbitrary command-line URLs cannot bypass policy. + bootstrap = CatalogBootstrapConfig.from_dict(bootstrap.to_dict()) + models = tuple( + replace(model, manifest_urls=(f"{base}/manifests/{model.manifest_digest.removeprefix('sha256:')}.json",)) + for model in envelope.signed.models + ) + envelope = replace(envelope, signed=replace(envelope.signed, models=models)).add_signature(key) + manifests = tuple(ModelManifest.load(p) for p in sorted((candidate / "manifests").glob("*.json"))) + write_catalog_publication_bundle(output, bootstrap, envelope, manifests) + print( + json.dumps( + { + "bundle": str(output.resolve()), + "key_id": key.key_id, + "catalog_digest": envelope.signed.digest, + "sequence": envelope.signed.sequence, + "published": False, + "complete_release_qualification": False, + } + ) + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("candidate", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--publication-base-url", required=True) + parser.add_argument("--replace-trust-root") + parser.add_argument( + "--signer-registry", + type=Path, + default=Path(os.environ.get("LOCALAPPDATA", Path.home() / ".local/share")) + / "CommunityAI/publisher-keys/active-catalog.json", + ) + args = parser.parse_args() + seal(args.candidate, args.output, args.signer_registry, args.publication_base_url, args.replace_trust_root) diff --git a/scripts/smoke_tinyllama_local_swarm.py b/scripts/smoke_tinyllama_local_swarm.py index 963ff7c29..8580f3974 100644 --- a/scripts/smoke_tinyllama_local_swarm.py +++ b/scripts/smoke_tinyllama_local_swarm.py @@ -75,6 +75,23 @@ def qualification_lm_head_chunking(model) -> bool | str | None: return getattr(model.get_output_embeddings(), "use_chunked_forward", None) +def manifest_quant_type(manifest: ModelManifest | None) -> QuantType: + """Use the exact manifest execution profile for the hosted blocks.""" + return QuantType.NONE if manifest is None else QuantType[manifest.runtime.quantization.upper()] + + +def create_qualification_worker( + *, + manifest: ModelManifest | None, + server_info_kwargs: dict, + container_kwargs: dict, +): + """Create a worker whose advertised and effective quantization profiles agree.""" + quant_type = manifest_quant_type(manifest) + server_info = ServerInfo(quant_type=quant_type.name.lower(), **server_info_kwargs) + return ModuleContainer.create(server_info=server_info, quant_type=quant_type, **container_kwargs) + + def parse_block_indices(value: str) -> list[int]: try: start_block, end_block = [int(index.strip()) for index in value.split(":")] @@ -252,8 +269,6 @@ def main(argv=None) -> None: revision=None, dht_prefix=None, ) - if manifest.runtime.quantization != "none": - raise ValueError("The local qualification smoke currently supports only manifest quantization='none'") if args.torch_dtype is None: args.torch_dtype = manifest.runtime.dtype elif args.torch_dtype != manifest.runtime.dtype: @@ -378,55 +393,55 @@ def main(argv=None) -> None: serving_identities.append(NodeIdentity.load(worker_identity_path) if worker_identity_path else None) for replica_index, (serving_dht, serving_identity) in enumerate(zip(serving_dhts, serving_identities)): - server_info = ServerInfo( - state=ServerState.JOINING, - throughput=1.0, - manifest_digest=manifest.digest if manifest is not None else None, - torch_dtype=str(torch_dtype).removeprefix("torch."), - quant_type=QuantType.NONE.name.lower(), - using_relay=False, - ) model_info = ModelInfo(num_blocks=block_config.num_hidden_layers, repository=model_name) log(f"starting module container replica={replica_index}") - container = ModuleContainer.create( - dht=serving_dht, - dht_prefix=dht_prefix, - converted_model_name_or_path=model_name, - block_config=block_config, - attn_cache_bytes=attn_cache_bytes, - server_info=server_info, - model_info=model_info, - block_indices=block_indices, - num_handlers=1, - min_batch_size=1, - max_batch_size=64, - max_chunk_size_bytes=16 * 1024 * 1024, - max_alloc_timeout=30, - paged_cache=args.cache == "paged", - page_size=args.page_size, - inference_max_length=64, - torch_dtype=torch_dtype, - cache_dir=args.cache_dir, - max_disk_space=None, - device=device, - compression=CompressionType.NONE, - stats_report_interval=None, - update_period=2 if args.test_failover else 5, - expiration=max(10, MAX_DHT_TIME_DISCREPANCY_SECONDS), - request_timeout=args.failover_request_timeout if args.test_failover else 60, - session_timeout=60, - step_timeout=30, - prefetch_batches=1, - sender_threads=1, - revision=revision, - token=None, - model_manifest=manifest, - protocol_identity=serving_identity, - manifest_execution_profile=manifest.runtime.to_dict() if manifest is not None else None, - quant_type=QuantType.NONE, - tensor_parallel_devices=(device,), - start=True, + container = create_qualification_worker( + manifest=manifest, + server_info_kwargs=dict( + state=ServerState.JOINING, + throughput=1.0, + manifest_digest=manifest.digest if manifest is not None else None, + torch_dtype=str(torch_dtype).removeprefix("torch."), + using_relay=False, + ), + container_kwargs=dict( + dht=serving_dht, + dht_prefix=dht_prefix, + converted_model_name_or_path=model_name, + block_config=block_config, + attn_cache_bytes=attn_cache_bytes, + model_info=model_info, + block_indices=block_indices, + num_handlers=1, + min_batch_size=1, + max_batch_size=64, + max_chunk_size_bytes=16 * 1024 * 1024, + max_alloc_timeout=30, + paged_cache=args.cache == "paged", + page_size=args.page_size, + inference_max_length=64, + torch_dtype=torch_dtype, + cache_dir=args.cache_dir, + max_disk_space=None, + device=device, + compression=CompressionType.NONE, + stats_report_interval=None, + update_period=2 if args.test_failover else 5, + expiration=max(10, MAX_DHT_TIME_DISCREPANCY_SECONDS), + request_timeout=args.failover_request_timeout if args.test_failover else 60, + session_timeout=60, + step_timeout=30, + prefetch_batches=1, + sender_threads=1, + revision=revision, + token=None, + model_manifest=manifest, + protocol_identity=serving_identity, + manifest_execution_profile=manifest.runtime.to_dict() if manifest is not None else None, + tensor_parallel_devices=(device,), + start=True, + ), ) containers.append(container) assert container.ready.wait(timeout=30), "module container did not become ready" diff --git a/scripts/validate_manifest_resume.py b/scripts/validate_manifest_resume.py index ff4bc72a8..bdd7038bb 100644 --- a/scripts/validate_manifest_resume.py +++ b/scripts/validate_manifest_resume.py @@ -17,6 +17,36 @@ from huggingface_hub import hf_hub_download from drift.model_manifest import ManifestArtifactVerifier, ModelManifest +from drift.utils.hub_ranges import RANGE_BYTES + + +def validate_resume_observations(observations, *, prefix_size, artifact_size): + if artifact_size > RANGE_BYTES: + spans = [ + (start, min(artifact_size, start + RANGE_BYTES) - 1) + for start in range(prefix_size, artifact_size, RANGE_BYTES) + ] + expected = [ + { + "requested_range": f"bytes={start}-{end}", + "status": 206, + "content_range": f"bytes {start}-{end}/{artifact_size}", + } + for start, end in spans + ] + else: + expected = [ + { + "requested_range": f"bytes={prefix_size}-", + "status": 206, + "content_range": f"bytes {prefix_size}-{artifact_size - 1}/{artifact_size}", + } + ] + # Parallel bounded requests can finish in either order; every exact span must + # still appear once, with an HTTP 206 and matching Content-Range/total size. + key = lambda item: item.get("requested_range") or "" + if sorted(observations, key=key) != sorted(expected, key=key): + raise RuntimeError(f"Hub did not honor the exact resume requests: {observations}") def main() -> None: @@ -76,15 +106,7 @@ def observed_get(*call_args, **call_kwargs): finally: requests.get = original_get - expected_range = f"bytes={prefix_size}-" - if observations != [ - { - "requested_range": expected_range, - "status": 206, - "content_range": f"bytes {prefix_size}-{artifact.size - 1}/{artifact.size}", - } - ]: - raise RuntimeError(f"Hub did not honor the exact resume request: {observations}") + validate_resume_observations(observations, prefix_size=prefix_size, artifact_size=artifact.size) if result != final or partial.exists(): raise RuntimeError("resumed artifact was not atomically promoted from its partial path") diff --git a/src/drift/api/server.py b/src/drift/api/server.py index 767ee905c..515fe8ffc 100644 --- a/src/drift/api/server.py +++ b/src/drift/api/server.py @@ -17,6 +17,7 @@ import asyncio import json import secrets +import threading import time import uuid from queue import Empty @@ -26,8 +27,8 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse from hivemind.utils.logging import get_logger -from pydantic import BaseModel -from transformers import TextIteratorStreamer +from pydantic import BaseModel, StrictBool +from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer from drift.node.model_manager import ( AmbiguousModelError, @@ -38,6 +39,15 @@ ModelNotFoundError, ) + +class _RequestCancelled(StoppingCriteria): + def __init__(self): + self.event = threading.Event() + + def __call__(self, input_ids, scores, **kwargs): + return self.event.is_set() + + logger = get_logger(__name__) DEFAULT_MAX_TOKENS = 512 @@ -76,6 +86,9 @@ class ChatCompletionRequest(BaseModel): stop: Optional[Union[str, List[str]]] = None stream: bool = False n: int = 1 + # Optional extension for verified templates that expose a reasoning switch. + # Omission preserves the model's own template default. + enable_thinking: Optional[StrictBool] = None class CompletionRequest(BaseModel): @@ -264,6 +277,19 @@ def generate_sync( ) return output_ids + def validate_generation(loaded, input_ids, gen_kwargs): + validator = getattr(loaded.runtime.model, "validate_generation", None) + if validator is not None: + try: + validator(input_ids, gen_kwargs) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + def request_cancellation(gen_kwargs): + cancelled = _RequestCancelled() + gen_kwargs["stopping_criteria"] = StoppingCriteriaList([cancelled]) + return cancelled + def usage(input_ids: torch.Tensor, output_ids: torch.Tensor) -> Dict[str, int]: prompt_tokens = input_ids.shape[1] completion_tokens = output_ids.shape[1] - prompt_tokens @@ -293,6 +319,7 @@ def chunk(payload: Dict[str, Any], reason: Optional[str] = None) -> str: future = None release_deferred = False + cancelled = request_cancellation(gen_kwargs) try: async with semaphore: loop = asyncio.get_running_loop() @@ -323,6 +350,7 @@ def chunk(payload: Dict[str, Any], reason: Optional[str] = None) -> str: yield chunk({"delta": {}} if chat else {"text": ""}, reason) yield "data: [DONE]\n\n" finally: + cancelled.event.set() # Cancelling an HTTP stream does not stop model.generate() in its executor # thread. Keep the residency lease until that thread really exits so an # unload or LRU eviction cannot close the runtime underneath generation. @@ -366,20 +394,28 @@ async def chat_completions(body: ChatCompletionRequest, request: Request): if body.n != 1: raise HTTPException(status_code=400, detail="n > 1 is not supported") loaded = await load_model(body.model) + if loaded.runtime.text_client is not None: + from drift.api.text_response import text_peer_response + + return await text_peer_response(loaded, body, chat=True, semaphore=semaphore) selected_model = loaded.descriptor.model_id selected_tokenizer = loaded.runtime.tokenizer try: messages = [{"role": m.role, "content": message_text(m.content)} for m in body.messages] + template_options = {} if body.enable_thinking is None else {"enable_thinking": body.enable_thinking} input_ids = selected_tokenizer.apply_chat_template( - messages, add_generation_prompt=True, return_dict=True, return_tensors="pt" + messages, add_generation_prompt=True, return_dict=True, return_tensors="pt", **template_options )["input_ids"] gen_kwargs = build_generate_kwargs( max_tokens=body.max_tokens if body.max_tokens is not None else body.max_completion_tokens, temperature=body.temperature, top_p=body.top_p, stop=body.stop, - default_max_tokens=default_max_tokens, + default_max_tokens=min( + default_max_tokens, getattr(loaded.runtime.model, "generation_token_limit", default_max_tokens) + ), ) + validate_generation(loaded, input_ids, gen_kwargs) except BaseException: loaded.release() raise @@ -391,12 +427,14 @@ async def chat_completions(body: ChatCompletionRequest, request: Request): future = None release_deferred = False + cancelled = request_cancellation(gen_kwargs) try: async with semaphore: loop = asyncio.get_running_loop() future = loop.run_in_executor(None, lambda: generate_sync(loaded, input_ids, gen_kwargs)) output_ids = await asyncio.shield(future) finally: + cancelled.event.set() if future is not None and not future.done(): future.add_done_callback(lambda _: loaded.release()) release_deferred = True @@ -425,6 +463,10 @@ async def completions(body: CompletionRequest, request: Request): if body.n != 1: raise HTTPException(status_code=400, detail="n > 1 is not supported") loaded = await load_model(body.model) + if loaded.runtime.text_client is not None: + from drift.api.text_response import text_peer_response + + return await text_peer_response(loaded, body, chat=False, semaphore=semaphore) selected_model = loaded.descriptor.model_id selected_tokenizer = loaded.runtime.tokenizer try: @@ -438,8 +480,11 @@ async def completions(body: CompletionRequest, request: Request): temperature=body.temperature, top_p=body.top_p, stop=body.stop, - default_max_tokens=default_max_tokens, + default_max_tokens=min( + default_max_tokens, getattr(loaded.runtime.model, "generation_token_limit", default_max_tokens) + ), ) + validate_generation(loaded, input_ids, gen_kwargs) except BaseException: loaded.release() raise @@ -451,12 +496,14 @@ async def completions(body: CompletionRequest, request: Request): future = None release_deferred = False + cancelled = request_cancellation(gen_kwargs) try: async with semaphore: loop = asyncio.get_running_loop() future = loop.run_in_executor(None, lambda: generate_sync(loaded, input_ids, gen_kwargs)) output_ids = await asyncio.shield(future) finally: + cancelled.event.set() if future is not None and not future.done(): future.add_done_callback(lambda _: loaded.release()) release_deferred = True diff --git a/src/drift/api/text_response.py b/src/drift/api/text_response.py new file mode 100644 index 000000000..21d63aa0d --- /dev/null +++ b/src/drift/api/text_response.py @@ -0,0 +1,103 @@ +"""OpenAI responses assembled from peer-produced text, without local tensors.""" + +import json +import time +import uuid + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from drift.text_mesh import TextPeerUnavailable + + +async def text_peer_response(loaded, body, *, chat, semaphore): + request_id = ("chatcmpl-" if chat else "cmpl-") + uuid.uuid4().hex[:24] + created = int(time.time()) + model_id = loaded.descriptor.model_id + request = body.model_dump(exclude_none=True) + request["model"] = loaded.descriptor.manifest_digest + + def chunk(text=None, *, done=None, role=False): + choice = {"index": 0, "finish_reason": None if done is None else done.get("finish_reason", "stop")} + if chat: + choice["delta"] = {"role": "assistant", "content": ""} if role else ({"content": text} if text else {}) + else: + choice["text"] = text or "" + result = { + "id": request_id, + "object": "chat.completion.chunk" if chat else "text_completion", + "created": created, + "model": model_id, + "choices": [choice], + } + if done is not None: + result["usage"] = done["usage"] + return "data: " + json.dumps(result) + "\n\n" + + async def events(): + iterator = loaded.runtime.text_client.stream(request, chat=chat) + try: + async with semaphore: + async for frame in iterator: + yield frame + finally: + try: + await iterator.aclose() + finally: + loaded.release() + + async def sse(): + frames = events() + try: + if chat: + yield chunk(role=True) + async for frame in frames: + if frame["type"] == "delta": + yield chunk(frame["text"]) + elif frame["type"] == "done": + yield chunk(done=frame) + else: + yield ": waiting for community\n\n" + yield "data: [DONE]\n\n" + except (TextPeerUnavailable, ValueError, TimeoutError) as exc: + yield "data: " + json.dumps({"error": {"message": str(exc), "type": "server_error"}}) + "\n\n" + yield "data: [DONE]\n\n" + finally: + try: + await frames.aclose() + finally: + # Also release if the caller disconnects after the role chunk, + # before the peer event iterator has started. + loaded.release() + + if body.stream: + return StreamingResponse(sse(), media_type="text/event-stream") + parts, done = [], None + frames = events() + try: + async for frame in frames: + if frame["type"] == "delta": + parts.append(frame["text"]) + elif frame["type"] == "done": + done = frame + if done is None: + raise TextPeerUnavailable("The community answer did not finish") + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except (TextPeerUnavailable, TimeoutError) as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + finally: + await frames.aclose() + from drift.api.server import trim_stop_strings + + text = trim_stop_strings("".join(parts), body.stop) + choice = {"index": 0, "finish_reason": done.get("finish_reason", "stop")} + choice.update({"message": {"role": "assistant", "content": text}} if chat else {"text": text}) + return { + "id": request_id, + "object": "chat.completion" if chat else "text_completion", + "created": created, + "model": model_id, + "choices": [choice], + "usage": done["usage"], + } diff --git a/src/drift/catalog_release.py b/src/drift/catalog_release.py index f395749cd..7743907ea 100644 --- a/src/drift/catalog_release.py +++ b/src/drift/catalog_release.py @@ -14,6 +14,7 @@ import secrets import shutil import stat +import time from pathlib import Path from typing import Any, Dict, Mapping, Sequence from urllib.parse import urlsplit @@ -213,6 +214,11 @@ def verify_catalog_publication_bundle( selectors: dict[str, str] = {} for model in catalog.models: manifest = manifest_by_digest[model.manifest_digest] + if manifest.model.gated: + raise CatalogBootstrapError( + f"Publication manifest {manifest.digest_id} is gated; CommunityAI catalogs require " + "unauthenticated, no-consent artifact access" + ) declared_weight_bytes = sum(artifact.size for artifact in manifest.artifacts if artifact.role == "weight") if model.weight_bytes != declared_weight_bytes: raise CatalogBootstrapError( @@ -229,10 +235,10 @@ def verify_catalog_publication_bundle( selectors[folded] = manifest.digest_id for rung in catalog.rungs: - if rung.minimum_replicas < 1 or rung.minimum_independent_routes < 1 or rung.minimum_surviving_replicas < 1: + if rung.minimum_replicas < 1 or rung.minimum_independent_routes < 1 or rung.minimum_surviving_replicas < 0: raise CatalogBootstrapError( f"Publication rung {rung.rung_id!r} must require at least one complete replica, " - "one route, and one surviving replica" + "one route, and a nonnegative surviving-replica requirement" ) return { @@ -519,6 +525,19 @@ def load_catalog_publication_bundle( return index +def _replace_bundle_directory(source: Path, destination: Path) -> None: + # Windows scanners can briefly retain a handle after verification closes it. + # Retry the same atomic operation; never fall back to a partially copied bundle. + for attempt in range(6): + try: + os.replace(source, destination) + return + except PermissionError as exc: + if getattr(exc, "winerror", None) not in (5, 32, 33) or attempt == 5: + raise + time.sleep(0.1) + + def write_catalog_publication_bundle( output_directory: Path | str, bootstrap: CatalogBootstrapConfig, @@ -601,12 +620,12 @@ def write_catalog_publication_bundle( raise CatalogBootstrapError("Catalog publication bundle validation returned an unexpected index") if resolved.exists(): - os.replace(resolved, backup) + _replace_bundle_directory(resolved, backup) try: - os.replace(staging, resolved) + _replace_bundle_directory(staging, resolved) except OSError: if backup.exists() and not resolved.exists(): - os.replace(backup, resolved) + _replace_bundle_directory(backup, resolved) raise if backup.exists(): shutil.rmtree(backup) diff --git a/src/drift/cli/__main__.py b/src/drift/cli/__main__.py index 89fa53db4..20b8dc97a 100644 --- a/src/drift/cli/__main__.py +++ b/src/drift/cli/__main__.py @@ -32,6 +32,7 @@ "manifest", "catalog", "identity", + "text-peer", ) _USAGE = """usage: drift [options] @@ -53,6 +54,7 @@ manifest Validate and inspect a content-addressed ModelManifest v1 catalog Create signing keys, trust roots, and threshold-signed model catalogs identity Create, inspect, rotate, revoke, and verify public-swarm identities + text-peer Serve input/output processing and generation for text-only mesh clients Run `drift --help` for command-specific options. """ @@ -92,7 +94,9 @@ def main() -> int: elif command == "catalog": from drift.cli.run_catalog import main as run elif command == "identity": - from drift.cli.run_identity import main as run + from drift.cli.run_identity import main as run + elif command == "text-peer": + from drift.cli.run_text_peer import main as run else: # dht from drift.cli.run_dht import main as run diff --git a/src/drift/cli/run_bootstrap.py b/src/drift/cli/run_bootstrap.py index 39be0e5a8..bd6cd7d80 100644 --- a/src/drift/cli/run_bootstrap.py +++ b/src/drift/cli/run_bootstrap.py @@ -6,7 +6,13 @@ import json from pathlib import Path -from drift.node.catalog_bootstrap import CatalogBootstrapError, bootstrap_node_from_catalog +from drift.node.catalog_bootstrap import ( + CatalogBootstrapConfig, + CatalogBootstrapError, + CatalogBootstrapInstaller, + bootstrap_node_from_catalog, +) +from drift.node.config import NodeConfig DEFAULT_NODE_DATA_DIR = Path.home() / ".drift" / "node" @@ -20,6 +26,14 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("bootstrap_config", type=Path, help="Trusted release bootstrap JSON") parser.add_argument("--data_dir", type=Path, default=DEFAULT_NODE_DATA_DIR) parser.add_argument("--node_config", type=Path, help="Generated NodeConfig v1 path") + parser.add_argument( + "--refresh", action="store_true", help="Authenticate a newer catalog while preserving existing user settings" + ) + parser.add_argument( + "--refresh_if_needed", + action="store_true", + help="Migrate a legacy installation to authenticated periodic refresh", + ) return parser @@ -29,7 +43,23 @@ def main() -> None: data_dir = args.data_dir.expanduser().resolve() config_path = (args.node_config or data_dir / "node-config.json").expanduser().resolve() try: - result = bootstrap_node_from_catalog(args.bootstrap_config, data_dir=data_dir, config_path=config_path) + if args.refresh or args.refresh_if_needed: + installer = CatalogBootstrapInstaller( + CatalogBootstrapConfig.load(args.bootstrap_config), data_dir=data_dir, config_path=config_path + ) + existing = NodeConfig.load(config_path) + needs_refresh = existing.catalog_path is None + if existing.catalog_bootstrap_path is not None: + installed = CatalogBootstrapConfig.load(existing.catalog_bootstrap_path) + # Only a locally installed application bundle can supply a new + # root. A catalog fetched from the network cannot change it. + needs_refresh = ( + installer.bootstrap.trust_root != installed.trust_root + and installer.bootstrap.permits_replacement_of(installed) + ) + result = installer.refresh() if args.refresh or needs_refresh else installer.repair_existing_config() + else: + result = bootstrap_node_from_catalog(args.bootstrap_config, data_dir=data_dir, config_path=config_path) except CatalogBootstrapError as exc: parser.error(str(exc)) print(json.dumps(result.to_dict(), sort_keys=True)) diff --git a/src/drift/cli/run_edge_acquisition.py b/src/drift/cli/run_edge_acquisition.py index 541a0bb84..721d8e39d 100644 --- a/src/drift/cli/run_edge_acquisition.py +++ b/src/drift/cli/run_edge_acquisition.py @@ -3,7 +3,9 @@ from __future__ import annotations import argparse +import hashlib import json +import re import sys from pathlib import Path @@ -12,6 +14,9 @@ from drift.model_manifest import ManifestError, ModelManifest from drift.node.edge_acquisition import acquire_client_artifacts +MAX_MANIFEST_STDIN_BYTES = 65_536 +_MANIFEST_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( @@ -19,9 +24,19 @@ def build_parser() -> argparse.ArgumentParser: description="Acquire and verify exact client-selected artifacts into one empty persistent cache", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument("model_manifest", type=Path, help="Path to one exact ModelManifest v1") + parser.add_argument("model_manifest", type=Path, nargs="?", help="Path to one exact ModelManifest v1") + parser.add_argument( + "--manifest_stdin_sha256", + help="Read the manifest from stdin and require this exact sha256: digest", + ) parser.add_argument("--cache_dir", type=Path, required=True, help="Empty persistent cache to populate") - parser.add_argument("--token", default=None, help="Hugging Face token for a gated repository") + credentials = parser.add_mutually_exclusive_group() + credentials.add_argument("--token", default=None, help="Hugging Face token for a gated repository") + credentials.add_argument( + "--no_token", + action="store_true", + help="Disable explicit and implicit Hugging Face authentication", + ) parser.add_argument("--max_resumptions", type=int, default=3, choices=range(4)) parser.add_argument( "--require_direct_upstream", @@ -32,18 +47,39 @@ def build_parser() -> argparse.ArgumentParser: return parser +def _load_manifest(args: argparse.Namespace, parser: argparse.ArgumentParser) -> ModelManifest: + if (args.model_manifest is None) == (args.manifest_stdin_sha256 is None): + parser.error("provide exactly one manifest path or --manifest_stdin_sha256") + if args.model_manifest is not None: + return ModelManifest.load(args.model_manifest) + + expected = args.manifest_stdin_sha256 + if _MANIFEST_DIGEST_RE.fullmatch(expected) is None: + parser.error("--manifest_stdin_sha256 must be sha256:<64 lowercase hex characters>") + payload = sys.stdin.buffer.read(MAX_MANIFEST_STDIN_BYTES + 1) + if not 1 <= len(payload) <= MAX_MANIFEST_STDIN_BYTES: + parser.error("stdin manifest is empty or exceeds its byte limit") + observed = "sha256:" + hashlib.sha256(payload).hexdigest() + if observed != expected: + parser.error("stdin manifest digest does not match --manifest_stdin_sha256") + try: + return ModelManifest.from_json(payload.decode("utf-8")) + except UnicodeError as exc: + raise ManifestError("Manifest stdin is not valid UTF-8") from exc + + def main() -> None: parser = build_parser() args = parser.parse_args() try: - manifest = ModelManifest.load(args.model_manifest) + manifest = _load_manifest(args, parser) manifest.validate_runtime(drift.__version__) if manifest.runtime.adapter_profile != "none": raise ManifestError("Content-addressed adapter profiles are not executable in this release") result = acquire_client_artifacts( manifest, cache_dir=args.cache_dir.expanduser().resolve(), - token=args.token, + token=False if args.no_token else args.token, max_resumptions=args.max_resumptions, require_direct_upstream=args.require_direct_upstream, ) diff --git a/src/drift/cli/run_manifest.py b/src/drift/cli/run_manifest.py index dc824178f..d0e088aff 100644 --- a/src/drift/cli/run_manifest.py +++ b/src/drift/cli/run_manifest.py @@ -80,7 +80,7 @@ def _generate(argv=None) -> None: parser.add_argument("--maximum_version_exclusive", default="2.4.0") parser.add_argument("--attention_implementation", choices=("auto", "eager", "sdpa"), default="auto") parser.add_argument("--dtype", choices=("float32", "float16", "bfloat16"), default="bfloat16") - parser.add_argument("--quantization", choices=("none", "int8", "nf4"), default="none") + parser.add_argument("--quantization", choices=("none", "int8", "nf4", "fp8_dequant"), default="none") parser.add_argument("--token", help="Hugging Face token for gated repositories") parser.add_argument("--cache_dir", help="Hugging Face cache directory") parser.add_argument("--output", help="Write pretty, deterministic JSON to this path instead of stdout") diff --git a/src/drift/cli/run_node.py b/src/drift/cli/run_node.py index 3eedb184b..76e247157 100644 --- a/src/drift/cli/run_node.py +++ b/src/drift/cli/run_node.py @@ -4,6 +4,7 @@ import argparse import math +import secrets import sys import time from dataclasses import replace @@ -15,18 +16,21 @@ from hivemind.utils.timed_storage import get_dht_time import drift -from drift.model_manifest import ManifestError, ModelManifest +from drift.model_manifest import ManifestArtifactVerifier, ManifestError, ModelManifest, select_manifest_block_artifacts +from drift.node.catalog_refresh import CatalogRefreshService, load_configured_catalog from drift.node.config import NODE_CONFIG_SCHEMA_VERSION, NodeConfig, NodeConfigError, NodeModelConfig from drift.node.contribution_planner import ( AutomaticContributionPlanner, AutomaticPlacementService, + PlacementArtifactPlan, PlacementCandidate, PlacementPlan, PlacementRegistry, ) from drift.node.discovery import CoverageTarget, ModelCoverageDiscovery, PeerCache from drift.node.keys import ApiKeyStore, ApiKeyStoreError, load_or_create_api_key, load_or_create_control_key -from drift.node.loading import make_manifest_loader +from drift.node.loading import make_text_peer_loader, validate_manifest_execution +from drift.node.local_inference import local_route_observer, make_local_manifest_loader from drift.node.model_manager import ModelDescriptor, ModelManager, ModelNotFoundError from drift.node.native_credentials import ( DEFAULT_CREDENTIAL_ACCOUNT, @@ -50,6 +54,8 @@ create_intent_lease, create_route_demand, ) +from drift.utils.auto_config import AutoDistributedConfig +from drift.utils.disk_cache import DEFAULT_CACHE_DIR from drift.utils.hardware import auto_detect_device, get_device_total_memory, is_accelerator, normalize_device from drift.utils.process_lifetime import tie_child_processes_to_this_process @@ -163,17 +169,7 @@ def _load_node_config(args: argparse.Namespace, *, persisted_config: NodeConfig configured = persisted_config if persisted_config is not None else NodeConfig.load(args.config) if args.max_loaded_models is None: return configured - return NodeConfig( - schema_version=configured.schema_version, - max_loaded_models=args.max_loaded_models, - models=configured.models, - auto_model_priority=configured.auto_model_priority, - route_demand_authority_roots=configured.route_demand_authority_roots, - workers=configured.workers, - contribution_policy=configured.contribution_policy, - discovery_update_period=configured.discovery_update_period, - discovery_startup_timeout=configured.discovery_startup_timeout, - ) + return replace(configured, max_loaded_models=args.max_loaded_models) manifest_path = Path(args.model_manifest).expanduser().resolve() cache_dir = Path(args.cache_dir).expanduser().resolve() if args.cache_dir else None @@ -243,6 +239,7 @@ def _build_model_manager( for model_config in config.models: manifest = ModelManifest.load(model_config.manifest_path) manifest.validate_runtime(drift.__version__) + validate_manifest_execution(manifest, model_config.execution) if manifest.runtime.adapter_profile != "none": raise ManifestError("Content-addressed adapter profiles are not executable in this release") configured_manifests.append((model_config, manifest)) @@ -258,31 +255,39 @@ def _build_model_manager( ), ) for model_config, manifest in configured_manifests + if model_config.execution == "distributed" ], update_period=config.discovery_update_period, startup_timeout=config.discovery_startup_timeout, peer_cache=peer_cache, replay_history_dir=replay_history_dir, route_demand_authority_roots=config.route_demand_authority_roots, + discover_text=True, ) manager.add_shutdown_callback(discovery.close) for model_config, manifest in configured_manifests: - descriptors.append( - manager.register_manifest( - manifest, - make_manifest_loader( - manifest, - initial_peers=model_config.initial_peers, - token=token, - cache_dir=str(model_config.cache_dir) if model_config.cache_dir is not None else None, - revocation_files=tuple(str(path) for path in model_config.revocation_files), - request_timeout=model_config.request_timeout, - max_retries=model_config.max_retries, - ), - route_health=discovery.observer(manifest.digest_id), + if model_config.execution == "local": + descriptor = replace(ModelDescriptor.from_manifest(manifest), execution="local") + manager.register( + descriptor, + make_local_manifest_loader(manifest, model_config), + route_health=local_route_observer(manifest, model_config), ) + descriptors.append(descriptor) + continue + descriptor = replace(ModelDescriptor.from_manifest(manifest), selected_whole_shard_bytes=0) + manager.register( + descriptor, + make_text_peer_loader( + manifest, + initial_peers=model_config.initial_peers, + revocation_files=tuple(str(path) for path in model_config.revocation_files), + request_timeout=model_config.request_timeout, + ), + route_health=discovery.observer(manifest.digest_id), ) - manager.configure_auto_selection(config.auto_model_priority) + descriptors.append(descriptor) + manager.configure_auto_selection(config.auto_model_priority, local_only=config.inference_mode == "local_only") except BaseException: manager.shutdown() raise @@ -307,6 +312,58 @@ def _resolve_policy_models( return resolved +def _resolved_automatic_cache_root(worker, model_config: NodeModelConfig) -> Path: + configured = worker.cache_dir if worker.cache_dir is not None else model_config.cache_dir + return Path(DEFAULT_CACHE_DIR if configured is None else configured).expanduser().resolve() + + +def _manifest_artifact_plans( + manifest: ModelManifest, + worker, + *, + token: str | None, + cache_dir: Path | None, + max_disk_space: int, +) -> tuple[PlacementArtifactPlan, ...]: + if worker.num_blocks is None or worker.num_blocks > manifest.model.num_blocks: + raise ManifestError("Automatic worker block count exceeds the manifested model") + source_token = token if manifest.model.gated else False + verifier = ManifestArtifactVerifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=source_token, + cache_dir=cache_dir, + max_disk_space=max_disk_space, + ) + config_root = verifier.ensure_startup_metadata() + block_config = AutoDistributedConfig.from_pretrained( + config_root, + token=source_token, + local_files_only=True, + ) + manifest.validate_model_config(block_config) + weight_map = verifier.load_weight_map() + return tuple( + PlacementArtifactPlan( + start_block=start, + end_block=start + worker.num_blocks, + artifact_bytes=plan.artifact_bytes, + artifact_set_digest=plan.artifact_set_digest, + ) + for start in range(manifest.model.num_blocks - worker.num_blocks + 1) + for plan in ( + select_manifest_block_artifacts( + manifest, + block_prefix=block_config.block_prefix, + start_block=start, + end_block=start + worker.num_blocks, + weight_map=weight_map, + ), + ) + ) + + def _automatic_placement_candidates( config: NodeConfig, manager: ModelManager, @@ -316,6 +373,7 @@ def _automatic_placement_candidates( token: str | None, route_outcomes: RouteOutcomeTracker | None = None, allow_remote_route_demand: bool = False, + artifact_plan_cache: dict | None = None, ) -> tuple[PlacementCandidate, ...]: policy = config.contribution_policy allowed = _resolve_policy_models(manager, policy.allowed_models, "allowed_models") @@ -329,18 +387,22 @@ def _automatic_placement_candidates( ordered_keys.append(key) manifested = [] for model_config in config.models: + if model_config.execution == "local": + continue manifest = ModelManifest.load(model_config.manifest_path) + if not manager.catalog_allows_contribution(manifest.digest_id): + continue descriptor = manager.resolve(manifest.digest_id) key = _model_key(descriptor) if key not in ordered_keys: ordered_keys.append(key) - manifested.append((manifest, descriptor, key)) + manifested.append((model_config, manifest, descriptor, key)) priority = {key: index for index, key in enumerate(ordered_keys)} disk_limits = [value for value in (worker.max_disk_bytes, policy.max_disk_bytes) if value is not None] effective_disk_bytes = min(disk_limits, default=None) candidates = [] - for manifest, descriptor, key in manifested: + for model_config, manifest, descriptor, key in manifested: artifact_bytes = sum(artifact.size for artifact in manifest.artifacts) if key in denied: reason = f"model {descriptor.model_id!r} is denied by contribution policy" @@ -350,13 +412,30 @@ def _automatic_placement_candidates( reason = f"model {descriptor.model_id!r} requires gated artifact authorization" elif effective_disk_bytes is None: reason = "automatic placement requires a finite disk budget" - elif artifact_bytes > effective_disk_bytes: - reason = ( - f"manifested artifacts require {artifact_bytes} bytes, above the " - f"{effective_disk_bytes}-byte disk budget" - ) else: reason = None + artifact_plans = () + if reason is None: + try: + cache_root = _resolved_automatic_cache_root(worker, model_config) + plan_key = (manifest.digest_id, worker.num_blocks, str(cache_root), effective_disk_bytes) + cached = None if artifact_plan_cache is None else artifact_plan_cache.get(plan_key) + if cached is None: + artifact_plans = _manifest_artifact_plans( + manifest, worker, token=token, cache_dir=cache_root, max_disk_space=effective_disk_bytes + ) + if artifact_plan_cache is not None: + # Plans are immutable claims derived from a verified pinned + # index. Reopening that index on every discovery tick can + # interrupt an admitted worker during cache materialization. + # Worker startup still independently verifies all artifacts. + if len(artifact_plan_cache) >= 128: + artifact_plan_cache.pop(next(iter(artifact_plan_cache))) + artifact_plan_cache[plan_key] = artifact_plans + else: + artifact_plans = cached + except (ManifestError, OSError, RuntimeError, TypeError, ValueError) as exc: + reason = f"exact block artifact planning failed: {type(exc).__name__}" candidates.append( PlacementCandidate( model_id=descriptor.model_id, @@ -371,6 +450,8 @@ def _automatic_placement_candidates( discovery.route_demand_snapshot(manifest.digest_id) if allow_remote_route_demand else None ), policy_reason=reason, + artifact_plans=artifact_plans, + max_artifact_bytes=effective_disk_bytes, ) ) return tuple(candidates) @@ -425,10 +506,20 @@ def _prepare_worker_supervisor_settings( placement = automatic_placements.get(worker.worker_id.casefold()) decision = None if placement is None else placement.decision if automatic: - fallback_selector = ( - config.auto_model_priority[0] - if config.auto_model_priority - else ModelManifest.load(config.models[0].manifest_path).digest_id + distributed = [ + digest + for digest, (model_config, _) in manifested_models.items() + if model_config.execution == "distributed" and manager.catalog_allows_contribution(digest) + ] + if not distributed: + continue + fallback_selector = next( + ( + selector + for selector in config.auto_model_priority + if manager.resolve(selector).manifest_digest in distributed + ), + distributed[0], ) selector = fallback_selector if decision is None else decision.manifest_digest else: @@ -438,6 +529,8 @@ def _prepare_worker_supervisor_settings( except ModelNotFoundError as exc: raise NodeConfigError(f"worker {worker.worker_id!r} selects {exc}") from exc model_config, manifest = manifested_models[descriptor.manifest_digest] + if model_config.execution == "local": + raise NodeConfigError(f"worker {worker.worker_id!r} cannot host a standalone local inference profile") if automatic: selected_num_blocks = None selected_block_indices = f"0:{worker.num_blocks}" if decision is None else decision.block_indices @@ -459,12 +552,20 @@ def _prepare_worker_supervisor_settings( ) if worker.public_ip is not None and worker.port is None: raise NodeConfigError(f"worker {worker.worker_id!r} public_ip requires port") + if worker.public_port is not None and (worker.port is None or worker.public_ip is None): + raise NodeConfigError(f"worker {worker.worker_id!r} public_port requires port and public_ip") resolved_model = _model_key(descriptor) + intent_published = bool(automatic and placement is not None and placement.intent_published) + remote_acknowledged = bool(automatic and placement is not None and placement.remote_acknowledged) if not policy.sharing_enabled: policy_reason = "sharing is disabled by contribution policy" elif automatic and decision is None: policy_reason = placement_reason + elif automatic and not (intent_published and remote_acknowledged): + policy_reason = "automatic placement intent is not remotely acknowledged" + elif automatic and decision.artifact_set_digest is None: + policy_reason = "automatic placement has no exact artifact-set binding" elif resolved_model in denied_models: policy_reason = f"model {descriptor.model_id!r} is denied by contribution policy" elif allowed_models and resolved_model not in allowed_models: @@ -529,6 +630,8 @@ def resolve_vram_limit(limit): size, fraction = limit return size if size is not None else math.floor(total_vram * fraction) + # Sharing owns its configured fraction of the physical device. + # An optional local fallback must not reduce this budget. policy_vram_bytes = min(total_vram, resolve_vram_limit(policy_vram_limit)) effective_vram_bytes = policy_vram_bytes if worker_vram_limit != (None, None): @@ -557,21 +660,54 @@ def resolve_vram_limit(limit): command.extend(("--num_blocks", str(selected_num_blocks))) else: command.extend(("--block_indices", selected_block_indices)) + cache_dir = ( + _resolved_automatic_cache_root(worker, model_config) + if automatic + else worker.cache_dir + if worker.cache_dir is not None + else model_config.cache_dir + ) + if automatic and decision is not None and decision.artifact_set_digest is not None: + command.extend( + ( + "--expected_manifest_digest", + decision.manifest_digest, + "--expected_block_indices", + selected_block_indices, + "--expected_artifact_bytes", + str(decision.artifact_bytes), + "--expected_artifact_set_digest", + decision.artifact_set_digest, + "--expected_cache_root", + str(cache_dir), + ) + ) if worker.device is not None: command.extend(("--device", worker.device)) - cache_dir = worker.cache_dir if worker.cache_dir is not None else model_config.cache_dir if cache_dir is not None: command.extend(("--cache_dir", str(cache_dir))) if effective_disk_space is not None: command.extend(("--max_disk_space", effective_disk_space)) if effective_vram_bytes is not None: command.extend(("--max_device_memory", str(effective_vram_bytes))) + command.extend(("--max_processing_percent", str(policy.max_processing_percent))) + if policy.max_processing_percent < 100: + # One stable lock for all this node's workers, across model/device + # changes. Never remove a live lock file during a policy update. + budget_path = config.workers[0].identity_path.with_name( + f".{config.workers[0].identity_path.name}.processing-budget" + ) + command.extend(("--processing_budget_path", str(budget_path))) if worker.port is not None: command.extend(("--port", str(worker.port))) if worker.public_ip is not None: - command.extend(("--public_ip", worker.public_ip)) + if worker.public_port is None: + command.extend(("--public_ip", worker.public_ip)) + else: + command.extend(("--announce_maddrs", f"/ip4/{worker.public_ip}/tcp/{worker.public_port}")) for revocation_file in model_config.revocation_files: command.extend(("--revocation_file", str(revocation_file))) + placement_binding = decision if decision is not None and decision.artifact_set_digest is not None else None launches.append( WorkerLaunch( @@ -587,6 +723,14 @@ def resolve_vram_limit(limit): automatic=automatic, block_indices=selected_block_indices if automatic else None, placement_reason=placement_reason, + intent_published=intent_published, + remote_acknowledged=remote_acknowledged, + placement_manifest_digest=(None if placement_binding is None else placement_binding.manifest_digest), + placement_artifact_bytes=(None if placement_binding is None else placement_binding.artifact_bytes), + placement_artifact_set_digest=( + None if placement_binding is None else placement_binding.artifact_set_digest + ), + placement_cache_root=None if placement_binding is None else str(cache_dir), max_disk_bytes=effective_disk_bytes, max_vram_bytes=effective_vram_bytes, vram_device=vram_device, @@ -665,6 +809,95 @@ def _prepare_route_identity( return None +def _intent_lease_binding(worker, decision, identity_key_id: str) -> tuple[int, int, dict, tuple]: + start_block, end_block = (int(value) for value in decision.block_indices.split(":")) + throughput = None if isinstance(worker.throughput, str) else max(1, round(worker.throughput * 1000)) + resource_claims = { + "schema_version": INTENT_RESOURCE_CLAIMS_SCHEMA_VERSION, + "artifact_bytes": decision.artifact_bytes, + "block_count": end_block - start_block, + "throughput_milli_rps": throughput, + } + decision_key = ( + decision.manifest_digest, + start_block, + end_block, + decision.artifact_set_digest, + str(worker.identity_path), + identity_key_id, + tuple(sorted(resource_claims.items())), + ) + return start_block, end_block, resource_claims, decision_key + + +def _placement_decision_key(worker, decision, identity_key_id: str) -> tuple: + return _intent_lease_binding(worker, decision, identity_key_id)[3] + + +def _can_retain_acknowledged_plan( + previous: PlacementPlan | None, + current_decision, + worker, + identity_key_id: str | None, + lease: Mapping | None, + *, + now: float, +) -> bool: + if previous is None or previous.decision is None or identity_key_id is None: + return False + previous_key = _placement_decision_key(worker, previous.decision, identity_key_id) + current_key = _placement_decision_key(worker, current_decision, identity_key_id) + return bool( + previous.remote_acknowledged + and lease is not None + and lease.get("decision_key") == previous_key == current_key + and isinstance(lease.get("expires_at"), (int, float)) + and not isinstance(lease.get("expires_at"), bool) + and math.isfinite(lease["expires_at"]) + and lease["expires_at"] > now + ) + + +def _recent_gap_preserves_artifact_claim(candidate, decision, num_blocks, *, maximum_age): + if candidate is None or candidate.policy_reason is not None: + return False + health = candidate.health + age = health.get("last_updated_age") + if ( + health.get("status") != "unknown" + or health.get("last_known_status") not in ("complete", "incomplete") + or isinstance(age, bool) + or not isinstance(age, (int, float)) + or not math.isfinite(age) + or not 0 <= age <= maximum_age + ): + return False + start, end = map(int, decision.block_indices.split(":")) + return bool( + end - start == num_blocks + and candidate.max_artifact_bytes is not None + and decision.artifact_bytes <= candidate.max_artifact_bytes + and any( + p.start_block == start + and p.end_block == end + and p.artifact_bytes == decision.artifact_bytes + and p.artifact_set_digest == decision.artifact_set_digest + for p in candidate.artifact_plans + ) + ) + + +def _automatic_placement_seed(worker) -> str: + try: + return NodeIdentity.ensure(worker.identity_path).key_id + except (OSError, ProtocolSecurityError, RuntimeError, TypeError, ValueError): + # A temporarily unavailable contribution key must not stop local + # inference. Intent publication still denies worker admission until the + # key is usable; an ephemeral seed keeps this session independently + # dispersed if permissions recover before the node is restarted. + return secrets.token_hex(32) + + def _build_automatic_placement_service( config: NodeConfig, manager: ModelManager, @@ -684,16 +917,18 @@ def _build_automatic_placement_service( planners = { worker.worker_id.casefold(): AutomaticContributionPlanner( num_blocks=worker.num_blocks, - jitter_seed=str(worker.identity_path), + # Installation paths are identical on many machines. Disperse by + # the persistent public identity, not by the key's filename. + jitter_seed=_automatic_placement_seed(worker), maximum_observation_age_seconds=max(90.0, config.discovery_update_period * 3), ) for worker in automatic_workers } intent_ttl_seconds = 10 * 60 intent_refresh_seconds = 2 * 60 - intent_identities = {} intent_sequences = {} intent_leases = {} + artifact_plan_cache = {} route_identity = _prepare_route_identity(discovery, route_identity_path, config.route_demand_authority_roots) route_sequences = {} route_leases = {} @@ -735,9 +970,16 @@ def publish_route_demand(manifest: ModelManifest) -> None: "expires_at": expires_at, } - def publish_intent(worker_id, worker, decision) -> bool: - identity_path = str(worker.identity_path) - decision_key = (decision.manifest_digest, decision.block_indices, identity_path) + def publish_intent(worker_id, worker, decision) -> tuple[bool, str | None]: + if decision.artifact_set_digest is None: + return False, None + try: + identity = NodeIdentity.ensure(worker.identity_path) + start_block, end_block, resource_claims, decision_key = _intent_lease_binding( + worker, decision, identity.key_id + ) + except (OSError, ProtocolSecurityError, RuntimeError, TypeError, ValueError): + return False, None now = get_dht_time() current_lease = intent_leases.get(worker_id) if ( @@ -745,16 +987,8 @@ def publish_intent(worker_id, worker, decision) -> bool: and current_lease["decision_key"] == decision_key and current_lease["expires_at"] - now > intent_refresh_seconds ): - return True + return True, identity.key_id try: - identity_entry = intent_identities.get(worker_id) - if identity_entry is None or identity_entry[0] != identity_path: - identity = NodeIdentity.ensure(worker.identity_path) - intent_identities[worker_id] = (identity_path, identity) - else: - identity = identity_entry[1] - start_block, end_block = (int(value) for value in decision.block_indices.split(":")) - throughput = None if isinstance(worker.throughput, str) else max(1, round(worker.throughput * 1000)) sequence = max(intent_sequences.get(worker_id, 0) + 1, time.time_ns()) intent_sequences[worker_id] = sequence expires_at = now + intent_ttl_seconds @@ -763,28 +997,27 @@ def publish_intent(worker_id, worker, decision) -> bool: manifest_digest=decision.manifest_digest.removeprefix("sha256:"), start_block=start_block, end_block=end_block, - resource_claims={ - "schema_version": INTENT_RESOURCE_CLAIMS_SCHEMA_VERSION, - "artifact_bytes": decision.artifact_bytes, - "block_count": end_block - start_block, - "throughput_milli_rps": throughput, - }, + resource_claims=resource_claims, issued_at=now, expires_at=expires_at, sequence=sequence, ) except (OSError, ProtocolSecurityError, RuntimeError, TypeError, ValueError): - return False + return False, identity.key_id if not discovery.publish_intent(decision.manifest_digest, record.to_dict()): - return False + return False, identity.key_id intent_leases[worker_id] = { "decision_key": decision_key, "expires_at": expires_at, } - return True + return True, identity.key_id def reconcile() -> None: current = config if config_path is None else NodeConfig.load(config_path) + if current.catalog_path != config.catalog_path: + # A verified update is waiting for the current request leases to + # drain. Keep the old working contribution until activation. + return current = _reuse_runtime_initial_peers(current, config) current_workers = { worker.worker_id.casefold(): worker for worker in current.workers if worker.model.casefold() == "auto" @@ -794,7 +1027,10 @@ def reconcile() -> None: route_demand_authorities_unchanged = current.route_demand_authority_roots == config.route_demand_authority_roots if current.contribution_policy.sharing_enabled and route_demand_authorities_unchanged: for model_config in current.models: - publish_route_demand(ModelManifest.load(model_config.manifest_path)) + if model_config.execution == "distributed": + manifest = ModelManifest.load(model_config.manifest_path) + if manager.catalog_allows_contribution(manifest.digest_id): + publish_route_demand(manifest) for worker_id, planner in planners.items(): worker = current_workers.get(worker_id) if worker is None: @@ -809,25 +1045,70 @@ def reconcile() -> None: allow_remote_route_demand=( route_demand_authorities_unchanged and bool(config.route_demand_authority_roots) ), + artifact_plan_cache=artifact_plan_cache, ) proposal = planner.propose( candidates, sharing_enabled=current.contribution_policy.sharing_enabled, ) + if proposal.decision is None and current.contribution_policy.sharing_enabled: + previous = previous_plans.get(worker_id) + if previous is not None and previous.decision is not None: + # An interrupted lookup is not a revocation of an admitted + # span. Keep that exact claim only within the ordinary + # coverage freshness window and its existing remote lease. + candidate = next( + (c for c in candidates if c.manifest_digest == previous.decision.manifest_digest), None + ) + if _recent_gap_preserves_artifact_claim( + candidate, + previous.decision, + worker.num_blocks, + maximum_age=max(90.0, current.discovery_update_period * 3), + ): + try: + identity_key_id = NodeIdentity.load(worker.identity_path).key_id + except (OSError, ProtocolSecurityError, RuntimeError, TypeError, ValueError): + identity_key_id = None + if _can_retain_acknowledged_plan( + previous, + previous.decision, + worker, + identity_key_id, + intent_leases.get(worker_id), + now=get_dht_time(), + ): + plans[worker_id] = previous + continue if proposal.decision is not None: - if not publish_intent(worker_id, worker, proposal.decision): + published, identity_key_id = publish_intent(worker_id, worker, proposal.decision) + if not published: previous = previous_plans.get(worker_id) + lease = intent_leases.get(worker_id) + now = get_dht_time() plans[worker_id] = ( previous - if previous is not None and previous.decision is not None + if _can_retain_acknowledged_plan( + previous, + proposal.decision, + worker, + identity_key_id, + lease, + now=now, + ) else PlacementPlan( None, - "signed placement intent could not be published to a remote peer", + "signed placement intent could not be published with a live matching lease", proposal.evaluated_models, ) ) continue planner.commit(proposal) + proposal = replace( + proposal, + intent_published=True, + remote_acknowledged=True, + ) plans[worker_id] = proposal registry.replace(plans) settings = _prepare_worker_supervisor_settings( @@ -904,6 +1185,12 @@ def main() -> None: args = parser.parse_args() _validate_args(parser, args) + while _serve_once(args, parser): + logger.info("Activating the authenticated catalog update after all active generations finished") + + +def _serve_once(args, parser) -> bool: + try: persisted_config, configured = _load_persisted_and_runtime_config(args) peer_cache = PeerCache(args.data_dir / "discovery-peers.json") @@ -916,6 +1203,12 @@ def main() -> None: peer_cache_scopes=peer_cache_scopes, replay_history_dir=args.data_dir / "replay-history", ) + catalog = load_configured_catalog(config) + if catalog is not None: + manager.set_catalog_models(model.manifest_digest for model in catalog.models if model.execution != "local") + # Product chat uses authenticated text peers. It must not download local + # input/output weights or wait for a local synthetic generation probe. + # Complete end-to-end availability is checked by the manager/discovery. placement_registry = PlacementRegistry() route_outcomes = RouteOutcomeTracker() worker_supervisor = _build_worker_supervisor( @@ -993,6 +1286,9 @@ def main() -> None: # Arm this before a lazy request can create the model client's p2pd child. tie_child_processes_to_this_process() + from drift.node.hardware_status import HardwareStatus + + hardware_status = HardwareStatus(config) app = create_node_app( manager, api_key_store=key_store, @@ -1005,6 +1301,7 @@ def main() -> None: contribution_policy=config.contribution_policy, contribution_policy_store=policy_store, route_outcome_observer=route_outcomes.record, + hardware_status=hardware_status.snapshot, ) model_names = ", ".join(repr(descriptor.model_id) for descriptor in descriptors) logger.info( @@ -1014,13 +1311,28 @@ def main() -> None: worker_supervisor.start_service() if placement_service is not None: placement_service.start() + server = uvicorn.Server(uvicorn.Config(app, host=args.host, port=args.port, log_level="info")) + restart_requested = False + + def restart(): + nonlocal restart_requested + restart_requested = True + server.should_exit = True + + refresh_service = None + if config.catalog_path is not None and args.config is not None: + refresh_service = CatalogRefreshService(config, args.config, args.data_dir, manager, restart) + refresh_service.start() try: - uvicorn.run(app, host=args.host, port=args.port, log_level="info") + server.run() finally: + if refresh_service is not None: + refresh_service.close() if placement_service is not None: placement_service.close() worker_supervisor.shutdown() manager.shutdown() + return restart_requested if __name__ == "__main__": diff --git a/src/drift/cli/run_server.py b/src/drift/cli/run_server.py index cb36d4f33..af1eb5efa 100644 --- a/src/drift/cli/run_server.py +++ b/src/drift/cli/run_server.py @@ -3,6 +3,8 @@ import logging import os import signal +import sys +from pathlib import Path from typing import Callable, Optional import configargparse @@ -31,11 +33,26 @@ from drift.server.server import Server from drift.utils.convert_block import QuantType from drift.utils.process_lifetime import tie_child_processes_to_this_process +from drift.utils.resource_limits import DEVICE_MEMORY_BUDGET_EXIT_CODE, DeviceMemoryBudgetError from drift.utils.server_registry import register_server, unregister_server from drift.utils.version import log_version logger = get_logger(__name__) +_BOUND_WORKER_CLAIM_FLAGS = ( + "--expected_manifest_digest", + "--expected_block_indices", + "--expected_artifact_bytes", + "--expected_artifact_set_digest", + "--expected_cache_root", +) + + +def _uses_bound_worker_parser(argv) -> bool: + return any( + value == option or value.startswith(f"{option}=") for value in argv for option in _BOUND_WORKER_CLAIM_FLAGS + ) + def _install_graceful_sigterm() -> None: """Make ``drift down`` (SIGTERM) shut down as cleanly as Ctrl+C (SIGINT). @@ -74,11 +91,13 @@ def serve(server: Server, *, model: Optional[str], on_ready: Optional[Callable[[ server.shutdown() -def build_parser() -> configargparse.ArgParser: +def build_parser(*, bound_worker: bool = False) -> configargparse.ArgParser: # fmt:off - parser = configargparse.ArgParser(default_config_files=["config.yml"], + parser = configargparse.ArgParser(default_config_files=[] if bound_worker else ["config.yml"], formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add('-c', '--config', required=False, is_config_file=True, help='config file path') + parser.set_defaults(_bound_worker_parser=bound_worker) + if not bound_worker: + parser.add('-c', '--config', required=False, is_config_file=True, help='config file path') group = parser.add_mutually_exclusive_group(required=False) group.add_argument('--converted_model_name_or_path', type=str, default=None, @@ -94,6 +113,11 @@ def build_parser() -> configargparse.ArgParser: parser.add_argument('--num_blocks', type=int, default=None, help="The number of blocks to serve") parser.add_argument('--block_indices', type=str, default=None, help="Specific block indices to serve") + parser.add_argument('--expected_manifest_digest', type=str, default=None, help=argparse.SUPPRESS) + parser.add_argument('--expected_block_indices', type=str, default=None, help=argparse.SUPPRESS) + parser.add_argument('--expected_artifact_bytes', type=int, default=None, help=argparse.SUPPRESS) + parser.add_argument('--expected_artifact_set_digest', type=str, default=None, help=argparse.SUPPRESS) + parser.add_argument('--expected_cache_root', type=str, default=None, help=argparse.SUPPRESS) parser.add_argument('--dht_prefix', type=str, default=None, help="Announce all blocks with this DHT prefix") parser.add_argument('--port', type=int, required=False, @@ -191,9 +215,13 @@ def build_parser() -> configargparse.ArgParser: "for a long time and caches all model blocks after a number of rebalancings. " "However, this worst case is unlikely, expect the server to consume " "the disk space equal to 2-4x of your GPU memory on average.") - parser.add_argument("--max_device_memory", type=str, default=None, + parser.add_argument("--max_device_memory", type=str, default=None, help="Hard per-accelerator memory ceiling. Example: 8GiB. " - "The node resolves percentage contribution limits to bytes before launch.") + "The node resolves percentage contribution limits to bytes before launch.") + parser.add_argument("--max_processing_percent", type=float, default=100, + help="Contribution compute time percentage (1-100); paced between synchronized steps.") + parser.add_argument("--processing_budget_path", type=str, default=None, + help="Local shared processing-budget lock identity, supplied by the node.") parser.add_argument('--device', type=str, default=None, required=False, help='all blocks will use this device in torch notation; ' @@ -284,7 +312,42 @@ def server_from_args(args: dict) -> Server: # Arm this before anything can spawn a p2pd, so a hard-killed server does not orphan its daemons tie_child_processes_to_this_process() + bound_worker_parser = args.pop("_bound_worker_parser", False) requested_model = args.pop("model") or args["converted_model_name_or_path"] + artifact_claims = ( + args.get("expected_manifest_digest"), + args.get("expected_block_indices"), + args.get("expected_artifact_bytes"), + args.get("expected_artifact_set_digest"), + args.get("expected_cache_root"), + ) + if any(value is not None for value in artifact_claims): + if bound_worker_parser is not True: + raise ManifestError("Worker artifact-plan claims require the source-bound internal parser") + if any(value is None for value in artifact_claims): + raise ManifestError("Worker manifest, span, cache, and artifact-plan claims must be supplied together") + if ( + args.get("model_manifest") is None + or args.get("block_indices") is None + or args.get("num_blocks") is not None + ): + raise ManifestError("Worker artifact-plan claims require --model_manifest and explicit --block_indices") + if args["block_indices"] != args["expected_block_indices"]: + raise ManifestError("Worker block span does not match the acknowledged placement decision") + cache_dir = args.get("cache_dir") + if not isinstance(cache_dir, str) or not cache_dir: + raise ManifestError("Worker artifact-plan claims require an explicit canonical --cache_dir") + canonical_cache_root = str(Path(cache_dir).expanduser().resolve()) + if cache_dir != canonical_cache_root: + raise ManifestError("Worker artifact-plan claims require a canonical absolute --cache_dir") + if args["expected_cache_root"] != canonical_cache_root: + raise ManifestError("Worker cache root does not match the acknowledged placement decision") + if ( + args.get("custom_module_path") is not None + or args.get("allow_training_rpcs") is not False + or args.get("token") is not None + ): + raise ManifestError("Placement-bound workers forbid custom modules, training RPCs, and credential flags") admission_values = { "max_active_sessions": args.pop("admission_max_active_sessions"), "max_active_sessions_per_peer": args.pop("admission_max_active_sessions_per_peer"), @@ -301,6 +364,8 @@ def server_from_args(args: dict) -> Server: if manifest_path is not None: manifest = ModelManifest.load(manifest_path) manifest.validate_runtime(drift.__version__) + if args.get("expected_manifest_digest") is not None and args["expected_manifest_digest"] != manifest.digest_id: + raise ManifestError("Worker manifest digest does not match the acknowledged placement decision") if requested_model is None: requested_model = manifest.source.repository args["converted_model_name_or_path"] = requested_model @@ -424,12 +489,14 @@ def server_from_args(args: dict) -> Server: def main(): - parser = build_parser() + parser = build_parser(bound_worker=_uses_bound_worker_parser(sys.argv[1:])) args = vars(parser.parse_args()) args.pop("config", None) try: server = server_from_args(args) + except DeviceMemoryBudgetError as exc: + parser.exit(DEVICE_MEMORY_BUDGET_EXIT_CODE, f"{parser.prog}: {exc}\n") except (ManifestError, ValueError) as exc: parser.error(str(exc)) serve(server, model=server.converted_model_name_or_path) diff --git a/src/drift/cli/run_text_peer.py b/src/drift/cli/run_text_peer.py new file mode 100644 index 000000000..d01d25318 --- /dev/null +++ b/src/drift/cli/run_text_peer.py @@ -0,0 +1,64 @@ +"""Run an optional peer that owns input/output processing for text-only consumers.""" + +import argparse +import signal +import time +from pathlib import Path + +from hivemind import DHT + +from drift.model_manifest import ModelManifest +from drift.protocol_identity import NodeIdentity +from drift.server.text_peer import TextPeerService + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("--initial_peers", nargs="+", required=True) + parser.add_argument("--identity_path", type=Path, required=True) + parser.add_argument("--cache_dir", type=Path, required=True) + parser.add_argument("--host_maddrs", nargs="+", default=["/ip4/0.0.0.0/tcp/31337"]) + parser.add_argument("--announce_maddrs", nargs="+") + parser.add_argument("--max_context_tokens", type=int, default=2048) + parser.add_argument("--max_output_tokens", type=int, default=512) + args = parser.parse_args() + manifest = ModelManifest.load(args.manifest) + identity = NodeIdentity.ensure(args.identity_path) + dht = DHT( + initial_peers=args.initial_peers, + identity_path=str(args.identity_path), + host_maddrs=args.host_maddrs, + announce_maddrs=args.announce_maddrs, + tls=True, + start=True, + ) + service = TextPeerService( + dht, + identity, + manifest, + initial_peers=args.initial_peers, + cache_dir=str(args.cache_dir), + max_context_tokens=args.max_context_tokens, + max_output_tokens=args.max_output_tokens, + ).start() + stopped = False + + def stop(_signum, _frame): + nonlocal stopped + stopped = True + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + try: + while not stopped: + if service.error: + raise RuntimeError("Text peer startup failed; check the log") + time.sleep(1) + finally: + service.close() + dht.shutdown() + + +if __name__ == "__main__": + main() diff --git a/src/drift/client/lm_head.py b/src/drift/client/lm_head.py index efbac43e5..5b2b55363 100644 --- a/src/drift/client/lm_head.py +++ b/src/drift/client/lm_head.py @@ -1,5 +1,4 @@ import dataclasses -import platform from typing import Union import torch @@ -15,7 +14,7 @@ @dataclasses.dataclass class LMHeadConfig: # This settings matter for running the client with dtype bfloat16 on CPU. - # If the CPU doesn't support AVX512, chunked_forward() significantly speeds up computations. + # If the CPU lacks native BF16 arithmetic, chunked_forward() speeds up computations. use_chunked_forward: Union[str, bool] = "auto" chunked_forward_step: int = 16384 @@ -32,15 +31,12 @@ def __init__(self, config: PretrainedConfig): self.use_chunked_forward = config.use_chunked_forward if self.use_chunked_forward == "auto": - if platform.machine() == "x86_64": - # Import of cpufeature may crash on non-x86_64 machines - from cpufeature import CPUFeature - - # If the CPU supports AVX512, plain bfloat16 is ~10x faster than chunked_forward(). - # Otherwise, it's ~8x slower. - self.use_chunked_forward = not (CPUFeature["AVX512f"] and CPUFeature["OS_AVX512"]) - else: - self.use_chunked_forward = True + # cpufeature's native import can raise SIGFPE in virtualized hosts, + # killing the entire peer before Python can handle the error. + # Use PyTorch's capability probe and require actual BF16 support; + # AVX512 alone does not imply fast BF16 arithmetic. + supports_bf16 = getattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False) + self.use_chunked_forward = not supports_bf16() self.chunked_forward_step = config.chunked_forward_step self._chunked_warning_shown = False diff --git a/src/drift/client/routing/sequence_manager.py b/src/drift/client/routing/sequence_manager.py index 3643fb498..d256017c3 100644 --- a/src/drift/client/routing/sequence_manager.py +++ b/src/drift/client/routing/sequence_manager.py @@ -346,7 +346,13 @@ def __getitem__(self, ix: Union[int, slice]) -> RemoteSequenceManager: ix = slice(int(ix), int(ix) + 1, 1) return type(self)(self.config, self.block_uids[ix], dht=self.dht, state=self.state[ix]) - def update(self, *, wait: bool): + def start_discovery(self): + """Observe route availability before the first inference request.""" + with self._thread_start_lock: + if not self.is_alive(): + self._thread.start() + + def update(self, *, wait: bool): """Run an asynchronous update in background as soon as possible""" self.ready.clear() self._thread.trigger.set() diff --git a/src/drift/model_catalog.py b/src/drift/model_catalog.py index 87e0e2c25..a52918aed 100644 --- a/src/drift/model_catalog.py +++ b/src/drift/model_catalog.py @@ -383,7 +383,7 @@ def from_dict(cls, source: Mapping[str, Any], *, index: int) -> "CatalogRung": raise ModelCatalogError(f"{name}.id must match {_RUNG_ID_RE.pattern}") minimum_replicas = _require_int(source["minimum_replicas"], f"{name}.minimum_replicas", minimum=1) minimum_surviving = _require_int( - source["minimum_surviving_replicas"], f"{name}.minimum_surviving_replicas", minimum=1 + source["minimum_surviving_replicas"], f"{name}.minimum_surviving_replicas", minimum=0 ) if minimum_surviving > minimum_replicas: raise ModelCatalogError(f"{name}.minimum_surviving_replicas cannot exceed minimum_replicas") @@ -432,6 +432,7 @@ class CatalogModel: total_parameters: int active_parameters: int weight_bytes: int + execution: Optional[str] = None @classmethod def from_dict(cls, source: Mapping[str, Any], *, index: int) -> "CatalogModel": @@ -446,7 +447,10 @@ def from_dict(cls, source: Mapping[str, Any], *, index: int) -> "CatalogModel": "active_parameters", "weight_bytes", ) - _strict_fields(source, name, required=fields) + _strict_fields(source, name, required=fields, optional=("execution",)) + execution = source.get("execution") + if "execution" in source and execution not in ("local", "distributed"): + raise ModelCatalogError(f"{name}.execution must be local or distributed") urls_value = source["manifest_urls"] if not isinstance(urls_value, list) or not urls_value: raise ModelCatalogError(f"{name}.manifest_urls must be a non-empty array") @@ -468,10 +472,11 @@ def from_dict(cls, source: Mapping[str, Any], *, index: int) -> "CatalogModel": total_parameters=total_parameters, active_parameters=active_parameters, weight_bytes=_require_int(source["weight_bytes"], f"{name}.weight_bytes", minimum=1), + execution=execution, ) def to_dict(self) -> Dict[str, Any]: - return { + result = { "manifest_digest": self.manifest_digest, "manifest_urls": list(self.manifest_urls), "rung": self.rung_id, @@ -480,6 +485,9 @@ def to_dict(self) -> Dict[str, Any]: "active_parameters": self.active_parameters, "weight_bytes": self.weight_bytes, } + if self.execution is not None: + result["execution"] = self.execution + return result @dataclass(frozen=True) @@ -525,8 +533,8 @@ def from_dict(cls, source: Mapping[str, Any]) -> "ModelCatalog": raise ModelCatalogError("every catalog model must reference a declared rung") for rung_id in rung_ids: rung_models = [model for model in models if model.rung_id == rung_id] - if len(rung_models) < 2: - raise ModelCatalogError(f"catalog rung {rung_id!r} must approve at least two model options") + if not rung_models: + raise ModelCatalogError(f"catalog rung {rung_id!r} must approve at least one model") primary_count = sum(model.role == "primary" for model in rung_models) if primary_count != 1: raise ModelCatalogError(f"catalog rung {rung_id!r} must declare exactly one primary model") @@ -838,7 +846,7 @@ def select_highest_eligible_model( *, now: Optional[float] = None, ) -> Tuple[Optional[CatalogModel], Tuple[ModelEligibility, ...]]: - """Select the highest safe rung, preferring its primary over its standby. + """Select the highest safe rung, preferring its primary over any optional standby. This only selects a manifest for a *new* request. It does not mutate aliases, stop workers, or move an in-flight request between manifests. diff --git a/src/drift/model_manifest.py b/src/drift/model_manifest.py index 293458d55..c3c624d41 100644 --- a/src/drift/model_manifest.py +++ b/src/drift/model_manifest.py @@ -16,6 +16,7 @@ import unicodedata from dataclasses import dataclass, field from pathlib import Path, PurePosixPath +from types import MappingProxyType from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, Set, Tuple, Union from packaging.version import InvalidVersion, Version @@ -36,7 +37,7 @@ "weight_index", } _DTYPES = {"bfloat16", "float16", "float32"} -_QUANTIZATIONS = {"int8", "nf4", "none"} +_QUANTIZATIONS = {"fp8_dequant", "int8", "nf4", "none"} _ATTENTION_IMPLEMENTATIONS = {"auto", "eager", "sdpa"} _CHECKPOINT_ROLES = {"converted_weight", "quantized_weight", "weight"} _TOKENIZER_FILENAMES = { @@ -53,6 +54,7 @@ } _CHECKPOINT_INDEX_PREFERENCE = ("model.safetensors.index.json", "pytorch_model.bin.index.json") _CHECKPOINT_PREFERENCE = ("model.safetensors", "pytorch_model.bin") +_MAX_CHECKPOINT_INDEX_BYTES = 16 * 1024 * 1024 class ManifestError(ValueError): @@ -331,6 +333,8 @@ def from_dict(cls, source: Mapping[str, Any]) -> "ModelManifest": paths = [artifact.path for artifact in artifacts] if len(set(paths)) != len(paths): raise ManifestError("artifacts must not contain duplicate paths") + if len({path.casefold() for path in paths}) != len(paths): + raise ManifestError("artifacts must not contain paths that collide case-insensitively") roles = {artifact.role for artifact in artifacts} missing_roles = {"config", "tokenizer"} - roles if not ({"weight", "converted_weight", "quantized_weight"} & roles): @@ -453,6 +457,17 @@ def validate_model_config(self, config: Any) -> None: raise ManifestError( f"Manifest declares context length {self.model.context_length} but config declares {context_length!r}" ) + source_quantization = getattr(config, "_source_quantization_method", None) + compatible_source_profile = (source_quantization, self.runtime.quantization) in { + ("fp8", "fp8_dequant"), + } + if self.runtime.quantization == "fp8_dequant" and not compatible_source_profile: + raise ManifestError("Manifest runtime profile 'fp8_dequant' requires source config quant_method='fp8'") + if source_quantization is not None and not compatible_source_profile: + raise ManifestError( + f"Source config declares pre-quantized {source_quantization!r} weights, but the manifest runtime " + f"profile declares {self.runtime.quantization!r}; this checkpoint needs an explicit compatible profile" + ) def resolve_manifest_loading( @@ -486,6 +501,26 @@ def _artifact_path_below_root(root: Path | str, relative_path: str) -> Path: return candidate +def _windows_safe_path(path: Path) -> Path: + """Opt long manifest-cache paths into the Win32 extended namespace. + + Full manifest and artifact SHA-256 identifiers make resumable lock and + partial paths exceed the legacy Win32 path limit under an ordinary user + profile. Python then reports a misleading ``FileNotFoundError`` even when + the parent directory exists. Keep the audited on-disk layout unchanged, + but use an extended-length spelling for filesystem operations. + """ + absolute = path.absolute() + if os.name != "nt": + return absolute + rendered = str(absolute) + if rendered.startswith("\\\\?\\") or len(rendered) < 248: + return absolute + if rendered.startswith("\\\\"): + return Path("\\\\?\\UNC\\" + rendered[2:]) + return Path("\\\\?\\" + rendered) + + def _validate_artifact_file(artifact: ManifestArtifact, candidate: Path) -> os.stat_result: try: stat_result = candidate.stat() @@ -512,6 +547,113 @@ def _verify_artifact_file(artifact: ManifestArtifact, candidate: Path) -> os.sta return stat_result +@dataclass(frozen=True) +class ManifestBlockArtifactPlan: + """Exact manifested artifacts required to serve one contiguous block span.""" + + start_block: int + end_block: int + artifacts: Tuple[ManifestArtifact, ...] + + def __post_init__(self) -> None: + if self.start_block < 0 or self.end_block <= self.start_block: + raise ManifestError("Block artifact plan must select a non-empty non-negative range") + if not self.artifacts: + raise ManifestError("Block artifact plan must contain at least one artifact") + paths = tuple(artifact.path for artifact in self.artifacts) + if paths != tuple(sorted(paths)) or len(set(paths)) != len(paths): + raise ManifestError("Block artifact plan paths must be unique and sorted") + + @property + def artifact_bytes(self) -> int: + return sum(artifact.size for artifact in self.artifacts) + + @property + def artifact_paths(self) -> Tuple[str, ...]: + return tuple(artifact.path for artifact in self.artifacts) + + @property + def artifact_set_digest(self) -> str: + payload = json.dumps( + [artifact.to_dict() for artifact in self.artifacts], + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def select_manifest_block_artifacts( + manifest: ModelManifest, + *, + block_prefix: str, + start_block: int, + end_block: int, + weight_map: Optional[Mapping[str, str]] = None, +) -> ManifestBlockArtifactPlan: + """Select metadata and deduplicated checkpoint shards for an exact block span.""" + + if not isinstance(block_prefix, str) or not block_prefix or block_prefix.endswith("."): + raise ManifestError("Block prefix must be a non-empty dotted name without a trailing separator") + if ( + isinstance(start_block, bool) + or isinstance(end_block, bool) + or not isinstance(start_block, int) + or not isinstance(end_block, int) + or start_block < 0 + or end_block <= start_block + or end_block > manifest.model.num_blocks + ): + raise ManifestError(f"Block span must be within 0:{manifest.model.num_blocks}") + + index_artifacts = manifest.artifacts_for_roles({"weight_index"}) + if len(index_artifacts) > 1: + raise ManifestError("Manifest declares more than one checkpoint index") + selected = set(manifest.artifacts_for_roles({"config", "weight_index"})) + if index_artifacts: + if not isinstance(weight_map, Mapping) or not weight_map: + raise ManifestError("Manifested sharded checkpoint requires a non-empty weight_map") + prefixes = tuple(f"{block_prefix}.{index}." for index in range(start_block, end_block)) + matched_prefixes = set() + for parameter_name, shard_path in weight_map.items(): + if not isinstance(parameter_name, str) or not parameter_name: + raise ManifestError("Checkpoint index contains an invalid parameter name") + if not isinstance(shard_path, str) or not shard_path: + raise ManifestError("Checkpoint index contains an invalid shard path") + parsed_path = PurePosixPath(shard_path) + if ( + parsed_path.is_absolute() + or parsed_path == PurePosixPath(".") + or "\\" in shard_path + or shard_path != parsed_path.as_posix() + or ".." in parsed_path.parts + ): + raise ManifestError(f"Checkpoint index contains a non-normalized shard path {shard_path!r}") + artifact = manifest.get_artifact(shard_path) + if artifact.role not in _CHECKPOINT_ROLES: + raise ManifestError(f"Checkpoint index shard {shard_path!r} has non-checkpoint role {artifact.role!r}") + for prefix in prefixes: + if parameter_name.startswith(prefix): + selected.add(artifact) + matched_prefixes.add(prefix) + missing = sorted(set(prefixes) - matched_prefixes) + if missing: + raise ManifestError(f"Checkpoint index contains no parameters for block prefix(es) {missing}") + else: + if weight_map is not None: + raise ManifestError("Unsharded manifested checkpoint must not provide a weight_map") + checkpoints = {artifact.path: artifact for artifact in manifest.artifacts_for_roles(_CHECKPOINT_ROLES)} + checkpoint_path = next((path for path in _CHECKPOINT_PREFERENCE if path in checkpoints), None) + if checkpoint_path is None: + raise ManifestError("Manifest does not declare a checkpoint format supported by the block loader") + selected.add(checkpoints[checkpoint_path]) + + if not any(artifact.role == "config" for artifact in selected): + raise ManifestError("Manifest has no configuration artifact") + return ManifestBlockArtifactPlan(start_block, end_block, tuple(sorted(selected, key=lambda item: item.path))) + + @dataclass class ManifestArtifactVerifier: """Download declared artifacts at the pinned revision and verify them before use. @@ -529,10 +671,17 @@ class ManifestArtifactVerifier: cache_dir: Optional[Union[str, os.PathLike]] = None max_disk_space: Optional[int] = None artifact_root: Optional[Union[str, os.PathLike]] = None + allowed_paths: Optional[Iterable[str]] = None _verified: Dict[Tuple[str, int, int, str], bool] = field(default_factory=dict, init=False, repr=False) _snapshot_root: Optional[Path] = field(default=None, init=False, repr=False) + _weight_map: Optional[Mapping[str, str]] = field(default=None, init=False, repr=False) + _weight_map_loaded: bool = field(default=False, init=False, repr=False) + _progress: Any = field(default=None, init=False, repr=False) def __post_init__(self) -> None: + from drift.utils.download_progress import current_progress + + self._progress = current_progress() if self.repository != self.manifest.source.repository: raise ManifestError( f"Artifact verifier repository is {self.repository!r}, expected {self.manifest.source.repository!r}" @@ -541,6 +690,8 @@ def __post_init__(self) -> None: raise ManifestError( f"Artifact verifier revision is {self.revision!r}, expected {self.manifest.source.revision!r}" ) + if self.allowed_paths is not None: + self.allowed_paths = self._normalize_allowed_paths(self.allowed_paths) if self.artifact_root is not None: self._snapshot_root = Path(self.artifact_root).absolute() elif self.cache_dir is None: @@ -548,12 +699,113 @@ def __post_init__(self) -> None: self.cache_dir = DEFAULT_CACHE_DIR + def _normalize_allowed_paths(self, paths: Iterable[str]) -> frozenset[str]: + normalized = set() + for path in paths: + if not isinstance(path, str) or not path: + raise ManifestError("Artifact allowlist paths must be non-empty strings") + artifact = self.manifest.get_artifact(path) + if path != artifact.path: + raise ManifestError(f"Artifact allowlist path {path!r} is not normalized") + normalized.add(path) + if not normalized: + raise ManifestError("Artifact allowlist must not be empty") + return frozenset(normalized) + + def restrict_to_paths(self, paths: Iterable[str]) -> None: + """Narrow this verifier to an exact manifested artifact set.""" + normalized = self._normalize_allowed_paths(paths) + if self.allowed_paths is not None and not normalized.issubset(self.allowed_paths): + raise ManifestError("Artifact verifier allowlist cannot be widened") + self.allowed_paths = normalized + + def _require_allowed(self, artifact: ManifestArtifact) -> None: + if self.allowed_paths is not None and artifact.path not in self.allowed_paths: + raise ManifestError(f"Artifact {artifact.path!r} is outside this worker artifact plan") + @property def snapshot_root(self) -> Path: if self._snapshot_root is None: raise ManifestError("No manifest artifacts have been materialized yet") return self._snapshot_root + def load_weight_map(self) -> Optional[Mapping[str, str]]: + """Return one immutable parse of the exact verified checkpoint index.""" + indices = self.manifest.artifacts_for_roles({"weight_index"}) + if len(indices) > 1: + raise ManifestError("Manifest declares more than one checkpoint index") + if not indices: + self._weight_map_loaded = True + self._weight_map = None + return None + artifact = indices[0] + self._require_allowed(artifact) + if self._weight_map_loaded: + return self._weight_map + if artifact.size > _MAX_CHECKPOINT_INDEX_BYTES: + raise ManifestError( + f"Checkpoint index {artifact.path} exceeds the {_MAX_CHECKPOINT_INDEX_BYTES}-byte planning limit" + ) + path = self.ensure_path(artifact.path, allowed_roles={"weight_index"}) + try: + payload = path.read_bytes() + if len(payload) != artifact.size or hashlib.sha256(payload).hexdigest() != artifact.sha256: + raise ManifestError(f"Checkpoint index {artifact.path} changed while being planned") + + def reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ManifestError(f"Checkpoint index contains duplicate object key {key!r}") + result[key] = value + return result + + def reject_non_finite(value): + raise ManifestError(f"Checkpoint index contains non-finite number {value}") + + document = json.loads( + payload.decode("utf-8"), + object_pairs_hook=reject_duplicate_keys, + parse_constant=reject_non_finite, + ) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ManifestError(f"Could not parse checkpoint index {artifact.path}: {exc}") from exc + if not isinstance(document, dict) or not isinstance(document.get("weight_map"), dict): + raise ManifestError(f"Checkpoint index {artifact.path} has no weight_map object") + weight_map = document["weight_map"] + if not weight_map: + raise ManifestError(f"Checkpoint index {artifact.path} has an empty weight_map") + self._weight_map = MappingProxyType(dict(weight_map)) + self._weight_map_loaded = True + return self._weight_map + + def plan_block_artifacts(self, *, block_prefix: str, start_block: int, end_block: int) -> ManifestBlockArtifactPlan: + self.ensure_startup_metadata() + return select_manifest_block_artifacts( + self.manifest, + block_prefix=block_prefix, + start_block=start_block, + end_block=end_block, + weight_map=self.load_weight_map(), + ) + + def bind_block_artifact_plan( + self, *, block_prefix: str, start_block: int, end_block: int + ) -> ManifestBlockArtifactPlan: + """Expand a metadata-only scope to the exact plan derived from that metadata.""" + if self.allowed_paths is None: + raise ManifestError("Block artifact binding requires an exact startup-metadata allowlist") + artifact_plan = self.plan_block_artifacts( + block_prefix=block_prefix, + start_block=start_block, + end_block=end_block, + ) + planned_paths = frozenset(artifact_plan.artifact_paths) + if not self.allowed_paths.issubset(planned_paths): + raise ManifestError("Block artifact plan omitted required startup metadata") + self.allowed_paths = planned_paths + return artifact_plan + def ensure_startup_metadata(self, *, include_tokenizer: bool = False) -> Path: roles: Set[str] = {"config", "weight_index"} if include_tokenizer: @@ -567,6 +819,23 @@ def ensure_startup_metadata(self, *, include_tokenizer: bool = False) -> Path: def ensure_path(self, path: str, *, allowed_roles: Optional[Iterable[str]] = None) -> Path: artifact = self.manifest.get_artifact(path) + self._require_allowed(artifact) + self._report(artifact, "checking") + try: + candidate = self._ensure_path(path, allowed_roles=allowed_roles) + except Exception: + self._report(artifact, "failed") + raise + self._report(artifact, "verified") + return candidate + + def _report(self, artifact, state, **values): + if self._progress is not None: + self._progress.event(self.manifest, artifact, state, **values) + + def _ensure_path(self, path: str, *, allowed_roles: Optional[Iterable[str]] = None) -> Path: + artifact = self.manifest.get_artifact(path) + self._require_allowed(artifact) if allowed_roles is not None and artifact.role not in set(allowed_roles): raise ManifestError( f"Artifact {artifact.path!r} has role {artifact.role!r}, expected one of {sorted(set(allowed_roles))}" @@ -598,7 +867,7 @@ def ensure_path(self, path: str, *, allowed_roles: Optional[Iterable[str]] = Non Path(self.cache_dir).mkdir(parents=True, exist_ok=True) with allow_cache_writes(self.cache_dir): free_disk_space_for( - artifact.size, + max(0, artifact.size - self.partial_size(artifact.path)), cache_dir=self.cache_dir, max_disk_space=self.max_disk_space, ) @@ -623,6 +892,7 @@ def ensure_path(self, path: str, *, allowed_roles: Optional[Iterable[str]] = Non if resolved_candidate != candidate: self._promote_cached_artifact(artifact, resolved_candidate, candidate) + self._report(artifact, "verifying", received=artifact.size) self._verify(artifact, candidate) return candidate @@ -657,6 +927,7 @@ def _promote_cached_artifact(self, artifact: ManifestArtifact, source: Path, des def partial_size(self, path: str) -> int: """Return resumable bytes retained for one declared artifact, without exposing its local path.""" artifact = self.manifest.get_artifact(path) + self._require_allowed(artifact) if self.cache_dir is None: return 0 partial, _, _ = self._resumable_paths(artifact) @@ -670,9 +941,9 @@ def _resumable_paths(self, artifact: ManifestArtifact) -> Tuple[Path, Path, Path cache_root = Path(self.cache_dir).absolute() manifest_root = cache_root / "manifest-artifacts" / self.manifest.digest name_digest = hashlib.sha256(artifact.path.encode("utf-8")).hexdigest() - partial = manifest_root / "partial" / f"{name_digest}.part" - final = _artifact_path_below_root(manifest_root / "snapshot", artifact.path) - lock = manifest_root / "locks" / f"{name_digest}.lock" + partial = _windows_safe_path(manifest_root / "partial" / f"{name_digest}.part") + final = _windows_safe_path(_artifact_path_below_root(manifest_root / "snapshot", artifact.path)) + lock = _windows_safe_path(manifest_root / "locks" / f"{name_digest}.lock") return partial, final, lock def _resumable_hub_download(self, artifact: ManifestArtifact, *, destination: Optional[Path] = None) -> str: @@ -702,6 +973,7 @@ def _resumable_hub_download(self, artifact: ManifestArtifact, *, destination: Op if partial.exists() and partial.stat().st_size > artifact.size: partial.unlink() offset = partial.stat().st_size if partial.exists() else 0 + self._report(artifact, "downloading", received=offset, resumed=offset) if offset == artifact.size: try: _verify_artifact_file(artifact, partial) @@ -714,6 +986,30 @@ def _resumable_hub_download(self, artifact: ManifestArtifact, *, destination: Op url = hf_hub_url(self.repository, artifact.path, revision=self.revision) headers = build_hf_headers(token=self.token, library_name="drift", library_version="2") + from drift.utils.hub_ranges import RANGE_BYTES, download_ranges + + if artifact.size > RANGE_BYTES: + try: + download_ranges( + url, + headers, + partial, + size=artifact.size, + offset=offset, + progress=lambda state, **values: self._report(artifact, state, **values), + ) + self._report(artifact, "verifying", received=artifact.size) + _verify_artifact_file(artifact, partial) + except (OSError, requests.RequestException) as exc: + raise ManifestTransferInterrupted( + f"Interrupted download of {artifact.path} at byte {offset}: {type(exc).__name__}" + ) from exc + except ManifestError: + if partial.exists() and partial.stat().st_size >= artifact.size: + partial.unlink() + raise + _replace_verified_artifact(partial, final) + return str(final) if offset: headers["Range"] = f"bytes={offset}-" @@ -733,9 +1029,12 @@ def _resumable_hub_download(self, artifact: ManifestArtifact, *, destination: Op mode = "wb" with partial.open(mode) as stream: + received = offset for chunk in response.iter_content(chunk_size=1024 * 1024): if chunk: stream.write(chunk) + received += len(chunk) + self._report(artifact, "downloading", received=received, transferred=len(chunk)) stream.flush() os.fsync(stream.fileno()) except (OSError, requests.RequestException) as exc: @@ -776,6 +1075,7 @@ def verify_resolved_file( f"Resolved checkpoint file {candidate} does not map uniquely to a declared manifest artifact" ) artifact = matches[0] + self._require_allowed(artifact) if allowed_roles is not None and artifact.role not in set(allowed_roles): raise ManifestError( f"Resolved file {artifact.path!r} has role {artifact.role!r}, expected one of " diff --git a/src/drift/models/qwen3_5/block.py b/src/drift/models/qwen3_5/block.py index 745e9bc27..ee9ebaab7 100644 --- a/src/drift/models/qwen3_5/block.py +++ b/src/drift/models/qwen3_5/block.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple import torch -from transformers.cache_utils import DynamicCache +from transformers.cache_utils import Cache, DynamicCache from transformers.masking_utils import create_causal_mask, create_recurrent_attention_mask from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5TextRotaryEmbedding @@ -55,11 +55,19 @@ def forward( position_embeddings = self.rotary_emb(hidden_states, text_position_ids) if self.block_type == "full_attention": + # This worker owns one layer's cache, not the complete HF model cache. + # Mask helpers select the first full-attention layer by default, which + # is empty when this is a later layer (e.g. block 7 instead of 3). + # Give them a view of this layer so both prefix offset and KV length + # remain correct across multi-token prefill chunks. + mask_cache = ( + Cache(layers=[past_key_values.layers[self.global_layer_idx]]) if past_key_values is not None else None + ) causal_mask = create_causal_mask( self.config, hidden_states, attention_mask, - past_key_values, + mask_cache, position_ids=text_position_ids, ) else: diff --git a/src/drift/models/qwen3_5/config.py b/src/drift/models/qwen3_5/config.py index 3ff4daec9..23331aff0 100644 --- a/src/drift/models/qwen3_5/config.py +++ b/src/drift/models/qwen3_5/config.py @@ -27,7 +27,7 @@ def _peek_top_level_model_type(model_name_or_path: Union[str, os.PathLike, None] try: config_dict, _ = PretrainedConfig.get_config_dict(model_name_or_path, **peek_kwargs) except Exception as exc: - raise RuntimeError(f"Could not inspect Qwen3.5 checkpoint config at {model_name_or_path}") from exc + raise RuntimeError(f"Could not inspect Qwen3.5/Qwen3.8 checkpoint config at {model_name_or_path}") from exc return config_dict.get("model_type") diff --git a/src/drift/node/catalog_bootstrap.py b/src/drift/node/catalog_bootstrap.py index 97bd41d15..150d80674 100644 --- a/src/drift/node/catalog_bootstrap.py +++ b/src/drift/node/catalog_bootstrap.py @@ -7,11 +7,15 @@ from __future__ import annotations +import hashlib import ipaddress import json import os import re import secrets +import stat +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Dict, Mapping, Optional, Tuple @@ -28,12 +32,35 @@ ) from drift.model_manifest import ManifestError, ModelManifest from drift.node.config import NODE_CONFIG_SCHEMA_VERSION, NodeConfig, NodeConfigError -from drift.node.config_lock import NodeConfigWriteLockError, node_config_write_lock +from drift.node.config_lock import ( + NodeConfigWriteLockError, + _acquire as _acquire_process_lock, + _release as _release_process_lock, + node_config_write_lock, +) CATALOG_BOOTSTRAP_SCHEMA_VERSION = 1 MAX_CATALOG_BYTES = 4 * 1024 * 1024 MAX_MANIFEST_BYTES = 8 * 1024 * 1024 DEFAULT_FETCH_TIMEOUT = (5.0, 20.0) +# The first public alpha predated immutable installed-catalog history. Its two +# managed entries survived the v2 migration even though v2 withdrew them. Keep +# this compatibility record exact: names or directories alone are not evidence +# that an advanced user's model belongs to our retired catalog. +_LEGACY_PUBLIC_ALPHA_CATALOG_ID = "communityai-public-alpha-v1" +_LEGACY_PUBLIC_ALPHA_ROOT = "sha256:9388a51a4c3856256e9db2c53838045e6be34202c72f1c5abd84cd33391a6b31" +_LEGACY_PUBLIC_ALPHA_MODELS = frozenset( + ( + "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + ) +) +_DEFAULT_CONTRIBUTION_POLICY = { + "sharing_enabled": False, + "max_vram": "100%", + "max_processing_percent": 100, + "max_disk_space": "20GiB", +} _DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") _PUBLIC_PEER_RE = re.compile(r"^/(ip4|ip6|dns4|dns6)/([^/]+)/tcp/([1-9][0-9]{0,4})/p2p/([^/]{20,128})$") _SPECIAL_USE_DNS_SUFFIXES = ( @@ -60,6 +87,66 @@ def _absolute_path(path: Path | str) -> Path: return Path(os.path.abspath(os.fspath(Path(path).expanduser()))) +def _unsafe_lock_metadata(metadata: os.stat_result) -> bool: + return stat.S_ISLNK(metadata.st_mode) or bool( + getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + + +@contextmanager +def _catalog_bootstrap_lock(path: Path) -> Iterator[None]: + """Hold a kernel lock, released even if the bootstrap process is killed. + + Keep the sidecar: unlinking it lets concurrent installers lock different + files. An empty marker left by an older interrupted bootstrap is reusable. + """ + descriptor = None + acquired = False + try: + for parent in (path.parent, *path.parent.parents): + if _unsafe_lock_metadata(parent.lstat()): + raise CatalogBootstrapError("Refusing unsafe catalog bootstrap lock directory") + try: + existing = path.lstat() + except FileNotFoundError: + existing = None + if existing is not None and ( + _unsafe_lock_metadata(existing) or not stat.S_ISREG(existing.st_mode) or existing.st_nlink != 1 + ): + raise CatalogBootstrapError("Refusing unsafe catalog bootstrap lock file") + flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags, 0o600) + opened = os.fstat(descriptor) + current = path.lstat() + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or _unsafe_lock_metadata(current) + or not os.path.samestat(opened, current) + ): + raise CatalogBootstrapError("Refusing unsafe catalog bootstrap lock file") + try: + _acquire_process_lock(descriptor) + except OSError as exc: + raise CatalogBootstrapError("Another first-install catalog bootstrap is already in progress") from exc + acquired = True + current = path.lstat() + if _unsafe_lock_metadata(current) or not os.path.samestat(opened, current): + raise CatalogBootstrapError("Catalog bootstrap lock changed while it was acquired") + except OSError as exc: + raise CatalogBootstrapError(f"Could not lock catalog bootstrap in {path.parent}: {exc}") from exc + else: + yield + finally: + if descriptor is not None: + if acquired: + try: + _release_process_lock(descriptor) + except OSError: + pass + os.close(descriptor) + + def _require_mapping(value: Any, field: str) -> Mapping[str, Any]: if not isinstance(value, dict): raise CatalogBootstrapError(f"{field} must be a JSON object") @@ -251,6 +338,17 @@ class CatalogBootstrapConfig: catalog_mirrors: Tuple[str, ...] initial_peers: Tuple[str, ...] max_loaded_models: int = 1 + replaces_trust_roots: Tuple[str, ...] = () + + @property + def trust_root_digest(self) -> str: + rendered = json.dumps(self.trust_root.to_dict(), sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(rendered.encode("utf-8")).hexdigest() + + def permits_replacement_of(self, previous: "CatalogBootstrapConfig") -> bool: + return self.trust_root.catalog_id == previous.trust_root.catalog_id and ( + self.trust_root == previous.trust_root or previous.trust_root_digest in self.replaces_trust_roots + ) @classmethod def from_dict(cls, source: Mapping[str, Any]) -> "CatalogBootstrapConfig": @@ -259,7 +357,7 @@ def from_dict(cls, source: Mapping[str, Any]) -> "CatalogBootstrapConfig": source, "catalog bootstrap config", required=("schema_version", "trust_root", "catalog_mirrors", "initial_peers"), - optional=("max_loaded_models",), + optional=("max_loaded_models", "replaces_trust_roots"), ) schema_version = _require_positive_int(source["schema_version"], "schema_version") if schema_version != CATALOG_BOOTSTRAP_SCHEMA_VERSION: @@ -271,12 +369,23 @@ def from_dict(cls, source: Mapping[str, Any]) -> "CatalogBootstrapConfig": trust_root = CatalogTrustRoot.from_dict(_require_mapping(source["trust_root"], "trust_root")) except ModelCatalogError as exc: raise CatalogBootstrapError(f"Invalid catalog trust root: {exc}") from exc + replacements = source.get("replaces_trust_roots", []) + if ( + not isinstance(replacements, list) + or len(replacements) > 16 + or any( + not isinstance(value, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", value) for value in replacements + ) + or len(set(replacements)) != len(replacements) + ): + raise CatalogBootstrapError("replaces_trust_roots must be at most 16 distinct SHA-256 root digests") return cls( schema_version=schema_version, trust_root=trust_root, catalog_mirrors=_require_https_urls(source["catalog_mirrors"], "catalog_mirrors"), initial_peers=_require_initial_peers(source["initial_peers"]), max_loaded_models=_require_positive_int(source.get("max_loaded_models", 1), "max_loaded_models"), + replaces_trust_roots=tuple(replacements), ) @classmethod @@ -295,13 +404,16 @@ def load(cls, path: Path | str) -> "CatalogBootstrapConfig": return cls.from_json(source) def to_dict(self) -> Dict[str, Any]: - return { + result = { "schema_version": self.schema_version, "trust_root": self.trust_root.to_dict(), "catalog_mirrors": list(self.catalog_mirrors), "initial_peers": list(self.initial_peers), "max_loaded_models": self.max_loaded_models, } + if self.replaces_trust_roots: + result["replaces_trust_roots"] = list(self.replaces_trust_roots) + return result @dataclass(frozen=True) @@ -409,7 +521,7 @@ def __init__( now: Optional[float] = None, ) -> None: self.bootstrap = bootstrap - self.data_dir = Path(data_dir).expanduser().resolve() + self.data_dir = _absolute_path(data_dir) self.config_path = _absolute_path(config_path) self.fetch_text = fetch_text self.now = now @@ -419,6 +531,10 @@ def __init__( self.manifest_dir = self.data_dir / "manifests" self.cache_dir = self.data_dir / "model-cache" self.lock_path = self.data_dir / ".catalog-bootstrap.lock" + bootstrap_bytes = json.dumps(bootstrap.to_dict(), sort_keys=True, separators=(",", ":")).encode("utf-8") + self.installed_bootstrap_path = ( + self.catalog_dir / f"bootstrap-{hashlib.sha256(bootstrap_bytes).hexdigest()}.json" + ) def _existing_result(self) -> CatalogBootstrapResult: try: @@ -442,7 +558,91 @@ def _load_catalog(self, source: str, rendered: str, guard: CatalogRollbackGuard) except ModelCatalogError as exc: raise CatalogBootstrapError(f"Rejected model catalog from {source}: {exc}") from exc + def _installed_catalog(self, config: NodeConfig) -> Optional[ModelCatalog]: + if config.catalog_path is None: + return None + installed = CatalogBootstrapConfig.load(config.catalog_bootstrap_path) + if not self.bootstrap.permits_replacement_of(installed): + raise CatalogBootstrapError("The application bootstrap does not authorize this installed catalog") + envelope = SignedModelCatalog.load(config.catalog_path) + # Historical membership remains meaningful after expiry. This never + # authorizes a network update or bypasses its current-time/rollback gate. + return envelope.verify(installed.trust_root, now=envelope.signed.issued_at_ms / 1000) + + def _retired_managed_paths(self, config: NodeConfig, catalog: ModelCatalog) -> set[Path]: + managed = set() + previous = self._installed_catalog(config) + if previous is not None: + managed.update(model.manifest_digest for model in previous.models) + if ( + catalog.catalog_id == _LEGACY_PUBLIC_ALPHA_CATALOG_ID + and catalog.sequence >= 2 + and _LEGACY_PUBLIC_ALPHA_ROOT in self.bootstrap.replaces_trust_roots + ): + managed.update(_LEGACY_PUBLIC_ALPHA_MODELS) + retired = managed.difference(model.manifest_digest for model in catalog.models) + pinned = {worker.model.casefold() for worker in config.workers if worker.model.casefold() != "auto"} + paths = set() + for model in config.models: + manifest = ModelManifest.load(model.manifest_path) + if ( + manifest.digest_id in retired + and model.manifest_path == self.manifest_dir / f"{manifest.digest}.json" + and not pinned.intersection( + value.casefold() for value in (manifest.digest_id, manifest.name, *manifest.aliases) + ) + ): + paths.add(model.manifest_path) + return paths + + def _repair_existing_config(self) -> CatalogBootstrapResult: + """Remove proven retired managed entries, retaining model files and user settings.""" + if not self.config_path.is_file() or self.config_path.is_symlink(): + raise CatalogBootstrapError("Catalog migration requires a safe existing node configuration") + try: + with node_config_write_lock(self.config_path): + original = self.config_path.read_text(encoding="utf-8") + config = NodeConfig.from_json(original, base_dir=self.config_path.parent) + catalog = self._installed_catalog(config) + retired = set() if catalog is None else self._retired_managed_paths(config, catalog) + previous = json.loads(original) + missing_policy = catalog is not None and previous.get("contribution_policy") is None + if not retired and not missing_policy: + return self._existing_result() + if missing_policy: + previous["contribution_policy"] = dict(_DEFAULT_CONTRIBUTION_POLICY) + previous["models"] = [ + entry + for model, entry in zip(config.models, previous["models"]) + if model.manifest_path not in retired + ] + NodeConfig.from_dict(previous, base_dir=self.config_path.parent) + _atomic_write( + self.config_path, + json.dumps(previous, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + overwrite=True, + ) + except NodeConfigWriteLockError as exc: + raise CatalogBootstrapError("Another node configuration writer is active") from exc + return CatalogBootstrapResult( + config_path=self.config_path, + catalog_id=catalog.catalog_id, + catalog_sequence=catalog.sequence, + catalog_digest=catalog.digest, + model_count=len(previous["models"]), + source="existing-config-migration", + created=True, + ) + + def repair_existing_config(self) -> CatalogBootstrapResult: + """Apply offline application migrations before starting an existing node.""" + with _catalog_bootstrap_lock(self.lock_path): + return self._repair_existing_config() + def _install_manifests(self, catalog: ModelCatalog) -> Tuple[Path, ...]: + import drift + from drift.node.loading import validate_manifest_execution + installed = [] selectors: Dict[str, str] = {} for model in catalog.models: @@ -475,6 +675,11 @@ def _install_manifests(self, catalog: ModelCatalog) -> Tuple[Path, ...]: if manifest is None: detail = "; ".join(errors) if errors else "no manifest mirror was attempted" raise CatalogBootstrapError(f"Could not install manifest {model.manifest_digest}: {detail}") + try: + manifest.validate_runtime(drift.__version__) + validate_manifest_execution(manifest, model.execution or "distributed") + except ManifestError as exc: + raise CatalogBootstrapError(f"Manifest {manifest.digest_id} is not executable: {exc}") from exc for selector in (manifest.name, *manifest.aliases): folded = selector.casefold() @@ -492,15 +697,18 @@ def _install_manifests(self, catalog: ModelCatalog) -> Tuple[Path, ...]: def _render_node_config(self, catalog: ModelCatalog, manifest_paths: Tuple[Path, ...]) -> str: models = [] - for path in manifest_paths: + for model, path in zip(catalog.models, manifest_paths): digest = path.stem - models.append( - { - "manifest": str(path), - "initial_peers": list(self.bootstrap.initial_peers), - "cache_dir": str(self.cache_dir / digest), - } - ) + entry = { + "manifest": str(path), + "initial_peers": [] if model.execution == "local" else list(self.bootstrap.initial_peers), + "cache_dir": str(self.cache_dir / digest), + } + if model.execution is not None: + entry["execution"] = model.execution + if model.execution == "distributed": + entry["request_timeout"] = 180 + models.append(entry) rung_order = {rung.rung_id: rung.order for rung in catalog.rungs} ranked_models = sorted( enumerate(catalog.models), @@ -515,7 +723,12 @@ def _render_node_config(self, catalog: ModelCatalog, manifest_paths: Tuple[Path, "max_loaded_models": self.bootstrap.max_loaded_models, "models": models, "auto_model_priority": [model.manifest_digest for _, model in ranked_models], + "catalog_path": str( + self.catalog_dir / f"{catalog.sequence}-{catalog.digest.removeprefix('sha256:')}.signed.json" + ), + "catalog_bootstrap_path": str(self.installed_bootstrap_path), "route_demand_authority_roots": list(catalog.route_demand_authority_roots or ()), + "contribution_policy": dict(_DEFAULT_CONTRIBUTION_POLICY), "workers": [ { "id": "automatic", @@ -524,7 +737,9 @@ def _render_node_config(self, catalog: ModelCatalog, manifest_paths: Tuple[Path, "num_blocks": 1, "enabled": True, } - ], + ] + if any(model.execution != "local" for model in catalog.models) + else [], } try: NodeConfig.from_dict(source, base_dir=self.config_path.parent) @@ -533,13 +748,22 @@ def _render_node_config(self, catalog: ModelCatalog, manifest_paths: Tuple[Path, return json.dumps(source, ensure_ascii=False, indent=2, sort_keys=True) + "\n" def _try_candidate( - self, source: str, rendered: str, persisted_guard: CatalogRollbackGuard + self, source: str, rendered: str, persisted_guard: CatalogRollbackGuard, *, refresh: bool = False ) -> CatalogBootstrapResult: guard = CatalogRollbackGuard.from_dict(persisted_guard.to_dict()) catalog = self._load_catalog(source, rendered, guard) manifest_paths = self._install_manifests(catalog) config_text = self._render_node_config(catalog, manifest_paths) + # Configuration points at an immutable envelope. A later failed refresh + # cannot silently change the policy used by the still-running old config. + _atomic_write( + self.catalog_dir / f"{catalog.sequence}-{catalog.digest.removeprefix('sha256:')}.signed.json", + rendered, + overwrite=True, + ) + _atomic_write(self.installed_bootstrap_path, json.dumps(self.bootstrap.to_dict()), overwrite=True) + _atomic_write( self.cached_catalog_path, json.dumps(SignedModelCatalog.from_json(rendered).to_dict(), ensure_ascii=False, indent=2, sort_keys=True) @@ -550,7 +774,61 @@ def _try_candidate( self.config_path.parent.mkdir(parents=True, exist_ok=True) try: with node_config_write_lock(self.config_path): - _atomic_write(self.config_path, config_text, overwrite=False) + if refresh: + original = self.config_path.read_text(encoding="utf-8") + old_config = NodeConfig.from_json(original, base_dir=self.config_path.parent) + retired_paths = self._retired_managed_paths(old_config, catalog) + previous = json.loads(original) + generated = json.loads(config_text) + if previous.get("contribution_policy") is None: + previous["contribution_policy"] = generated["contribution_policy"] + old_entries = { + model.manifest_path: entry for model, entry in zip(old_config.models, previous["models"]) + } + old_by_digest = {} + for model in old_config.models: + digest = ModelManifest.load(model.manifest_path).digest_id + if digest in old_by_digest: + raise CatalogBootstrapError("Existing models have ambiguous duplicate manifest identities") + old_by_digest[digest] = old_entries[model.manifest_path] + entries = [] + current_paths = set() + current_selectors = set() + for entry in generated["models"]: + path = Path(entry["manifest"]) + current_paths.add(path) + manifest = ModelManifest.load(path) + current_selectors.update(s.casefold() for s in (manifest.name, *manifest.aliases)) + prior = old_by_digest.get(manifest.digest_id) + # Preserve explicit per-model resource/cache preferences; + # execution mode itself remains a signed catalog choice. + if prior is not None: + entry = dict(prior, manifest=str(path), execution=entry.get("execution", "distributed")) + if entry["execution"] != "local": + entry = {k: v for k, v in entry.items() if not k.startswith("local_")} + entries.append(entry) + for model in old_config.models: + if model.manifest_path in current_paths or model.manifest_path in retired_paths: + continue + old_manifest = ModelManifest.load(model.manifest_path) + if current_selectors.intersection( + s.casefold() for s in (old_manifest.name, *old_manifest.aliases) + ): + continue + # User-added manifests stay explicit choices, but leave + # automatic selection and contribution approval. + entries.append(old_entries[model.manifest_path]) + previous["models"] = entries + for field in ( + "auto_model_priority", + "route_demand_authority_roots", + "catalog_path", + "catalog_bootstrap_path", + ): + previous[field] = generated[field] + NodeConfig.from_dict(previous, base_dir=self.config_path.parent) + config_text = json.dumps(previous, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + _atomic_write(self.config_path, config_text, overwrite=refresh) except NodeConfigWriteLockError as exc: raise CatalogBootstrapError("Another node configuration writer is active") from exc return CatalogBootstrapResult( @@ -563,6 +841,37 @@ def _try_candidate( created=True, ) + def refresh(self) -> CatalogBootstrapResult: + """Authenticate a newer sequence while preserving user policy and local settings.""" + if not self.config_path.is_file() or self.config_path.is_symlink(): + raise CatalogBootstrapError("Catalog refresh requires a safe existing node configuration") + with _catalog_bootstrap_lock(self.lock_path): + guard = CatalogRollbackGuard.load(self.rollback_path) + existing = NodeConfig.load(self.config_path) + current = None + if existing.catalog_path is not None: + installed_bootstrap = CatalogBootstrapConfig.load(existing.catalog_bootstrap_path) + if not self.bootstrap.permits_replacement_of(installed_bootstrap): + raise CatalogBootstrapError( + "The supplied application bootstrap does not authorize this trust-root replacement" + ) + # Its identity is only an optimization. A newly fetched candidate + # must still pass current-time signature and rollback validation; + # an expired installed catalog must not prevent renewal. + current = SignedModelCatalog.from_json(existing.catalog_path.read_text(encoding="utf-8")).signed + errors = [] + for url in self.bootstrap.catalog_mirrors: + try: + rendered = self.fetch_text(url, MAX_CATALOG_BYTES) + candidate = self._load_catalog(url, rendered, CatalogRollbackGuard.from_dict(guard.to_dict())) + if current is not None and candidate.digest == current.digest: + return self._repair_existing_config() + return self._try_candidate(url, rendered, guard, refresh=True) + except (CatalogBootstrapError, ModelCatalogError, ManifestError, OSError) as exc: + errors.append(str(exc)) + guard = CatalogRollbackGuard.load(self.rollback_path) + raise CatalogBootstrapError("No trusted catalog update could be activated: " + "; ".join(errors)) + def install(self) -> CatalogBootstrapResult: if self.config_path.is_symlink(): raise CatalogBootstrapError(f"Refusing unsafe node configuration symlink {self.config_path}") @@ -570,15 +879,7 @@ def install(self) -> CatalogBootstrapResult: return self._existing_result() self.data_dir.mkdir(parents=True, exist_ok=True) - try: - descriptor = os.open(self.lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - except FileExistsError as exc: - raise CatalogBootstrapError("Another first-install catalog bootstrap is already in progress") from exc - except OSError as exc: - raise CatalogBootstrapError(f"Could not lock catalog bootstrap in {self.data_dir}: {exc}") from exc - - os.close(descriptor) - try: + with _catalog_bootstrap_lock(self.lock_path): if self.config_path.is_file() and not self.config_path.is_symlink(): return self._existing_result() try: @@ -614,11 +915,6 @@ def install(self) -> CatalogBootstrapResult: errors.append(f"Could not use last-known-good catalog: {exc}") detail = "; ".join(errors) if errors else "no catalog source was available" raise CatalogBootstrapError(f"No trusted usable model catalog could be installed: {detail}") - finally: - try: - self.lock_path.unlink() - except FileNotFoundError: - pass def bootstrap_node_from_catalog( diff --git a/src/drift/node/catalog_refresh.py b/src/drift/node/catalog_refresh.py new file mode 100644 index 000000000..5258d7946 --- /dev/null +++ b/src/drift/node/catalog_refresh.py @@ -0,0 +1,55 @@ +"""Periodic authenticated catalog refresh with idle-only node reconfiguration.""" + +import logging +import threading + +from drift.model_catalog import SignedModelCatalog +from drift.node.catalog_bootstrap import CatalogBootstrapConfig, CatalogBootstrapInstaller + + +def load_configured_catalog(config): + if config.catalog_path is None: + return None + bootstrap = CatalogBootstrapConfig.load(config.catalog_bootstrap_path) + envelope = SignedModelCatalog.from_json(config.catalog_path.read_text(encoding="utf-8")) + # Keep verified local fallback usable offline after catalog expiry. Community + # selection separately requires current validity; refresh can renew the policy. + return envelope.verify(bootstrap.trust_root, now=envelope.signed.issued_at_ms / 1000) + + +class CatalogRefreshService: + def __init__(self, config, config_path, data_dir, manager, restart): + self.config = config + self.manager = manager + self.restart = restart + self.installer = CatalogBootstrapInstaller( + CatalogBootstrapConfig.load(config.catalog_bootstrap_path), + data_dir=data_dir, + config_path=config_path, + ) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, name="drift-catalog-refresh", daemon=True) + + def start(self): + self._thread.start() + + def close(self): + self._stop.set() + if self._thread.is_alive(): + self._thread.join(timeout=1) + + def _run(self): + while not self._stop.wait(self.config.catalog_refresh_seconds): + try: + result = self.installer.refresh() + if not result.created: + continue + # Active generations own leases. Reconfiguration waits for all + # leases and loads, then atomically closes admission before restart. + while not self._stop.is_set(): + if self.manager.begin_idle_restart(): + self.restart() + return + self._stop.wait(1) + except Exception: + logging.getLogger(__name__).exception("Catalog refresh failed; retaining the active configuration") diff --git a/src/drift/node/config.py b/src/drift/node/config.py index 02c9b5eda..883a3c630 100644 --- a/src/drift/node/config.py +++ b/src/drift/node/config.py @@ -160,6 +160,13 @@ class NodeModelConfig: revocation_files: Tuple[Path, ...] = () request_timeout: float = 30.0 max_retries: int = 3 + execution: str = "distributed" + local_device: str = "auto" + local_max_memory_bytes: int = 3 * 1024**3 + local_max_disk_bytes: int = 8 * 1024**3 + local_max_context: int = 2048 + local_max_new_tokens: int = 256 + local_max_seconds: float = 120.0 @classmethod def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path, index: int) -> "NodeModelConfig": @@ -169,20 +176,59 @@ def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path, index: int) -> source, field, required=("manifest", "initial_peers"), - optional=("cache_dir", "revocation_files", "request_timeout", "max_retries"), + optional=( + "cache_dir", + "revocation_files", + "request_timeout", + "max_retries", + "execution", + "local_device", + "local_max_memory", + "local_max_disk_space", + "local_max_context", + "local_max_new_tokens", + "local_max_seconds", + ), ) + execution = source.get("execution", "distributed") + if execution not in ("distributed", "local"): + raise NodeConfigError(f"{field}.execution must be distributed or local") + device = source.get("local_device", "auto") + if not isinstance(device, str) or re.fullmatch(r"auto|cpu|cuda(?::[0-9]+)?", device) is None: + raise NodeConfigError(f"{field}.local_device must be auto, cpu, or cuda[:index]") + if execution != "local" and any(key.startswith("local_") for key in source): + raise NodeConfigError(f"{field} local settings require local execution") cache_value = source.get("cache_dir") cache_dir = None if cache_value is None else _resolve_path(cache_value, f"{field}.cache_dir", base_dir) revocation_values = _require_string_list(source.get("revocation_files", []), f"{field}.revocation_files") return cls( manifest_path=_resolve_path(source["manifest"], f"{field}.manifest", base_dir), - initial_peers=_require_string_list(source["initial_peers"], f"{field}.initial_peers", nonempty=True), + initial_peers=_require_string_list( + source["initial_peers"], f"{field}.initial_peers", nonempty=execution == "distributed" + ), cache_dir=cache_dir, revocation_files=tuple( _resolve_path(value, f"{field}.revocation_files[]", base_dir) for value in revocation_values ), request_timeout=_require_positive_number(source.get("request_timeout", 30), f"{field}.request_timeout"), max_retries=_require_positive_int(source.get("max_retries", 3), f"{field}.max_retries"), + execution=execution, + local_device=device, + local_max_memory_bytes=_require_size(source.get("local_max_memory", "3GiB"), f"{field}.local_max_memory")[ + 1 + ], + local_max_disk_bytes=_require_size( + source.get("local_max_disk_space", "8GiB"), f"{field}.local_max_disk_space" + )[1], + local_max_context=_require_positive_int( + source.get("local_max_context", 2048), f"{field}.local_max_context" + ), + local_max_new_tokens=_require_positive_int( + source.get("local_max_new_tokens", 256), f"{field}.local_max_new_tokens" + ), + local_max_seconds=_require_positive_number( + source.get("local_max_seconds", 120), f"{field}.local_max_seconds" + ), ) @@ -285,6 +331,7 @@ class ContributionPolicyConfig: max_vram: Optional[str] = None max_vram_bytes: Optional[int] = None max_vram_fraction: Optional[float] = None + max_processing_percent: float = 100.0 max_bandwidth_mbps: Optional[float] = None max_power_watts: Optional[float] = None pause_timeout: float = 10.0 @@ -304,6 +351,7 @@ def from_dict(cls, source: Mapping[str, Any]) -> "ContributionPolicyConfig": "denied_models", "max_disk_space", "max_vram", + "max_processing_percent", "max_bandwidth_mbps", "max_power_watts", "pause_timeout", @@ -333,6 +381,14 @@ def from_dict(cls, source: Mapping[str, Any]) -> "ContributionPolicyConfig": else: max_vram, max_vram_bytes, max_vram_fraction = _require_vram_limit(max_vram_value, f"{field}.max_vram") sharing_enabled = _require_bool(source["sharing_enabled"], f"{field}.sharing_enabled") + processing = source.get("max_processing_percent", 100.0) + if ( + isinstance(processing, bool) + or not isinstance(processing, (int, float)) + or not math.isfinite(processing) + or not 1 <= processing <= 100 + ): + raise NodeConfigError(f"{field}.max_processing_percent must be between 1 and 100") if sharing_enabled and max_disk_bytes is None: raise NodeConfigError(f"{field}.max_disk_space is required when sharing_enabled is true") return cls( @@ -345,6 +401,7 @@ def from_dict(cls, source: Mapping[str, Any]) -> "ContributionPolicyConfig": max_vram=max_vram, max_vram_bytes=max_vram_bytes, max_vram_fraction=max_vram_fraction, + max_processing_percent=float(processing), max_bandwidth_mbps=( None if source.get("max_bandwidth_mbps") is None @@ -370,6 +427,7 @@ def to_dict(self) -> Mapping[str, Any]: "denied_models": list(self.denied_models), "max_disk_space": self.max_disk_space, "max_vram": self.max_vram, + "max_processing_percent": self.max_processing_percent, "max_bandwidth_mbps": self.max_bandwidth_mbps, "max_power_watts": self.max_power_watts, "pause_timeout": self.pause_timeout, @@ -401,6 +459,7 @@ class WorkerConfig: throughput: float | str = "auto" port: Optional[int] = None public_ip: Optional[str] = None + public_port: Optional[int] = None @classmethod def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path, index: int) -> "WorkerConfig": @@ -425,6 +484,7 @@ def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path, index: int) -> "throughput", "port", "public_ip", + "public_port", ), ) worker_id = _require_string(source["id"], f"{field}.id") @@ -460,6 +520,12 @@ def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path, index: int) -> port = None if port_value is None else _require_positive_int(port_value, f"{field}.port") if port is not None and port > 65535: raise NodeConfigError(f"{field}.port must be <= 65535") + public_port_value = source.get("public_port") + public_port = ( + None if public_port_value is None else _require_positive_int(public_port_value, f"{field}.public_port") + ) + if public_port is not None and (public_port > 65535 or port is None or source.get("public_ip") is None): + raise NodeConfigError(f"{field}.public_port requires port, public_ip and a value <= 65535") def optional_string(name: str) -> Optional[str]: value = source.get(name) @@ -508,6 +574,7 @@ def optional_string(name: str) -> Optional[str]: throughput=throughput, port=port, public_ip=optional_string("public_ip"), + public_port=public_port, ) @@ -524,6 +591,10 @@ class NodeConfig: contribution_policy: ContributionPolicyConfig = ContributionPolicyConfig() discovery_update_period: float = 30.0 discovery_startup_timeout: float = 15.0 + inference_mode: str = "auto" + catalog_path: Optional[Path] = None + catalog_bootstrap_path: Optional[Path] = None + catalog_refresh_seconds: float = 300.0 @classmethod def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path) -> "NodeConfig": @@ -540,9 +611,18 @@ def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path) -> "NodeConfig" "contribution_policy", "auto_model_priority", "route_demand_authority_roots", + "inference_mode", + "catalog_path", + "catalog_bootstrap_path", + "catalog_refresh_seconds", ), ) schema_version = _require_positive_int(source["schema_version"], "schema_version") + inference_mode = source.get("inference_mode", "auto") + if inference_mode not in ("auto", "local_only"): + raise NodeConfigError("inference_mode must be auto or local_only") + if (source.get("catalog_path") is None) != (source.get("catalog_bootstrap_path") is None): + raise NodeConfigError("catalog_path and catalog_bootstrap_path must be configured together") if schema_version != NODE_CONFIG_SCHEMA_VERSION: raise NodeConfigError(f"Unsupported schema_version {schema_version}; expected {NODE_CONFIG_SCHEMA_VERSION}") models_value = source["models"] @@ -581,6 +661,16 @@ def from_dict(cls, source: Mapping[str, Any], *, base_dir: Path) -> "NodeConfig" return cls( schema_version=schema_version, max_loaded_models=_require_positive_int(source.get("max_loaded_models", 1), "max_loaded_models"), + inference_mode=inference_mode, + catalog_path=None + if source.get("catalog_path") is None + else _resolve_path(source["catalog_path"], "catalog_path", base_dir), + catalog_bootstrap_path=None + if source.get("catalog_bootstrap_path") is None + else _resolve_path(source["catalog_bootstrap_path"], "catalog_bootstrap_path", base_dir), + catalog_refresh_seconds=_require_positive_number( + source.get("catalog_refresh_seconds", 300), "catalog_refresh_seconds" + ), models=models, auto_model_priority=_require_model_list(source.get("auto_model_priority", []), "auto_model_priority"), route_demand_authority_roots=_require_route_demand_authority_roots( diff --git a/src/drift/node/contribution_planner.py b/src/drift/node/contribution_planner.py index aa22aad29..1bddfae33 100644 --- a/src/drift/node/contribution_planner.py +++ b/src/drift/node/contribution_planner.py @@ -17,6 +17,26 @@ MODEL_DISPERSION_POINTS = 32.0 +@dataclass(frozen=True) +class PlacementArtifactPlan: + """Content-bound resource claim for one possible contiguous span.""" + + start_block: int + end_block: int + artifact_bytes: int + artifact_set_digest: str + + def __post_init__(self) -> None: + if self.start_block < 0 or self.end_block <= self.start_block or self.artifact_bytes < 0: + raise ValueError("placement artifact plan range and byte count are invalid") + if ( + len(self.artifact_set_digest) != 64 + or self.artifact_set_digest.lower() != self.artifact_set_digest + or any(character not in "0123456789abcdef" for character in self.artifact_set_digest) + ): + raise ValueError("placement artifact plan digest must be lowercase SHA-256") + + @dataclass(frozen=True) class PlacementCandidate: """One exact manifested model evaluated against local policy and live coverage.""" @@ -31,6 +51,8 @@ class PlacementCandidate: route_observation: Optional[Mapping[str, Any]] = None remote_route_observation: Optional[Mapping[str, Any]] = None policy_reason: Optional[str] = None + artifact_plans: Tuple[PlacementArtifactPlan, ...] = () + max_artifact_bytes: Optional[int] = None def __post_init__(self) -> None: if not self.model_id or not self.manifest_digest: @@ -39,6 +61,16 @@ def __post_init__(self) -> None: raise ValueError("placement candidate sizes and priority must be non-negative") if self.total_blocks > MAX_AUTOMATIC_PLACEMENT_BLOCKS: raise ValueError("placement candidate exceeds the automatic placement block limit") + if self.max_artifact_bytes is not None and self.max_artifact_bytes < 0: + raise ValueError("placement candidate artifact budget must be non-negative") + ranges = set() + for plan in self.artifact_plans: + if plan.end_block > self.total_blocks: + raise ValueError("placement artifact plan exceeds the candidate block range") + key = (plan.start_block, plan.end_block) + if key in ranges: + raise ValueError("placement candidate contains duplicate artifact-plan ranges") + ranges.add(key) @dataclass(frozen=True) @@ -52,6 +84,7 @@ class PlacementDecision: replica_counts: Tuple[int, ...] score: float reason: str + artifact_set_digest: Optional[str] = None @dataclass(frozen=True) @@ -61,6 +94,16 @@ class PlacementPlan: decision: Optional[PlacementDecision] reason: str evaluated_models: int + intent_published: bool = False + remote_acknowledged: bool = False + + def __post_init__(self) -> None: + if type(self.intent_published) is not bool or type(self.remote_acknowledged) is not bool: + raise ValueError("placement intent publication fields must be booleans") + if self.intent_published != self.remote_acknowledged: + raise ValueError("placement intent publication requires a remote acknowledgement") + if self.decision is None and self.intent_published: + raise ValueError("an empty placement cannot carry an acknowledged intent") class PlacementRegistry: @@ -176,9 +219,35 @@ def _evaluate(self, candidate: PlacementCandidate) -> tuple[Optional[PlacementDe ): return None, "coverage observation has invalid replica counts" - # Find the least-covered contiguous window in one bounded pass. Equal - # windows use a node-specific rendezvous rank instead of numeric start, so - # a cohort sharing one snapshot does not all announce range zero. + artifact_plans = {(plan.start_block, plan.end_block): plan for plan in candidate.artifact_plans} + if not artifact_plans and candidate.max_artifact_bytes is not None: + if candidate.artifact_bytes > candidate.max_artifact_bytes: + return None, ( + f"manifested artifacts require {candidate.artifact_bytes} bytes, above the " + f"{candidate.max_artifact_bytes}-byte disk budget" + ) + if artifact_plans: + expected_ranges = { + (start, start + self.num_blocks) for start in range(candidate.total_blocks - self.num_blocks + 1) + } + if set(artifact_plans) != expected_ranges: + return None, f"exact artifact plans are unavailable for every {self.num_blocks}-block span" + + # Break equal-coverage ties by how many workers of this capacity would + # still be needed to fill the remaining gaps. Random interior windows + # can strand short gaps, preventing four 16-block workers from covering + # 64 blocks despite having enough total capacity. Prefix/suffix costs + # keep this bounded scan linear; rendezvous still disperses equal fits. + prefix_cost = [0] * (len(counts) + 1) + suffix_cost = [0] * (len(counts) + 1) + gap = 0 + for index, count in enumerate(counts): + gap = gap + 1 if count == 0 else 0 + prefix_cost[index + 1] = prefix_cost[index] + int(gap > 0 and (gap - 1) % self.num_blocks == 0) + gap = 0 + for index in range(len(counts) - 1, -1, -1): + gap = gap + 1 if counts[index] == 0 else 0 + suffix_cost[index] = suffix_cost[index + 1] + int(gap > 0 and (gap - 1) % self.num_blocks == 0) maxima: deque[int] = deque() window_sum = 0 best_key = None @@ -197,16 +266,33 @@ def _evaluate(self, candidate: PlacementCandidate) -> tuple[Optional[PlacementDe continue start = index - self.num_blocks + 1 end = start + self.num_blocks + artifact_plan = artifact_plans.get((start, end)) + if ( + artifact_plan is not None + and candidate.max_artifact_bytes is not None + and artifact_plan.artifact_bytes > candidate.max_artifact_bytes + ): + continue key = ( counts[maxima[0]], window_sum, + prefix_cost[start] + suffix_cost[end], self._range_jitter(candidate.manifest_digest, start, end), start, ) if best_key is None or key < best_key: best_key = key best_start = start + if best_key is None: + return None, ( + f"every {self.num_blocks}-block artifact set exceeds the " + f"{candidate.max_artifact_bytes}-byte disk budget" + ) start = best_start + end = start + self.num_blocks + selected_artifacts = artifact_plans.get((start, end)) + artifact_bytes = candidate.artifact_bytes if selected_artifacts is None else selected_artifacts.artifact_bytes + artifact_set_digest = None if selected_artifacts is None else selected_artifacts.artifact_set_digest window = tuple(counts[start : start + self.num_blocks]) minimum_replicas = min(window) coverage_pressure = max(0, 2 - minimum_replicas) * 100.0 @@ -225,7 +311,6 @@ def _evaluate(self, candidate: PlacementCandidate) -> tuple[Optional[PlacementDe + remote_signal + self._jitter(candidate.manifest_digest) ) - end = start + self.num_blocks reason = f"selected {start}:{end} from fresh verified coverage; minimum replicas {minimum_replicas}" if local_observation is not None: reason += ( @@ -244,10 +329,11 @@ def _evaluate(self, candidate: PlacementCandidate) -> tuple[Optional[PlacementDe model_id=candidate.model_id, manifest_digest=candidate.manifest_digest, block_indices=f"{start}:{end}", - artifact_bytes=candidate.artifact_bytes, + artifact_bytes=artifact_bytes, replica_counts=window, score=score, reason=reason, + artifact_set_digest=artifact_set_digest, ), "", ) @@ -290,13 +376,52 @@ def propose( ), None, ) - if current is not None and self._assigned_at is not None: + current_assignment_is_eligible = current is not None + if current_assignment_is_eligible: + current_candidate = next( + candidate for candidate in candidates if candidate.manifest_digest == self._current.manifest_digest + ) + if current_candidate.artifact_plans: + start, end = (int(value) for value in self._current.block_indices.split(":")) + plan = next( + ( + plan + for plan in current_candidate.artifact_plans + if (plan.start_block, plan.end_block) == (start, end) + ), + None, + ) + current_assignment_is_eligible = ( + plan is not None + and ( + current_candidate.max_artifact_bytes is None + or plan.artifact_bytes <= current_candidate.max_artifact_bytes + ) + and plan.artifact_bytes == self._current.artifact_bytes + and plan.artifact_set_digest == self._current.artifact_set_digest + ) + if current_assignment_is_eligible and self._assigned_at is not None: residency_elapsed = now - self._assigned_at cooldown_elapsed = math.inf if self._last_switch_at is None else now - self._last_switch_at if residency_elapsed < self._minimum_residency or cooldown_elapsed < self._cooldown: best = self._current elif best.manifest_digest != current.manifest_digest and best.score < current.score + self._switch_margin: best = self._current + elif best.manifest_digest == self._current.manifest_digest: + counts = current_candidate.health["replica_counts"] + old_start, old_end = map(int, self._current.block_indices.split(":")) + new_start, new_end = map(int, best.block_indices.split(":")) + # Slow joins must not move a coverage gap around the model. + # Permit abandoning unique blocks only for a net coverage gain; + # redundant overlapping workers can still move to fill a gap. + lost = sum( + counts[index] == 1 and not new_start <= index < new_end for index in range(old_start, old_end) + ) + gained = sum( + counts[index] == 0 for index in range(new_start, new_end) if not old_start <= index < old_end + ) + if lost and gained <= lost: + best = self._current return PlacementPlan(best, best.reason, len(candidates)) diff --git a/src/drift/node/discovery.py b/src/drift/node/discovery.py index 3ca596713..db944bd9e 100644 --- a/src/drift/node/discovery.py +++ b/src/drift/node/discovery.py @@ -11,6 +11,7 @@ import secrets import threading import time +import weakref from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple @@ -260,6 +261,10 @@ def _default_peer_snapshot(dht: Any) -> Sequence[str]: return dht.run_coroutine(_connected_peer_addresses) +async def _routing_peer_count(_dht: Any, node: Any) -> int: + return len(node.protocol.routing_table.uid_to_peer_id) + + @dataclass(frozen=True) class CoverageTarget: manifest: ModelManifest @@ -280,10 +285,10 @@ class _TargetState: remote_route_updated: Optional[float] = None -def _default_dht_factory(**kwargs): - from hivemind import DHT +def _default_dht_factory(*, start=True, startup_timeout=15.0, **kwargs): + from drift.utils.client_dht import create_client_dht - return DHT(**kwargs) + return create_client_dht(start=start, startup_timeout=startup_timeout, **kwargs) class ModelCoverageDiscovery: @@ -306,6 +311,7 @@ def __init__( replay_history_dir: Optional[Path | str] = None, route_demand_authority_roots: Sequence[str] = (), peer_snapshot: Callable[[Any], Sequence[str]] = _default_peer_snapshot, + discover_text: bool = False, ) -> None: if update_period <= 0 or startup_timeout <= 0: raise ValueError("discovery periods must be positive") @@ -313,6 +319,7 @@ def __init__( self._startup_timeout = startup_timeout self._dht_factory = dht_factory self._lookup = lookup + self._discover_text = discover_text self._peer_cache = peer_cache authority_roots = tuple(route_demand_authority_roots) if authority_roots and not 2 <= len(authority_roots) <= _MAX_ROUTE_DEMAND_AUTHORITY_ROOTS: @@ -356,7 +363,7 @@ def __init__( self._stop = threading.Event() self._threads: list[threading.Thread] = [] self._dhts: Dict[Tuple[str, ...], Any] = {} - self._shutdown_dht_ids: set[int] = set() + self._shutdown_dhts = weakref.WeakSet() self._local_route_demand_keys: set[str] = set() self._started = False self._closed = False @@ -405,6 +412,15 @@ def snapshot(self, digest_id: str) -> Dict[str, Any]: result["status"] = "unknown" result["source"] = "discovery" result["last_error"] = state.last_error + if self._discover_text: + text_peers = [ + peer for peer in result.get("text_peers", []) if peer["expires_at_ms"] > time.time() * 1000 + ] + result["text_peers"] = text_peers + result["text_peer_count"] = len(text_peers) + result["chat_ready"] = result["status"] == "complete" and bool(text_peers) + if isinstance(result.get("reservations"), list): + result["reservations"] = [r for r in result["reservations"] if r["expires_at"] > time.time()] return result def register_local_route_demand_key(self, key_id: str) -> None: @@ -494,10 +510,9 @@ def _set_error(self, states: Sequence[_TargetState], exc: Exception) -> None: def _shutdown_dht_once(self, dht: Any) -> None: with self._lock: - identity = id(dht) - if identity in self._shutdown_dht_ids: + if dht in self._shutdown_dhts: return - self._shutdown_dht_ids.add(identity) + self._shutdown_dhts.add(dht) if dht.is_alive(): dht.shutdown() @@ -510,7 +525,7 @@ def _run_group(self, initial_peers: Tuple[str, ...], states: Tuple[_TargetState, dht = self._dht_factory( initial_peers=list(initial_peers), client_mode=True, - num_workers=min(max(state.target.manifest.model.num_blocks for state in states), 32), + num_workers=min(max(state.target.manifest.model.num_blocks for state in states), 4), startup_timeout=self._startup_timeout, start=True, tls=True, @@ -545,7 +560,31 @@ def _run_group(self, initial_peers: Tuple[str, ...], states: Tuple[_TargetState, replay_guard=state.replay_guard, latest=True, ) - self._set_success(state, module_infos_route_health(module_infos)) + if ( + callable(getattr(dht, "run_coroutine", None)) + and dht.run_coroutine(_routing_peer_count) == 0 + ): + self._set_error( + states, RuntimeError("Discovery lost all routing peers; reconnecting to seeds") + ) + self._shutdown_dht_once(dht) + dht = None + break + health = module_infos_route_health(module_infos) + if self._discover_text: + from drift.text_mesh import discover_text_peers + + with self._group_io_locks[initial_peers]: + health["text_peers"] = discover_text_peers( + dht, manifest, revocations=state.revocations, replay_guard=state.replay_guard + ) + if callable(getattr(dht, "get", None)): + try: + with self._group_io_locks[initial_peers]: + health["reservations"] = self._read_intents(state, dht) + except Exception: + health["reservations"] = None + self._set_success(state, health) any_success = True if callable(getattr(dht, "get", None)): try: @@ -559,7 +598,7 @@ def _run_group(self, initial_peers: Tuple[str, ...], states: Tuple[_TargetState, self._set_error((state,), exc) logger.warning("Coverage discovery failed for %s: %s", manifest.digest_id, exc) - if any_success and self._peer_cache is not None: + if any_success and dht is not None and self._peer_cache is not None: try: connected_peers = self._peer_snapshot(dht) for cache_scope in dict.fromkeys(state.target.cache_scope or initial_peers for state in states): @@ -578,6 +617,38 @@ def _run_group(self, initial_peers: Tuple[str, ...], states: Tuple[_TargetState, except Exception: logger.exception("Failed to close a coverage-discovery DHT") + def _read_intents(self, state, dht): + wrapped = dht.get(f"{state.target.manifest.dht_prefix}.intent-v1", latest=True) + container = getattr(wrapped, "value", wrapped) + if container is None: + return [] + if not isinstance(container, Mapping) or len(container) > 256: + return None + reservations = [] + for subkey, value in container.items(): + source = getattr(value, "value", value) + try: + if not isinstance(source, Mapping) or not _bounded_route_demand_source(source): + continue + record = verify_intent_lease( + source, expected_manifest_digest=state.target.manifest.digest, revocations=state.revocations + ) + payload = record.payload + if subkey != record.key_id or payload["end_block"] > state.target.manifest.model.num_blocks: + continue + reservations.append( + { + "peer_id": payload["peer_id"], + "start_block": payload["start_block"], + "end_block": payload["end_block"], + "expires_at": payload["expires_at_ms"] / 1000, + "artifact_bytes": payload["resource_claims"]["artifact_bytes"], + } + ) + except (ProtocolSecurityError, TypeError, ValueError): + continue + return reservations + def publish_intent(self, digest_id: str, source: Mapping[str, Any]) -> bool: """Publish one verified, expiring intent to at least one remote DHT peer.""" diff --git a/src/drift/node/hardware_status.py b/src/drift/node/hardware_status.py new file mode 100644 index 000000000..a94f4d0f3 --- /dev/null +++ b/src/drift/node/hardware_status.py @@ -0,0 +1,83 @@ +"""Small, read-only hardware inventory for the desktop's resource summary.""" + +from __future__ import annotations + +import math +import platform +from pathlib import Path + + +def cpu_name() -> str: + if platform.system() == "Windows": + try: + import winreg + + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"HARDWARE\DESCRIPTION\System\CentralProcessor\0") as key: + return str(winreg.QueryValueEx(key, "ProcessorNameString")[0]).strip() + except OSError: + pass + elif platform.system() == "Linux": + try: + for line in Path("/proc/cpuinfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("model name"): + return line.partition(":")[2].strip() + except OSError: + pass + return platform.processor().strip() or "CPU model unavailable" + + +class HardwareStatus: + """Cache device identity once; calculate saved budgets without allocating tensors.""" + + def __init__(self, config): + import torch + + from drift.utils.hardware import auto_detect_device, get_device_total_memory, normalize_device + + self.inventory = { + "cpu_name": cpu_name(), + "gpu_name": None, + "gpu_total_bytes": None, + "gpu_device": None, + "device": "cpu", + } + self.shared_pool = None + try: + selected = next((worker.device for worker in config.workers if worker.device), None) + detected = normalize_device(torch.device(auto_detect_device())) + device = normalize_device(torch.device(selected)) if selected else detected + self.inventory["device"] = str(device) + gpu = detected if device.type == "cpu" else device + if gpu.type == "cpu": + return + backend = getattr(torch, gpu.type, None) + name = getattr(backend, "get_device_name", None) + self.inventory["gpu_name"] = name(gpu) if name is not None else gpu.type.upper() + self.inventory["gpu_device"] = str(gpu) + total = int(get_device_total_memory(gpu)) + self.inventory["gpu_total_bytes"] = total + if device.type == "cpu": + self.shared_pool = 0 + return + self.shared_pool = total + except (RuntimeError, ValueError, OSError, AssertionError): + # Hardware reporting must not prevent the API or desktop from starting. + self.inventory["device"] = "unknown" + + def snapshot(self, policy): + from drift.node.config import ContributionPolicyConfig + + parsed = ContributionPolicyConfig.from_dict(policy) + total = self.inventory["gpu_total_bytes"] + budget = None + if total is not None and self.shared_pool is not None: + requested = parsed.max_vram_bytes + if requested is None: + requested = math.floor(total * (parsed.max_vram_fraction or 1.0)) + budget = min(self.shared_pool, requested) + return { + **self.inventory, + "sharing_vram_bytes": budget, + "sharing_vram_available_bytes": self.shared_pool, + "processing_percent": parsed.max_processing_percent, + } diff --git a/src/drift/node/loading.py b/src/drift/node/loading.py index 814ec8ce2..160430756 100644 --- a/src/drift/node/loading.py +++ b/src/drift/node/loading.py @@ -8,13 +8,55 @@ from hivemind.utils.logging import get_logger -from drift.model_manifest import ManifestArtifactVerifier, ModelManifest +from drift.model_manifest import ManifestArtifactVerifier, ManifestError, ModelManifest from drift.node.model_manager import ModelRuntime from drift.node.route_health import sequence_manager_route_health logger = get_logger(__name__) +def make_text_peer_loader(manifest, *, initial_peers, revocation_files=(), request_timeout=30): + """Build a consumer with no tokenizer, model tensors or weight downloads.""" + + def load(): + from drift.protocol_identity import RevocationStore + from drift.text_mesh import TextPeerClient + + client = TextPeerClient( + manifest, + initial_peers=initial_peers, + revocations=RevocationStore.from_files(revocation_files), + request_timeout=request_timeout, + ) + return ModelRuntime(model=None, tokenizer=None, text_client=client, close=client.close) + + return load + + +def validate_manifest_execution(manifest: ModelManifest, execution: str) -> None: + """Reject catalog architectures absent from this runtime without fetching weights.""" + from transformers.models.auto.modeling_auto import ( + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES, + ) + + from drift.utils.auto_config import _CLASS_MAPPING + + if execution == "local": + from drift.node.local_inference import local_weight_bytes + + local_weight_bytes(manifest) + mappings = [*MODEL_FOR_CAUSAL_LM_MAPPING_NAMES.values(), *MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES.values()] + architecture = manifest.model.architecture + else: + mappings = [MODEL_FOR_CAUSAL_LM_MAPPING_NAMES.get(model_type, ()) for model_type in _CLASS_MAPPING] + # Supported multimodal checkpoints use the registered text-only adapter. + architecture = manifest.model.architecture.replace("ForConditionalGeneration", "ForCausalLM") + supported = {name for value in mappings for name in ((value,) if isinstance(value, str) else value)} + if architecture not in supported: + raise ManifestError(f"This runtime cannot execute {manifest.model.architecture!r} in {execution} mode") + + def make_manifest_loader( manifest: ModelManifest, *, diff --git a/src/drift/node/local_inference.py b/src/drift/node/local_inference.py new file mode 100644 index 000000000..48edb60bb --- /dev/null +++ b/src/drift/node/local_inference.py @@ -0,0 +1,217 @@ +"""Verified standalone inference with finite context, residency and generation budgets.""" + +from __future__ import annotations + +import gc +import threading +from typing import Callable + +from drift.model_manifest import ManifestArtifactVerifier, ManifestError, ModelManifest +from drift.node.config import NodeModelConfig +from drift.node.model_manager import ModelRuntime + +_GIB = 1024**3 +_WEIGHT_ROLES = {"weight", "quantized_weight", "converted_weight"} + + +def local_weight_bytes(manifest: ModelManifest) -> int: + if manifest.runtime.quantization != "none": + raise ManifestError("Standalone execution requires an explicitly qualified unquantized manifest") + if manifest.runtime.adapter_profile != "none": + raise ManifestError("Standalone adapters are not supported") + return sum(artifact.size for artifact in manifest.artifacts_for_roles(_WEIGHT_ROLES)) + + +def local_device(config: NodeModelConfig, manifest: ModelManifest) -> str: + """Admit against available memory before any weight download or CUDA allocation.""" + import psutil + import torch + + weights = local_weight_bytes(manifest) + # The reserve covers temporary activations/cache and allocator workspace. Admission + # is also checked against actual resident bytes after loading and per generation. + required = weights + _GIB + if required > config.local_max_memory_bytes: + raise MemoryError("The local model and runtime reserve exceed the configured memory budget") + if sum(a.size for a in manifest.artifacts) > config.local_max_disk_bytes: + raise OSError("The verified local model exceeds the configured download/storage budget") + requested = config.local_device + if requested != "cpu" and torch.cuda.is_available(): + device = "cuda:0" if requested == "auto" else str(torch.device(requested)) + free, _total = torch.cuda.mem_get_info(device) + if free >= required + 256 * 1024**2: + return device + if requested != "auto": + raise MemoryError("Insufficient free GPU memory for the local model") + elif requested.startswith("cuda"): + raise RuntimeError("The configured local CUDA device is unavailable") + if psutil.virtual_memory().available < required + 256 * 1024**2: + raise MemoryError("Insufficient available RAM for the local model") + return "cpu" + + +def local_route_observer(manifest: ModelManifest, config: NodeModelConfig) -> Callable[[], dict]: + def observe() -> dict: + try: + device = local_device(config, manifest) + except (RuntimeError, ValueError, OSError, MemoryError) as exc: + return {"status": "unavailable", "source": "local", "last_error": str(exc)} + return { + "status": "complete", + "source": "local", + "device": device, + "total_blocks": manifest.model.num_blocks, + "covered_blocks": manifest.model.num_blocks, + "missing_blocks": [], + "minimum_replicas": 1, + "replica_counts": [1] * manifest.model.num_blocks, + "peer_count": 0, + "last_updated_age": 0.0, + } + + return observe + + +class LocalInferenceModel: + """Own model residency and serialize bounded local generations.""" + + def __init__( + self, model, config: NodeModelConfig, manifest: ModelManifest, device: str, *, previous_cuda_fraction=None + ): + self._model = model + self.config = config + self.manifest = manifest + self.device = device + self._lock = threading.Lock() + self._closed = threading.Event() + self._previous_cuda_fraction = previous_cuda_fraction + + @property + def generation_token_limit(self): + return self.config.local_max_new_tokens + + def validate_generation(self, input_ids, kwargs): + new_tokens = kwargs.get("max_new_tokens", self.config.local_max_new_tokens) + if type(new_tokens) is not int or not 1 <= new_tokens <= self.config.local_max_new_tokens: + raise ValueError("Requested generation exceeds the local token budget") + context_limit = min(self.config.local_max_context, self.manifest.model.context_length) + if input_ids.ndim != 2 or input_ids.shape[0] != 1 or input_ids.shape[1] + new_tokens > context_limit: + raise ValueError("Requested conversation exceeds the local context budget") + + def generate(self, input_ids, *, streamer=None, **kwargs): + import torch + from transformers import StoppingCriteria, StoppingCriteriaList + + owner = self + + class StopOnClose(StoppingCriteria): + def __call__(self, input_ids, scores, **unused): + return owner._closed.is_set() + + with self._lock: + if self._closed.is_set(): + raise RuntimeError("Local runtime is closed") + self.validate_generation(input_ids, kwargs) + kwargs.setdefault("max_new_tokens", self.config.local_max_new_tokens) + kwargs["max_time"] = self.config.local_max_seconds + kwargs["stopping_criteria"] = StoppingCriteriaList([*kwargs.get("stopping_criteria", ()), StopOnClose()]) + with torch.inference_mode(): + output = self._model.generate(input_ids.to(self.device), streamer=streamer, **kwargs) + return output.cpu() + + def close(self): + import torch + + self._closed.set() + with self._lock: + self._model = None + gc.collect() + if self.device.startswith("cuda"): + with torch.cuda.device(self.device): + torch.cuda.empty_cache() + if self._previous_cuda_fraction is not None: + torch.cuda.set_per_process_memory_fraction(self._previous_cuda_fraction, self.device) + self._previous_cuda_fraction = None + + def route_health(self): + import torch + + result = { + "status": "unavailable" if self._closed.is_set() else "complete", + "source": "local", + "device": self.device, + "total_blocks": self.manifest.model.num_blocks, + "covered_blocks": self.manifest.model.num_blocks, + "peer_count": 0, + "last_updated_age": 0.0, + } + if self.device.startswith("cuda") and not self._closed.is_set(): + result["memory"] = { + "budget_bytes": self.config.local_max_memory_bytes, + "allocated_bytes": torch.cuda.memory_allocated(self.device), + "reserved_bytes": torch.cuda.memory_reserved(self.device), + "peak_allocated_bytes": torch.cuda.max_memory_allocated(self.device), + "peak_reserved_bytes": torch.cuda.max_memory_reserved(self.device), + } + return result + + +def make_local_manifest_loader(manifest: ModelManifest, config: NodeModelConfig) -> Callable[[], ModelRuntime]: + def load(): + import torch + from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForImageTextToText, AutoTokenizer + + local_device(config, manifest) + verifier = ManifestArtifactVerifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=False, + cache_dir=str(config.cache_dir) if config.cache_dir else None, + ) + verifier.ensure_startup_metadata(include_tokenizer=True) + for artifact in manifest.artifacts: + verifier.ensure_path(artifact.path) + stock_config = AutoConfig.from_pretrained( + verifier.snapshot_root, local_files_only=True, trust_remote_code=False + ) + model_class = ( + AutoModelForImageTextToText if getattr(stock_config, "text_config", None) else AutoModelForCausalLM + ) + # Downloads can take minutes. Choose from the memory available now, + # without reserving GPU memory or reducing contribution limits. + device = local_device(config, manifest) + model = None + previous_fraction = None + try: + if device.startswith("cuda"): + previous_fraction = torch.cuda.get_per_process_memory_fraction(device) + total = torch.cuda.get_device_properties(device).total_memory + limit = (torch.cuda.memory_reserved(device) + config.local_max_memory_bytes) / total + torch.cuda.set_per_process_memory_fraction(min(previous_fraction, limit), device) + model = model_class.from_pretrained( + verifier.snapshot_root, + config=stock_config, + local_files_only=True, + trust_remote_code=False, + torch_dtype=getattr(torch, manifest.runtime.dtype), + attn_implementation=manifest.runtime.attention_implementation, + device_map={"": device}, + ).eval() + if model.get_memory_footprint() + _GIB > config.local_max_memory_bytes: + raise MemoryError("Loaded local model exceeds the configured resident-memory budget") + tokenizer = AutoTokenizer.from_pretrained( + verifier.snapshot_root, local_files_only=True, trust_remote_code=False + ) + owned = LocalInferenceModel(model, config, manifest, device, previous_cuda_fraction=previous_fraction) + return ModelRuntime(model=owned, tokenizer=tokenizer, close=owned.close, route_health=owned.route_health) + except BaseException: + model = None + gc.collect() + if device.startswith("cuda"): + torch.cuda.empty_cache() + if previous_fraction is not None: + torch.cuda.set_per_process_memory_fraction(previous_fraction, device) + raise + + return load diff --git a/src/drift/node/model_manager.py b/src/drift/node/model_manager.py index 3f2b9ed3b..33fe2c866 100644 --- a/src/drift/node/model_manager.py +++ b/src/drift/node/model_manager.py @@ -15,6 +15,7 @@ from typing import Any, Callable, Dict, Iterable, Optional, Tuple from drift.model_manifest import ModelManifest +from drift.utils.download_progress import DownloadProgress logger = logging.getLogger(__name__) @@ -66,6 +67,7 @@ class ModelRuntime: close: Optional[Callable[[], None]] = None route_health: Optional[Callable[[], Dict[str, Any]]] = None cleanup_health: Optional[Callable[[], Dict[str, Any]]] = None + text_client: Any = None @dataclass(frozen=True) @@ -78,8 +80,11 @@ class ModelDescriptor: repository: Optional[str] = None name: Optional[str] = None selected_whole_shard_bytes: Optional[int] = None + execution: str = "distributed" def __post_init__(self) -> None: + if self.execution not in ("distributed", "local"): + raise ValueError("execution must be distributed or local") identifiers = (self.model_id, *self.aliases) if any(not isinstance(value, str) or not value.strip() for value in identifiers): raise ValueError("model identifiers must be non-empty strings") @@ -89,10 +94,10 @@ def __post_init__(self) -> None: if self.selected_whole_shard_bytes is not None and ( isinstance(self.selected_whole_shard_bytes, bool) or not isinstance(self.selected_whole_shard_bytes, int) - or not 1 <= self.selected_whole_shard_bytes <= MAX_SELECTED_WHOLE_SHARD_BYTES + or not 0 <= self.selected_whole_shard_bytes <= MAX_SELECTED_WHOLE_SHARD_BYTES ): raise ValueError( - "selected_whole_shard_bytes must be None or an integer between 1 and " + "selected_whole_shard_bytes must be None or an integer between 0 and " f"{MAX_SELECTED_WHOLE_SHARD_BYTES}" ) @@ -131,6 +136,7 @@ class ModelSnapshot: last_used_at: Optional[float] active_requests: int route: Optional[Dict[str, Any]] + progress: Optional[Dict[str, Any]] = None def to_dict(self) -> Dict[str, Any]: return { @@ -141,6 +147,7 @@ def to_dict(self) -> Dict[str, Any]: "download": { "schema_version": MODEL_DOWNLOAD_SCHEMA_VERSION, "selected_whole_shard_bytes": self.selected_whole_shard_bytes, + **({"progress": self.progress} if self.progress is not None else {}), }, "state": self.state.value, "last_error": self.last_error, @@ -196,6 +203,7 @@ class _ModelRecord: active_requests: int = 0 close_failed: bool = False load_lock: threading.Lock = field(default_factory=threading.Lock) + progress: Optional[DownloadProgress] = None class ModelManager: @@ -213,7 +221,44 @@ def __init__(self, *, max_loaded_models: Optional[int] = None) -> None: self._max_loaded_models = max_loaded_models self._shutdown_callbacks: list[Callable[[], None]] = [] self._auto_priority: Tuple[str, ...] = () + self._local_only = False + self._selection_policy: Optional[Callable[[ModelDescriptor, Dict[str, Any]], bool]] = None self._closed = False + self._draining = False + self._catalog_models: Optional[frozenset[str]] = None + + def set_catalog_models(self, digests: Iterable[str]) -> None: + with self._lock: + self._catalog_models = frozenset(digests) + + @property + def inference_mode(self) -> str: + with self._lock: + return "local_only" if self._local_only else "auto" + + def set_inference_mode(self, mode: str) -> None: + if mode not in ("auto", "local_only"): + raise ValueError("inference mode must be auto or local_only") + with self._lock: + self._local_only = mode == "local_only" + + def catalog_allows_contribution(self, digest: str) -> bool: + with self._lock: + return self._catalog_models is None or digest in self._catalog_models + + def begin_idle_restart(self) -> bool: + """Atomically stop admission only after existing leases and loads finish.""" + with self._capacity_changed: + if self._closed or self._draining: + return False + if any( + record.active_requests or record.state in (ModelState.LOADING, ModelState.UNLOADING) + for record in self._records.values() + ): + return False + self._draining = True + self._capacity_changed.notify_all() + return True def register( self, @@ -261,10 +306,13 @@ def add_shutdown_callback(self, callback: Callable[[], None]) -> None: raise ModelManagerClosedError("model manager is shutting down") self._shutdown_callbacks.append(callback) - def configure_auto_selection(self, identifiers: Iterable[str]) -> None: + def configure_auto_selection(self, identifiers: Iterable[str], *, local_only: bool = False) -> None: """Bind auto to catalog priority while keeping exact selectors unchanged.""" requested = tuple(identifiers) + if not isinstance(local_only, bool): + raise ValueError("local_only must be boolean") with self._lock: + self._local_only = local_only if self._closed: raise ModelManagerClosedError("model manager is shutting down") if not requested: @@ -284,6 +332,11 @@ def configure_auto_selection(self, identifiers: Iterable[str]) -> None: raise ValueError("auto model priority must not select the same model more than once") self._auto_priority = tuple(resolved) + def set_selection_policy(self, policy: Optional[Callable[[ModelDescriptor, Dict[str, Any]], bool]]) -> None: + """Install the catalog eligibility gate; status reads never run network probes.""" + with self._lock: + self._selection_policy = policy + def register_loaded( self, model_id: str, model: Any, tokenizer: Any, *, aliases: Iterable[str] = () ) -> ModelDescriptor: @@ -302,7 +355,7 @@ def register_loaded( def _record_for(self, identifier: Optional[str]) -> _ModelRecord: with self._lock: - if self._closed: + if self._closed or self._draining: raise ModelManagerClosedError("model manager is shutting down") if identifier is None: if len(self._records) == 1: @@ -340,7 +393,7 @@ def _reserve_runtime_slot(self, target: _ModelRecord) -> None: candidate: Optional[_ModelRecord] = None candidate_runtime: Optional[ModelRuntime] = None with self._capacity_changed: - if self._closed: + if self._closed or self._draining: raise ModelManagerClosedError("model manager is shutting down") if self._max_loaded_models is None or self._resident_count_locked() < self._max_loaded_models: target.state = ModelState.LOADING @@ -425,9 +478,11 @@ def load(self, identifier: Optional[str]) -> LoadedModel: The returned lease must be released when the request finishes. """ record = self._record_for(identifier) + if self._local_only and record.descriptor.execution != "local": + raise AutoModelUnavailableError("This installation is configured for local-only inference") with record.load_lock: with self._lock: - if self._closed: + if self._closed or self._draining: raise ModelManagerClosedError("model manager is shutting down") if record.runtime is not None: if record.close_failed: @@ -436,16 +491,20 @@ def load(self, identifier: Optional[str]) -> LoadedModel: ) return self._lease_locked(record, record.runtime) self._reserve_runtime_slot(record) + record.progress = DownloadProgress() try: - runtime = record.loader() + with record.progress.observe(): + runtime = record.loader() if not isinstance(runtime, ModelRuntime): raise TypeError("model loader must return ModelRuntime") except BaseException as exc: + record.progress.finish("failed") with self._lock: record.state = ModelState.STOPPING if self._closed else ModelState.UNAVAILABLE record.last_error = f"{type(exc).__name__}: {exc}" self._capacity_changed.notify_all() raise + record.progress.finish("ready") with self._lock: if self._closed: record.state = ModelState.STOPPING @@ -526,6 +585,15 @@ def _read_route(self, record: _ModelRecord) -> Optional[Dict[str, Any]]: route = reader() if not isinstance(route, dict): raise TypeError("route health reader must return a dictionary") + if record.runtime is not None and record.route_health is not None and reader is not record.route_health: + try: + discovery = record.route_health() + except Exception: + discovery = None + if isinstance(discovery, dict): + route = {**route, "reservations": discovery.get("reservations")} + if "peers" not in route: + route["peers"] = discovery.get("peers", []) return dict(route) except Exception: logger.exception("Failed to read route health for model %r", record.descriptor.model_id) @@ -544,8 +612,15 @@ def _auto_selection_locked(self) -> Dict[str, Any]: "peer_count": None, "source": None, } - for priority, model_id in enumerate(self._auto_priority, start=1): + # A standalone fallback is always considered after the community candidates. + priorities = sorted( + self._auto_priority, key=lambda model_id: self._records[model_id].descriptor.execution == "local" + ) + for priority, model_id in enumerate(priorities, start=1): record = self._records[model_id] + local = record.descriptor.execution == "local" + if self._local_only and not local: + continue route = self._read_route(record) if route is None: continue @@ -562,8 +637,11 @@ def _auto_selection_locked(self) -> Dict[str, Any]: and covered == total and isinstance(peers, int) and not isinstance(peers, bool) - and peers > 0 + and (peers == 0 if local else peers > 0) + and (local or route.get("chat_ready", True)) ) + if complete and not local and self._selection_policy is not None: + complete = self._selection_policy(record.descriptor, route) if complete: peer_label = "peer" if peers == 1 else "peers" return { @@ -571,7 +649,9 @@ def _auto_selection_locked(self) -> Dict[str, Any]: "status": "selected", "model": model_id, "manifest_digest": record.descriptor.manifest_digest, - "reason": ( + "reason": "Selected a verified standalone model on this computer." + if local + else ( f"Selected catalog priority {priority}: live discovery reports a complete " f"{covered}/{total}-block route from {peers} verified {peer_label}." ), @@ -618,6 +698,7 @@ def snapshots(self) -> Tuple[ModelSnapshot, ...]: last_used_at=record.last_used_at, active_requests=record.active_requests, route=route, + progress=None if record.progress is None else record.progress.snapshot(), ) ) return tuple(snapshots) diff --git a/src/drift/node/model_selection.py b/src/drift/node/model_selection.py new file mode 100644 index 000000000..72afe28f3 --- /dev/null +++ b/src/drift/node/model_selection.py @@ -0,0 +1,252 @@ +"""Local measured catalog eligibility; no request text or peer identities retained.""" + +from __future__ import annotations + +import math +import threading +import time +from collections import Counter +from dataclasses import dataclass, field +from typing import Callable, Optional + +from drift.model_catalog import CapacityObservation, ModelCatalog, ModelCatalogError, select_highest_eligible_model + + +@dataclass +class _Measurements: + fingerprint: str + stable_since: float + observed_at: float + measured_at: Optional[float] = None + ttft_histogram: Counter = field(default_factory=Counter) + completion_tokens: int = 0 + generation_seconds: float = 0.0 + samples: int = 0 + + +class MeasuredModelSelector: + """Require local probes plus fresh authenticated coverage for new auto requests.""" + + def __init__(self, catalog: ModelCatalog, health_reader: Callable[[str], dict], *, clock=time.time): + self.catalog = catalog + self._health_reader = health_reader + self._clock = clock + self._measurements = {} + self._lock = threading.RLock() + + def _observe(self, digest, health): + now = self._clock() + fingerprint = health.get("coverage_fingerprint") + age = health.get("last_updated_age") + if ( + health.get("status") != "complete" + or not isinstance(fingerprint, str) + or len(fingerprint) != 64 + or isinstance(age, bool) + or not isinstance(age, (int, float)) + or not math.isfinite(age) + or age < 0 + ): + self._measurements.pop(digest, None) + return None + model = next((m for m in self.catalog.models if m.manifest_digest == digest), None) + if model is None: + return None + rung = next(r for r in self.catalog.rungs if r.rung_id == model.rung_id) + if age > rung.maximum_observation_age_seconds: + self._measurements.pop(digest, None) + return None + previous = self._measurements.get(digest) + if ( + previous is None + or previous.fingerprint != fingerprint + or now - previous.observed_at > rung.maximum_observation_age_seconds + ): + previous = _Measurements(fingerprint, now - age, now) + self._measurements[digest] = previous + previous.observed_at = now + # Expire old performance evidence without discarding continuous coverage. + if previous.measured_at is not None and now - previous.measured_at > 600: + previous.measured_at = None + previous.ttft_histogram.clear() + previous.completion_tokens = 0 + previous.generation_seconds = 0 + previous.samples = 0 + return previous + + def probe_target(self): + """Return one complete candidate needing measurement; never start I/O here.""" + with self._lock: + try: + self.catalog.validate_time(now=self._clock()) + except ModelCatalogError: + return None + for model in sorted( + self.catalog.models, key=lambda m: next(r.order for r in self.catalog.rungs if r.rung_id == m.rung_id) + ): + if getattr(model, "execution", None) == "local": + continue + state = self._observe(model.manifest_digest, self._health_reader(model.manifest_digest)) + if state is not None and state.measured_at is None: + return model.manifest_digest, state.fingerprint + return None + + def record_probe(self, digest, fingerprint, *, first_token_seconds, completion_tokens, duration_seconds): + if any( + isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v <= 0 + for v in (first_token_seconds, duration_seconds) + ): + raise ValueError("Probe durations must be finite and positive") + if type(completion_tokens) is not int or completion_tokens < 1 or first_token_seconds > duration_seconds: + raise ValueError("Probe must generate tokens and report consistent timing") + with self._lock: + state = self._observe(digest, self._health_reader(digest)) + if state is None or state.fingerprint != fingerprint: + return False + # Fixed-width latency buckets retain only aggregates and conservatively + # round latency up. No prompts, outputs, request ids or users are retained. + bucket_ms = math.ceil(first_token_seconds * 10) * 100 + if bucket_ms > 3_600_000 or state.samples >= 4096: + return False + state.ttft_histogram[bucket_ms] += 1 + state.completion_tokens += completion_tokens + state.generation_seconds += duration_seconds + state.samples += 1 + state.measured_at = self._clock() + return True + + def selection(self): + with self._lock: + try: + self.catalog.validate_time(now=self._clock()) + except ModelCatalogError: + return None, () + observations = [] + for model in self.catalog.models: + if getattr(model, "execution", None) == "local": + continue + health = self._health_reader(model.manifest_digest) + state = self._observe(model.manifest_digest, health) + if state is None or state.measured_at is None: + continue + rank = math.ceil(state.samples * 0.95) + p95 = 0 + for upper, count in sorted(state.ttft_histogram.items()): + rank -= count + if rank <= 0: + p95 = upper + break + observations.append( + CapacityObservation( + manifest_digest=model.manifest_digest, + observed_at_ms=int((self._clock() - health["last_updated_age"]) * 1000), + stable_since_ms=int(state.stable_since * 1000), + bottleneck_replicas=health.get("minimum_replicas", 0), + independent_routes=health.get("independent_routes", 0), + replicas_after_largest_peer_loss=health.get("replicas_after_largest_peer_loss", 0), + p95_first_token_ms=p95, + tokens_per_minute=math.floor(state.completion_tokens * 60 / state.generation_seconds), + ) + ) + return select_highest_eligible_model(self.catalog, observations, now=self._clock()) + + def allows(self, descriptor, _health): + selected, _evaluations = self.selection() + return selected is not None and selected.manifest_digest == descriptor.manifest_digest + + +class FirstTokenTimer: + """HF streamer measuring first generated token, excluding the initial prompt put.""" + + def __init__(self): + self.started = time.monotonic() + self.first_token_seconds = None + self._prompt_seen = False + + def put(self, value): + if not self._prompt_seen: + self._prompt_seen = True + elif self.first_token_seconds is None: + self.first_token_seconds = max(1e-9, time.monotonic() - self.started) + + def end(self): + pass + + +class RouteProbeService: + """Bounded synthetic probes populate eligibility without retaining user content.""" + + def __init__(self, manager, selector: MeasuredModelSelector, *, period=5): + self.manager, self.selector, self.period = manager, selector, period + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, name="drift-route-probes", daemon=True) + self._observer_thread = threading.Thread( + target=self._observe_routes, name="drift-route-observations", daemon=True + ) + + def start(self): + self._observer_thread.start() + self._thread.start() + + def close(self): + self._stop.set() + if self._observer_thread.is_alive(): + self._observer_thread.join(timeout=1) + if self._thread.is_alive(): + self._thread.join(timeout=1) + + def _observe_routes(self): + import logging + + # Generations and probes can exceed the catalog's freshness window. + # Read discovery independently so continuous soak does not depend on UI + # status polling or end when a long answer occupies the probe thread. + while not self._stop.wait(self.period): + try: + self.selector.selection() + except Exception: + logging.getLogger(__name__).exception("Community route observation failed") + + def _run(self): + import logging + + import torch + + while not self._stop.wait(self.period): + if self.manager.inference_mode == "local_only": + continue + if any(snapshot.active_requests for snapshot in self.manager.snapshots()): + continue + target = self.selector.probe_target() + if target is None: + continue + digest, fingerprint = target + try: + with self.manager.load(digest) as loaded: + if self._stop.is_set(): + return + inputs = loaded.runtime.tokenizer("The capital of France is", return_tensors="pt").input_ids + timer = FirstTokenTimer() + with torch.inference_mode(): + outputs = loaded.runtime.model.generate( + inputs, max_new_tokens=3, do_sample=False, streamer=timer + ) + duration = time.monotonic() - timer.started + if timer.first_token_seconds is not None: + accepted = self.selector.record_probe( + digest, + fingerprint, + first_token_seconds=timer.first_token_seconds, + completion_tokens=outputs.shape[1] - inputs.shape[1], + duration_seconds=duration, + ) + logging.getLogger(__name__).info( + "Community route measurement: retained=%s first_token_seconds=%.3f " + "completion_tokens=%s duration_seconds=%.3f", + accepted, + timer.first_token_seconds, + outputs.shape[1] - inputs.shape[1], + duration, + ) + except Exception: + logging.getLogger(__name__).exception("Community route probe failed") diff --git a/src/drift/node/policy_store.py b/src/drift/node/policy_store.py index 402c1c268..8063efbb9 100644 --- a/src/drift/node/policy_store.py +++ b/src/drift/node/policy_store.py @@ -275,6 +275,22 @@ def _atomic_replace_locked(self, payload: bytes, *, expected_revision: str) -> N except OSError: pass + def update_inference_mode(self, mode: str, *, expected_revision: str) -> dict[str, Any]: + if mode not in ("auto", "local_only"): + raise NodeConfigError("inference mode must be auto or local_only") + with self._lock: + if expected_revision != self._revision: + raise ContributionPolicyConflictError("node config changed; refresh before saving") + document, payload = self._read() + if _revision(payload) != self._revision: + raise ContributionPolicyConflictError("node config changed; refresh before saving") + document["inference_mode"] = mode + NodeConfig.from_dict(document, base_dir=self.path.parent) + encoded = (json.dumps(document, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8") + self._atomic_replace(encoded, expected_revision=self._revision) + self._revision = _revision(encoded) + return {"inference_mode": mode, "config_revision": self._revision} + def update(self, source: Mapping[str, Any], *, expected_revision: str) -> dict[str, Any]: if not isinstance(expected_revision, str) or not expected_revision.startswith("sha256:"): raise ContributionPolicyConflictError("policy update has an invalid config revision") diff --git a/src/drift/node/route_health.py b/src/drift/node/route_health.py index 4a4e4aa7f..a7b766d4d 100644 --- a/src/drift/node/route_health.py +++ b/src/drift/node/route_health.py @@ -2,7 +2,10 @@ from __future__ import annotations +import hashlib +import json import time +from collections import Counter from typing import Any, Dict, Sequence from drift.data_structures import RemoteModuleInfo, ServerState @@ -20,6 +23,38 @@ def _coverage_health(replica_sets: Sequence[set], *, updated_age: float) -> Dict total_blocks = len(replica_sets) covered_blocks = total_blocks - len(missing_blocks) + peer_coverage = Counter(peer for peers in replica_sets for peer in peers) + largest_count = max(peer_coverage.values(), default=0) + surviving = min( + ( + min((len(peers - {peer}) for peers in replica_sets), default=0) + for peer, count in peer_coverage.items() + if count == largest_count + ), + default=0, + ) + available = set(peer_ids) + independent = 0 + # Greedy disjoint routes are a conservative lower bound. A peer used anywhere + # in one route cannot be counted as an independent peer in another route. + while replica_sets and all(peers & available for peers in replica_sets): + used = set() + cursor = 0 + while cursor < total_blocks: + candidates = [] + for peer in replica_sets[cursor] & available: + end = cursor + 1 + while end < total_blocks and peer in replica_sets[end]: + end += 1 + candidates.append((end, str(peer), peer)) + end, _label, peer = max(candidates, key=lambda item: item[:2]) + used.add(peer) + cursor = end + independent += 1 + available.difference_update(used) + fingerprint = hashlib.sha256( + json.dumps([sorted(str(peer) for peer in peers) for peers in replica_sets], separators=(",", ":")).encode() + ).hexdigest() return { "status": "complete" if not missing_blocks else "incomplete", "total_blocks": total_blocks, @@ -29,6 +64,9 @@ def _coverage_health(replica_sets: Sequence[set], *, updated_age: float) -> Dict "replica_counts": replica_counts, "peer_count": len(peer_ids), "last_updated_age": max(0.0, updated_age), + "independent_routes": independent, + "replicas_after_largest_peer_loss": surviving, + "coverage_fingerprint": fingerprint, } @@ -38,7 +76,35 @@ def module_infos_route_health(module_infos: Sequence[RemoteModuleInfo]) -> Dict[ {peer_id for peer_id, server_info in module_info.servers.items() if server_info.state is ServerState.ONLINE} for module_info in module_infos ] - return _coverage_health(replica_sets, updated_age=0.0) + return {**_coverage_health(replica_sets, updated_age=0.0), **_peer_details(module_infos)} + + +def _peer_details(module_infos): + peers = {} + joining = [0] * len(module_infos) + offline = [0] * len(module_infos) + for index, module in enumerate(module_infos): + for peer_id, info in module.servers.items(): + state = info.state.name.lower() + if state == "joining": + joining[index] += 1 + elif state == "offline": + offline[index] += 1 + peer = peers.setdefault( + str(peer_id), + {"peer_id": str(peer_id), "online_blocks": [], "joining_blocks": [], "offline_blocks": []}, + ) + peer[f"{state}_blocks"].append(index) + for field in ("public_name", "version", "torch_dtype", "quant_type"): + value = getattr(info, field, None) + peer[field] = " ".join(value.split())[:128] if isinstance(value, str) else None + peer["using_relay"] = getattr(info, "using_relay", None) + return { + "peers": [peers[key] for key in sorted(peers)[:256]], + "peer_details_truncated": len(peers) > 256, + "joining_counts": joining, + "offline_counts": offline, + } def sequence_manager_route_health(sequence_manager) -> Dict[str, Any]: @@ -64,4 +130,7 @@ def sequence_manager_route_health(sequence_manager) -> Dict[str, Any]: } replica_sets = [{span.peer_id for span in spans} for spans in sequence_info.spans_containing_block] - return _coverage_health(replica_sets, updated_age=time.perf_counter() - sequence_info.last_updated_time) + result = _coverage_health(replica_sets, updated_age=time.perf_counter() - sequence_info.last_updated_time) + if hasattr(sequence_info, "block_infos"): + result.update(_peer_details(sequence_info.block_infos)) + return result diff --git a/src/drift/node/server.py b/src/drift/node/server.py index 9c054d73a..68425a529 100644 --- a/src/drift/node/server.py +++ b/src/drift/node/server.py @@ -6,7 +6,7 @@ import math import secrets import time -from typing import Callable, List, Optional +from typing import Callable, List, Literal, Optional from fastapi import HTTPException, Request from pydantic import BaseModel @@ -33,11 +33,17 @@ WorkerReconfigurationBusyError, WorkerSupervisor, ) +from drift.utils.download_progress import public_progress CONTROL_API_VERSION = 1 CONTRIBUTION_STATUS_SCHEMA_VERSION = 3 +class InferenceModeRequest(BaseModel): + inference_mode: Literal["auto", "local_only"] + expected_config_revision: str + + def _bounded_text(value, fallback: str, *, limit: int = 300) -> str: if not isinstance(value, str): return fallback @@ -88,6 +94,8 @@ def _contribution_status(worker_snapshots, *, configured: bool, editable: bool, else "unknown" ), "desired_running": snapshot.get("desired_running") is True, + "operator_paused": snapshot.get("operator_paused") is True, + "download_progress": public_progress(snapshot.get("download_progress")), "placement": { "automatic": snapshot.get("automatic") is True, "block_indices": ( @@ -157,6 +165,7 @@ def create_node_app( contribution_policy: Optional[ContributionPolicyConfig] = None, contribution_policy_store: Optional[ContributionPolicyStore] = None, route_outcome_observer: Optional[Callable[..., None]] = None, + hardware_status: Optional[Callable[[dict], dict]] = None, ): """Compose the OpenAI API and authenticated local control surface.""" if api_key_store is None and (not api_keys or any(not isinstance(key, str) or not key for key in api_keys)): @@ -221,6 +230,9 @@ async def node_status(request: Request): "started_at": started_at, "openai_base_url": f"http://{'[' + host + ']' if ':' in host else host}:{port}/v1", "runtime_budget": model_manager.residency(), + "hardware": hardware_status(policy_snapshot["policy"]) if hardware_status is not None else {}, + "inference_mode": model_manager.inference_mode, + "inference_mode_editable": contribution_policy_store is not None, "auto_selection": model_manager.auto_selection_snapshot(), "models": [snapshot.to_dict() for snapshot in model_manager.snapshots()], "workers": [ @@ -240,6 +252,20 @@ async def get_contribution_policy(request: Request): check_control_auth(request) return require_policy_store().snapshot() + @app.put("/control/v1/inference-mode") + async def update_inference_mode(body: InferenceModeRequest, request: Request): + check_control_auth(request) + try: + result = require_policy_store().update_inference_mode( + body.inference_mode, expected_revision=body.expected_config_revision + ) + model_manager.set_inference_mode(body.inference_mode) + return result + except ContributionPolicyConflictError as exc: + raise HTTPException(status_code=412, detail=str(exc)) from exc + except ContributionPolicyPersistenceError as exc: + raise HTTPException(status_code=503, detail="inference mode could not be saved") from exc + @app.put("/control/v1/contribution-policy") async def update_contribution_policy(request: Request): check_control_auth(request) diff --git a/src/drift/node/worker_supervisor.py b/src/drift/node/worker_supervisor.py index 8b1bd7a07..b7f996b54 100644 --- a/src/drift/node/worker_supervisor.py +++ b/src/drift/node/worker_supervisor.py @@ -3,16 +3,23 @@ from __future__ import annotations import collections +import json import logging import math import os +import signal import subprocess +import sys +import tempfile import threading import time from dataclasses import dataclass, field from enum import Enum +from pathlib import Path from typing import Any, Callable, Deque, Dict, Optional, Sequence, Tuple +from drift.utils.resource_limits import DEVICE_MEMORY_BUDGET_EXIT_CODE + logger = logging.getLogger(__name__) @@ -135,7 +142,15 @@ class WorkerLaunch: preferred: bool = False automatic: bool = False block_indices: Optional[str] = None - placement_reason: Optional[str] = None + # Coverage/demand explanations change without changing the worker assignment. + # They must not make the placement reconciler stop a healthy worker. + placement_reason: Optional[str] = field(default=None, compare=False) + intent_published: bool = False + remote_acknowledged: bool = False + placement_manifest_digest: Optional[str] = None + placement_artifact_bytes: Optional[int] = None + placement_artifact_set_digest: Optional[str] = None + placement_cache_root: Optional[str] = None max_disk_bytes: Optional[int] = None max_vram_bytes: Optional[int] = None vram_device: Optional[str] = None @@ -157,6 +172,99 @@ def __post_init__(self) -> None: raise ValueError("automatic workers require a block range and placement reason") if not self.automatic and (self.block_indices is not None or self.placement_reason is not None): raise ValueError("manual workers must not carry automatic placement metadata") + if type(self.intent_published) is not bool or type(self.remote_acknowledged) is not bool: + raise ValueError("placement intent publication fields must be booleans") + if self.intent_published != self.remote_acknowledged: + raise ValueError("placement intent publication requires a remote acknowledgement") + if not self.automatic and self.intent_published: + raise ValueError("manual workers must not carry an acknowledged automatic intent") + if self.automatic and self.policy_admitted and not self.remote_acknowledged: + raise ValueError("admitted automatic workers require a remotely acknowledged intent") + placement_claims = ( + self.placement_manifest_digest, + self.placement_artifact_bytes, + self.placement_artifact_set_digest, + self.placement_cache_root, + ) + if any(value is not None for value in placement_claims) and not all( + value is not None for value in placement_claims + ): + raise ValueError("automatic placement artifact claims must be configured together") + if not self.automatic and any(value is not None for value in placement_claims): + raise ValueError("manual workers must not carry automatic placement artifact claims") + if self.automatic and self.policy_admitted and not all(value is not None for value in placement_claims): + raise ValueError("admitted automatic workers require an exact placement artifact binding") + if self.placement_manifest_digest is not None and ( + not isinstance(self.placement_manifest_digest, str) + or len(self.placement_manifest_digest) != 71 + or not self.placement_manifest_digest.startswith("sha256:") + or any(character not in "0123456789abcdef" for character in self.placement_manifest_digest[7:]) + ): + raise ValueError("placement manifest digest must be canonical sha256") + if self.placement_artifact_bytes is not None and ( + isinstance(self.placement_artifact_bytes, bool) + or not isinstance(self.placement_artifact_bytes, int) + or self.placement_artifact_bytes < 0 + ): + raise ValueError("placement artifact bytes must be a non-negative integer") + if self.placement_artifact_set_digest is not None and ( + not isinstance(self.placement_artifact_set_digest, str) + or len(self.placement_artifact_set_digest) != 64 + or any(character not in "0123456789abcdef" for character in self.placement_artifact_set_digest) + ): + raise ValueError("placement artifact-set digest must be lowercase SHA-256") + if self.placement_cache_root is not None: + if not isinstance(self.placement_cache_root, str) or not self.placement_cache_root: + raise ValueError("placement cache root must be a canonical absolute path") + canonical_cache_root = os.path.realpath(os.path.abspath(os.path.expanduser(self.placement_cache_root))) + if self.placement_cache_root != canonical_cache_root: + raise ValueError("placement cache root must be a canonical absolute path") + + command = self.command + if any(not isinstance(value, str) or not value for value in command): + raise ValueError("placement-bound worker command arguments must be non-empty strings") + if command[0] != sys.executable or os.path.realpath(command[0]) != os.path.realpath(sys.executable): + raise ValueError("placement-bound worker command must use the current node executable") + forbidden_options = ( + "-c", + "--config", + "--custom_module_path", + "--allow_training_rpcs", + "--token", + "--use_auth_token", + ) + if any( + value == option or value.startswith(f"{option}=") or (option == "-c" and value.startswith("-c")) + for value in command + for option in forbidden_options + ): + raise ValueError("placement-bound worker command contains a forbidden server option") + module_entrypoint = len(command) >= 4 and command[1:4] == ("-m", "drift.cli", "server") + frozen_entrypoint = len(command) >= 2 and command[1] == "server" + if not module_entrypoint and not frozen_entrypoint: + raise ValueError("placement-bound worker command must invoke the drift server entrypoint") + if any(value == "--num_blocks" or value.startswith("--num_blocks=") for value in command): + raise ValueError("placement-bound worker command must not use --num_blocks") + + def bound_option(option: str) -> str: + positions = [ + index for index, value in enumerate(command) if value == option or value.startswith(f"{option}=") + ] + if len(positions) != 1 or command[positions[0]] != option or positions[0] + 1 >= len(command): + raise ValueError(f"placement-bound worker command requires exactly one {option}") + return command[positions[0] + 1] + + for option, expected in ( + ("--block_indices", self.block_indices), + ("--expected_block_indices", self.block_indices), + ("--expected_manifest_digest", self.placement_manifest_digest), + ("--expected_artifact_bytes", str(self.placement_artifact_bytes)), + ("--expected_artifact_set_digest", self.placement_artifact_set_digest), + ("--cache_dir", self.placement_cache_root), + ("--expected_cache_root", self.placement_cache_root), + ): + if bound_option(option) != expected: + raise ValueError(f"placement-bound worker command has a mismatched {option}") if self.max_disk_bytes is not None and ( isinstance(self.max_disk_bytes, bool) or not isinstance(self.max_disk_bytes, int) or self.max_disk_bytes < 1 ): @@ -208,6 +316,7 @@ def __post_init__(self) -> None: @dataclass class _WorkerRecord: + progress_directory: Any = field(default=None, init=False, repr=False) launch: WorkerLaunch state: WorkerState = WorkerState.PAUSED desired_running: bool = False @@ -220,6 +329,7 @@ class _WorkerRecord: next_restart_at: float = 0.0 schedule_suspended: bool = False resource_suspended: bool = False + memory_rejected_command: Optional[Tuple[str, ...]] = None last_power_watts: Optional[float] = None suspension_stop_thread: Optional[threading.Thread] = field(default=None, repr=False) recent_logs: Deque[str] = field(default_factory=lambda: collections.deque(maxlen=50)) @@ -323,6 +433,8 @@ def _measured_budget_status_locked( def _resource_status_locked(self, record: _WorkerRecord) -> Tuple[bool, Optional[str]]: launch = record.launch + if record.memory_rejected_command == launch.command: + return False, "selected blocks exceed the VRAM budget; increase VRAM or contribute fewer blocks" if launch.max_vram_bytes is not None: reserved = sum( other.launch.max_vram_bytes @@ -387,6 +499,15 @@ def _spawn_locked( environment = os.environ.copy() environment.update(record.launch.environment) environment["PYTHONUNBUFFERED"] = "1" + environment.pop("DRIFT_DOWNLOAD_PROGRESS", None) + try: + if record.progress_directory is not None: + record.progress_directory.cleanup() + record.progress_directory = tempfile.TemporaryDirectory(prefix="communityai-download-") + environment["DRIFT_DOWNLOAD_PROGRESS"] = str(Path(record.progress_directory.name) / "progress.json") + except OSError: + record.progress_directory = None + logger.warning("Local download progress is unavailable for worker %s", record.launch.worker_id) try: process = self._popen( list(record.launch.command), @@ -399,6 +520,7 @@ def _spawn_locked( bufsize=1, env=environment, creationflags=self._creation_flags(), + **({"start_new_session": True} if sys.platform.startswith("linux") else {}), ) except Exception as exc: record.process = None @@ -447,9 +569,15 @@ def _refresh_locked(self, record: _WorkerRecord) -> None: exit_code = process.poll() if exit_code is None: return + self._kill_linux_worker_group(process) record.process = None record.last_exit_code = exit_code - if record.desired_running: + if exit_code == DEVICE_MEMORY_BUDGET_EXIT_CODE: + record.memory_rejected_command = record.launch.command + record.state = WorkerState.PAUSED + record.resource_suspended = record.desired_running + record.last_error = self._resource_status_locked(record)[1] + elif record.desired_running: record.state = WorkerState.CRASHED record.last_error = f"worker exited with code {exit_code}" record.next_restart_at = time.monotonic() + record.launch.restart_backoff @@ -579,19 +707,43 @@ def start_worker(self, worker_id: str) -> bool: with self._lock: if not record.launch.policy_admitted: record.desired_running = False + if record.launch.automatic: + # A user's Start clears an earlier Pause even while placement + # is pending. The reconciler may start it only after every + # policy and signed-placement check admits its next launch. + record.operator_paused = False + return False raise WorkerPolicyError(record.launch.policy_reason) record.operator_paused = False record.desired_running = True - return self._spawn_locked(record) + return self._spawn_locked( + record, + defer_outside_schedule=record.launch.automatic, + defer_unavailable_resources=record.launch.automatic, + ) - def _terminate(self, process: subprocess.Popen) -> int: - if process.poll() is None: - process.terminate() + @staticmethod + def _kill_linux_worker_group(process: subprocess.Popen) -> None: + if sys.platform.startswith("linux"): + # Each worker owns a new session. Its multiprocessing DHT children + # can survive the direct child's exit and otherwise retain p2pd's + # identity/port, preventing the replacement worker from starting. try: - return process.wait(timeout=self._stop_timeout) - except subprocess.TimeoutExpired: - process.kill() - return process.wait(timeout=self._stop_timeout) + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def _terminate(self, process: subprocess.Popen) -> int: + try: + if process.poll() is None: + process.terminate() + try: + return process.wait(timeout=self._stop_timeout) + except subprocess.TimeoutExpired: + process.kill() + return process.wait(timeout=self._stop_timeout) + finally: + self._kill_linux_worker_group(process) def pause_worker(self, worker_id: str) -> bool: """Pause a worker and persist the operator's explicit stopped intent.""" @@ -660,6 +812,7 @@ def snapshots(self) -> Tuple[Dict[str, Any], ...]: "id": record.launch.worker_id, "model": record.launch.model_id, "state": record.state.value, + "download_progress": self._download_snapshot(record), "desired_running": record.desired_running, "operator_paused": record.operator_paused, "auto_restart": record.launch.auto_restart, @@ -675,6 +828,8 @@ def snapshots(self) -> Tuple[Dict[str, Any], ...]: "automatic": record.launch.automatic, "block_indices": record.launch.block_indices, "placement_reason": record.launch.placement_reason, + "intent_published": record.launch.intent_published, + "remote_acknowledged": record.launch.remote_acknowledged, "max_disk_bytes": record.launch.max_disk_bytes, "max_vram_bytes": record.launch.max_vram_bytes, "vram_pool_bytes": record.launch.vram_pool_bytes, @@ -690,7 +845,31 @@ def snapshots(self) -> Tuple[Dict[str, Any], ...]: "recent_logs": list(record.recent_logs), } ) - return tuple(result) + return tuple(result) + + @staticmethod + def _download_snapshot(record): + from drift.utils.download_progress import public_progress + + if record.progress_directory is None: + return None + try: + with (Path(record.progress_directory.name) / "progress.json").open("rb") as stream: + payload = stream.read(16385) + if len(payload) > 16384: + return None + result = json.loads(payload) + if not isinstance(result, dict) or result.get("schema_version") != 1: + return None + # The fresh per-launch directory binds this report to the worker. + # Windows venv launchers may write from a child PID, and frozen + # workers may use their own PID; neither changes that ownership. + if record.state in (WorkerState.PAUSED, WorkerState.CRASHED, WorkerState.STOPPING): + result["state"] = "failed" if record.state is WorkerState.CRASHED else "paused" + result["bytes_per_second"] = 0 + return public_progress(result) + except (OSError, ValueError): + return None def snapshot(self, worker_id: str) -> Dict[str, Any]: record = self._record(worker_id) @@ -823,3 +1002,6 @@ def shutdown(self) -> None: record.state = WorkerState.PAUSED if monitor is not None: monitor.join(timeout=5) + for record in records: + if record.process is None and record.progress_directory is not None: + record.progress_directory.cleanup() diff --git a/src/drift/server/block_utils.py b/src/drift/server/block_utils.py index b043ae78e..e11af6cda 100644 --- a/src/drift/server/block_utils.py +++ b/src/drift/server/block_utils.py @@ -37,9 +37,9 @@ def get_block_size( n_params = sum(param.numel() for param in block.parameters()) if location == "memory": - if quant_type == QuantType.NONE: - dtype = resolve_block_dtype(config, dtype) - bytes_per_value = get_size_in_bytes(dtype) + if quant_type in (QuantType.NONE, QuantType.FP8_DEQUANT): + dtype = resolve_block_dtype(config, dtype) + bytes_per_value = get_size_in_bytes(dtype) elif quant_type == QuantType.INT8: bytes_per_value = 1 elif quant_type == QuantType.NF4: diff --git a/src/drift/server/from_pretrained.py b/src/drift/server/from_pretrained.py index 356cab94f..c9eff1f23 100644 --- a/src/drift/server/from_pretrained.py +++ b/src/drift/server/from_pretrained.py @@ -26,6 +26,7 @@ from drift.model_manifest import ManifestArtifactVerifier, ManifestError from drift.server.block_utils import get_model_block, resolve_block_dtype from drift.utils.auto_config import AutoDistributedConfig +from drift.utils.convert_block import QuantType from drift.utils.disk_cache import ( DEFAULT_CACHE_DIR, allow_cache_reads, @@ -58,6 +59,56 @@ def _find_unconsumed_checkpoint_keys(block: nn.Module, state_dict: "StateDict") ) +def _dequantize_finegrained_fp8_tensor( + weight: torch.Tensor, scale_inv: torch.Tensor, *, output_dtype: torch.dtype +) -> torch.Tensor: + """Dequantize one block-scaled FP8 matrix using its checkpoint scale grid.""" + if weight.ndim < 2 or scale_inv.ndim < 2: + raise ValueError( + f"Fine-grained FP8 weight and scale must be matrices, got {tuple(weight.shape)} and " + f"{tuple(scale_inv.shape)}" + ) + rows, cols = weight.shape[-2:] + scale_rows, scale_cols = scale_inv.shape[-2:] + if scale_rows < 1 or scale_cols < 1 or rows % scale_rows or cols % scale_cols: + raise ValueError( + f"Fine-grained FP8 weight shape ({rows}, {cols}) is not divisible by scale grid " + f"({scale_rows}, {scale_cols})" + ) + if weight.shape[:-2] != scale_inv.shape[:-2]: + raise ValueError( + f"Fine-grained FP8 weight and scale batch dimensions differ: {tuple(weight.shape[:-2])} " + f"versus {tuple(scale_inv.shape[:-2])}" + ) + + block_rows, block_cols = rows // scale_rows, cols // scale_cols + quantized = weight.to(torch.float32).reshape(-1, scale_rows, block_rows, scale_cols, block_cols) + scales = scale_inv.to(torch.float32).reshape(-1, scale_rows, scale_cols).unsqueeze(2).unsqueeze(-1) + return (quantized * scales).to(output_dtype).reshape(weight.shape) + + +def dequantize_finegrained_fp8_state_dict(state_dict: "StateDict", *, output_dtype: torch.dtype) -> "StateDict": + """Replace checkpoint FP8 matrices and ``weight_scale_inv`` grids with dense tensors.""" + result = dict(state_dict) + scale_names = sorted(name for name in state_dict if name.endswith(".weight_scale_inv")) + if not scale_names: + raise ValueError("Fine-grained FP8 checkpoint block contains no weight_scale_inv tensors") + + for scale_name in scale_names: + weight_name = scale_name.removesuffix("_scale_inv") + if weight_name not in state_dict: + raise ValueError(f"Fine-grained FP8 scale {scale_name!r} has no matching {weight_name!r}") + result[weight_name] = _dequantize_finegrained_fp8_tensor( + state_dict[weight_name], state_dict[scale_name], output_dtype=output_dtype + ) + del result[scale_name] + + orphan_fp8 = sorted(name for name, tensor in result.items() if str(tensor.dtype).startswith("torch.float8")) + if orphan_fp8: + raise ValueError(f"Fine-grained FP8 checkpoint tensors have no scale grid: {orphan_fp8}") + return result + + def load_pretrained_block( model_name: str, block_index: int, @@ -69,6 +120,7 @@ def load_pretrained_block( cache_dir: Optional[str] = None, max_disk_space: Optional[int] = None, artifact_verifier: Optional[ManifestArtifactVerifier] = None, + quant_type: QuantType = QuantType.NONE, ) -> nn.Module: if config is None: config_source = artifact_verifier.ensure_startup_metadata() if artifact_verifier is not None else model_name @@ -98,6 +150,17 @@ def load_pretrained_block( artifact_verifier=artifact_verifier, ) + source_quantization = getattr(config, "_source_quantization_method", None) + if source_quantization == "fp8": + if quant_type != QuantType.FP8_DEQUANT: + raise ValueError( + "This checkpoint contains pre-quantized fine-grained FP8 weights; load it with " + "quant_type=QuantType.FP8_DEQUANT" + ) + state_dict = dequantize_finegrained_fp8_state_dict(state_dict, output_dtype=torch_dtype) + elif quant_type == QuantType.FP8_DEQUANT: + raise ValueError("FP8_DEQUANT requires a checkpoint whose config declares quant_method='fp8'") + # transformers >=5.0 may restructure weights when loading (e.g. Mixtral fuses per-expert # weights). DRIFT-LLM loads block weights by name, so apply the same conversion here. state_dict = maybe_convert_block_state_dict(config, state_dict, block) @@ -157,28 +220,27 @@ def _load_state_dict_from_repo( artifact_verifier=artifact_verifier, ) if index_file.endswith(".index.json"): # Sharded model - path = ( - str(artifact_verifier.ensure_path(index_file, allowed_roles={"weight_index"})) - if artifact_verifier is not None - else get_file_from_repo( + if artifact_verifier is not None: + weight_map = artifact_verifier.load_weight_map() + if weight_map is None: # pragma: no cover - maintained by the index-file branch + raise ManifestError("Manifested sharded checkpoint lost its verified weight map") + else: + path = get_file_from_repo( model_name, filename=index_file, revision=revision, use_auth_token=token, cache_dir=cache_dir, ) - ) - if path is None: - # _find_index_file() told that a file exists but we can't get it (e.g., it just disappeared) - raise ValueError(f"Failed to get file {index_file}") - - with open(path) as f: - index = json.load(f) - filenames = { - filename for param_name, filename in index["weight_map"].items() if param_name.startswith(block_prefix) - } + if path is None: + # _find_index_file() told that a file exists but we can't get it (e.g., it just disappeared) + raise ValueError(f"Failed to get file {index_file}") + with open(path) as f: + index = json.load(f) + weight_map = index["weight_map"] + filenames = {filename for param_name, filename in weight_map.items() if param_name.startswith(block_prefix)} if not filenames: - raise RuntimeError(f"Block {block_prefix}* not found in the index: {index['weight_map']}") + raise RuntimeError(f"Block {block_prefix}* not found in the index: {weight_map}") else: # Non-sharded model filenames = {index_file} logger.debug(f"Loading {block_prefix}* from {filenames}") diff --git a/src/drift/server/processing_budget.py b/src/drift/server/processing_budget.py new file mode 100644 index 000000000..e3e272aed --- /dev/null +++ b/src/drift/server/processing_budget.py @@ -0,0 +1,65 @@ +"""Pace contribution compute while leaving discovery and control threads responsive.""" + +from __future__ import annotations + +import errno +import math +import time +from contextlib import nullcontext +from pathlib import Path +from threading import Event +from typing import Callable + +from drift.node.config_lock import NodeConfigWriteLockError, node_config_write_lock + + +class ProcessingBudget: + """Bound compute duty cycle, including synchronized accelerator work. + + A node's capped workers share a lock across compute and cooldown. Concurrent + workers therefore cannot each consume the whole node budget. One compute step + may burst above the percentage; its mandatory cooldown repays that time before + any next step. Loading, discovery and the user's local inference are separate. + """ + + def __init__(self, percent=100, *, path=None, stop=None, clock=time.monotonic): + if isinstance(percent, bool) or not isinstance(percent, (int, float)) or not math.isfinite(percent): + raise ValueError("processing percentage must be a finite number from 1 to 100") + if not 1 <= percent <= 100: + raise ValueError("processing percentage must be from 1 to 100") + self.percent = float(percent) + self.path = None if path is None else Path(path) + self.stop = stop if stop is not None else Event() + self.clock = clock + + def run(self, operation: Callable, *, synchronize: Callable = lambda: None): + if self.percent == 100: + return operation() + while not self.stop.is_set(): + lock = nullcontext() if self.path is None else node_config_write_lock(self.path) + try: + lock.__enter__() + break + except NodeConfigWriteLockError as exc: + if not isinstance(exc.__cause__, OSError) or exc.__cause__.errno not in ( + errno.EACCES, + errno.EAGAIN, + errno.EDEADLK, + ): + raise + self.stop.wait(0.01) + else: + raise InterruptedError("contribution processing stopped") + try: + if self.stop.is_set(): + raise InterruptedError("contribution processing stopped") + synchronize() + started = self.clock() + try: + return operation() + finally: + synchronize() + elapsed = max(0.0, self.clock() - started) + self.stop.wait(elapsed * (100.0 / self.percent - 1.0)) + finally: + lock.__exit__(None, None, None) diff --git a/src/drift/server/server.py b/src/drift/server/server.py index 67a504eea..128a198c3 100644 --- a/src/drift/server/server.py +++ b/src/drift/server/server.py @@ -24,7 +24,7 @@ import drift from drift.constants import DTYPE_MAP from drift.data_structures import CHAIN_DELIMITER, UID_DELIMITER, ModelInfo, ServerInfo, ServerState, parse_uid -from drift.model_manifest import ManifestArtifactVerifier, ModelManifest +from drift.model_manifest import ManifestArtifactVerifier, ManifestError, ModelManifest from drift.protocol_identity import ( MAX_SIGNED_RECORD_TTL_SECONDS, NodeIdentity, @@ -46,6 +46,7 @@ write_public_worker_health, ) from drift.server.memory_cache import MemoryCache +from drift.server.processing_budget import ProcessingBudget from drift.server.reachability import ReachabilityProtocol, check_direct_reachability from drift.server.throughput import get_dtype_name, get_server_throughput from drift.utils.auto_config import AutoDistributedConfig @@ -65,6 +66,7 @@ from drift.utils.misc import format_all_thread_stacks, get_size_in_bytes from drift.utils.ping import PingAggregator from drift.utils.random import sample_up_to +from drift.utils.resource_limits import DeviceMemoryBudgetError from drift.utils.version import get_compatible_model_repo logger = get_logger(__name__) @@ -80,6 +82,124 @@ def parse_block_indices(value: str, num_hidden_layers: int) -> range: return range(start_block, end_block) +def _validate_artifact_plan_claim( + manifest: Optional[ModelManifest], + artifact_plan, + *, + cache_dir: Optional[str], + expected_manifest_digest: Optional[str], + expected_block_indices: Optional[str], + expected_artifact_bytes: Optional[int], + expected_artifact_set_digest: Optional[str], + expected_cache_root: Optional[str], +) -> None: + claims = ( + expected_manifest_digest, + expected_block_indices, + expected_artifact_bytes, + expected_artifact_set_digest, + expected_cache_root, + ) + if all(value is None for value in claims): + return + if any(value is None for value in claims): + raise ManifestError("Worker manifest, span, cache, and artifact-plan claims must be supplied together") + if manifest is None or artifact_plan is None: + raise ManifestError("Worker artifact-plan claims require a manifested explicit block span") + if not isinstance(expected_manifest_digest, str) or expected_manifest_digest != manifest.digest_id: + raise ManifestError("Worker manifest digest does not match the acknowledged placement decision") + actual_block_indices = f"{artifact_plan.start_block}:{artifact_plan.end_block}" + if expected_block_indices != actual_block_indices: + raise ManifestError("Worker block span does not match the acknowledged placement decision") + if not isinstance(cache_dir, str) or not cache_dir: + raise ManifestError("Worker artifact-plan claims require an explicit canonical cache root") + canonical_cache_root = os.path.realpath(os.path.abspath(os.path.expanduser(cache_dir))) + if cache_dir != canonical_cache_root: + raise ManifestError("Worker artifact-plan claims require a canonical absolute cache root") + if expected_cache_root != canonical_cache_root: + raise ManifestError("Worker cache root does not match the acknowledged placement decision") + if ( + isinstance(expected_artifact_bytes, bool) + or not isinstance(expected_artifact_bytes, int) + or expected_artifact_bytes < 0 + ): + raise ManifestError("Worker artifact-plan byte count must be a non-negative integer") + if ( + not isinstance(expected_artifact_set_digest, str) + or len(expected_artifact_set_digest) != 64 + or expected_artifact_set_digest.lower() != expected_artifact_set_digest + or any(character not in "0123456789abcdef" for character in expected_artifact_set_digest) + ): + raise ManifestError("Worker artifact-plan digest must be lowercase SHA-256") + if artifact_plan.artifact_bytes != expected_artifact_bytes: + raise ManifestError("Worker artifact plan byte count does not match the acknowledged placement decision") + if artifact_plan.artifact_set_digest != expected_artifact_set_digest: + raise ManifestError("Worker artifact plan digest does not match the acknowledged placement decision") + + +def _scoped_manifest_artifact_verifier( + manifest: Optional[ModelManifest], + *, + repository: str, + revision: Optional[str], + token, + cache_dir: str, + max_disk_space: int, + block_prefix: str, + block_indices: Sequence[int], + expected_manifest_digest: Optional[str] = None, + expected_block_indices: Optional[str] = None, + expected_artifact_bytes: Optional[int] = None, + expected_artifact_set_digest: Optional[str] = None, + expected_cache_root: Optional[str] = None, + artifact_root=None, +) -> Optional[ManifestArtifactVerifier]: + if manifest is None: + _validate_artifact_plan_claim( + manifest, + None, + cache_dir=cache_dir, + expected_manifest_digest=expected_manifest_digest, + expected_block_indices=expected_block_indices, + expected_artifact_bytes=expected_artifact_bytes, + expected_artifact_set_digest=expected_artifact_set_digest, + expected_cache_root=expected_cache_root, + ) + return None + if not block_indices: + raise ManifestError("Manifested module container requires at least one block") + start_block, end_block = min(block_indices), max(block_indices) + 1 + if list(block_indices) != list(range(start_block, end_block)): + raise ManifestError("Manifested module container requires one contiguous block span") + startup_paths = tuple(artifact.path for artifact in manifest.artifacts_for_roles({"config", "weight_index"})) + verifier = ManifestArtifactVerifier( + manifest, + repository=repository, + revision=revision, + token=token, + cache_dir=cache_dir, + max_disk_space=max_disk_space, + artifact_root=artifact_root, + allowed_paths=startup_paths, + ) + artifact_plan = verifier.bind_block_artifact_plan( + block_prefix=block_prefix, + start_block=start_block, + end_block=end_block, + ) + _validate_artifact_plan_claim( + manifest, + artifact_plan, + cache_dir=cache_dir, + expected_manifest_digest=expected_manifest_digest, + expected_block_indices=expected_block_indices, + expected_artifact_bytes=expected_artifact_bytes, + expected_artifact_set_digest=expected_artifact_set_digest, + expected_cache_root=expected_cache_root, + ) + return verifier + + def _probe_quantization(quant_type: QuantType, device: torch.device) -> Optional[str]: """Return None if ``quant_type`` can actually run on ``device``, else a short reason why not. @@ -88,7 +208,7 @@ def _probe_quantization(quant_type: QuantType, device: torch.device) -> Optional weights are already downloaded. A tiny quantization exercises the shared native library that both the nf4 and int8 paths depend on, so we can catch the problem before loading anything. """ - if quant_type == QuantType.NONE: + if quant_type in (QuantType.NONE, QuantType.FP8_DEQUANT): return None try: import bitsandbytes as bnb @@ -115,6 +235,11 @@ def __init__( throughput: Union[float, str], num_blocks: Optional[int] = None, block_indices: Optional[str] = None, + expected_manifest_digest: Optional[str] = None, + expected_block_indices: Optional[str] = None, + expected_artifact_bytes: Optional[int] = None, + expected_artifact_set_digest: Optional[str] = None, + expected_cache_root: Optional[str] = None, num_handlers: int = 1, inference_max_length: Optional[int] = None, min_batch_size: int = 1, @@ -132,7 +257,9 @@ def __init__( revocation_files: Sequence[str] = (), cache_dir: Optional[str] = None, max_disk_space: Optional[int] = None, - max_device_memory: Optional[int] = None, + max_device_memory: Optional[int] = None, + max_processing_percent: float = 100, + processing_budget_path: Optional[str] = None, device: Optional[Union[str, torch.device]] = None, compression=CompressionType.NONE, stats_report_interval: Optional[int] = None, @@ -163,7 +290,8 @@ def __init__( converted_model_name_or_path = get_compatible_model_repo(converted_model_name_or_path) self.converted_model_name_or_path = converted_model_name_or_path - self.num_handlers = num_handlers + self.num_handlers = num_handlers + self.processing_budget = ProcessingBudget(max_processing_percent, path=processing_budget_path) self.compression = compression self.stats_report_interval, self.update_period = stats_report_interval, update_period self.prefetch_batches, self.sender_threads = prefetch_batches, sender_threads @@ -266,7 +394,7 @@ def __init__( self.dht = DHT( initial_peers=initial_peers, start=True, - num_workers=self.block_config.num_hidden_layers, + num_workers=min(self.block_config.num_hidden_layers, 4), use_relay=use_relay, use_auto_relay=use_auto_relay, client_mode=reachable_via_relay, @@ -408,13 +536,45 @@ def __init__( if block_indices is not None: block_indices = parse_block_indices(block_indices, self.block_config.num_hidden_layers) num_blocks = len(block_indices) + artifact_claims = ( + expected_manifest_digest, + expected_block_indices, + expected_artifact_bytes, + expected_artifact_set_digest, + expected_cache_root, + ) + artifact_plan = None + if all(value is not None for value in artifact_claims): + if artifact_verifier is not None and block_indices is not None: + artifact_plan = artifact_verifier.plan_block_artifacts( + block_prefix=self.block_config.block_prefix, + start_block=block_indices.start, + end_block=block_indices.stop, + ) + _validate_artifact_plan_claim( + model_manifest, + artifact_plan, + cache_dir=cache_dir, + expected_manifest_digest=expected_manifest_digest, + expected_block_indices=expected_block_indices, + expected_artifact_bytes=expected_artifact_bytes, + expected_artifact_set_digest=expected_artifact_set_digest, + expected_cache_root=expected_cache_root, + ) + ( + self.expected_manifest_digest, + self.expected_block_indices, + self.expected_artifact_bytes, + self.expected_artifact_set_digest, + self.expected_cache_root, + ) = artifact_claims self.strict_block_indices, self.num_blocks = block_indices, num_blocks if self.device_memory_limits is not None: required_memory = self._estimate_device_memory(num_blocks, block_indices) available_memory = min(self.device_memory_limits) * len(self.device_memory_limits) if required_memory > available_memory: - raise ValueError( + raise DeviceMemoryBudgetError( "Configured blocks require an estimated " f"{required_memory / 1024**3:.2f} GiB, exceeding the enforced " f"--max_device_memory budget of {available_memory / 1024**3:.2f} GiB" @@ -570,7 +730,9 @@ def _create_module_container(self, block_indices: List[int]) -> ModuleContainer: server_info=self.server_info, model_info=self.model_info, block_indices=block_indices, - num_handlers=self.num_handlers, + num_handlers=self.num_handlers, + max_processing_percent=self.processing_budget.percent, + processing_budget_path=self.processing_budget.path, min_batch_size=self.min_batch_size, max_batch_size=self.max_batch_size, max_chunk_size_bytes=self.max_chunk_size_bytes, @@ -594,6 +756,11 @@ def _create_module_container(self, block_indices: List[int]) -> ModuleContainer: revision=self.revision, token=self.token, model_manifest=self.model_manifest, + expected_manifest_digest=self.expected_manifest_digest, + expected_block_indices=self.expected_block_indices, + expected_artifact_bytes=self.expected_artifact_bytes, + expected_artifact_set_digest=self.expected_artifact_set_digest, + expected_cache_root=self.expected_cache_root, admission_policy=self.admission_policy, protocol_identity=self.protocol_identity, manifest_execution_profile=self.manifest_execution_profile, @@ -606,15 +773,21 @@ def _create_module_container(self, block_indices: List[int]) -> ModuleContainer: start=True, ) - def _run_module_container(self, block_indices: List[int]) -> bool: + def _run_module_container(self, block_indices: List[int]) -> bool: self.module_container = self._create_module_container(block_indices) try: self.module_container.ready.wait() if self.stop.wait(0): return True - if self.health_state_path is not None and not self.module_container.is_healthy(): + if self.health_state_path is not None and not self.module_container.is_healthy(): logger.warning("Public worker failed its initial aggregate health check") - return False + return False + + from drift.utils.download_progress import current_progress + + progress = current_progress() + if progress is not None: + progress.finish("ready") while True: timeout = random.random() * 2 * self.mean_balance_check_period @@ -731,6 +904,11 @@ def create( revision: Optional[str], token: Optional[Union[str, bool]], model_manifest: Optional[ModelManifest] = None, + expected_manifest_digest: Optional[str] = None, + expected_block_indices: Optional[str] = None, + expected_artifact_bytes: Optional[int] = None, + expected_artifact_set_digest: Optional[str] = None, + expected_cache_root: Optional[str] = None, protocol_identity: Optional[NodeIdentity] = None, manifest_execution_profile: Optional[Dict[str, object]] = None, revocations: Optional[RevocationStore] = None, @@ -740,6 +918,21 @@ def create( **kwargs, ) -> ModuleContainer: module_uids = [f"{dht_prefix}{UID_DELIMITER}{block_index}" for block_index in block_indices] + artifact_verifier = _scoped_manifest_artifact_verifier( + model_manifest, + repository=converted_model_name_or_path, + revision=revision, + token=token, + cache_dir=cache_dir, + max_disk_space=max_disk_space, + block_prefix=block_config.block_prefix, + block_indices=block_indices, + expected_manifest_digest=expected_manifest_digest, + expected_block_indices=expected_block_indices, + expected_artifact_bytes=expected_artifact_bytes, + expected_artifact_set_digest=expected_artifact_set_digest, + expected_cache_root=expected_cache_root, + ) memory_cache = MemoryCache(attn_cache_bytes, max_alloc_timeout, paged=paged_cache, page_size=page_size) server_info.state = ServerState.JOINING @@ -765,18 +958,6 @@ def create( blocks = {} try: - artifact_verifier = ( - ManifestArtifactVerifier( - model_manifest, - repository=converted_model_name_or_path, - revision=revision, - token=token, - cache_dir=cache_dir, - max_disk_space=max_disk_space, - ) - if model_manifest is not None - else None - ) for module_uid, block_index in zip(module_uids, block_indices): block = load_pretrained_block( converted_model_name_or_path, @@ -788,6 +969,7 @@ def create( cache_dir=cache_dir, max_disk_space=max_disk_space, artifact_verifier=artifact_verifier, + quant_type=quant_type, ) block = convert_block( block, @@ -1045,8 +1227,8 @@ def is_healthy(self) -> bool: ready = self.ready.is_set() healthy = previous_health and ready try: - payload = build_public_worker_health( - manifest_digest=self.server_info.manifest_digest, + payload = build_public_worker_health( + manifest_digest=f"sha256:{self.server_info.manifest_digest}", start_block=self.server_info.start_block, end_block=self.server_info.end_block, admission_snapshot=admission_snapshot, @@ -1056,8 +1238,8 @@ def is_healthy(self) -> bool: pools_alive=pools_alive, ) write_public_worker_health(self.health_state_path, payload) - except HealthStateError: - logger.error("Machine-readable public health is unavailable; the worker will restart") + except HealthStateError as exc: + logger.error("Machine-readable public health is unavailable; the worker will restart: %s", exc) return False return healthy @@ -1226,12 +1408,31 @@ def _ping_next_servers(self) -> Dict[PeerID, float]: self.ping_aggregator.ping(list(pinged_servers)) -class RuntimeWithDeduplicatedPools(Runtime): +class RuntimeWithDeduplicatedPools(Runtime): """A version of hivemind.moe.server.runtime.Runtime that allows multiple backends to reuse a task pool""" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.pools = tuple(set(self.pools)) + def __init__(self, *args, max_processing_percent=100, processing_budget_path=None, **kwargs): + super().__init__(*args, **kwargs) + self.pools = tuple(set(self.pools)) + self.processing_budget = ProcessingBudget( + max_processing_percent, path=processing_budget_path, stop=self.shutdown_trigger + ) + + def process_batch(self, pool, batch_index, *batch): + def synchronize(): + devices = {device for backend in self.module_backends.values() for device in backend.module.devices} + for device in devices: + if device.type == "cuda": + torch.cuda.synchronize(device) + elif device.type == "xpu": + torch.xpu.synchronize(device) + elif device.type == "mps": + torch.mps.synchronize() + + return self.processing_budget.run( + lambda: super(RuntimeWithDeduplicatedPools, self).process_batch(pool, batch_index, *batch), + synchronize=synchronize, + ) def iterate_minibatches_from_pools(self, timeout=None): # multiprocessing.connection.wait() delegates to WaitForMultipleObjects on Windows, which diff --git a/src/drift/server/text_generation.py b/src/drift/server/text_generation.py new file mode 100644 index 000000000..aee502bd9 --- /dev/null +++ b/src/drift/server/text_generation.py @@ -0,0 +1,210 @@ +"""Input/output processing and generation owned by a contributing text peer.""" + +import asyncio +import logging +import queue +import threading +import time + +from drift.api.server import ( + ChatCompletionRequest, + CompletionRequest, + _RequestCancelled, + build_generate_kwargs, + message_text, +) + +logger = logging.getLogger(__name__) + + +class _PrefillCancelled(Exception): + pass + + +class TextGenerationEngine: + def __init__(self, runtime, *, max_context_tokens=2048, max_output_tokens=512, request_timeout=900): + self.runtime = runtime + self.max_context_tokens = max_context_tokens + self.max_output_tokens = max_output_tokens + self.request_timeout = request_timeout + self._lock = threading.Lock() + self._generation_lock = threading.Lock() + self._active = {} + + def cancel(self, request_id, remote_peer): + with self._lock: + cancel = self._active.get((remote_peer, request_id)) + if cancel is None: + return False + cancel.event.set() + return True + + def close(self): + with self._lock: + for cancel in self._active.values(): + cancel.event.set() + + async def stream(self, payload, remote_peer): + request_id = payload.get("request_id") + if not isinstance(request_id, str) or len(request_id) != 32 or type(payload.get("chat")) is not bool: + yield {"type": "error", "code": "invalid_request", "message": "Invalid text request"} + return + cancel = _RequestCancelled() + key = remote_peer, request_id + with self._lock: + # A chat application may request an answer and a conversation title + # together. Keep one generation running and at most two waiting. + busy = key in self._active or len(self._active) >= 3 + if not busy: + self._active[key] = cancel + if busy: + yield {"type": "error", "code": "busy", "message": "This community peer is busy"} + return + out = queue.Queue(maxsize=8) + task = asyncio.create_task(asyncio.to_thread(self._run_generation, payload, key, cancel, out)) + deadline = time.monotonic() + self.request_timeout + last_heartbeat = 0 + try: + while time.monotonic() < deadline: + try: + frame = await asyncio.to_thread(out.get, True, 1) + except queue.Empty: + if task.done(): + await task + return + if time.monotonic() - last_heartbeat >= 5: + yield {"type": "heartbeat"} + last_heartbeat = time.monotonic() + continue + yield frame + if frame["type"] in ("done", "error"): + return + yield {"type": "error", "code": "timeout", "message": "The community answer took too long"} + finally: + cancel.event.set() + # Admission remains occupied until the real generation thread exits. + # A disconnect must not allow a second request onto a still-busy model. + + def _run_generation(self, payload, key, cancel, out): + try: + while not cancel.event.is_set(): + if self._generation_lock.acquire(timeout=0.1): + try: + if not cancel.event.is_set(): + self._generate(payload, key, cancel, out) + finally: + self._generation_lock.release() + return + finally: + with self._lock: + self._active.pop(key, None) + + def _generate(self, payload, key, cancel, out): + import torch + from transformers import StoppingCriteriaList, TextIteratorStreamer + + def put(frame): + while not cancel.event.is_set(): + try: + out.put(frame, timeout=0.1) + return + except queue.Full: + continue + + class Streamer(TextIteratorStreamer): + def on_finalized_text(self, text, stream_end=False): + if text: + put({"type": "delta", "text": text}) + + try: + body = (ChatCompletionRequest if payload["chat"] else CompletionRequest).model_validate(payload.get("body")) + if body.n != 1: + raise ValueError("Only one answer at a time is supported") + max_tokens = body.max_tokens + if payload["chat"] and max_tokens is None: + max_tokens = body.max_completion_tokens + max_tokens = min(128, self.max_output_tokens) if max_tokens is None else max_tokens + if type(max_tokens) is not int or not 1 <= max_tokens <= self.max_output_tokens: + raise ValueError(f"Choose between 1 and {self.max_output_tokens} output tokens") + if body.temperature is not None and not 0 <= body.temperature <= 2: + raise ValueError("Temperature must be between 0 and 2") + if body.top_p is not None and not 0 < body.top_p <= 1: + raise ValueError("top_p must be greater than 0 and at most 1") + stops = [body.stop] if isinstance(body.stop, str) else body.stop or [] + if len(stops) > 16 or any(not s or len(s) > 128 for s in stops): + raise ValueError("Stop sequences are too long or empty") + tokenizer = self.runtime.tokenizer + if payload["chat"]: + if not body.messages or len(body.messages) > 128: + raise ValueError("Provide between 1 and 128 messages") + messages = [{"role": m.role, "content": message_text(m.content)} for m in body.messages] + options = {"enable_thinking": body.enable_thinking is True} + input_ids = tokenizer.apply_chat_template( + messages, add_generation_prompt=True, return_dict=True, return_tensors="pt", **options + )["input_ids"] + else: + prompt = body.prompt + if isinstance(prompt, list): + if len(prompt) != 1: + raise ValueError("Batched prompts are not supported") + prompt = prompt[0] + input_ids = tokenizer(prompt, return_tensors="pt").input_ids + model_device = getattr(self.runtime.model, "device", None) + if model_device is not None: + input_ids = input_ids.to(model_device) + if input_ids.shape[1] + max_tokens > self.max_context_tokens: + raise ValueError(f"This community peer supports {self.max_context_tokens} tokens including the answer") + kwargs = build_generate_kwargs( + max_tokens=max_tokens, temperature=body.temperature, top_p=body.top_p, stop=body.stop + ) + # A Qwen 27B prompt with 533 tokens expands to more than 5 MiB + # of activations. Prefill in small pieces to stay below the public + # worker's 4 MiB message bound while retaining the entire prompt. + kwargs["prefill_chunk_size"] = 64 + if payload["chat"] and tokenizer.eos_token_id is not None: + # The tokenizer's chat end marker may differ from the base + # model's end-of-text token. Stop on either, rather than letting + # generation invent another user/assistant turn after im_end. + configured = getattr(self.runtime.model.generation_config, "eos_token_id", None) + endings = list(configured) if isinstance(configured, (list, tuple)) else [configured] + kwargs["eos_token_id"] = list( + dict.fromkeys(value for value in [*endings, tokenizer.eos_token_id] if value is not None) + ) + kwargs["stopping_criteria"] = StoppingCriteriaList([cancel]) + if stops: + kwargs["tokenizer"] = tokenizer + streamer = Streamer(tokenizer, skip_prompt=True, skip_special_tokens=True) + if cancel.event.is_set(): + return + + def check_cancel(_module, _inputs): + if cancel.event.is_set(): + raise _PrefillCancelled() + + # Transformers checks stopping criteria during decoding, but not + # between prefill chunks. Stop there too after a caller disconnects. + hook = self.runtime.model.register_forward_pre_hook(check_cancel) + try: + with torch.inference_mode(): + output = self.runtime.model.generate(input_ids, streamer=streamer, **kwargs) + finally: + hook.remove() + count = output.shape[1] - input_ids.shape[1] + put( + { + "type": "done", + "finish_reason": "length" if count >= max_tokens else "stop", + "usage": { + "prompt_tokens": input_ids.shape[1], + "completion_tokens": count, + "total_tokens": output.shape[1], + }, + } + ) + except _PrefillCancelled: + return + except ValueError as exc: + put({"type": "error", "code": "invalid_request", "message": str(exc)[:256]}) + except Exception: + logger.exception("Text peer generation failed") + put({"type": "error", "code": "unavailable", "message": "This community peer could not finish the answer"}) diff --git a/src/drift/server/text_peer.py b/src/drift/server/text_peer.py new file mode 100644 index 000000000..aeff1210f --- /dev/null +++ b/src/drift/server/text_peer.py @@ -0,0 +1,133 @@ +"""A contributor role for complete text inference through the existing block mesh.""" + +import asyncio +import contextlib +import threading +import time + +from hivemind.utils import get_logger + +from drift.node.loading import make_manifest_loader +from drift.server.text_generation import TextGenerationEngine +from drift.text_mesh import ANNOUNCEMENT_TTL, TextPeerProtocol, announcement_key, create_text_announcement + +logger = get_logger(__name__) + + +class TextPeerService: + def __init__( + self, + dht, + identity, + manifest, + *, + initial_peers, + cache_dir, + max_context_tokens=2048, + max_output_tokens=512, + request_timeout=900 + ): + if dht.peer_id != identity.peer_id: + raise ValueError("Text service must use its signing identity's peer transport") + if not 1 <= max_output_tokens < max_context_tokens <= manifest.model.context_length: + raise ValueError("Invalid text peer context/output limits") + self.dht, self.identity, self.manifest = dht, identity, manifest + self.initial_peers, self.cache_dir = initial_peers, cache_dir + self.max_context_tokens, self.max_output_tokens = max_context_tokens, max_output_tokens + self.request_timeout = request_timeout + self._stop = threading.Event() + self._thread = None + self.engine = None + self.error = None + self.ready = threading.Event() + + def start(self): + self._thread = threading.Thread(target=self._run, name="community-text-peer", daemon=True) + self._thread.start() + return self + + def close(self): + self._stop.set() + if self.engine is not None: + self.engine.close() + if self._thread is not None: + self._thread.join(timeout=30) + + def _run(self): + runtime = None + try: + runtime = make_manifest_loader( + self.manifest, + initial_peers=self.initial_peers, + cache_dir=self.cache_dir, + request_timeout=180, + max_retries=1, + )() + if not self._stop.is_set(): + # The tensor client normally starts discovery on first inference. + # Text peers must establish readiness before admitting that request. + runtime.model.transformer.h.sequence_manager.start_discovery() + self.engine = TextGenerationEngine( + runtime, + max_context_tokens=self.max_context_tokens, + max_output_tokens=self.max_output_tokens, + request_timeout=self.request_timeout, + ) + asyncio.run(self._serve(runtime)) + except Exception as exc: + self.error = type(exc).__name__ + logger.exception("Community text peer could not start") + finally: + self.ready.clear() + if runtime is not None and runtime.close is not None: + runtime.close() + + async def _serve(self, runtime): + p2p = await self.dht.replicate_p2p() + protocol = TextPeerProtocol(self.manifest, self.engine) + try: + await protocol.add_p2p_handlers(p2p) + while not self._stop.is_set(): + health = runtime.route_health() + complete = ( + health.get("status") == "complete" + and health.get("covered_blocks") == self.manifest.model.num_blocks + ) + if complete: + record = create_text_announcement( + self.manifest, + self.identity, + max_context_tokens=self.max_context_tokens, + max_output_tokens=self.max_output_tokens, + ) + await asyncio.to_thread( + self.dht.store, + announcement_key(self.manifest), + record.to_dict(), + subkey=str(self.identity.peer_id), + expiration_time=time.time() + ANNOUNCEMENT_TTL, + ) + self.ready.set() + else: + self.ready.clear() + await asyncio.to_thread( + self.dht.store, + announcement_key(self.manifest), + None, + subkey=str(self.identity.peer_id), + expiration_time=time.time() + ANNOUNCEMENT_TTL, + ) + await asyncio.to_thread(self._stop.wait, 10) + finally: + self.ready.clear() + self.engine.close() + with contextlib.suppress(Exception): + await asyncio.to_thread( + self.dht.store, + announcement_key(self.manifest), + None, + subkey=str(self.identity.peer_id), + expiration_time=time.time() + ANNOUNCEMENT_TTL, + ) + await protocol.remove_p2p_handlers(p2p) + await p2p.shutdown() diff --git a/src/drift/server/throughput.py b/src/drift/server/throughput.py index dc2f0c06d..313899df8 100644 --- a/src/drift/server/throughput.py +++ b/src/drift/server/throughput.py @@ -236,8 +236,10 @@ def step(cache_): return device_rps -def get_dtype_name(dtype: torch.dtype, quant_type: QuantType) -> str: - name = str(dtype).replace("torch.", "") - if quant_type != QuantType.NONE: - name += f", quantized to {quant_type.name.lower()}" - return name +def get_dtype_name(dtype: torch.dtype, quant_type: QuantType) -> str: + name = str(dtype).replace("torch.", "") + if quant_type == QuantType.FP8_DEQUANT: + name += ", loaded from fine-grained fp8" + elif quant_type != QuantType.NONE: + name += f", quantized to {quant_type.name.lower()}" + return name diff --git a/src/drift/text_mesh.py b/src/drift/text_mesh.py new file mode 100644 index 000000000..b280e26b5 --- /dev/null +++ b/src/drift/text_mesh.py @@ -0,0 +1,272 @@ +"""Authenticated discovery and text-only inference over the public peer transport. + +Consumers never construct a tokenizer, tensor model or artifact downloader here. +Text peers own the input/output weights and route generation through block peers. +""" + +import asyncio +import contextlib +import json +import logging +import time +import uuid +from typing import AsyncIterator + +from hivemind.p2p import P2PContext, PeerID, ServicerBase +from hivemind.proto import runtime_pb2 + +from drift.protocol_identity import TRANSPORT_SECURITY, ProtocolSecurityError, SignedRecord, _validate_lifetime +from drift.utils.client_dht import create_client_dht + +MAX_REQUEST_BYTES = 128 * 1024 +MAX_FRAME_BYTES = 64 * 1024 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +ANNOUNCEMENT_TTL = 40 +logger = logging.getLogger(__name__) + + +def encode(value, limit=MAX_FRAME_BYTES): + data = json.dumps(value, allow_nan=False, separators=(",", ":")).encode("utf-8") + if len(data) > limit: + raise ValueError("Text request or response is too large") + return data + + +def decode(data, limit=MAX_FRAME_BYTES): + if len(data) > limit: + raise ValueError("Text request or response is too large") + + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Duplicate field") + result[key] = value + return result + + def invalid(_value): + raise ValueError("Non-finite value") + + result = json.loads(data, object_pairs_hook=pairs, parse_constant=invalid) + if not isinstance(result, dict): + raise ValueError("Expected a text protocol object") + return result + + +def announcement_key(manifest): + return f"{manifest.dht_prefix}.text-v1" + + +def create_text_announcement(manifest, identity, *, max_context_tokens, max_output_tokens, now=None): + now = time.time() if now is None else now + return SignedRecord.create( + "text_peer_announcement", + { + "peer_id": str(identity.peer_id), + "manifest_digest": manifest.digest, + "dht_prefix": manifest.dht_prefix, + "execution_profile": manifest.runtime.to_dict(), + "transport_security": TRANSPORT_SECURITY, + "protocol": "text-v1", + "max_context_tokens": max_context_tokens, + "max_output_tokens": max_output_tokens, + "issued_at_ms": int(now * 1000), + "expires_at_ms": int((now + ANNOUNCEMENT_TTL) * 1000), + "sequence": time.time_ns(), + }, + identity, + ) + + +def verify_text_announcement(source, manifest, peer_id, *, revocations=None, replay_guard=None, now=None): + # Bound before RSA/JSON work, including already decoded DHT values. + encode(source, 16 * 1024) + record = SignedRecord.from_dict(source) + record.verify(expected_kind="text_peer_announcement") + p = record.payload + if str(record.peer_id) != str(peer_id) or p.get("peer_id") != str(peer_id): + raise ProtocolSecurityError("Text peer identity mismatch") + expected = { + "manifest_digest": manifest.digest, + "dht_prefix": manifest.dht_prefix, + "execution_profile": manifest.runtime.to_dict(), + "protocol": "text-v1", + "transport_security": TRANSPORT_SECURITY, + } + if any(p.get(key) != value for key, value in expected.items()): + raise ProtocolSecurityError("Text peer model or protocol mismatch") + for field in ("max_context_tokens", "max_output_tokens"): + if type(p.get(field)) is not int or not 1 <= p[field] <= 262144: + raise ProtocolSecurityError("Invalid text peer capacity") + if type(p.get("sequence")) is not int or p["sequence"] < 0: + raise ProtocolSecurityError("Invalid text peer sequence") + _validate_lifetime(p, now=now) + if revocations is not None: + revocations.require_active(record.key_id) + if replay_guard is not None: + replay_guard.check(record) + return record + + +def discover_text_peers(dht, manifest, *, revocations=None, replay_guard=None): + found = dht.get(announcement_key(manifest), latest=True) + values = getattr(found, "value", None) + if not isinstance(values, dict) or len(values) > 256: + return [] + peers = [] + for peer_id, wrapped in list(values.items())[:256]: + try: + record = verify_text_announcement( + getattr(wrapped, "value", wrapped), + manifest, + peer_id, + revocations=revocations, + replay_guard=replay_guard, + ) + peers.append(dict(record.payload)) + except (ValueError, TypeError, KeyError, ProtocolSecurityError): + continue + return sorted(peers, key=lambda p: p["peer_id"]) + + +class TextPeerProtocol(ServicerBase): + def __init__(self, manifest=None, engine=None): + self.manifest, self.engine = manifest, engine + + async def rpc_generate( + self, request: runtime_pb2.ExpertRequest, context: P2PContext + ) -> AsyncIterator[runtime_pb2.ExpertResponse]: + if request.uid != self.manifest.digest_id or request.tensors: + raise ValueError("Text request model mismatch") + body = decode(request.metadata, MAX_REQUEST_BYTES) + frames = self.engine.stream(body, str(context.remote_id)) + try: + async for frame in frames: + yield runtime_pb2.ExpertResponse(metadata=encode(dict(frame, manifest_digest=self.manifest.digest_id))) + finally: + await frames.aclose() + + async def rpc_cancel( + self, request: runtime_pb2.ExpertRequest, context: P2PContext + ) -> AsyncIterator[runtime_pb2.ExpertResponse]: + if request.uid != self.manifest.digest_id or request.tensors: + raise ValueError("Text request model mismatch") + body = decode(request.metadata, 1024) + cancelled = self.engine.cancel(body.get("request_id"), str(context.remote_id)) + yield runtime_pb2.ExpertResponse(metadata=encode({"cancelled": cancelled})) + + +class TextPeerUnavailable(RuntimeError): + pass + + +class TextPeerClient: + """One lightweight consumer. Retry another peer only before receiving answer text.""" + + def __init__(self, manifest, *, initial_peers, revocations=None, request_timeout=30, total_timeout=900, dht=None): + self.manifest = manifest + self.revocations = revocations + self.request_timeout = max(15, min(request_timeout, 120)) + self.total_timeout = min(total_timeout, 900) + self.dht = ( + dht + if dht is not None + else create_client_dht(initial_peers=initial_peers, client_mode=True, tls=True, startup_timeout=30) + ) + self._owns_dht = dht is None + + async def stream(self, body, *, chat): + request_id = uuid.uuid4().hex + payload = encode({"request_id": request_id, "chat": chat, "body": body}, MAX_REQUEST_BYTES) + peers = await asyncio.to_thread(discover_text_peers, self.dht, self.manifest, revocations=self.revocations) + if not peers: + raise TextPeerUnavailable("No community peer is ready to answer yet. Please try again shortly.") + p2p = await self.dht.replicate_p2p() + deadline = time.monotonic() + self.total_timeout + last_peer_error = None + try: + for candidate in peers[:3]: + emitted, complete, received = False, False, 0 + stub = TextPeerProtocol.get_stub(p2p, PeerID.from_base58(candidate["peer_id"])) + responses = None + try: + responses = await asyncio.wait_for( + stub.rpc_generate(runtime_pb2.ExpertRequest(uid=self.manifest.digest_id, metadata=payload)), + self.request_timeout, + ) + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("Community answer exceeded its time limit") + response = await asyncio.wait_for(anext(responses), min(self.request_timeout, remaining)) + received += len(response.metadata) + if response.tensors or received > MAX_RESPONSE_BYTES: + raise ValueError("Invalid community response") + frame = decode(response.metadata) + if frame.get("manifest_digest") != self.manifest.digest_id: + raise ProtocolSecurityError("Community response model mismatch") + if frame.get("type") == "error": + if frame.get("code") == "invalid_request": + raise ValueError(str(frame.get("message", "Invalid request"))[:256]) + raise TextPeerUnavailable(str(frame.get("message", "Community peer unavailable"))[:256]) + if frame.get("type") == "delta": + if not isinstance(frame.get("text"), str): + raise ValueError("Invalid community text") + emitted = emitted or bool(frame["text"]) + elif frame.get("type") == "done": + usage = frame.get("usage", {}) + if ( + frame.get("finish_reason") not in ("stop", "length") + or not isinstance(usage, dict) + or any( + type(usage.get(k)) is not int or not 0 <= usage[k] <= 1048576 + for k in ("prompt_tokens", "completion_tokens", "total_tokens") + ) + ): + raise ValueError("Invalid community usage") + complete = True + elif frame.get("type") != "heartbeat": + raise ValueError("Invalid community response type") + yield frame + if complete: + return + except (ValueError, ProtocolSecurityError): + raise + except Exception as exc: + if emitted: + raise TextPeerUnavailable( + "The community connection stopped during the answer. Please retry." + ) from exc + if isinstance(exc, TextPeerUnavailable): + last_peer_error = exc + else: + logger.warning("Community peer request failed (%s)", type(exc).__name__) + finally: + if not complete: + try: + # Use the same stream transport as generation, including + # on desktop daemons that do not support unary handlers. + async with asyncio.timeout(3): + cancelled = await stub.rpc_cancel( + runtime_pb2.ExpertRequest( + uid=self.manifest.digest_id, metadata=encode({"request_id": request_id}) + ) + ) + try: + await anext(cancelled) + finally: + await cancelled.aclose() + except Exception as exc: + logger.warning("Could not cancel community generation: %s", exc) + if responses is not None: + with contextlib.suppress(Exception): + await responses.aclose() + if last_peer_error is not None: + raise last_peer_error + raise TextPeerUnavailable("Could not connect to a community peer. Please try again shortly.") + finally: + await p2p.shutdown() + + def close(self): + if self._owns_dht and self.dht.is_alive(): + self.dht.shutdown() diff --git a/src/drift/utils/auto_config.py b/src/drift/utils/auto_config.py index ba548ec5b..138084a0f 100644 --- a/src/drift/utils/auto_config.py +++ b/src/drift/utils/auto_config.py @@ -40,6 +40,11 @@ def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike, None], *arg config = AutoConfig.from_pretrained(model_name_or_path, *args, **kwargs) model_type = config.model_type source_architectures = None + source_quantization = getattr(config, "quantization_config", None) + if isinstance(source_quantization, dict): + source_quantization_method = source_quantization.get("quant_method") + else: + source_quantization_method = getattr(source_quantization, "quant_method", None) if model_type not in _CLASS_MAPPING: # Multimodal wrappers (e.g. Gemma4ForConditionalGeneration) carry the language model in a # nested text_config; fall back to it so we serve the text tower of a multimodal checkpoint. @@ -58,6 +63,9 @@ def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike, None], *arg if cls._mapping_field == "config" and source_architectures: loaded_config = result[0] if isinstance(result, tuple) else result loaded_config._source_architectures = source_architectures + if cls._mapping_field == "config" and source_quantization_method is not None: + loaded_config = result[0] if isinstance(result, tuple) else result + loaded_config._source_quantization_method = source_quantization_method return result diff --git a/src/drift/utils/client_dht.py b/src/drift/utils/client_dht.py new file mode 100644 index 000000000..b4c9bf847 --- /dev/null +++ b/src/drift/utils/client_dht.py @@ -0,0 +1,35 @@ +"""Bounded client discovery startup when saved peers have gone offline.""" + +from hivemind.utils import get_logger + +logger = get_logger(__name__) + + +def create_client_dht(*, initial_peers=(), start=True, startup_timeout=15.0, **kwargs): + from hivemind import DHT + + peers = tuple(dict.fromkeys(initial_peers)) + if not start: + return DHT(initial_peers=list(peers), start=False, startup_timeout=startup_timeout, **kwargs) + # Joining a set containing dead cached addresses can time out even when + # the configured bootstrap is healthy. One successful seed is sufficient + # to discover the rest of the mesh. Keep later seeds as bounded fallbacks. + candidates = [(peer,) for peer in peers] or [()] + last_error = None + for candidate in candidates: + dht = None + try: + dht = DHT(initial_peers=list(candidate), start=False, startup_timeout=startup_timeout, **kwargs) + dht.run_in_background(timeout=startup_timeout) + return dht + except BaseException as exc: + if dht is not None: + try: + dht.shutdown() + except Exception: + logger.exception("Failed to close unsuccessful discovery startup") + if not isinstance(exc, Exception): + raise + last_error = exc + assert last_error is not None + raise last_error diff --git a/src/drift/utils/convert_block.py b/src/drift/utils/convert_block.py index 3b93f592c..88984293e 100644 --- a/src/drift/utils/convert_block.py +++ b/src/drift/utils/convert_block.py @@ -17,10 +17,11 @@ logger = get_logger(__name__) -class QuantType(Enum): - NONE = 0 - INT8 = 1 # 8-bit as in the LLM.int8() paper - NF4 = 2 # 4-bit as in the QLoRA paper +class QuantType(Enum): + NONE = 0 + INT8 = 1 # 8-bit as in the LLM.int8() paper + NF4 = 2 # 4-bit as in the QLoRA paper + FP8_DEQUANT = 3 # Official fine-grained FP8 checkpoint, dequantized to the manifested dtype on load def convert_block( @@ -53,8 +54,8 @@ def convert_block( block = make_tensor_parallel(block, config, tensor_parallel_devices, output_device=output_device) - if quant_type != QuantType.NONE: - block = quantize_module(block, quant_type=quant_type) + if quant_type in (QuantType.INT8, QuantType.NF4): + block = quantize_module(block, quant_type=quant_type) for shard, device in zip(block.module_shards, block.devices): shard.to(device) diff --git a/src/drift/utils/dht.py b/src/drift/utils/dht.py index d022cc6fa..92f563404 100644 --- a/src/drift/utils/dht.py +++ b/src/drift/utils/dht.py @@ -3,6 +3,7 @@ """ from __future__ import annotations +import asyncio import math from functools import partial from typing import Any, Dict, List, Mapping, Optional, Sequence, Union @@ -10,6 +11,7 @@ from hivemind.dht import DHT, DHTNode, DHTValue from hivemind.p2p import PeerID from hivemind.utils import DHTExpiration, MPFuture, get_dht_time, get_logger +from hivemind.utils.multiaddr import Multiaddr from drift.data_structures import ( CHAIN_DELIMITER, @@ -26,6 +28,59 @@ logger = get_logger(__name__) +async def _reconnect_dht_if_isolated(dht: DHT, node: DHTNode) -> None: + """Rejoin configured seeds without replacing a worker's live RPC identity.""" + seeds = getattr(dht, "initial_peers", None) + if not seeds: + return + + def usable_peers(): + return sum(peer not in node.blacklist for peer in node.protocol.routing_table.uid_to_peer_id.values()) + + if usable_peers(): + return + lock = getattr(node, "_drift_reconnect_lock", None) + if lock is None: + lock = node._drift_reconnect_lock = asyncio.Lock() + async with lock: + if usable_peers(): + return + peers = {} + for seed in seeds: + address = Multiaddr(seed) + peer = PeerID.from_base58(address["p2p"]) + endpoint = address.decapsulate(Multiaddr(f"/p2p/{peer}")) + peers.setdefault(peer, set()).add(endpoint) + + async def rejoin(peer, endpoints): + # A lost connection can also leave the daemon without usable peer + # addresses. Reintroduce the configured endpoints before the RPC. + await asyncio.wait_for(node.p2p._client.connect(peer, endpoints), timeout=10) + response = await node.protocol.call_ping(peer, validate=True, strict=True) + if response is not None: + # Rejoining the routing table alone does not clear Hivemind's + # independent failed-query backoff. A validated live response + # must make the recovered seed usable by subsequent lookups. + node.blacklist.register_success(peer) + return response + + # call_ping applies the same reachability and clock validation as initial + # bootstrap. Its routing-table update runs as a scheduled coroutine. + results = await asyncio.gather( + *(rejoin(peer, endpoints) for peer, endpoints in peers.items()), return_exceptions=True + ) + await asyncio.sleep(0) + count = usable_peers() + if count: + logger.info("Reconnected isolated DHT to %d routing peer(s) using configured seeds", count) + else: + failures = sorted({type(result).__name__ for result in results if isinstance(result, BaseException)}) + logger.warning( + "DHT has no routing peers after retrying configured seeds (%s)", + ", ".join(failures) or "no validated response", + ) + + def declare_active_modules( dht: DHT, uids: Sequence[ModuleUID], @@ -62,7 +117,8 @@ async def _declare_active_modules( server_info: ServerInfo, expiration_time: DHTExpiration, ) -> Dict[ModuleUID, bool]: - num_workers = len(uids) if dht.num_workers is None else min(len(uids), dht.num_workers) + await _reconnect_dht_if_isolated(dht, node) + num_workers = min(len(uids), 4 if dht.num_workers is None else dht.num_workers, 4) return await node.store_many( keys=uids, subkeys=[dht.peer_id.to_base58()] * len(uids), @@ -84,12 +140,12 @@ def get_remote_module_infos( *, latest: bool = False, return_future: bool = False, -) -> Union[List[RemoteModuleInfo], MPFuture]: - if manifest_digest is not None and manifest_execution_profile is None: - raise ValueError("Manifested DHT reads require the exact manifest execution profile") - if manifest_digest is None and manifest_execution_profile is not None: - raise ValueError("manifest_execution_profile is only valid with manifest_digest") - return dht.run_coroutine( +) -> Union[List[RemoteModuleInfo], MPFuture]: + if manifest_digest is not None and manifest_execution_profile is None: + raise ValueError("Manifested DHT reads require the exact manifest execution profile") + if manifest_digest is None and manifest_execution_profile is not None: + raise ValueError("manifest_execution_profile is only valid with manifest_digest") + return dht.run_coroutine( partial( _get_remote_module_infos, uids=uids, @@ -117,15 +173,19 @@ async def _get_remote_module_infos( expiration_time: Optional[DHTExpiration], latest: bool, ) -> List[RemoteModuleInfo]: + await _reconnect_dht_if_isolated(dht, node) if latest: assert expiration_time is None, "You should define either `expiration_time` or `latest`, not both" expiration_time = math.inf elif expiration_time is None: expiration_time = get_dht_time() - num_workers = len(uids) if dht.num_workers is None else min(len(uids), dht.num_workers) + # A tiny swarm can have one responding seed and many stale peer IDs. Issuing + # one traversal per model block floods that seed and times out healthy RPCs. + num_workers = min(len(uids), 4 if dht.num_workers is None else dht.num_workers, 4) found: Dict[ModuleUID, DHTValue] = await node.get_many(uids, expiration_time, num_workers=num_workers) modules = [RemoteModuleInfo(uid=uid, servers={}) for uid in uids] + signed_candidates = {} for module_info in modules: metadata = found[module_info.uid] if metadata is None or not isinstance(metadata.value, dict): @@ -138,7 +198,7 @@ async def _get_remote_module_infos( peer_id = PeerID.from_base58(peer_id) server_info = ServerInfo.from_tuple(server_info.value) - if active_adapter and active_adapter not in server_info.adapters: + if manifest_digest is None and active_adapter and active_adapter not in server_info.adapters: logger.debug(f"Skipped server {peer_id} since it does not have adapter {active_adapter}") continue @@ -152,7 +212,7 @@ async def _get_remote_module_infos( if server_info.signed_announcement is None: raise ValueError("manifested server announcement is unsigned") dht_prefix, _ = parse_uid(module_info.uid) - verify_worker_announcement( + record = verify_worker_announcement( server_info.signed_announcement, expected_peer_id=peer_id, expected_dht_prefix=dht_prefix, @@ -160,7 +220,7 @@ async def _get_remote_module_infos( expected_server_info=server_info.signed_payload(), expected_execution_profile=manifest_execution_profile, revocations=revocations, - replay_guard=replay_guard, + replay_guard=None, ) _, block_index = parse_uid(module_info.uid) if ( @@ -169,10 +229,37 @@ async def _get_remote_module_infos( or not server_info.start_block <= block_index < server_info.end_block ): raise ValueError("signed worker announcement does not cover this DHT block key") + signed_candidates.setdefault(peer_id, []).append((record, server_info)) + continue module_info.servers[peer_id] = server_info except (TypeError, ValueError) as e: logger.warning(f"Incorrect peer entry for uid={module_info.uid}, peer_id={peer_id}: {e}") + # A worker publishes one signed span under several DHT keys. get_many can + # observe a mixture of two renewal generations; key order must not create + # fictitious holes or retain blocks removed by a newer announcement. Choose + # the newest verified record once per peer, then enforce the replay watermark + # and apply only that record's authenticated span to the requested keys. + for peer_id, candidates in signed_candidates.items(): + order = lambda item: (item[0].payload["issued_at_ms"], item[0].payload["sequence"]) + record, server_info = max(candidates, key=order) + newest_order = order((record, server_info)) + try: + if len({item[0].digest for item in candidates if order(item) == newest_order}) != 1: + raise ValueError("identity equivocated within the DHT snapshot") + if replay_guard is not None: + replay_guard.check(record) + if active_adapter and active_adapter not in server_info.adapters: + continue + for module_info in modules: + prefix, block_index = parse_uid(module_info.uid) + if ( + prefix == record.payload["dht_prefix"] + and server_info.start_block <= block_index < server_info.end_block + ): + module_info.servers[peer_id] = server_info + except (TypeError, ValueError) as exc: + logger.warning(f"Rejected signed span for peer_id={peer_id}: {exc}") return modules diff --git a/src/drift/utils/disk_cache.py b/src/drift/utils/disk_cache.py index 864f49e17..e01e23e33 100644 --- a/src/drift/utils/disk_cache.py +++ b/src/drift/utils/disk_cache.py @@ -65,6 +65,53 @@ def allow_cache_writes(cache_dir: Optional[str]): return file_lock(Path(cache_dir, BLOCKS_LOCK_FILE), exclusive=True) +def _cache_file_identity(path, info): + # Some filesystems do not expose stable inode numbers. Count those paths + # separately rather than accidentally treating every file as inode zero. + return (info.st_dev, info.st_ino) if info.st_ino else ("path", os.path.normcase(os.path.abspath(path))) + + +def _manifest_cache_usage(cache_dir, cache_info): + """Count owned artifacts/partials once, including files shared with the Hub cache.""" + root = Path(cache_dir) / "manifest-artifacts" + if root.is_symlink() or getattr(root, "is_junction", lambda: False)(): + raise RuntimeError("Cannot account for a linked manifest artifact cache directory") + if not root.exists(): + return 0, set() + + def identity(path): + info = path.stat() + return _cache_file_identity(path, info) + + seen = { + identity(file.blob_path) for repo in cache_info.repos for revision in repo.revisions for file in revision.files + } + protected = set() + additional_bytes = 0 + pending = [root] + while pending: + directory = pending.pop() + if directory.is_symlink() or getattr(directory, "is_junction", lambda: False)(): + raise RuntimeError("Cannot account for a linked manifest artifact cache directory") + with os.scandir(directory) as entries: + for entry in entries: + if entry.is_symlink(): + raise RuntimeError("Cannot account for a linked manifest artifact cache entry") + if entry.is_dir(follow_symlinks=False): + pending.append(Path(entry.path)) + elif entry.is_file(follow_symlinks=False): + # DirEntry.stat() reports zero inode/device fields on Windows. + info = Path(entry.path).stat(follow_symlinks=False) + key = _cache_file_identity(entry.path, info) + protected.add(key) + if key not in seen: + seen.add(key) + additional_bytes += info.st_size + else: + raise RuntimeError("Cannot account for a non-file manifest artifact cache entry") + return additional_bytes, protected + + def free_disk_space_for( size: int, *, @@ -77,8 +124,11 @@ def free_disk_space_for( cache_info = huggingface_hub.scan_cache_dir(cache_dir) available_space = shutil.disk_usage(cache_dir).free - os_quota + manifest_bytes, protected = (0, set()) + if max_disk_space is not None or size > available_space: + manifest_bytes, protected = _manifest_cache_usage(cache_dir, cache_info) if max_disk_space is not None: - available_space = min(available_space, max_disk_space - cache_info.size_on_disk) + available_space = min(available_space, max_disk_space - cache_info.size_on_disk - manifest_bytes) gib = 1024**3 logger.debug(f"Disk space: required {size / gib:.1f} GiB, available {available_space / gib:.1f} GiB") @@ -92,6 +142,9 @@ def free_disk_space_for( freed_space = 0 extra_space_needed = size - available_space for file in sorted(cached_files, key=lambda file: file.blob_last_accessed): + info = file.blob_path.stat() + if _cache_file_identity(file.blob_path, info) in protected: + continue # A manifest snapshot still owns these bytes; deleting its Hub alias frees nothing. os.remove(file.file_path) # Remove symlink os.remove(file.blob_path) # Remove contents @@ -103,8 +156,10 @@ def free_disk_space_for( logger.info(f"Removed {len(removed_files)} files to free {freed_space / gib:.1f} GiB of disk space") logger.debug(f"Removed paths: {[str(file.file_path) for file in removed_files]}") - if freed_space < extra_space_needed: + remaining_os_shortfall = max(0, size - (shutil.disk_usage(cache_dir).free - os_quota)) + shortfall = max(extra_space_needed - freed_space, remaining_os_shortfall) + if shortfall > 0: raise RuntimeError( - f"Insufficient disk space to load a block. Please free {(extra_space_needed - freed_space) / gib:.1f} GiB " + f"Insufficient disk space to load a block. Please free {shortfall / gib:.1f} GiB " f"on the volume for {cache_dir} or increase --max_disk_space if you set it manually" ) diff --git a/src/drift/utils/download_progress.py b/src/drift/utils/download_progress.py new file mode 100644 index 000000000..1586bb574 --- /dev/null +++ b/src/drift/utils/download_progress.py @@ -0,0 +1,146 @@ +"""Content-free local artifact progress; reporting never authorizes cached bytes.""" + +from __future__ import annotations + +import contextlib +import contextvars +import json +import math +import os +import threading +import time +from pathlib import Path + +_observer = contextvars.ContextVar("artifact_progress", default=None) +_process_observer = None + + +class DownloadProgress: + def __init__(self, *, output=None): + self._lock = threading.RLock() + self._files = {} + self._state = "waiting" + self._current = None + self._retries = 0 + self._network_bytes = 0 + self._samples = [] + self._output = output + self._last_write = 0.0 + self._updated = time.time() + + @contextlib.contextmanager + def observe(self): + token = _observer.set(self) + try: + yield self + finally: + _observer.reset(token) + + def event(self, manifest, artifact, state, *, received=None, transferred=0, resumed=0): + with self._lock: + key = (manifest.digest, artifact.path) + item = self._files.setdefault(key, {"size": artifact.size, "received": 0, "verified": False, "resumed": 0}) + if received is not None: + item["received"] = min(artifact.size, max(0, received)) + item["resumed"] = max(item["resumed"], resumed) + item["verified"] = state == "verified" + if item["verified"]: + item["received"] = artifact.size + self._current = artifact.path + self._state = "loading" if state == "verified" else state + self._retries += state == "retrying" + self._network_bytes += max(0, transferred) + now = time.monotonic() + self._samples.append((now, self._network_bytes)) + self._samples = [sample for sample in self._samples[-256:] if now - sample[0] <= 5] + self._updated = time.time() + self._write(force=state not in ("downloading",)) + + def finish(self, state): + with self._lock: + self._state = state + self._updated = time.time() + self._write(force=True) + + def snapshot(self): + with self._lock: + now = time.monotonic() + samples = [sample for sample in self._samples if now - sample[0] <= 5] + speed = 0.0 + if len(samples) > 1 and samples[-1][0] - samples[0][0] > 0: + speed = (samples[-1][1] - samples[0][1]) / (samples[-1][0] - samples[0][0]) + current = next((item for key, item in self._files.items() if key[1] == self._current), None) + return { + "schema_version": 1, + "state": self._state, + "artifact": self._current, + "artifact_bytes": None if current is None else current["size"], + "artifact_received_bytes": None if current is None else current["received"], + "selected_bytes": sum(item["size"] for item in self._files.values()), + "received_bytes": sum(item["received"] for item in self._files.values()), + "verified_bytes": sum(item["size"] for item in self._files.values() if item["verified"]), + "verified_files": sum(item["verified"] for item in self._files.values()), + "selected_files": len(self._files), + "resumed_bytes": sum(item["resumed"] for item in self._files.values()), + "bytes_per_second": speed if self._state == "downloading" else 0.0, + "retries": self._retries, + "updated_at": self._updated, + } + + def _write(self, *, force=False): + if self._output is None or (not force and time.monotonic() - self._last_write < 0.25): + return + try: + target = Path(self._output) + temporary = target.with_suffix(".tmp") + temporary.write_text(json.dumps({**self.snapshot(), "pid": os.getpid()}), encoding="utf-8") + os.replace(temporary, target) + self._last_write = time.monotonic() + except OSError: + pass # A display failure must not interrupt a verified transfer. + + +def current_progress(): + global _process_observer + observer = _observer.get() + if observer is not None: + return observer + if _process_observer is None: + output = os.environ.pop("DRIFT_DOWNLOAD_PROGRESS", None) + if output: + _process_observer = DownloadProgress(output=output) + return _process_observer + + +def public_progress(value): + """Allow only content-free fields from a worker's bounded local status file.""" + states = {"waiting", "checking", "downloading", "retrying", "verifying", "loading", "ready", "failed", "paused"} + if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("state") not in states: + return None + result = {"schema_version": 1, "state": value["state"]} + artifact = value.get("artifact") + result["artifact"] = " ".join(artifact.split())[:256] if isinstance(artifact, str) else None + for key in ( + "artifact_bytes", + "artifact_received_bytes", + "selected_bytes", + "received_bytes", + "verified_bytes", + "verified_files", + "selected_files", + "resumed_bytes", + "retries", + "bytes_per_second", + "updated_at", + ): + item = value.get(key) + numeric_type = (int, float) if key in ("bytes_per_second", "updated_at") else int + result[key] = ( + item + if not isinstance(item, bool) + and isinstance(item, numeric_type) + and math.isfinite(item) + and 0 <= item <= 64 * 1024**4 + else None + ) + return result diff --git a/src/drift/utils/hub_ranges.py b/src/drift/utils/hub_ranges.py new file mode 100644 index 000000000..cfdbb68b1 --- /dev/null +++ b/src/drift/utils/hub_ranges.py @@ -0,0 +1,154 @@ +"""Bounded HTTP ranges for large immutable model artifacts.""" + +import os +import time +from concurrent.futures import ThreadPoolExecutor + +import requests + +RANGE_BYTES = 8 * 1024**2 +RANGE_WORKERS = 4 + + +def download_ranges(url, headers, partial, *, size, offset, progress=None): + """Append contiguous bytes only; the caller verifies SHA-256 before promotion. + + A first 200 response safely falls back to a complete sequential download. + A range-supporting origin is read in bounded batches of four 8 MiB ranges. + Failed ranges never leave gaps or preallocated unverified tails. + """ + from drift.model_manifest import ManifestError + + started = time.monotonic() + + def report(state, **values): + if progress is not None: + progress(state, **values) + + def retry_transfer(operation): + for attempt in range(3): + try: + return operation() + except requests.RequestException as exc: + status = None if exc.response is None else exc.response.status_code + if ( + attempt == 2 + or time.monotonic() - started > 3600 + or (status is not None and status < 500 and status != 429) + ): + raise + report("retrying") + time.sleep(0.5 * 2**attempt) + + def open_range(start, end): + response = requests.get( + url, + headers=dict(headers, Range=f"bytes={start}-{end}"), + stream=True, + allow_redirects=True, + timeout=(10, 60), + ) + try: + response.raise_for_status() + if response.status_code == 206: + expected = f"bytes {start}-{end}/{size}" + if response.headers.get("Content-Range") != expected: + raise ManifestError("Hub returned an inconsistent bounded Content-Range") + elif response.status_code != 200: + raise ManifestError("Hub returned an unsupported artifact response") + return response + except BaseException: + response.close() + raise + + def read_exact(response, length): + body = bytearray() + until = time.monotonic() + 120 + for chunk in response.iter_content(chunk_size=min(1024**2, length)): + if time.monotonic() > until: + raise requests.Timeout("Bounded artifact range exceeded its transfer deadline") + if len(body) + len(chunk) > length: + raise ManifestError("Hub returned more bytes than the declared range") + body.extend(chunk) + report("downloading", transferred=len(chunk)) + if len(body) != length: + raise requests.ConnectionError("Hub ended a bounded artifact range early") + return body + + def fetch_once(start, end): + with open_range(start, end) as response: + if response.status_code != 206: + raise ManifestError("Hub stopped honoring artifact ranges during transfer") + return read_exact(response, end - start + 1) + + def fetch(start, end): + return retry_transfer(lambda: fetch_once(start, end)) + + end = min(size, offset + RANGE_BYTES) - 1 + + def first_transfer(): + nonlocal offset, end + # A failed 200 fallback truncates/replaces the old partial. An origin + # may honor Range again on retry, so bind its next request to the prefix + # actually present, not the offset from before that truncation. + offset = partial.stat().st_size if partial.exists() else 0 + if offset == size: + return None # The caller still verifies the complete hash. + if offset > size: + raise ManifestError("Artifact partial exceeds its declared length") + end = min(size, offset + RANGE_BYTES) - 1 + with open_range(offset, end) as response: + if response.status_code == 200: + # Never append a complete response to an existing partial. + written = 0 + with partial.open("wb") as stream: + for chunk in response.iter_content(chunk_size=1024**2): + if time.monotonic() - started > 3600: + raise requests.Timeout("Artifact transfer exceeded its deadline") + if written + len(chunk) > size: + raise ManifestError("Hub returned more bytes than the manifested artifact") + stream.write(chunk) + written += len(chunk) + report("downloading", received=written, transferred=len(chunk)) + stream.flush() + os.fsync(stream.fileno()) + if written != size: + raise requests.ConnectionError("Hub ended the artifact before its declared length") + return None + return read_exact(response, end - offset + 1) + + first = retry_transfer(first_transfer) + if first is None: + return + + with partial.open("ab" if offset else "wb") as stream: + stream.write(first) + del first + stream.flush() + os.fsync(stream.fileno()) + offset = end + 1 + report("downloading", received=offset) + with ThreadPoolExecutor(max_workers=RANGE_WORKERS, thread_name_prefix="drift-artifact-range") as pool: + while offset < size: + if time.monotonic() - started > 3600: + raise requests.Timeout("Artifact transfer exceeded its deadline") + ranges = [ + (start, min(size, start + RANGE_BYTES) - 1) + for start in range(offset, min(size, offset + RANGE_WORKERS * RANGE_BYTES), RANGE_BYTES) + ] + futures = [pool.submit(fetch, start, end) for start, end in ranges] + try: + for (start, end), future in zip(ranges, futures): + body = future.result() + assert start == offset + stream.write(body) + del body + stream.flush() + os.fsync(stream.fileno()) + offset = end + 1 + report("downloading", received=offset) + finally: + for future in futures: + future.cancel() + futures.clear() + del future diff --git a/src/drift/utils/resource_limits.py b/src/drift/utils/resource_limits.py new file mode 100644 index 000000000..e93063dbc --- /dev/null +++ b/src/drift/utils/resource_limits.py @@ -0,0 +1,7 @@ +"""Stable worker exit signal for a budget that cannot admit the selected blocks.""" + +DEVICE_MEMORY_BUDGET_EXIT_CODE = 78 + + +class DeviceMemoryBudgetError(ValueError): + """The selected blocks cannot fit within the configured device-memory ceiling.""" diff --git a/tests/test_automatic_formation.py b/tests/test_automatic_formation.py new file mode 100644 index 000000000..1ed053b9b --- /dev/null +++ b/tests/test_automatic_formation.py @@ -0,0 +1,70 @@ +"""Regression coverage for a network with just enough multi-block capacity.""" + +from drift.node.contribution_planner import AutomaticContributionPlanner, PlacementCandidate + + +def candidate(counts): + return PlacementCandidate( + model_id="qwen", + manifest_digest="sha256:" + "a" * 64, + priority=0, + preferred=False, + artifact_bytes=1, + total_blocks=len(counts), + health={ + "status": "complete" if all(counts) else "incomplete", + "last_updated_age": 0, + "replica_counts": counts.copy(), + }, + ) + + +def test_four_sixteen_block_contributors_fill_all_sixty_four_blocks(): + for cohort in range(32): + counts = [0] * 64 + allocations = [] + for index in range(4): + planner = AutomaticContributionPlanner(num_blocks=16, jitter_seed=f"{cohort}-{index}") + decision = planner.plan((candidate(counts),), sharing_enabled=True, now=0).decision + allocations.append(decision.block_indices) + start, end = map(int, decision.block_indices.split(":")) + for block in range(start, end): + counts[block] += 1 + assert counts == [1] * 64, allocations + + +def test_complete_route_does_not_move_a_sole_provider_after_residency(): + planner = AutomaticContributionPlanner(num_blocks=16, jitter_seed="node") + counts = [1] * 64 + counts[0:16] = [0] * 16 + original = planner.plan((candidate(counts),), sharing_enabled=True, now=0).decision + counts[:] = [1] * 64 + later = planner.plan((candidate(counts),), sharing_enabled=True, now=1800).decision + assert later.block_indices == original.block_indices + + +def test_slow_growth_does_not_move_existing_unique_blocks_after_residency(): + planners = [] + counts = [0] * 64 + for index in range(4): + now = index * 1800 + for planner, original in planners: + later = planner.plan((candidate(counts),), sharing_enabled=True, now=now).decision + assert later.block_indices == original.block_indices + planner = AutomaticContributionPlanner(num_blocks=16, jitter_seed=f"slow-{index}") + original = planner.plan((candidate(counts),), sharing_enabled=True, now=now).decision + start, end = map(int, original.block_indices.split(":")) + for block in range(start, end): + counts[block] += 1 + planners.append((planner, original)) + assert counts == [1] * 64 + + +def test_partial_overlap_may_move_when_it_increases_total_coverage(): + planner = AutomaticContributionPlanner(num_blocks=16, jitter_seed="overlap") + counts = [0] * 16 + [2] * 48 + original = planner.plan((candidate(counts),), sharing_enabled=True, now=0).decision + assert original.block_indices == "0:16" + counts = [2] * 8 + [1] * 8 + [0] * 48 + later = planner.plan((candidate(counts),), sharing_enabled=True, now=1800).decision + assert later.block_indices != original.block_indices diff --git a/tests/test_catalog_bootstrap.py b/tests/test_catalog_bootstrap.py index 181667e7e..da35ad44a 100644 --- a/tests/test_catalog_bootstrap.py +++ b/tests/test_catalog_bootstrap.py @@ -1,4 +1,8 @@ import json +import os +import subprocess +import sys +import time import pytest @@ -135,6 +139,9 @@ def fetch(url, maximum_bytes): assert automatic.model == "auto" assert automatic.num_blocks == 1 assert automatic.enabled is True + assert config.contribution_policy.sharing_enabled is False + assert config.contribution_policy.max_vram == "100%" + assert config.contribution_policy.max_processing_percent == 100 assert automatic.identity_path == (data_dir / "worker-identities" / "automatic.key").resolve() assert all(model.initial_peers == (PEER,) for model in config.models) assert all(model.manifest_path.parent == (data_dir / "manifests").resolve() for model in config.models) @@ -142,7 +149,156 @@ def fetch(url, maximum_bytes): assert (data_dir / "catalogs" / "communityai-test" / "rollback-state.json").is_file() assert calls[0] == (bootstrap["catalog_mirrors"][0], MAX_CATALOG_BYTES) assert any(limit == MAX_MANIFEST_BYTES for _, limit in calls) - assert not (data_dir / ".catalog-bootstrap.lock").exists() + assert (data_dir / ".catalog-bootstrap.lock").is_file() + + +@pytest.mark.parametrize("prior_failure", [False, True]) +def test_bootstrap_recovers_an_empty_legacy_lock_marker(tmp_path, prior_failure): + bootstrap, envelope, manifests = _release_documents() + bootstrap_path = _write_bootstrap(tmp_path, bootstrap) + data_dir = tmp_path / "data" + data_dir.mkdir() + lock_path = data_dir / ".catalog-bootstrap.lock" + lock_path.touch() + original = lock_path.stat() + if prior_failure: + with pytest.raises(CatalogBootstrapError, match="No trusted usable"): + bootstrap_node_from_catalog( + bootstrap_path, + data_dir=data_dir, + config_path=data_dir / "node-config.json", + fetch_text=lambda *_: (_ for _ in ()).throw(CatalogBootstrapError("offline")), + now=NOW, + ) + + result = bootstrap_node_from_catalog( + bootstrap_path, + data_dir=data_dir, + config_path=data_dir / "node-config.json", + fetch_text=lambda url, _: json.dumps(envelope.to_dict()) + if url in bootstrap["catalog_mirrors"] + else manifests[url], + now=NOW, + ) + + assert result.created is True + assert os.path.samestat(original, lock_path.stat()) + + +def test_bootstrap_lock_excludes_a_live_installer_and_recovers_after_process_kill(tmp_path): + bootstrap, envelope, manifests = _release_documents() + bootstrap_path = _write_bootstrap(tmp_path, bootstrap) + data_dir = tmp_path / "data" + config_path = data_dir / "node-config.json" + ready_path = tmp_path / "fetch-started" + child_code = """ +import sys +import time +from pathlib import Path +from drift.node.catalog_bootstrap import bootstrap_node_from_catalog + +def blocked_fetch(*_args): + Path(sys.argv[4]).touch() + while True: + time.sleep(1) + +bootstrap_node_from_catalog(sys.argv[1], data_dir=sys.argv[2], config_path=sys.argv[3], fetch_text=blocked_fetch) +""" + child = subprocess.Popen( + [sys.executable, "-c", child_code, str(bootstrap_path), str(data_dir), str(config_path), str(ready_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + deadline = time.monotonic() + 30 + while not ready_path.exists() and child.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + if child.poll() is not None: + stdout, stderr = child.communicate(timeout=5) + pytest.fail(f"bootstrap child exited before reaching the locked fetch: {stdout}\n{stderr}") + assert ready_path.exists(), "bootstrap child did not reach the locked fetch" + lock_path = data_dir / ".catalog-bootstrap.lock" + original = lock_path.stat() + with pytest.raises(CatalogBootstrapError, match="already in progress"): + bootstrap_node_from_catalog( + bootstrap_path, + data_dir=data_dir, + config_path=config_path, + fetch_text=lambda *_: pytest.fail("a concurrent installer must not fetch"), + now=NOW, + ) + assert os.path.samestat(original, lock_path.stat()) + child.kill() + child.communicate(timeout=10) + assert not config_path.exists() + + result = bootstrap_node_from_catalog( + bootstrap_path, + data_dir=data_dir, + config_path=config_path, + fetch_text=lambda url, _: ( + json.dumps(envelope.to_dict()) if url in bootstrap["catalog_mirrors"] else manifests[url] + ), + now=NOW, + ) + assert result.created is True + assert os.path.samestat(original, lock_path.stat()) + assert NodeConfig.load(config_path).models + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=10) + + +@pytest.mark.parametrize("unsafe_kind", ["symlink", "hardlink", "directory"]) +def test_bootstrap_refuses_unsafe_lock_targets_before_fetching(tmp_path, unsafe_kind): + bootstrap, _, _ = _release_documents() + bootstrap_path = _write_bootstrap(tmp_path, bootstrap) + data_dir = tmp_path / "data" + data_dir.mkdir() + lock_path = data_dir / ".catalog-bootstrap.lock" + target = tmp_path / "unrelated-file" + target.write_text("unchanged", encoding="utf-8") + try: + if unsafe_kind == "symlink": + lock_path.symlink_to(target) + elif unsafe_kind == "hardlink": + os.link(target, lock_path) + else: + lock_path.mkdir() + except OSError: + pytest.skip(f"{unsafe_kind} is unavailable on this test host") + with pytest.raises(CatalogBootstrapError, match="unsafe catalog bootstrap lock"): + bootstrap_node_from_catalog( + bootstrap_path, + data_dir=data_dir, + config_path=data_dir / "node-config.json", + fetch_text=lambda *_: pytest.fail("an unsafe lock target must not fetch"), + now=NOW, + ) + assert target.read_text(encoding="utf-8") == "unchanged" + + +def test_bootstrap_refuses_a_linked_lock_directory(tmp_path): + bootstrap, _, _ = _release_documents() + bootstrap_path = _write_bootstrap(tmp_path, bootstrap) + real_data = tmp_path / "real-data" + real_data.mkdir() + linked_data = tmp_path / "linked-data" + try: + linked_data.symlink_to(real_data, target_is_directory=True) + except OSError: + pytest.skip("directory symbolic links are unavailable on this test host") + with pytest.raises(CatalogBootstrapError, match="unsafe catalog bootstrap lock directory"): + bootstrap_node_from_catalog( + bootstrap_path, + data_dir=linked_data, + config_path=tmp_path / "node-config.json", + fetch_text=lambda *_: pytest.fail("a linked lock directory must not fetch"), + now=NOW, + ) + assert list(real_data.iterdir()) == [] def test_existing_config_is_preserved_without_network_access(tmp_path): diff --git a/tests/test_catalog_desktop_cleanup.py b/tests/test_catalog_desktop_cleanup.py new file mode 100644 index 000000000..3642ff1cb --- /dev/null +++ b/tests/test_catalog_desktop_cleanup.py @@ -0,0 +1,117 @@ +"""Owned-process fallback checks without launching Qt or a model runtime.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "desktop" / "src")) +import qualify_catalog_desktop as replay + + +class Process: + def __init__(self, pid, created, children=()): + self.pid, self.created, self.descendants = pid, created, children + self.alive = True + self.signals = [] + + def create_time(self): + return self.created + + def is_running(self): + return self.alive + + def status(self): + return replay.psutil.STATUS_RUNNING if self.alive else replay.psutil.STATUS_DEAD + + def children(self, recursive=False): + assert recursive + return self.descendants + + def terminate(self): + self.signals.append("terminate") + self.alive = False + + def kill(self): + self.signals.append("kill") + self.alive = False + + +def test_fallback_stops_owned_descendants_without_signaling_reused_pid(monkeypatch): + child = Process(12, 120) + owned = Process(10, 100, (child,)) + reused = Process(11, 999) + processes = {process.pid: process for process in (owned, reused, child)} + monkeypatch.setattr(replay.psutil, "Process", processes.__getitem__) + monkeypatch.setattr(replay.time, "sleep", lambda _: None) + identities = [(10, 100), (11, 110)] + + replay.force_stop_owned_tree(identities) + + assert owned.signals == child.signals == ["terminate"] + assert reused.signals == [] and reused.alive + assert (12, 120) in identities + + +def test_fallback_reports_failed_cleanup_when_ownership_cannot_be_inspected(monkeypatch): + def denied(_): + raise replay.psutil.AccessDenied() + + monkeypatch.setattr(replay.psutil, "Process", denied) + with pytest.raises(replay.psutil.AccessDenied): + replay.force_stop_owned_tree([(10, 100)]) + + +def test_fallback_has_a_finite_deadline_without_signaling_after_expiry(monkeypatch): + owned = Process(10, 100) + clock = iter((100, 116)) + monkeypatch.setattr(replay.psutil, "Process", lambda _: owned) + monkeypatch.setattr(replay.time, "monotonic", lambda: next(clock)) + with pytest.raises(TimeoutError, match="deadline"): + replay.force_stop_owned_tree([(10, 100)], timeout=15) + assert owned.signals == [] + + +def test_zombie_is_not_signaled_or_treated_as_an_executing_runtime(monkeypatch): + zombie = Process(10, 100) + monkeypatch.setattr(zombie, "status", lambda: replay.psutil.STATUS_ZOMBIE) + monkeypatch.setattr(replay.psutil, "Process", lambda _: zombie) + replay.wait_tree_gone([(10, 100)]) + replay.force_stop_owned_tree([(10, 100)]) + assert zombie.signals == [] + + +def test_status_permission_failure_is_not_misreported_as_gone(monkeypatch): + owned = Process(10, 100) + + def denied(): + raise replay.psutil.AccessDenied() + + monkeypatch.setattr(owned, "status", denied) + monkeypatch.setattr(replay.psutil, "Process", lambda _: owned) + with pytest.raises(replay.psutil.AccessDenied): + replay.wait_tree_gone([(10, 100)]) + + +def test_preflight_allows_zombie_but_refuses_live_desktop(monkeypatch): + process = Process(10, 100) + process.info = {"name": "CommunityAI"} + monkeypatch.setattr(replay.psutil, "process_iter", lambda _: [process]) + with pytest.raises(RuntimeError, match="existing CommunityAI"): + replay.require_desktop_stopped() + monkeypatch.setattr(process, "status", lambda: replay.psutil.STATUS_ZOMBIE) + replay.require_desktop_stopped() + + +def test_preflight_does_not_treat_unreadable_desktop_as_stopped(monkeypatch): + process = Process(10, 100) + process.info = {"name": "CommunityAI.exe"} + monkeypatch.setattr(replay.psutil, "process_iter", lambda _: [process]) + + def denied(): + raise replay.psutil.AccessDenied() + + monkeypatch.setattr(process, "status", denied) + with pytest.raises(replay.psutil.AccessDenied): + replay.require_desktop_stopped() diff --git a/tests/test_catalog_publication.py b/tests/test_catalog_publication.py index 977303d23..453f532b3 100644 --- a/tests/test_catalog_publication.py +++ b/tests/test_catalog_publication.py @@ -23,10 +23,55 @@ PEER_ID_TWO = "Qm" + "B" * 44 -def _manifest(name: str, alias: str) -> ModelManifest: +def test_bundle_directory_retries_transient_windows_lock_atomically(tmp_path, monkeypatch): + from drift import catalog_release + + source, target = tmp_path / "staging", tmp_path / "published" + source.mkdir() + (source / "member").write_bytes(b"verified") + replace = catalog_release.os.replace + attempts = [] + + def transient_lock(src, dst): + attempts.append((src, dst)) + if len(attempts) <= 2: + assert source.exists() and not target.exists() + error = PermissionError("temporary Windows sharing violation") + error.winerror = 32 + raise error + replace(src, dst) + + monkeypatch.setattr(catalog_release.os, "replace", transient_lock) + monkeypatch.setattr(catalog_release.time, "sleep", lambda _: None) + catalog_release._replace_bundle_directory(source, target) + assert len(attempts) == 3 and not source.exists() + assert (target / "member").read_bytes() == b"verified" + + +@pytest.mark.parametrize("winerror,expected_attempts", [(5, 6), (32, 6), (33, 6), (None, 1), (87, 1)]) +def test_bundle_directory_retry_is_bounded_and_preserves_failure(tmp_path, monkeypatch, winerror, expected_attempts): + from drift import catalog_release + + attempts = [] + failure = PermissionError("persistent access failure") + failure.winerror = winerror + + def fail(*args): + attempts.append(args) + raise failure + + monkeypatch.setattr(catalog_release.os, "replace", fail) + monkeypatch.setattr(catalog_release.time, "sleep", lambda _: None) + with pytest.raises(PermissionError) as caught: + catalog_release._replace_bundle_directory(tmp_path / "staging", tmp_path / "published") + assert caught.value is failure and len(attempts) == expected_attempts + + +def _manifest(name: str, alias: str, *, gated: bool = False) -> ModelManifest: source = ModelManifest.load("tests/data/model_manifest_v1_vector.json").to_dict() source["name"] = name source["aliases"] = [alias] + source["model"]["gated"] = gated return ModelManifest.from_dict(source) @@ -37,9 +82,10 @@ def _documents( weight_delta: int = 0, shared_alias: bool = False, best_effort_alpha: bool = False, + gated_role: str | None = None, ): - primary = _manifest("Primary Test", "shared" if shared_alias else "primary-test") - standby = _manifest("Standby Test", "shared" if shared_alias else "standby-test") + primary = _manifest("Primary Test", "shared" if shared_alias else "primary-test", gated=gated_role == "primary") + standby = _manifest("Standby Test", "shared" if shared_alias else "standby-test", gated=gated_role == "standby") now = time.time() models = [] for role, manifest in (("primary", primary), ("standby", standby)): @@ -67,7 +113,7 @@ def _documents( "order": 1, "minimum_replicas": 1 if best_effort_alpha else 2, "minimum_independent_routes": 1 if best_effort_alpha else 2, - "minimum_surviving_replicas": 1, + "minimum_surviving_replicas": 0 if best_effort_alpha else 1, "minimum_soak_seconds": 60, "maximum_observation_age_seconds": 30, "maximum_p95_first_token_ms": 2_000, @@ -139,6 +185,13 @@ def test_publication_preflight_accepts_explicit_best_effort_alpha_minimum(): assert "public-worker route redundancy and soak" in report["not_covered"] +def test_publication_preflight_rejects_gated_model_even_when_signed(): + bootstrap, envelope, manifests = _documents(gated_role="standby") + + with pytest.raises(CatalogBootstrapError, match="gated.*unauthenticated, no-consent"): + verify_catalog_publication_bundle(bootstrap, envelope, manifests) + + @pytest.mark.parametrize( "mirror_urls, initial_peers, message", [ diff --git a/tests/test_catalog_refresh.py b/tests/test_catalog_refresh.py new file mode 100644 index 000000000..f1b5df7f1 --- /dev/null +++ b/tests/test_catalog_refresh.py @@ -0,0 +1,546 @@ +import json +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path + +import pytest +from test_catalog_bootstrap import NOW, _release_documents + +from drift.model_catalog import CatalogSigningKey, ModelCatalogError, SignedModelCatalog +from drift.model_manifest import ModelManifest +from drift.node.catalog_bootstrap import CatalogBootstrapConfig, CatalogBootstrapError, CatalogBootstrapInstaller +from drift.node.catalog_refresh import CatalogRefreshService, load_configured_catalog +from drift.node.config import NodeConfig +from drift.node.model_manager import ModelDescriptor, ModelManager, ModelManagerClosedError, ModelRuntime + + +def installation(tmp_path): + bootstrap, envelope, manifests = _release_documents() + key = CatalogSigningKey.generate() + bootstrap["trust_root"]["keys"] = [key.trusted_key.to_dict()] + released = [replace(envelope, signatures=()).add_signature(key)] + + def fetch(url, _maximum): + return json.dumps(released[0].to_dict()) if url in bootstrap["catalog_mirrors"] else manifests[url] + + config_path = tmp_path / "node.json" + installer = CatalogBootstrapInstaller( + CatalogBootstrapConfig.from_dict(bootstrap), + data_dir=tmp_path, + config_path=config_path, + fetch_text=fetch, + now=NOW, + ) + installer.install() + return installer, config_path, released, key + + +def test_signed_refresh_preserves_user_policy_and_supports_existing_installations(tmp_path): + installer, path, released, key = installation(tmp_path) + original = json.loads(path.read_text()) + original["inference_mode"] = "local_only" + original["max_loaded_models"] = 2 + original["contribution_policy"] = {"sharing_enabled": False} + original["models"][0]["request_timeout"] = 47 + path.write_text(json.dumps(original)) + before = NodeConfig.load(path) + released[0] = SignedModelCatalog(1, replace(released[0].signed, sequence=2), ()).add_signature(key) + assert installer.refresh().created + after = NodeConfig.load(path) + assert after.inference_mode == "local_only" + assert after.max_loaded_models == 2 + assert after.contribution_policy == before.contribution_policy + assert after.workers == before.workers + assert after.models[0].request_timeout == 47 + assert after.catalog_path != before.catalog_path + assert before.catalog_path.is_file() + assert not installer.refresh().created + + +def test_catalog_withdrawal_removes_managed_entries_but_preserves_custom_model_and_files(tmp_path): + installer, path, released, key = installation(tmp_path) + original = json.loads(path.read_text()) + withdrawn = original["models"][1] + retired_path = Path(withdrawn["manifest"]) + cache = Path(withdrawn["cache_dir"]) + cache.mkdir(parents=True) + (cache / "retained-weights").write_bytes(b"downloaded model") + custom_source = ModelManifest.load(retired_path).to_dict() + custom_source.update(name="My advanced model", aliases=["my-advanced-model"]) + custom = tmp_path / "my-model.json" + custom.write_text(ModelManifest.from_dict(custom_source).canonical_json()) + original["models"].append(dict(withdrawn, manifest=str(custom))) + path.write_text(json.dumps(original)) + released[0] = SignedModelCatalog( + 1, replace(released[0].signed, sequence=2, models=(released[0].signed.models[0],)), () + ).add_signature(key) + + assert installer.refresh().created + refreshed = NodeConfig.load(path) + assert [model.manifest_path for model in refreshed.models] == [Path(original["models"][0]["manifest"]), custom] + assert refreshed.auto_model_priority == (released[0].signed.models[0].manifest_digest,) + assert retired_path.is_file() and custom.is_file() + assert (cache / "retained-weights").read_bytes() == b"downloaded model" + + +def legacy_public_alpha_installation(tmp_path): + """Reproduce the released app's v2 config with the two v1 leftovers and no v1 history.""" + public = Path("public-alpha") + bootstrap = CatalogBootstrapConfig.load(public / "catalog-qwen-v2/catalog-bootstrap.json") + envelope = SignedModelCatalog.load(public / "catalog-qwen-v2/catalog.signed.json") + path = tmp_path / "node-config.json" + installer = CatalogBootstrapInstaller( + bootstrap, + data_dir=tmp_path, + config_path=path, + fetch_text=lambda *_: json.dumps(envelope.to_dict()), + now=envelope.signed.issued_at_ms / 1000 + 60, + ) + installer.catalog_dir.mkdir(parents=True) + installer.manifest_dir.mkdir() + installer.installed_bootstrap_path.write_text(json.dumps(bootstrap.to_dict())) + catalog_path = installer.catalog_dir / f"2-{envelope.signed.digest.removeprefix('sha256:')}.signed.json" + catalog_path.write_text(json.dumps(envelope.to_dict())) + models = [] + for bundle in ("catalog-qwen-v2", "catalog-v1"): + released = SignedModelCatalog.load(public / bundle / "catalog.signed.json") + for model in released.signed.models: + digest = model.manifest_digest.removeprefix("sha256:") + manifest_path = installer.manifest_dir / f"{digest}.json" + manifest_path.write_bytes((public / bundle / "manifests" / manifest_path.name).read_bytes()) + entry = { + "manifest": str(manifest_path), + "cache_dir": str(installer.cache_dir / digest), + "initial_peers": [] if model.execution == "local" else list(bootstrap.initial_peers), + } + if model.execution is not None: + entry["execution"] = model.execution + models.append(entry) + path.write_text( + json.dumps( + { + "schema_version": 1, + "models": models, + "catalog_path": str(catalog_path), + "catalog_bootstrap_path": str(installer.installed_bootstrap_path), + "auto_model_priority": [model.manifest_digest for model in envelope.signed.models], + "inference_mode": "local_only", + "max_loaded_models": 2, + "workers": [ + { + "id": "automatic", + "model": "auto", + "identity_path": str(tmp_path / "identity.key"), + "num_blocks": 1, + } + ], + } + ) + ) + for name in ("identity.key", "api-keys.json", "local-api.key", "cache-sentinel"): + (tmp_path / name).write_bytes(b"private user data retained") + return installer, path, envelope + + +@pytest.mark.parametrize("method", ["repair_existing_config", "refresh", "cli"]) +def test_existing_v2_repairs_legacy_catalog_and_missing_limits_without_network_or_data_loss( + tmp_path, monkeypatch, method +): + installer, path, envelope = legacy_public_alpha_installation(tmp_path) + original = json.loads(path.read_text()) + preserved = {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file() and p != path} + if method == "cli": + from drift.cli import run_bootstrap + + def no_fetch(*_): + pytest.fail("An unchanged installed catalog must be repaired offline") + + monkeypatch.setattr(run_bootstrap, "CatalogBootstrapInstaller", lambda *_args, **_kwargs: installer) + installer.fetch_text = no_fetch + monkeypatch.setattr( + sys, + "argv", + [ + "bootstrap", + str(installer.installed_bootstrap_path), + "--data_dir", + str(tmp_path), + "--node_config", + str(path), + "--refresh_if_needed", + ], + ) + run_bootstrap.main() + else: + assert getattr(installer, method)().created + repaired = NodeConfig.load(path) + assert {ModelManifest.load(model.manifest_path).digest_id for model in repaired.models} == { + model.manifest_digest for model in envelope.signed.models + } + assert repaired.inference_mode == "local_only" and repaired.max_loaded_models == 2 + assert json.loads(path.read_text())["workers"] == original["workers"] + assert repaired.contribution_policy.sharing_enabled is False + assert repaired.contribution_policy.max_vram == "100%" + assert repaired.contribution_policy.max_processing_percent == 100 + assert repaired.contribution_policy.max_disk_space == "20GiB" + assert all(p.read_bytes() == content for p, content in preserved.items()) + assert not installer.repair_existing_config().created + + +@pytest.mark.parametrize("preservation", ["external_path", "explicit_worker", "custom_canonical"]) +def test_legacy_cleanup_preserves_advanced_models_and_explicit_preferences(tmp_path, preservation): + installer, path, _ = legacy_public_alpha_installation(tmp_path) + original = json.loads(path.read_text()) + old_entry = original["models"][2] + old_manifest = ModelManifest.load(old_entry["manifest"]) + if preservation == "external_path": + custom_path = tmp_path / "advanced.json" + custom_path.write_text(old_manifest.canonical_json()) + old_entry["manifest"] = str(custom_path) + elif preservation == "explicit_worker": + original["workers"][0]["model"] = old_manifest.aliases[0] + else: + custom_source = old_manifest.to_dict() + custom_source.update(name="My custom model", aliases=["my-custom"]) + custom = ModelManifest.from_dict(custom_source) + custom_path = installer.manifest_dir / f"{custom.digest}.json" + custom_path.write_text(custom.canonical_json()) + old_entry["manifest"] = str(custom_path) + original["contribution_policy"] = { + "sharing_enabled": True, + "max_disk_space": "35GiB", + "max_vram": "7GiB", + "max_processing_percent": 37, + } + path.write_text(json.dumps(original)) + + assert installer.repair_existing_config().created + repaired = json.loads(path.read_text()) + assert old_entry in repaired["models"] + assert repaired["contribution_policy"] == original["contribution_policy"] + assert repaired["workers"] == original["workers"] + + +def test_legacy_cleanup_rejects_tampered_catalog_and_requires_authorized_predecessor(tmp_path): + installer, path, _ = legacy_public_alpha_installation(tmp_path) + installer.bootstrap = replace(installer.bootstrap, replaces_trust_roots=()) + assert installer.repair_existing_config().created # Only supplies missing default limits. + assert len(NodeConfig.load(path).models) == 4 + accepted = path.read_bytes() + catalog_path = NodeConfig.load(path).catalog_path + envelope = SignedModelCatalog.load(catalog_path) + tampered = replace(envelope, signed=replace(envelope.signed, sequence=3)) + catalog_path.write_text(json.dumps(tampered.to_dict())) + with pytest.raises(ModelCatalogError, match="signature"): + installer.repair_existing_config() + assert path.read_bytes() == accepted + + +@pytest.mark.parametrize("same_identity", [True, False]) +def test_catalog_migration_preserves_preferences_by_exact_manifest_identity(tmp_path, same_identity): + installer, path, released, key = installation(tmp_path) + original = json.loads(path.read_text()) + manifest = ModelManifest.load(original["models"][0]["manifest"]) + if not same_identity: + source = manifest.to_dict() + source["source"]["revision"] = "f" * 40 + manifest = ModelManifest.from_dict(source) + old_path = tmp_path / "custom-manifest.json" + old_path.write_text(manifest.canonical_json()) + cache = tmp_path / "retained-cache" + cache.mkdir() + (cache / "sentinel").write_bytes(b"retained") + original["models"][0].update(manifest=str(old_path), cache_dir=str(cache), request_timeout=47) + original["auto_model_priority"][0] = manifest.digest_id + original.pop("catalog_path") + original.pop("catalog_bootstrap_path") + path.write_text(json.dumps(original)) + + assert installer.refresh().created + migrated = NodeConfig.load(path) + assert migrated.models[0].manifest_path != old_path + assert (migrated.models[0].cache_dir == cache) is same_identity + assert (migrated.models[0].request_timeout == 47) is same_identity + assert (cache / "sentinel").read_bytes() == b"retained" + old_path.unlink() + assert ( + ModelManifest.load(migrated.models[0].manifest_path).digest_id == released[0].signed.models[0].manifest_digest + ) + + +def test_refresh_rejects_tamper_rollback_and_equivocation_without_changing_active_config(tmp_path): + installer, path, released, key = installation(tmp_path) + old = released[0] + released[0] = SignedModelCatalog(1, replace(old.signed, sequence=2), ()).add_signature(key) + installer.refresh() + accepted = path.read_bytes() + for rejected in ( + old, + SignedModelCatalog( + 1, replace(old.signed, sequence=2, expires_at_ms=old.signed.expires_at_ms + 1000), () + ).add_signature(key), + replace(released[0], signed=replace(released[0].signed, sequence=3)), + ): + released[0] = rejected + with pytest.raises(CatalogBootstrapError): + installer.refresh() + assert path.read_bytes() == accepted + + +def test_expired_installed_catalog_can_be_renewed(tmp_path): + installer, path, released, key = installation(tmp_path) + installer.now = NOW + 4000 + released[0] = SignedModelCatalog( + 1, + replace( + released[0].signed, + sequence=2, + issued_at_ms=int((NOW + 3900) * 1000), + expires_at_ms=int((NOW + 7600) * 1000), + ), + (), + ).add_signature(key) + assert installer.refresh().created + assert NodeConfig.load(path).catalog_path.is_file() + + +def test_replacement_root_requires_application_authorization_and_preserves_rollback(tmp_path): + installer, path, released, old_key = installation(tmp_path) + old_config = NodeConfig.load(path) + old_bootstrap = installer.bootstrap + new_key = CatalogSigningKey.generate() + new_root = replace(old_bootstrap.trust_root, keys=(new_key.trusted_key,)) + old_catalog = released[0].signed + released[0] = SignedModelCatalog(1, replace(old_catalog, sequence=2), ()).add_signature(new_key) + with pytest.raises(CatalogBootstrapError, match="No trusted"): + installer.refresh() # A network response alone cannot install a root. + installer.bootstrap = replace(old_bootstrap, trust_root=new_root) + with pytest.raises(CatalogBootstrapError, match="does not authorize"): + installer.refresh() + authorized = replace(installer.bootstrap, replaces_trust_roots=(old_bootstrap.trust_root_digest,)) + installer = CatalogBootstrapInstaller( + authorized, data_dir=tmp_path, config_path=path, fetch_text=installer.fetch_text, now=NOW + ) + assert installer.refresh().created + assert load_configured_catalog(NodeConfig.load(path)).sequence == 2 + # The prior configuration remains self-verifying, including after a crash + # between staging the new trust file and committing the new configuration. + assert load_configured_catalog(old_config).sequence == 1 + assert old_config.catalog_bootstrap_path != NodeConfig.load(path).catalog_bootstrap_path + accepted = path.read_bytes() + released[0] = SignedModelCatalog(1, old_catalog, ()).add_signature(new_key) + with pytest.raises(CatalogBootstrapError): + installer.refresh() + assert path.read_bytes() == accepted + + +def test_catalog_architecture_rejection_does_not_advance_rollback_or_activate(tmp_path, monkeypatch): + import drift.node.loading as loading + from drift.model_manifest import ManifestError + + installer, path, released, key = installation(tmp_path) + accepted = path.read_bytes() + guard = installer.rollback_path.read_bytes() + released[0] = SignedModelCatalog(1, replace(released[0].signed, sequence=2), ()).add_signature(key) + + def reject(*_args): + raise ManifestError("unsupported architecture") + + monkeypatch.setattr(loading, "validate_manifest_execution", reject) + with pytest.raises(CatalogBootstrapError, match="unsupported architecture"): + installer.refresh() + assert installer.rollback_path.read_bytes() == guard + assert path.read_bytes() == accepted + + +def periodic_service(installer, path, manager, restart): + """Use production refresh/admission code; substitute only transport and time.""" + config = replace(NodeConfig.load(path), catalog_refresh_seconds=0.01) + service = CatalogRefreshService(config, path, installer.data_dir, manager, restart) + service.installer.fetch_text = installer.fetch_text + service.installer.now = installer.now + attempted, staged, busy = threading.Event(), threading.Event(), threading.Event() + real_refresh = service.installer.refresh + real_begin_restart = manager.begin_idle_restart + + def observe_refresh(): + try: + result = real_refresh() + if result.created: + staged.set() + return result + finally: + attempted.set() + + def observe_admission(): + claimed = real_begin_restart() + if not claimed: + busy.set() + return claimed + + service.installer.refresh = observe_refresh + manager.begin_idle_restart = observe_admission + return service, attempted, staged, busy + + +def test_periodic_signed_withdrawal_waits_for_last_lease_before_claiming_restart(tmp_path): + installer, path, released, key = installation(tmp_path) + old_config = NodeConfig.load(path) + old_catalog = load_configured_catalog(old_config) + withdrawn = old_catalog.models[1].manifest_digest + closed, callbacks = [], [] + manager = ModelManager() + manager.set_catalog_models(model.manifest_digest for model in old_catalog.models) + manager.register_manifest( + ModelManifest.load(old_config.models[1].manifest_path), + lambda: ModelRuntime(object(), object(), lambda: closed.append(True)), + ) + first, last = manager.load(withdrawn), manager.load(withdrawn) + restarted = threading.Event() + + def restart(): + # This callback is the server-restart boundary, after atomic admission + # closure. The old runtime must not admit a request in this gap. + try: + manager.load(withdrawn) + except ModelManagerClosedError: + callbacks.append("admission-closed") + else: + callbacks.append("admission-open") + restarted.set() + + released[0] = SignedModelCatalog( + 1, replace(old_catalog, sequence=2, models=(old_catalog.models[0],)), () + ).add_signature(key) + service, _, staged, busy = periodic_service(installer, path, manager, restart) + service.start() + try: + assert staged.wait(3) and busy.wait(3), "The signed update was not staged while leases were active" + assert load_configured_catalog(NodeConfig.load(path)).sequence == 2 + assert load_configured_catalog(old_config).sequence == 1 + assert manager.catalog_allows_contribution(withdrawn) + assert first.runtime is last.runtime and first.runtime.model is not None + assert not restarted.is_set() and not closed + first.release() + busy.clear() + assert busy.wait(3), "Restart did not observe the remaining lease" + assert manager.snapshots()[0].active_requests == 1 + assert not restarted.is_set() and not closed + last.release() + assert restarted.wait(3), "Restart did not follow the last lease release" + assert callbacks == ["admission-closed"] + updated = load_configured_catalog(NodeConfig.load(path)) + next_manager = ModelManager() + next_manager.set_catalog_models(model.manifest_digest for model in updated.models) + assert not next_manager.catalog_allows_contribution(withdrawn) + next_manager.shutdown() + finally: + first.release() + last.release() + service.close() + manager.shutdown() + assert not service._thread.is_alive() + assert closed == [True] + + +def test_periodic_refresh_waits_for_loading_then_its_returned_lease(tmp_path): + installer, path, released, key = installation(tmp_path) + manager = ModelManager() + loader_entered, allow_loader, restarted = threading.Event(), threading.Event(), threading.Event() + runtime = ModelRuntime(object(), object()) + + def loader(): + loader_entered.set() + assert allow_loader.wait(5), "Test did not release its controlled loader" + return runtime + + manager.register(ModelDescriptor("loading"), loader) + released[0] = SignedModelCatalog(1, replace(released[0].signed, sequence=2), ()).add_signature(key) + service, _, staged, busy = periodic_service(installer, path, manager, restarted.set) + lease = None + with ThreadPoolExecutor(max_workers=1) as executor: + pending_load = executor.submit(manager.load, "loading") + try: + assert loader_entered.wait(3) + service.start() + assert staged.wait(3) and busy.wait(3) + assert not restarted.is_set() + allow_loader.set() + lease = pending_load.result(timeout=3) + assert lease.runtime is runtime + busy.clear() + assert busy.wait(3), "The completed loader's lease did not hold the restart" + assert not restarted.is_set() + lease.release() + assert restarted.wait(3) + with pytest.raises(ModelManagerClosedError): + manager.load("loading") + finally: + allow_loader.set() + if lease is None: + lease = pending_load.result(timeout=3) + lease.release() + service.close() + manager.shutdown() + assert not service._thread.is_alive() + + +@pytest.mark.parametrize("rejection", ["tamper", "rollback", "equivocation"]) +def test_periodic_rejection_keeps_catalog_guard_and_request_admission(tmp_path, rejection): + installer, path, released, key = installation(tmp_path) + previous = released[0] + current = SignedModelCatalog(1, replace(previous.signed, sequence=2), ()).add_signature(key) + released[0] = current + installer.refresh() + before, guard = path.read_bytes(), installer.rollback_path.read_bytes() + if rejection == "tamper": + released[0] = replace(current, signed=replace(current.signed, sequence=3)) + elif rejection == "rollback": + released[0] = previous + else: + released[0] = SignedModelCatalog( + 1, replace(current.signed, expires_at_ms=current.signed.expires_at_ms + 1000), () + ).add_signature(key) + manager = ModelManager() + manager.register(ModelDescriptor("retained"), lambda: ModelRuntime(object(), object())) + restarted = threading.Event() + service, attempted, staged, _ = periodic_service(installer, path, manager, restarted.set) + service.start() + try: + assert attempted.wait(3), "The periodic refresh did not run" + service.close() + assert path.read_bytes() == before and installer.rollback_path.read_bytes() == guard + assert not staged.is_set() and not restarted.is_set() + with manager.load("retained") as lease: + assert lease.runtime.model is not None + finally: + service.close() + manager.shutdown() + assert not service._thread.is_alive() + + +def test_closing_periodic_service_during_drain_wait_preserves_active_lease(tmp_path): + installer, path, released, key = installation(tmp_path) + manager = ModelManager() + manager.register(ModelDescriptor("active"), lambda: ModelRuntime(object(), object())) + lease = manager.load("active") + restarted = threading.Event() + released[0] = SignedModelCatalog(1, replace(released[0].signed, sequence=2), ()).add_signature(key) + service, _, staged, busy = periodic_service(installer, path, manager, restarted.set) + service.start() + try: + assert staged.wait(3) and busy.wait(3) + service.close() + assert not service._thread.is_alive() and not restarted.is_set() + assert lease.runtime.model is not None + with manager.load("active") as another: + assert another.runtime is lease.runtime + # The accepted immutable update remains available for the next node + # start, without erasing the configuration or rollback guard. + assert load_configured_catalog(NodeConfig.load(path)).sequence == 2 + finally: + lease.release() + service.close() + manager.shutdown() diff --git a/tests/test_contribution_planner.py b/tests/test_contribution_planner.py index deeb89d99..c53fb4379 100644 --- a/tests/test_contribution_planner.py +++ b/tests/test_contribution_planner.py @@ -1,4 +1,9 @@ -from drift.node.contribution_planner import AutomaticContributionPlanner, PlacementCandidate, PlacementRegistry +from drift.node.contribution_planner import ( + AutomaticContributionPlanner, + PlacementArtifactPlan, + PlacementCandidate, + PlacementRegistry, +) def _candidate( @@ -12,6 +17,8 @@ def _candidate( policy_reason=None, route_observation=None, remote_route_observation=None, + artifact_plans=(), + max_artifact_bytes=None, ): return PlacementCandidate( model_id=name, @@ -28,6 +35,8 @@ def _candidate( route_observation=route_observation, remote_route_observation=remote_route_observation, policy_reason=policy_reason, + artifact_plans=artifact_plans, + max_artifact_bytes=max_artifact_bytes, ) @@ -74,6 +83,76 @@ def test_planner_selects_preferred_model_and_least_covered_contiguous_range(): assert "fresh verified coverage" in plan.reason +def test_planner_skips_over_budget_window_and_uses_exact_deduplicated_bytes(): + plans = ( + PlacementArtifactPlan(0, 2, 30, "a" * 64), + PlacementArtifactPlan(1, 3, 50, "b" * 64), + PlacementArtifactPlan(2, 4, 40, "c" * 64), + ) + planner = AutomaticContributionPlanner(num_blocks=2, jitter_seed="node-a") + candidate = _candidate( + "qwen", + digest="sha256:" + "1" * 64, + counts=(2, 0, 0, 2), + artifact_plans=plans, + max_artifact_bytes=45, + ) + + decision = planner.plan((candidate,), sharing_enabled=True, now=10).decision + + assert decision.block_indices != "1:3" + selected = next(plan for plan in plans if f"{plan.start_block}:{plan.end_block}" == decision.block_indices) + assert decision.artifact_bytes == selected.artifact_bytes + assert decision.artifact_set_digest == selected.artifact_set_digest + + rejected = _candidate( + "qwen", + digest="sha256:" + "1" * 64, + counts=(2, 0, 0, 2), + artifact_plans=plans, + max_artifact_bytes=29, + ) + result = AutomaticContributionPlanner(num_blocks=2, jitter_seed="node-a").plan( + (rejected,), sharing_enabled=True, now=10 + ) + assert result.decision is None + assert "every 2-block artifact set exceeds" in result.reason + + +def test_hysteresis_cannot_retain_a_now_over_budget_range(): + plans = ( + PlacementArtifactPlan(0, 1, 30, "a" * 64), + PlacementArtifactPlan(1, 2, 10, "b" * 64), + PlacementArtifactPlan(2, 3, 10, "c" * 64), + ) + planner = AutomaticContributionPlanner( + num_blocks=1, + jitter_seed="node-a", + minimum_residency_seconds=1_000, + cooldown_seconds=1_000, + ) + initial = _candidate( + "qwen", + digest="sha256:" + "1" * 64, + counts=(0, 2, 2), + artifact_plans=plans, + max_artifact_bytes=100, + ) + assert planner.plan((initial,), sharing_enabled=True, now=0).decision.block_indices == "0:1" + + reduced = _candidate( + "qwen", + digest="sha256:" + "1" * 64, + counts=(0, 2, 2), + artifact_plans=plans, + max_artifact_bytes=20, + ) + decision = planner.plan((reduced,), sharing_enabled=True, now=1).decision + + assert decision.block_indices != "0:1" + assert decision.artifact_bytes == 10 + + def test_completed_route_utility_breaks_only_comparable_coverage_ties(): planner = AutomaticContributionPlanner( num_blocks=1, diff --git a/tests/test_dht_reconnect.py b/tests/test_dht_reconnect.py new file mode 100644 index 000000000..862f6d36b --- /dev/null +++ b/tests/test_dht_reconnect.py @@ -0,0 +1,49 @@ +import asyncio +from types import SimpleNamespace + +import pytest +from hivemind.dht.node import Blacklist +from hivemind.p2p import PeerID + +from drift.utils.dht import _reconnect_dht_if_isolated + +SEED = "/dns4/bootstrap.communityai.flujo.com.co/tcp/31337/p2p/QmZhGcSVR6qPLZTq3TJPZEi734GbMkouv3kPxQLdDY2qUo" + + +@pytest.mark.parametrize("state", ["healthy", "isolated", "backed-off", "unreachable"]) +def test_reconnect_keeps_live_protocol_and_requires_validated_seed_response(state): + peers = {"existing": "peer"} if state == "healthy" else {} + calls = [] + + async def connect(peer, endpoints): + assert {str(endpoint) for endpoint in endpoints} == {SEED.rsplit("/p2p/", 1)[0]} + + async def ping(peer, *, validate, strict): + assert validate and strict + calls.append(peer) + await asyncio.sleep(0) + if state == "unreachable": + raise RuntimeError("seed did not validate") + # Match Hivemind's scheduled routing-table update. + asyncio.get_running_loop().call_soon(peers.update, {"seed": peer}) + return "seed" + + protocol = SimpleNamespace(routing_table=SimpleNamespace(uid_to_peer_id=peers), call_ping=ping) + node = SimpleNamespace( + protocol=protocol, p2p=SimpleNamespace(_client=SimpleNamespace(connect=connect)), blacklist=Blacklist(5, 2) + ) + dht = SimpleNamespace(initial_peers=[SEED, SEED]) + seed_peer = PeerID.from_base58(SEED.rsplit("/p2p/", 1)[1]) + if state == "backed-off": + peers["seed"] = seed_peer + node.blacklist.register_failure(seed_peer) + + async def run(): + await asyncio.gather(*(_reconnect_dht_if_isolated(dht, node) for _ in range(2))) + + asyncio.run(run()) + assert node.protocol is protocol + assert len(calls) == {"healthy": 0, "isolated": 1, "backed-off": 1, "unreachable": 2}[state] + assert bool(peers) is (state != "unreachable") + if state == "backed-off": + assert seed_peer not in node.blacklist diff --git a/tests/test_discovery.py b/tests/test_discovery.py index d9fe8fc83..2b5155980 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -15,6 +15,7 @@ ModelCoverageDiscovery, PeerCache, _connected_peer_addresses, + _default_dht_factory, ) from drift.protocol_identity import NodeIdentity, create_intent_lease, create_route_demand @@ -26,6 +27,79 @@ CACHE_SCOPE = ("shipped-seed",) +def test_stalled_dht_startup_has_a_parent_timeout_and_cleans_the_child(monkeypatch): + import hivemind + + instances = [] + + class StalledDHT: + def __init__(self, *, start, **kwargs): + instances.append(self) + self.stopped = False + if start: + self.run_in_background() + + def run_in_background(self, *, timeout=None): + if timeout is None: + raise AssertionError("DHT constructor waits forever without a parent timeout") + assert timeout == 0.01 + raise TimeoutError("startup readiness never arrived") + + def shutdown(self): + self.stopped = True + + monkeypatch.setattr(hivemind, "DHT", StalledDHT) + with pytest.raises(TimeoutError, match="readiness never arrived"): + _default_dht_factory(start=True, startup_timeout=0.01, initial_peers=[]) + assert instances[0].stopped + + +def test_dead_cached_peers_do_not_delay_a_healthy_configured_seed(monkeypatch): + import hivemind + + attempts = [] + + class SeedDHT: + def __init__(self, *, initial_peers, **kwargs): + self.peers = initial_peers + attempts.append(self) + + def run_in_background(self, *, timeout): + if any(peer == "dead-cached-peer" for peer in self.peers): + raise TimeoutError("dead address blocks startup of the entire peer set") + + def shutdown(self): + pass + + monkeypatch.setattr(hivemind, "DHT", SeedDHT) + result = _default_dht_factory(initial_peers=["healthy-seed", "dead-cached-peer"]) + assert result is attempts[0] + assert len(attempts) == 1 + + +def test_offline_primary_falls_back_to_another_seed_and_closes_failed_dht(monkeypatch): + import hivemind + + attempts = [] + + class SeedDHT: + def __init__(self, *, initial_peers, **kwargs): + self.peers, self.closed = initial_peers, False + attempts.append(self) + + def run_in_background(self, *, timeout): + if self.peers == ["offline-primary"]: + raise TimeoutError("primary unavailable") + + def shutdown(self): + self.closed = True + + monkeypatch.setattr(hivemind, "DHT", SeedDHT) + result = _default_dht_factory(initial_peers=["offline-primary", "healthy-cached-peer"]) + assert attempts[0].closed + assert result is attempts[1] and not result.closed + + class FakeDHT: def __init__(self): self.alive = True @@ -94,6 +168,24 @@ def test_intent_publication_requires_a_remote_dht_store(tmp_path): assert call["expiration_time"] == record.payload["expires_at_ms"] / 1000 assert call["value"] == record.to_dict() + observed_dht = SimpleNamespace( + get=lambda *args, **kwargs: SimpleNamespace( + value={ + record.key_id: SimpleNamespace(value=record.to_dict()), + "wrong-key": SimpleNamespace(value=record.to_dict()), + "malformed": SimpleNamespace(value={"payload": "not a signed record"}), + } + ) + ) + state = discovery._states[manifest.digest_id] + reservations = discovery._read_intents(state, observed_dht) + assert len(reservations) == 1 + assert reservations[0]["peer_id"] == identity.peer_id.to_base58() + assert reservations[0]["start_block"] == 1 + health = {"status": "incomplete", "reservations": [dict(reservations[0], expires_at=now - 1)]} + discovery._set_success(state, health) + assert discovery.snapshot(manifest.digest_id)["reservations"] == [] + dht.store_result = False record = create_intent_lease( identity, @@ -347,6 +439,43 @@ def lookup(selected_dht, uids, **kwargs): assert dht.shutdown_calls == 1 +def test_discovery_rejoins_seeds_when_live_dht_loses_all_routing_peers(): + manifest = ModelManifest.load("tests/data/model_manifest_v1_vector.json") + disconnected, replacement = FakeDHT(), FakeDHT() + disconnected.run_coroutine = lambda callback: 0 + replacement.run_coroutine = lambda callback: 1 + created = [] + refreshed = threading.Event() + + def factory(**kwargs): + assert kwargs["initial_peers"] == ["seed"] + current = disconnected if not created else replacement + created.append(current) + return current + + def lookup(dht, uids, **kwargs): + if dht is replacement: + refreshed.set() + return [RemoteModuleInfo(uid, {}) for uid in uids] + + discovery = ModelCoverageDiscovery( + [CoverageTarget(manifest, ("seed",))], + update_period=0.01, + startup_timeout=1, + dht_factory=factory, + lookup=lookup, + peer_snapshot=lambda dht: (), + ) + try: + discovery.start() + assert refreshed.wait(timeout=2) + assert created == [disconnected, replacement] + assert disconnected.shutdown_calls == 1 + finally: + discovery.close() + assert replacement.shutdown_calls == 1 + + def _route_demand_record(identity, manifest, *, now, attempts, successes, sequence): return create_route_demand( identity, diff --git a/tests/test_download_progress.py b/tests/test_download_progress.py new file mode 100644 index 000000000..5c075537f --- /dev/null +++ b/tests/test_download_progress.py @@ -0,0 +1,153 @@ +import hashlib +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from drift.model_manifest import ManifestArtifactVerifier, ManifestError, ModelManifest +from drift.node.model_manager import ModelDescriptor, ModelManager, ModelRuntime +from drift.node.worker_supervisor import WorkerLaunch, WorkerSupervisor +from drift.utils.download_progress import public_progress + + +@pytest.mark.parametrize("corrupt", [False, True]) +def test_live_resumed_http_download_reports_verification_and_failure(tmp_path, monkeypatch, corrupt): + payload = b"verified model artifact" * 16000 + source = ModelManifest.load("tests/data/model_manifest_v1_vector.json").to_dict() + artifact = next(item for item in source["artifacts"] if item["role"] == "weight") + artifact.update(size=len(payload), sha256=hashlib.sha256(payload).hexdigest()) + manifest = ModelManifest.from_dict(source) + waiting, release = threading.Event(), threading.Event() + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + start, end = map(int, self.headers["Range"].removeprefix("bytes=").split("-")) + requests.append((start, end)) + if len(requests) == 1: + self.send_response(429) + self.end_headers() + return + waiting.set() + release.wait(10) + self.send_response(206) + self.send_header("Content-Range", f"bytes {start}-{end}/{len(payload)}") + self.end_headers() + chunk = payload[start : end + 1] + self.wfile.write((b"!" + chunk[1:]) if corrupt else chunk) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + monkeypatch.setattr("drift.utils.hub_ranges.RANGE_BYTES", 32768) + monkeypatch.setattr( + "huggingface_hub.hf_hub_url", lambda *a, **kw: f"http://127.0.0.1:{server.server_port}/artifact" + ) + verifier = ManifestArtifactVerifier( + manifest, manifest.source.repository, manifest.source.revision, cache_dir=tmp_path + ) + partial, _, _ = verifier._resumable_paths(manifest.get_artifact(artifact["path"])) + partial.parent.mkdir(parents=True) + partial.write_bytes(payload[:8192]) + + def loader(): + active = ManifestArtifactVerifier( + manifest, manifest.source.repository, manifest.source.revision, cache_dir=tmp_path + ) + active.ensure_path(artifact["path"]) + return ModelRuntime(object(), object()) + + manager = ModelManager() + manager.register(ModelDescriptor("download-test"), loader) + try: + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(manager.load, "download-test") + try: + assert waiting.wait(10) + live = manager.snapshots()[0].to_dict()["download"]["progress"] + assert live["state"] in ("downloading", "retrying") + assert live["verified_bytes"] == 0 + assert live["resumed_bytes"] == 8192 + assert live["retries"] == 1 + finally: + release.set() + if corrupt: + with pytest.raises(ManifestError): + future.result(timeout=10) + else: + future.result(timeout=10).release() + result = manager.snapshots()[0].to_dict()["download"]["progress"] + assert result["state"] == ("failed" if corrupt else "ready") + assert result["verified_bytes"] == (0 if corrupt else len(payload)) + assert requests[0][0] == 8192 + if not corrupt: + count = len(requests) + manager.unload("download-test") + manager.load("download-test").release() + assert len(requests) == count + cached = manager.snapshots()[0].to_dict()["download"]["progress"] + assert cached["verified_bytes"] == len(payload) + assert cached["bytes_per_second"] == 0 + finally: + release.set() + manager.shutdown() + server.shutdown() + server.server_close() + thread.join(5) + + +def test_worker_progress_never_exposes_private_fields_or_nonfinite_values(): + result = public_progress( + { + "schema_version": 1, + "state": "failed", + "artifact": "model.safetensors", + "token": "private", + "path": "private", + "pid": 123, + "bytes_per_second": float("nan"), + } + ) + assert not {"token", "path", "pid"} & result.keys() + assert result["bytes_per_second"] is None + + +def test_supervised_process_reports_own_download_and_pause(tmp_path): + code = """ +import time +from types import SimpleNamespace +from drift.utils.download_progress import current_progress +progress = current_progress() +progress.event(SimpleNamespace(digest='a' * 64), SimpleNamespace(path='model.safetensors', size=1024), + 'downloading', received=512, transferred=512) +time.sleep(30) +""" + supervisor = WorkerSupervisor([WorkerLaunch("progress-worker", "model", (sys.executable, "-c", code))]) + try: + supervisor.start_worker("progress-worker") + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + value = supervisor.snapshot("progress-worker")["download_progress"] + if value is not None: + break + time.sleep(0.1) + snapshot = supervisor.snapshot("progress-worker") + assert value is not None, (snapshot["last_error"], snapshot["recent_logs"], snapshot["state"]) + assert value["state"] == "downloading" + assert value["received_bytes"] == 512 + assert value["verified_bytes"] == 0 + assert "pid" not in value + supervisor.pause_worker("progress-worker") + assert supervisor.snapshot("progress-worker")["download_progress"]["state"] == "paused" + directory = supervisor._records["progress-worker"].progress_directory.name + finally: + supervisor.shutdown() + from pathlib import Path + + assert not Path(directory).exists() diff --git a/tests/test_edge_acquisition.py b/tests/test_edge_acquisition.py index a5a7a87c3..2b482037e 100644 --- a/tests/test_edge_acquisition.py +++ b/tests/test_edge_acquisition.py @@ -1,9 +1,12 @@ import hashlib +import io import json +import sys from pathlib import Path import pytest +import drift.cli.run_edge_acquisition as run_edge_acquisition from drift.client.from_pretrained import select_checkpoint_shards from drift.model_manifest import ManifestError, ManifestTransferInterrupted, ModelManifest from drift.node.edge_acquisition import acquire_client_artifacts @@ -75,6 +78,154 @@ def ensure_path(self, path, *, allowed_roles=None): return destination +def test_cli_can_require_anonymous_acquisition(tmp_path, monkeypatch, capsys): + manifest = _manifest( + [ + ("config.json", "config", b"{}"), + ("tokenizer.json", "tokenizer", b"{}"), + ("weights.bin", "weight", b"weights"), + ] + ) + observed = {} + + def acquire(_manifest, **kwargs): + observed.update(kwargs) + return {"ok": True} + + monkeypatch.setattr(run_edge_acquisition.ModelManifest, "load", staticmethod(lambda _path: manifest)) + monkeypatch.setattr(run_edge_acquisition, "acquire_client_artifacts", acquire) + monkeypatch.setattr( + sys, + "argv", + [ + "drift edge-acquire", + str(tmp_path / "manifest.json"), + "--cache_dir", + str(tmp_path / "cache"), + "--no_token", + ], + ) + + run_edge_acquisition.main() + + assert observed["token"] is False + assert json.loads(capsys.readouterr().out) == {"ok": True} + with pytest.raises(SystemExit): + run_edge_acquisition.build_parser().parse_args( + [ + str(tmp_path / "manifest.json"), + "--cache_dir", + str(tmp_path / "cache"), + "--token", + "secret", + "--no_token", + ] + ) + + +def test_cli_can_bind_manifest_bytes_from_stdin(tmp_path, monkeypatch, capsys): + manifest = _manifest( + [ + ("config.json", "config", b"{}"), + ("tokenizer.json", "tokenizer", b"{}"), + ("weights.bin", "weight", b"weights"), + ] + ) + payload = (manifest.canonical_json() + "\n").encode("utf-8") + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + observed = {} + + def acquire(received_manifest, **kwargs): + observed["manifest"] = received_manifest + observed.update(kwargs) + return {"ok": True} + + monkeypatch.setattr(run_edge_acquisition, "acquire_client_artifacts", acquire) + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(payload), encoding="utf-8")) + monkeypatch.setattr( + sys, + "argv", + [ + "drift edge-acquire", + "--manifest_stdin_sha256", + digest, + "--cache_dir", + str(tmp_path / "cache"), + "--no_token", + ], + ) + + run_edge_acquisition.main() + + assert observed["manifest"].digest_id == manifest.digest_id + assert observed["token"] is False + assert json.loads(capsys.readouterr().out) == {"ok": True} + + +def test_cli_rejects_changed_or_ambiguous_stdin_manifest(tmp_path, monkeypatch): + payload = b"{}\n" + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(payload), encoding="utf-8")) + monkeypatch.setattr( + sys, + "argv", + [ + "drift edge-acquire", + "--manifest_stdin_sha256", + "sha256:" + "0" * 64, + "--cache_dir", + str(tmp_path / "cache"), + "--no_token", + ], + ) + with pytest.raises(SystemExit): + run_edge_acquisition.main() + + monkeypatch.setattr( + sys, + "argv", + [ + "drift edge-acquire", + str(tmp_path / "manifest.json"), + "--manifest_stdin_sha256", + "sha256:" + hashlib.sha256(payload).hexdigest(), + "--cache_dir", + str(tmp_path / "cache"), + ], + ) + with pytest.raises(SystemExit): + run_edge_acquisition.main() + + +@pytest.mark.parametrize( + "payload", + [ + b"", + b"x" * (run_edge_acquisition.MAX_MANIFEST_STDIN_BYTES + 1), + b"\xff", + b'{"schema_version":1,"schema_version":1}\n', + ], + ids=["empty", "oversized", "invalid-utf8", "duplicate-key"], +) +def test_cli_rejects_invalid_digest_bound_stdin_manifest(tmp_path, monkeypatch, payload): + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(payload), encoding="utf-8")) + monkeypatch.setattr( + sys, + "argv", + [ + "drift edge-acquire", + "--manifest_stdin_sha256", + digest, + "--cache_dir", + str(tmp_path / "cache"), + "--no_token", + ], + ) + + with pytest.raises(SystemExit): + run_edge_acquisition.main() + + def test_select_checkpoint_shards_matches_loader_filtering(): weight_map = { "model.embed_tokens.weight": "client-a.safetensors", diff --git a/tests/test_gate13_automated_playthrough.py b/tests/test_gate13_automated_playthrough.py new file mode 100644 index 000000000..343bd0cf8 --- /dev/null +++ b/tests/test_gate13_automated_playthrough.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_automated_playthrough as replay + +MODEL_ID = "Qwen3.5 2B" +DIGEST = "sha256:" + "a" * 64 + + +def config_document(root: Path, platform: str = "windows") -> dict: + executable = root / ("CommunityAI.exe" if platform == "windows" else "CommunityAI") + executable.write_bytes(b"packaged-desktop") + archive = root / f"communityai-desktop-{platform}.zip" + archive.write_bytes(b"verified-production-archive") + return { + "schema_version": 2, + "run_id": "gate13-automated-a", + "platform": platform, + "source_commit": "1" * 40, + "package_archive": str(archive.resolve()), + "package_sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "package_bytes": archive.stat().st_size, + "desktop_executable": str(executable.resolve()), + "work_root": str((root / ".gate13-playthrough-gate13-automated-a").resolve()), + "model_id": MODEL_ID, + "manifest_digest": DIGEST, + "total_blocks": 24, + "policy": { + "sharing_enabled": True, + "allowed_models": [MODEL_ID], + "preferred_models": [MODEL_ID], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": None, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + }, + }, + "session_timeout_seconds": 30.0, + "inference_timeout_seconds": 10.0, + } + + +def session_evidence(plan: dict) -> dict: + stage = plan["stage"] + platform = plan["platform"] + inference_required = (platform, stage) in { + ("windows", "initial"), + ("linux", "initial"), + ("linux", "restart"), + } + policy_session = (platform, stage) in { + ("windows", "restart"), + ("linux", "initial"), + } + resumed_session = platform == "linux" and stage == "restart" + pause_session = stage == "restart" + return { + "schema_version": 2, + "scope": "gate13-packaged-desktop-playthrough", + "run_id": plan["run_id"], + "platform": platform, + "stage": stage, + "result": "passed", + "model_id": plan["model_id"], + "manifest_digest": plan["manifest_digest"], + "duration_seconds": 1.25, + "route": { + "rendered_in_real_window": True, + "complete": True, + "covered_blocks": plan["total_blocks"], + "total_blocks": plan["total_blocks"], + }, + "inference": { + "passed": True, + "model_id": plan["model_id"], + "manifest_digest": plan["manifest_digest"], + "completion_count": 1, + "generated_token_count": 1, + "response_content_retained": False, + "token_identifiers_retained": False, + "temporary_key_removed": True, + } + if inference_required + else None, + "ui": { + "real_window_opened": True, + "policy_dialog_saved": policy_session, + "start_clicked": policy_session, + "pause_control_observed": policy_session or resumed_session, + "pause_clicked": pause_session, + "restart_resume_observed": resumed_session, + "sharing_intent_enabled_observed": policy_session or resumed_session, + "sharing_intent_disabled_observed": pause_session, + }, + "limits": { + "storage": policy_session, + "memory_or_vram": policy_session, + "bandwidth": policy_session, + "power": False, + "pause_timeout": policy_session, + "schedule": policy_session, + }, + "timing": { + "start_observation_seconds": 25.0 + if platform == "windows" and stage == "restart" + else (20.0 if platform == "linux" and stage == "initial" else 0.0), + "restart_observation_seconds": 15.0 if resumed_session else 0.0, + }, + "privacy": { + "prompt_retained": False, + "response_content_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + }, + } + + +@pytest.mark.parametrize("platform,expected_inferences,expected_resume", [("windows", 1, False), ("linux", 2, True)]) +def test_replay_runs_real_desktop_contract_twice_and_removes_temporaries( + tmp_path, platform, expected_inferences, expected_resume +): + document = config_document(tmp_path, platform) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + config = replay.load_config(config_path) + stages = [] + self_tests = [] + + def runner(argv, **kwargs): + assert kwargs["stdin"] is subprocess.DEVNULL + assert kwargs["stdout"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.DEVNULL + if "--gate13-ui-playthrough" not in argv: + self_tests.append(argv[1]) + return subprocess.CompletedProcess(argv, 0) + plan_path = Path(argv[argv.index("--gate13-ui-playthrough") + 1]) + evidence_path = Path(argv[argv.index("--gate13-ui-evidence") + 1]) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + stages.append(plan["stage"]) + evidence_path.write_text(json.dumps(session_evidence(plan)), encoding="utf-8") + return subprocess.CompletedProcess(argv, 0) + + result = replay.run_replay(config, runner=runner) + + assert stages == ["initial", "restart"] + assert self_tests == ["--check-runtime", "--self-test", "--ui-self-test", "--onboarding-ui-self-test"] + assert result["result"] == "passed" + assert result["real_window_sessions"] == 2 + assert result["localhost_inference_count"] == expected_inferences + assert result["restart_resume_observed"] is expected_resume + assert result["pause_control_observed"] is True + assert result["sharing_intent_paused"] is True + assert result["sequence_profile"] == replay.SEQUENCE_PROFILES[platform] + assert result["policy_profile"] == replay.POLICY_PROFILE + assert result["qualification_temporaries_removed"] is True + assert not config.work_root.exists() + + +def test_config_and_session_evidence_fail_closed(tmp_path): + document = config_document(tmp_path) + document["work_root"] = str((tmp_path / "wrong-root").resolve()) + config_path = tmp_path / "invalid.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay.load_config(config_path) + + invalid_power = config_document(tmp_path) + invalid_power["policy"]["max_power_watts"] = 250.0 + invalid_power_path = tmp_path / "invalid-power.json" + invalid_power_path.write_text(json.dumps(invalid_power), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay.load_config(invalid_power_path) + + valid = config_document(tmp_path) + valid_path = tmp_path / "valid.json" + valid_path.write_text(json.dumps(valid), encoding="utf-8") + config = replay.load_config(valid_path) + config.work_root.mkdir() + evidence = session_evidence({**valid, "stage": "restart"}) + evidence["ui"]["start_clicked"] = False + evidence_path = config.work_root / "evidence.json" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay._validate_session(evidence_path, config, "restart") + + evidence = session_evidence({**valid, "stage": "initial"}) + evidence["inference"]["generated_token_count"] = 2 + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay._validate_session(evidence_path, config, "initial") + + +def run_main_with_runner(tmp_path, monkeypatch, capsys, runner, platform="windows"): + document = config_document(tmp_path, platform) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + real_replay = replay.run_replay + monkeypatch.setattr(replay, "run_replay", lambda config: real_replay(config, runner=runner)) + assert replay.main(["--config", str(config_path)]) == 1 + return document, json.loads(capsys.readouterr().out) + + +@pytest.mark.parametrize("action", ["--check-runtime", "--self-test", "--ui-self-test", "--onboarding-ui-self-test"]) +@pytest.mark.parametrize("timed_out", [False, True]) +def test_self_test_failure_reports_the_exact_action_and_process_outcome( + tmp_path, monkeypatch, capsys, action, timed_out +): + actions = [] + + def runner(argv, **kwargs): + actions.append(argv[1]) + if argv[1] == action: + if timed_out: + raise subprocess.TimeoutExpired( + argv, kwargs["timeout"], output="private prompt", stderr="private secret" + ) + return subprocess.CompletedProcess(argv, 37, stdout="private prompt", stderr="private secret") + return subprocess.CompletedProcess(argv, 0) + + document, failure = run_main_with_runner(tmp_path, monkeypatch, capsys, runner) + + assert actions[-1] == action + assert failure["failed_step"] == action + assert failure["error_category"] == ("process_timeout" if timed_out else "process_exit") + assert failure["timeout_seconds" if timed_out else "exit_code"] == (120 if timed_out else 37) + assert "private" not in json.dumps(failure) + assert not Path(document["work_root"]).exists() + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +@pytest.mark.parametrize("stage", ["initial", "restart"]) +@pytest.mark.parametrize("outcome", ["exit", "timeout", "failed_evidence"]) +def test_session_failure_evidence_survives_temporary_cleanup(tmp_path, monkeypatch, capsys, platform, stage, outcome): + def runner(argv, **kwargs): + if "--gate13-ui-playthrough" not in argv: + return subprocess.CompletedProcess(argv, 0) + plan = json.loads(Path(argv[2]).read_text(encoding="utf-8")) + evidence_path = Path(argv[4]) + evidence = session_evidence(plan) + if plan["stage"] == stage: + evidence = { + key: item + for key, item in evidence.items() + if key not in ("route", "inference", "ui", "limits", "timing", "privacy") + } + evidence.update( + result="failed", + failure_code="inference_failed", + duration_seconds=29.25, + failure_phase="wait_ready", + failure_detail="bootstrap_failed" if outcome == "timeout" else "inference_http_503", + ) + else: + # Even unknown fields inside accepted inference evidence must not + # become retained diagnostics when a later session fails. + evidence["inference"]["unrecognized_private_value"] = "private model output" + evidence["unrecognized_private_value"] = "private credential" if plan["stage"] == stage else None + if plan["stage"] != stage: + del evidence["unrecognized_private_value"] + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + if plan["stage"] == stage: + if outcome == "timeout": + raise subprocess.TimeoutExpired(argv, kwargs["timeout"], stderr="private stderr") + return subprocess.CompletedProcess(argv, 19 if outcome == "exit" else 0) + return subprocess.CompletedProcess(argv, 0) + + document, failure = run_main_with_runner(tmp_path, monkeypatch, capsys, runner, platform) + + assert failure["failed_step"] == f"{stage}_session" + assert ( + failure["error_category"] + == {"exit": "process_exit", "timeout": "process_timeout", "failed_evidence": "session_evidence_invalid"}[ + outcome + ] + ) + if outcome == "exit": + assert failure["exit_code"] == 19 + elif outcome == "timeout": + assert failure["timeout_seconds"] == 90 + retained = failure["session_evidence"][stage] + assert retained["result"] == "failed" + assert retained["failure_code"] == "inference_failed" + assert retained["failure_phase"] == "wait_ready" + assert retained["failure_detail"] == ("bootstrap_failed" if outcome == "timeout" else "inference_http_503") + assert retained["duration_seconds"] == 29.25 + if stage == "restart": + assert failure["session_evidence"]["initial"]["result"] == "passed" + assert failure["session_evidence"]["initial"]["inference"]["passed"] is True + assert "private" not in json.dumps(failure) + assert failure["qualification_temporaries_removed"] is True + assert not Path(document["work_root"]).exists() + + +@pytest.mark.parametrize("result", ["failed", "passed"]) +def test_session_diagnostics_drop_arbitrary_failure_text_and_success_only_failure_fields(tmp_path, result): + document = config_document(tmp_path) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + config = replay.load_config(config_path) + evidence = session_evidence({**document, "stage": "initial"}) + evidence.update( + result=result, + failure_code="private credential" if result == "failed" else "playthrough_timed_out", + failure_phase={"private": "prompt"} if result == "failed" else "wait_ready", + failure_detail="inference_http_503 private response" if result == "failed" else "inference_timed_out", + ) + evidence_path = tmp_path / "session.json" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + + retained = replay._session_diagnostic(evidence_path, config, "initial") + + assert all(field not in retained for field in ("failure_code", "failure_phase", "failure_detail")) + assert "private" not in json.dumps(retained) + + +def test_cleanup_error_does_not_obscure_the_failed_session(tmp_path, monkeypatch, capsys): + def runner(argv, **kwargs): + return subprocess.CompletedProcess(argv, 23 if "--gate13-ui-playthrough" in argv else 0) + + def failed_cleanup(_path): + raise OSError("private filesystem path") + + monkeypatch.setattr(replay.shutil, "rmtree", failed_cleanup) + document, failure = run_main_with_runner(tmp_path, monkeypatch, capsys, runner) + + assert failure["failed_step"] == "initial_session" + assert failure["exit_code"] == 23 + assert failure["error_category"] == "process_exit" + assert failure["session_evidence_error"] == "required file is unavailable" + assert failure["cleanup_failure_code"] == "qualification_temporary_cleanup_failed" + assert failure["qualification_temporaries_removed"] is False + assert Path(document["work_root"]).exists() + assert "private" not in json.dumps(failure) + + +def test_bounded_session_log_reaches_host_stderr_before_temporary_cleanup(tmp_path, capsys): + document = config_document(tmp_path) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + config = replay.load_config(config_path) + message = ( + "gate13-playthrough: stage=initial phase=wait_ready " + "The signed model catalog could not be installed: " + "Another first-install catalog bootstrap is already in progress\n" + ) + + def runner(argv, **kwargs): + if "--gate13-ui-playthrough" not in argv: + return subprocess.CompletedProcess(argv, 0) + Path(argv[4]).with_suffix(".log").write_bytes(message.encode("utf-8")) + raise subprocess.TimeoutExpired(argv, kwargs["timeout"]) + + with pytest.raises(replay.ReplayError) as caught: + replay.run_replay(config, runner=runner) + + assert capsys.readouterr().err == message + assert caught.value.diagnostics["failed_step"] == "initial_session" + assert caught.value.diagnostics["qualification_temporaries_removed"] is True + assert not config.work_root.exists() + assert "signed model catalog" not in json.dumps(replay._failure(caught.value)) diff --git a/tests/test_gate13_client_startup.py b/tests/test_gate13_client_startup.py new file mode 100644 index 000000000..6f43009fe --- /dev/null +++ b/tests/test_gate13_client_startup.py @@ -0,0 +1,137 @@ +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WINDOWS = ROOT / "scripts" / "gate13_windows_client_startup.ps1" +LINUX = ROOT / "scripts" / "gate13_linux_client_startup.sh" +GIT_BASH = Path(r"C:\Program Files\Git\bin\bash.exe") +BASH = str(GIT_BASH) if GIT_BASH.is_file() else shutil.which("bash") + + +def test_windows_bootstrap_preserves_the_proven_interactive_boundary(): + source = WINDOWS.read_text(encoding="utf-8") + + assert "RandomNumberGenerator]::Create()" in source + assert "RandomNumberGenerator]::Fill" not in source + assert 'if ((Get-Service -Name sshd).Status -ne "Running") { Start-Service -Name sshd }' in source + assert source.count('Set-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -Enabled True -Profile Any') == 2 + assert 'New-LocalUser -Name "M"' in source + assert 'Remove-LocalGroupMember -Group "Administrators" -Member "M"' in source + assert 'New-LocalUser -Name "Gate13Admin"' in source + assert 'Set-ItemProperty -Path $winlogon -Name AutoAdminLogon -Value "1"' in source + assert 'New-ScheduledTaskTrigger -AtLogOn -User "M"' in source + assert "Remove-ItemProperty -Path $p -Name DefaultPassword" in source + assert "Restart-Computer -Force" in source + assert "2a52993092a19cfdffe126e2eeac46a4265e25705614546604ad44988e040c0f" in source + assert "communityai_gate13_m_authorized_keys" in source + + +@pytest.mark.skipif(sys.platform != "win32", reason="requires Windows PowerShell parser") +def test_windows_bootstrap_parses_natively(): + probe = ( + f"$source=Get-Content -Raw -LiteralPath '{WINDOWS}';" + "$tokens=$null;$errors=$null;" + "[Management.Automation.Language.Parser]::ParseInput($source,[ref]$tokens,[ref]$errors)|Out-Null;" + "if($errors.Count -ne 0){$errors|ForEach-Object{$_.Message};exit 2}" + ) + result = subprocess.run( + [ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + probe, + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_linux_bootstrap_contains_the_proven_x11_runtime_and_display(): + source = LINUX.read_text(encoding="utf-8") + for package in ( + "xvfb", + "dbus-x11", + "gnome-keyring", + "libsecret-tools", + "libxcb-cursor0", + "libxcb-icccm4", + "libxcb-keysyms1", + "libxcb-shape0", + "libxkbcommon-x11-0", + ): + assert package in source + assert "/usr/bin/Xvfb :99" in source + assert "-nolisten tcp" in source + assert "DISPLAY=:99 xdpyinfo" in source + + +def test_linux_bootstrap_bounds_apt_network_and_lock_waits(): + source = LINUX.read_text(encoding="utf-8") + + assert "apt_deadline=$(( $(date +%s) + 300 ))" in source + assert "for attempt in 1 2 3" in source + assert '"${attempt_timeout}s"' in source + assert "Acquire::ForceIPv4=true" in source + assert "Acquire::Retries=1" in source + assert "Acquire::http::Timeout=20" in source + assert "Acquire::https::Timeout=20" in source + assert "DPkg::Lock::Timeout=30" in source + assert "APT::Update::Error-Mode=any" in source + assert "update attempt ${attempt}/3" in source + assert 'apt-get "${apt_options[@]}" install -y' in source + assert "bootstrap_status=/var/lib/gate13-bootstrap-status" in source + + +@pytest.mark.skipif(BASH is None, reason="requires bash") +def test_linux_bootstrap_switches_only_security_transport_and_writes_newline(tmp_path): + source = LINUX.read_text(encoding="utf-8") + sources = tmp_path / "ubuntu.sources" + original = ( + "Types: deb\n" + "URIs: http://us-central1.gce.archive.ubuntu.com/ubuntu/\n" + "Suites: noble noble-updates noble-backports\n" + "Components: main universe restricted multiverse\n" + "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n\n" + "Types: deb\n" + "URIs: http://security.ubuntu.com/ubuntu/\n" + "Suites: noble-security\n" + "Components: main universe restricted multiverse\n" + "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n" + ) + sources.write_text(original, encoding="utf-8", newline="\n") + rewrite = source[source.index("sed -i ") : source.index("\n\napt_deadline=")] + rewrite = rewrite.replace("/etc/apt/sources.list.d/ubuntu.sources", '"$1"') + trap = next(line for line in source.splitlines() if line.startswith("trap ")) + result = subprocess.run( + [BASH, "-c", rewrite + "\n" + trap.replace(' >"$bootstrap_status"', "") + "\nexit 1", "--", sources.as_posix()], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 1, result.stderr + assert result.stdout == "failed\n" + assert sources.read_text(encoding="utf-8") == original.replace( + "http://security.ubuntu.com/ubuntu/", "https://security.ubuntu.com/ubuntu/" + ) + + +@pytest.mark.skipif(BASH is None, reason="requires bash parser") +def test_linux_bootstrap_parses_natively(): + result = subprocess.run( + [BASH, "-n", str(LINUX)], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/test_gate13_cloud_orchestrator.py b/tests/test_gate13_cloud_orchestrator.py new file mode 100644 index 000000000..f23671ad4 --- /dev/null +++ b/tests/test_gate13_cloud_orchestrator.py @@ -0,0 +1,199 @@ +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_cloud_orchestrator as cloud + + +def package(platform): + archive = "communityai-desktop-windows.zip" if platform == "windows" else "communityai-desktop-linux.tar.gz" + return cloud.PackageArtifact( + platform=platform, + source_commit="a" * 40, + workflow_run_id=123, + artifact_id=456 if platform == "windows" else 457, + artifact_name=f"communityai-desktop-install-{platform}", + wrapper_sha256="c" * 64, + wrapper_bytes=110, + archive_name=archive, + archive_sha256="b" * 64, + archive_bytes=100, + ) + + +class Packages: + def __init__(self): + self.prepare_calls = 0 + + def prepare(self): + self.prepare_calls += 1 + return {platform: package(platform) for platform in cloud.PLATFORMS} + + +class Provider: + name = "fake" + + def __init__(self, fail_at=None): + self.fail_at = fail_at + self.calls = [] + + def _call(self, name): + self.calls.append(name) + if name == self.fail_at: + raise RuntimeError(name) + + def preflight(self): + self._call("preflight") + return {"result": "passed"} + + def create_route(self): + self._call("create_route") + + def prepare_route(self): + self._call("prepare_route") + return {"result": "passed"} + + def fence_route(self, platform): + self._call(f"fence_{platform}") + return {"result": "passed", "target": platform} + + def create_client(self, platform, package): + self._call(f"create_{platform}") + + def prepare_client(self, platform, package): + self._call(f"prepare_{platform}") + return {"result": "passed"} + + def run_client(self, platform, package): + self._call(f"run_{platform}") + return json.dumps({"result": "passed", "platform": platform}).encode() + + def delete_client(self, platform): + self._call(f"delete_{platform}") + + def delete_route(self): + self._call("delete_route") + + def cleanup_all(self): + self._call("cleanup_all") + if self.fail_at == "cleanup_result": + return {"result": "failed"} + return {"result": "passed"} + + def verify_cleanup(self): + self._call("verify_cleanup") + return {"result": "passed", "all_absent": True} + + +def validate(platform, payload, package): + value = json.loads(payload) + assert value["platform"] == platform + assert package.platform == platform + return value + + +def test_complete_sequence_is_ordered_and_persisted(tmp_path): + provider = Provider() + result = cloud.Gate13CloudOrchestrator( + run_id="gate13-test-a", + package_source=Packages(), + provider=provider, + output_root=tmp_path, + evidence_validator=validate, + clock=lambda: 100, + ).run() + + assert result["result"] == "passed" + assert provider.calls == [ + "preflight", + "create_route", + "prepare_route", + "fence_windows", + "create_windows", + "prepare_windows", + "run_windows", + "delete_windows", + "fence_linux", + "create_linux", + "prepare_linux", + "run_linux", + "delete_linux", + "delete_route", + "cleanup_all", + "verify_cleanup", + ] + assert (tmp_path / "windows-evidence.json").is_file() + assert (tmp_path / "linux-evidence.json").is_file() + assert json.loads((tmp_path / "result.json").read_text())["cleanup"]["all_absent"] is True + + +@pytest.mark.parametrize( + "failure", + [ + "create_route", + "prepare_route", + "fence_windows", + "create_windows", + "prepare_windows", + "run_windows", + "delete_windows", + "fence_linux", + "create_linux", + "prepare_linux", + "run_linux", + "delete_linux", + "delete_route", + ], +) +def test_every_cloud_failure_attempts_cleanup_and_verifies_absence(tmp_path, failure): + provider = Provider(fail_at=failure) + result = cloud.Gate13CloudOrchestrator( + run_id="gate13-test-a", + package_source=Packages(), + provider=provider, + output_root=tmp_path, + evidence_validator=validate, + clock=lambda: 100, + ).run() + + assert result["result"] == "failed" + assert result["failure_reason"] + assert "cleanup_all" in provider.calls + assert provider.calls[-1] == "verify_cleanup" + + +def test_preflight_failure_does_not_mutate_but_still_verifies_absence(tmp_path): + provider = Provider(fail_at="preflight") + packages = Packages() + result = cloud.Gate13CloudOrchestrator( + run_id="gate13-test-a", + package_source=packages, + provider=provider, + output_root=tmp_path, + evidence_validator=validate, + clock=lambda: 100, + ).run() + + assert result["result"] == "failed" + assert provider.calls == ["preflight", "verify_cleanup"] + assert packages.prepare_calls == 0 + + +def test_cleanup_failure_result_cannot_be_overwritten_by_successful_verification(tmp_path): + provider = Provider(fail_at="cleanup_result") + result = cloud.Gate13CloudOrchestrator( + run_id="gate13-test-a", + package_source=Packages(), + provider=provider, + output_root=tmp_path, + evidence_validator=validate, + clock=lambda: 100, + ).run() + + assert result["result"] == "failed" + assert result["failure_code"] == "CleanupError" diff --git a/tests/test_gate13_gcp_provider.py b/tests/test_gate13_gcp_provider.py new file mode 100644 index 000000000..d9bcca15a --- /dev/null +++ b/tests/test_gate13_gcp_provider.py @@ -0,0 +1,800 @@ +import base64 +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_gcp_provider as gcp +from gate13_cloud_orchestrator import Gate13CloudOrchestrator, PackageArtifact +from gate13_gcp_provider import GcpConfig, GcpProvider, GitHubPackageSource, LoggedRunner + +RUN_ID = "g13-20260902-000000-abcd" + + +def artifact(platform): + return PackageArtifact( + platform=platform, + source_commit="a" * 40, + workflow_run_id=1, + artifact_id=2, + artifact_name=f"communityai-desktop-install-{platform}", + wrapper_sha256="c" * 64, + wrapper_bytes=130, + archive_name=( + "communityai-desktop-windows.zip" if platform == "windows" else "communityai-desktop-linux.tar.gz" + ), + archive_sha256="b" * 64, + archive_bytes=123, + ) + + +def provider(tmp_path, runner, signed_url=lambda package: "https://productionresults.example.invalid/artifact"): + return GcpProvider( + run_id=RUN_ID, + repository_root=ROOT, + output_root=tmp_path, + config=GcpConfig.load(ROOT / "config" / "gate13_gcp.json"), + runner=runner, + signed_url=signed_url, + ) + + +@pytest.mark.parametrize( + "existing_changes,expired_artifact,omitted_artifact,expected_run_id", + [ + ({}, None, None, 10), + ({"head_sha": "b" * 40}, None, None, 11), + ({"conclusion": "failure"}, None, None, 11), + ({"status": "in_progress", "conclusion": None}, None, None, 11), + ({}, "communityai-desktop-audit-linux", None, 11), + ({}, None, "communityai-desktop-install-windows", 11), + ], + ids=["reuse", "different-source", "failed", "unfinished", "expired-audit", "missing-install"], +) +def test_package_reuses_only_a_complete_successful_run_of_the_pushed_source( + tmp_path, existing_changes, expired_artifact, omitted_artifact, expected_run_id +): + calls = [] + + class Runner: + def run(self, argv, **_kwargs): + calls.append([str(item) for item in argv]) + return subprocess.CompletedProcess(argv, 0, "", "") + + def json(self, argv, **_kwargs): + existing_run = "/runs/10/" in str(argv[2]) + return { + "artifacts": [ + { + "id": number, + "name": name, + "expired": existing_run and name == expired_artifact, + } + for number, name in enumerate( + ( + f"communityai-desktop-{kind}-{platform}" + for kind in ("install", "audit") + for platform in ("windows", "linux") + ), + start=1, + ) + if not (existing_run and name == omitted_artifact) + ] + } + + source = GitHubPackageSource( + repository_root=ROOT, + output_root=tmp_path, + repository="flujo-app/CommunityAI", + workflow="desktop.yaml", + runner=Runner(), + sleeper=lambda _seconds: None, + ) + old = { + "id": 10, + "head_sha": "a" * 40, + "status": "completed", + "conclusion": "success", + **existing_changes, + } + fresh = { + "id": 11, + "head_sha": "a" * 40, + "status": "completed", + "conclusion": "success", + } + listings = iter([[old], [fresh]]) + source._workflow_runs = lambda _branch: next(listings) + + run_id, artifacts = source._select_or_build_run("a" * 40, "test-branch") + + assert run_id == expected_run_id + assert len(artifacts) == 4 + assert all(item["expired"] is False for item in artifacts.values()) + assert any(command[1:3] == ["workflow", "run"] for command in calls) is (expected_run_id == 11) + + +def test_logged_runner_does_not_retain_argv_or_output(tmp_path): + secret = "never-retain-this-secret" + runner = LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None) + result = runner.run( + [sys.executable, "-c", f"print('{secret}')"], + action="Safe public action", + sensitive_output=True, + ) + + assert secret in result.stdout + journal = (tmp_path / "journal.jsonl").read_text() + assert secret not in journal + assert "-c" not in journal + assert json.loads(journal)["output_retained"] is False + + +def test_windows_gcloud_bypasses_cmd_argument_parsing(tmp_path, monkeypatch): + sdk = tmp_path / "google-cloud-sdk" + launcher = sdk / "bin" / "gcloud.cmd" + python = sdk / "platform" / "bundledpython" / "python.exe" + entrypoint = sdk / "lib" / "gcloud.py" + for path in (launcher, python, entrypoint): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"test") + monkeypatch.setattr(gcp.sys, "platform", "win32") + monkeypatch.setattr(gcp.shutil, "which", lambda name: str(launcher) if name in {"gcloud", "gcloud.cmd"} else None) + remote = 'powershell.exe -Command "Get-Process explorer | Where-Object {$_.Id}"' + + command = gcp._command_for_subprocess(["gcloud", "compute", "ssh", "vm", "--command", remote]) + + assert command == [str(python), "-S", str(entrypoint), "compute", "ssh", "vm", "--command", remote] + + +def test_logged_runner_forces_noninteractive_putty_host_key_acceptance(tmp_path, monkeypatch): + observed = {} + + def fake_run(command, **kwargs): + observed["command"] = command + observed.update(kwargs) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(gcp.subprocess, "run", fake_run) + runner = LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None) + + runner.run([sys.executable, "-c", "pass"], action="Test command") + + assert observed["env"]["CLOUDSDK_CORE_DISABLE_PROMPTS"] == "1" + assert observed["env"]["CLOUDSDK_SSH_PUTTY_FORCE_CONNECT"] == "1" + + +class CreateRunner: + def __init__(self): + self.calls = [] + + def run(self, argv, **kwargs): + command = [str(item) for item in argv] + self.calls.append((command, kwargs)) + if "describe" in command and "instances" in command: + name = command[command.index("describe") + 1] + value = { + "name": name, + "labels": {"communityai_run": RUN_ID}, + "deletionProtection": False, + "disks": [ + { + "autoDelete": True, + "source": f"https://example.invalid/disks/{name}", + } + ], + } + return subprocess.CompletedProcess(command, 0, json.dumps(value), "") + return subprocess.CompletedProcess(command, 0, "", "") + + +def test_client_creation_uses_the_proven_private_route_relay(tmp_path, monkeypatch): + fake = CreateRunner() + public_key = tmp_path / "test.pub" + public_key.write_text("ssh-rsa test\n", encoding="ascii") + relay_calls = [] + item = provider(tmp_path, fake) + monkeypatch.setattr(item, "_ensure_ssh_key", lambda: public_key) + monkeypatch.setattr( + item, + "_prepare_route_relay", + lambda platform, package: relay_calls.append((platform, package.artifact_id)) + or "http://10.42.0.26:38081/artifact-wrapper.zip", + ) + + item.create_client("windows", artifact("windows")) + + flattened = "\n".join(" ".join(command) for command, _kwargs in fake.calls) + assert relay_calls == [("windows", 2)] + assert "package-url=http://10.42.0.26:38081/artifact-wrapper.zip" in flattened + assert "package-sha256=" + "b" * 64 in flattened + assert "package-bytes=123" in flattened + create = next(command for command, _kwargs in fake.calls if "instances" in command and "create" in command) + assert "--enable-display-device" in create + assert "--no-service-account" in create + assert "no-address" not in flattened + + +def test_route_creation_preserves_the_successful_private_relay_firewall(tmp_path): + fake = CreateRunner() + item = provider(tmp_path, fake) + + item.create_route() + + relay = next( + command for command, _kwargs in fake.calls if "firewall-rules" in command and item.relay_firewall in command + ) + assert relay[relay.index("--rules") + 1] == "tcp:38081" + assert relay[relay.index("--source-tags") + 1] == item.client_tag + assert relay[relay.index("--target-tags") + 1] == item.route + route = next(command for command, _kwargs in fake.calls if "instances" in command and "create" in command) + assert route[route.index("--machine-type") + 1] == "g2-standard-8" + assert route[route.index("--boot-disk-size") + 1] == "200GB" + assert route[route.index("--max-run-duration") + 1] == "57600s" + + +def test_route_relay_script_is_bound_to_both_wrapper_and_inner_archive(tmp_path): + item = provider( + tmp_path, + LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None), + ) + + source = item._relay_download_script("linux", artifact("linux")).read_text() + + assert "artifact-probe-url" in source + assert 'curl -fL --retry 4 --retry-delay 3 --silent --show-error "$url" -o "$wrapper"' in source + assert 'test "$(stat -c %s "$wrapper")" = 130' in source + assert "c" * 64 in source + assert "expected = 'communityai-desktop-linux.tar.gz'" in source + assert 'test "$(stat -c %s "$archive")" = 123' in source + assert "b" * 64 in source + + +def test_route_bundle_uses_fixed_runtime_and_unchanged_signed_catalog(tmp_path): + item = provider( + tmp_path, + LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None), + ) + + bundle = item._build_route_bundle() + + expected = { + "drift-2.3.0.dev2-py3-none-any.whl": ( + 389449, + "edfd4598c293719d4d7701c9613b64f47f9fd20c3a2dc2e4c0fcacacad3c493a", + ), + "gate13_route_setup.sh": ( + 3371, + "f8fb52f40133fdefcc137c4244e66bec81eb5c820cf902b46b85944a4d0229e1", + ), + "catalog-v1.tar": ( + 20480, + "2ecf7ecbe8159d6a6328eed9b59e2a3ae4543b6ee6b82d904de42676150b1452", + ), + } + for name, (byte_count, digest) in expected.items(): + payload = (bundle / name).read_bytes() + assert len(payload) == byte_count + assert hashlib.sha256(payload).hexdigest() == digest + + +def test_client_startup_scripts_are_taken_from_the_successful_run(tmp_path): + item = provider( + tmp_path, + LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None), + ) + + expected = { + "windows": ( + 8779, + "3f8600c42a3c0765e100963c2e28cdef7c6b248992924ff3406941aefce7cf47", + ), + "linux": ( + 3808, + "892c9d8568c491d67a7ef027177d11fc6737673f86eecd5eee9e8191ca1e8a55", + ), + } + for platform, (byte_count, digest) in expected.items(): + payload = item._client_startup_script(platform).read_bytes() + assert len(payload) == byte_count + assert hashlib.sha256(payload).hexdigest() == digest + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +def test_route_fencing_uses_1200_second_readiness_timeout(tmp_path, monkeypatch, platform): + item = provider(tmp_path, object()) + result = {"result": "passed", "target": platform} + calls = [] + + def ssh(resource, command, **kwargs): + calls.append((resource, command, kwargs)) + return subprocess.CompletedProcess([], 0, json.dumps(result), "") + + monkeypatch.setattr(item, "_ssh", ssh) + + assert item.fence_route(platform) == result + assert calls == [ + ( + item.route, + "sudo /opt/communityai/venv/bin/python " + f"/tmp/gate13_route_fence.py --target {platform} --timeout-seconds 1200 --settle-seconds 30", + {"action": f"Fencing the route for {platform}", "timeout": 2_100, "check": False}, + ) + ] + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +@pytest.mark.parametrize("failure", ["rejected", "malformed", "timeout"]) +def test_route_failure_logs_are_saved_before_cleanup_and_survive_it(tmp_path, monkeypatch, platform, failure): + item = provider(tmp_path, object()) + stage = f"{platform}-fence" + logs = tmp_path / "route-diagnostics" / stage + events = [] + command_output = "\n".join(f"fence output line {number}" for number in range(1_000)) + + def ssh(instance, command, **kwargs): + assert instance == item.route + if "gate13_route_fence.py" in command: + if f"--target {platform}" not in command: + return subprocess.CompletedProcess([], 0, '{"result":"passed","target":"windows"}', "") + events.append("fence-failed") + if failure == "timeout": + cause = subprocess.TimeoutExpired( + "ssh", 2_100, output=command_output.encode(), stderr=b"partial transport error" + ) + raise gcp.CommandError("Fencing could not run") from cause + payload = ( + '{"result":"failed","failure_code":"route did not become ready before the deadline"}' + if failure == "rejected" + else "this is not JSON" + ) + return subprocess.CompletedProcess([], 1, command_output + "\n" + payload, "actual fence stderr") + assert (logs / "command.log").is_file() + assert kwargs["check"] is False + assert kwargs["timeout"] == 90 + if "systemctl show" in command: + events.append("services-saved") + return subprocess.CompletedProcess([], 0, "ActiveState=failed\nExecMainStatus=1\n", "") + assert "journalctl -b" in command + assert "-u communityai-qwen.service -u communityai-gemma.service" in command + assert "-n " not in command and "tail" not in command + events.append("journal-saved") + return subprocess.CompletedProcess( + [], 0, "early worker error\n" + "download progress\n" * 1_000, "journal warning" + ) + + def cleanup(): + assert "early worker error" in (logs / "journal.log").read_text() + assert "ExecMainStatus=1" in (logs / "services.log").read_text() + events.append("cleanup") + return {"result": "passed"} + + monkeypatch.setattr(item, "_ssh", ssh) + monkeypatch.setattr(item, "preflight", lambda: {"result": "passed"}) + monkeypatch.setattr(item, "create_route", lambda: None) + monkeypatch.setattr(item, "prepare_route", lambda: {"result": "passed"}) + monkeypatch.setattr(item, "create_client", lambda *_args: None) + monkeypatch.setattr(item, "prepare_client", lambda *_args: {"result": "passed"}) + monkeypatch.setattr(item, "run_client", lambda *_args: b'{"result":"passed"}') + monkeypatch.setattr(item, "delete_client", lambda *_args: None) + monkeypatch.setattr(item, "cleanup_all", cleanup) + monkeypatch.setattr(item, "verify_cleanup", lambda: {"result": "passed"}) + + result = Gate13CloudOrchestrator( + run_id=RUN_ID, + package_source=SimpleNamespace(prepare=lambda: {target: artifact(target) for target in ("windows", "linux")}), + provider=item, + output_root=tmp_path, + evidence_validator=lambda _platform, payload, _package: json.loads(payload), + ).run() + + assert result["result"] == "failed" + assert result["cleanup"]["result"] == "passed" + assert str(logs) in result["failure_reason"] + assert events == ["fence-failed", "services-saved", "journal-saved", "cleanup"] + assert command_output in (logs / "command.log").read_text() + assert "journal warning" in (logs / "journal.log").read_text() + assert (logs / "collection.log").read_text() == "Collection completed\n" + if failure == "timeout": + assert "partial transport error" in (logs / "command.log").read_text() + assert "Fencing could not run" in result["failure_reason"] + else: + assert "actual fence stderr" in (logs / "command.log").read_text() + + +@pytest.mark.parametrize("read_failure", ["timeout", "exit"]) +def test_route_log_collection_failure_keeps_other_logs_and_original_error(tmp_path, monkeypatch, read_failure): + item = provider(tmp_path, object()) + logs = tmp_path / "route-diagnostics" / "linux-fence" + + def ssh(_instance, command, **_kwargs): + if "gate13_route_fence.py" in command: + return subprocess.CompletedProcess([], 1, '{"result":"failed","failure_code":"original failure"}', "") + if "systemctl show" in command: + if read_failure == "timeout": + raise subprocess.TimeoutExpired("ssh", 90, output=b"partial service status") + return subprocess.CompletedProcess([], 1, "", "service status unavailable") + return subprocess.CompletedProcess([], 0, "worker traceback", "") + + monkeypatch.setattr(item, "_ssh", ssh) + with pytest.raises(gcp.Gate13CloudError, match="original failure"): + item.fence_route("linux") + assert "worker traceback" in (logs / "journal.log").read_text() + assert "Collection incomplete: services:" in (logs / "collection.log").read_text() + assert (logs / "command.log").is_file() + + +def test_route_log_disk_failure_does_not_replace_original_fence_error(tmp_path, monkeypatch): + item = provider(tmp_path, object()) + + def fail(_platform): + raise gcp.Gate13CloudError("original route failure") + + def write(*_args): + raise OSError("disk full") + + monkeypatch.setattr(item, "_fence_route", fail) + monkeypatch.setattr(item, "_write_route_log", write) + with pytest.raises(gcp.Gate13CloudError, match="original route failure; route diagnostic collection failed"): + item.fence_route("windows") + + +def test_route_logs_redact_download_urls_and_bearer_tokens(tmp_path): + item = provider(tmp_path, object()) + path = item._write_route_log( + "setup", + "command", + "download failed https://example.invalid/file?signature=secret\nAuthorization: Bearer secret", + ) + content = path.read_text() + assert "secret" not in content + assert "download failed " in content + assert "Bearer " in content + + +@pytest.mark.parametrize("setup_failed", [False, True]) +def test_route_preparation_waits_five_minutes_and_for_ubuntu_installer(tmp_path, monkeypatch, setup_failed): + item = provider( + tmp_path, + LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None), + ) + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "catalog-v1.tar").write_bytes(b"catalog") + waits = [] + ssh_commands = [] + monkeypatch.setattr( + item, + "_describe_instance", + lambda _name: {"networkInterfaces": [{"accessConfigs": [{"natIP": "198.51.100.1"}]}]}, + ) + monkeypatch.setattr(item, "_build_route_bundle", lambda: bundle) + monkeypatch.setattr( + item, + "_wait_ssh", + lambda _name, command, **_kwargs: waits.append(command), + ) + monkeypatch.setattr(item, "_scp", lambda *_args, **_kwargs: None) + + def ssh(_name, command, **_kwargs): + ssh_commands.append(command) + if "gate13_route_setup.sh" in command: + return subprocess.CompletedProcess( + [], 1 if setup_failed else 0, "full setup output\n" * 1_000, "setup stderr" + ) + return subprocess.CompletedProcess([], 0, "", "") + + monkeypatch.setattr(item, "_ssh", ssh) + + if setup_failed: + with pytest.raises(gcp.Gate13CloudError, match="route services failed with exit code 1"): + item.prepare_route() + logs = tmp_path / "route-diagnostics" / "setup" + assert "full setup output\n" * 1_000 in (logs / "command.log").read_text() + assert "setup stderr" in (logs / "command.log").read_text() + assert (logs / "journal.log").is_file() + return + result = item.prepare_route() + + assert len(waits) == 1 + assert "/proc/uptime" in waits[0] + assert "-ge 300" in waits[0] + assert "/var/lib/dpkg/lock-frontend" in waits[0] + setup_command = next(command for command in ssh_commands if "gate13_route_setup.sh" in command) + assert "install -d -m 0755 /tmp/gate13-route/catalog-v1" in setup_command + assert len(ssh_commands) == 2 + assert result["result"] == "passed" + + +def test_client_readiness_cleans_the_route_relay_before_job_staging(tmp_path, monkeypatch): + item = provider( + tmp_path, + LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None), + ) + stage = tmp_path / "stage" + stage.mkdir() + stage_script = stage / "stage.sh" + stage_script.write_text("#!/usr/bin/env bash\n", encoding="utf-8") + events = [] + monkeypatch.setattr( + item, + "_wait_ssh", + lambda _name, command, **_kwargs: events.append(("ready", command)), + ) + monkeypatch.setattr( + item, + "_cleanup_route_relay", + lambda platform: events.append(("relay-cleaned", platform)), + ) + monkeypatch.setattr( + item, + "_build_client_stage", + lambda _platform, _package: (stage, stage_script), + ) + monkeypatch.setattr( + item, + "_scp", + lambda *_args, **_kwargs: events.append(("stage-copied", "linux")), + ) + monkeypatch.setattr( + item, + "_ssh", + lambda *_args, **_kwargs: subprocess.CompletedProcess( + [], + 0, + "Access granted. Press Return to begin session.\n" + "remote banner text\n" + "GATE13_STAGE_RESULT result=passed ready=true host_user=gate13\n" + "trailing transport text\n", + "non-fatal transport warning\n", + ), + ) + + result = item.prepare_client("linux", artifact("linux")) + + assert "gate13-bootstrap-ready" in events[0][1] + assert "sudo grep -qx failed /var/lib/gate13-bootstrap-status" in events[0][1] + assert [event[0] for event in events] == ["ready", "relay-cleaned", "stage-copied"] + assert result["package_relay_verified"] is True + captured = json.loads((tmp_path / "linux-stage-command-output.json").read_text()) + assert "remote banner text" in captured["stdout"] + assert captured["stderr"] == "non-fatal transport warning\n" + + +def test_generated_client_jobs_are_exactly_source_and_package_bound(tmp_path): + item = provider( + tmp_path, + LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None), + ) + for platform in ("windows", "linux"): + stage, stage_script = item._build_client_stage(platform, artifact(platform)) + lifecycle = json.loads((stage / f"gate13-{platform}-run.json").read_text()) + host = json.loads((stage / "host-job.json").read_text()) + assert lifecycle["source_commit"] == "a" * 40 + assert lifecycle["package_sha256"] == "sha256:" + "b" * 64 + assert lifecycle["package_bytes"] == 123 + assert host["source_commit"] == lifecycle["source_commit"] + assert host["lifecycle_run_id"] == f"{RUN_ID}-{platform}" + assert host["attempt_ordinal"] == 1 + assert stage_script.is_file() + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +def test_failed_client_stage_retains_output_even_with_a_success_marker(tmp_path, monkeypatch, platform): + item = provider(tmp_path, LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None)) + stage = tmp_path / "stage" + stage.mkdir() + monkeypatch.setattr(item, "_wait_ssh", lambda *_args, **_kwargs: None) + monkeypatch.setattr(item, "_cleanup_route_relay", lambda *_args: None) + monkeypatch.setattr(item, "_build_client_stage", lambda *_args: (stage, stage / "stage-script")) + monkeypatch.setattr(item, "_scp", lambda *_args, **_kwargs: None) + + def ssh(_instance, _command, **kwargs): + assert kwargs["check"] is False + return subprocess.CompletedProcess( + [], 1, "GATE13_STAGE_RESULT result=passed ready=true\n", "actual stage failure\n" + ) + + monkeypatch.setattr(item, "_ssh", ssh) + with pytest.raises(gcp.Gate13CloudError, match="stage failed with exit code 1"): + item.prepare_client(platform, artifact(platform)) + captured = json.loads((tmp_path / f"{platform}-stage-command-output.json").read_text()) + assert captured["exit_code"] == 1 + assert captured["stderr"] == "actual stage failure\n" + + +def test_client_status_poll_retries_one_temporary_connection_failure(tmp_path, monkeypatch): + item = provider(tmp_path, LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None)) + waits = [] + messages = [] + replies = iter( + [ + subprocess.CompletedProcess([], 0, '{"job_state":"running"}\n', ""), + subprocess.CompletedProcess([], 1, "", "temporary DNS failure"), + subprocess.CompletedProcess([], 0, '{"job_state":"passed"}\n', ""), + subprocess.CompletedProcess([], 0, '{"result":"passed"}\n', ""), + subprocess.CompletedProcess([], 0, "{}\n", ""), + ] + ) + monkeypatch.setattr(item, "_ssh", lambda *_args, **_kwargs: next(replies)) + item.sleeper = waits.append + item.progress = messages.append + + payload = item.run_client("windows", artifact("windows")) + + assert payload == b'{"result":"passed"}\n' + assert waits == [30] + assert messages == ["Checking the windows qualification job did not complete; trying again (2 of 5)"] + + +def test_linux_client_reads_json_after_the_ssh_greeting(tmp_path, monkeypatch): + item = provider(tmp_path, LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None)) + greeting = "Access granted. Press Return to begin session.\n" + replies = iter( + [ + subprocess.CompletedProcess([], 0, greeting + '{"job_state":"running"}\n', ""), + subprocess.CompletedProcess([], 0, greeting + '{"job_state":"passed"}\n', ""), + subprocess.CompletedProcess([], 0, greeting + '{"result":"passed"}\n', ""), + subprocess.CompletedProcess([], 0, greeting + "{}\n", ""), + ] + ) + monkeypatch.setattr(item, "_ssh", lambda *_args, **_kwargs: next(replies)) + item.sleeper = lambda _seconds: None + + payload = item.run_client("linux", artifact("linux")) + + assert payload == b'{"result":"passed"}\n' + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +def test_client_captures_terminal_and_stderr_before_raising(tmp_path, monkeypatch, platform): + item = provider(tmp_path, LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None)) + calls = [] + messages = [] + item.progress = messages.append + replies = iter( + [ + subprocess.CompletedProcess([], 0, '{"job_state":"running"}\n', ""), + subprocess.CompletedProcess([], 0, '{"job_state":"failed"}\n', ""), + subprocess.CompletedProcess([], 0, '{"failure_code":"lifecycle_failed"}\n', ""), + subprocess.CompletedProcess([], 0, "actual lifecycle error\n", ""), + subprocess.CompletedProcess([], 0, '{"result":"failed","phase":"launch"}\n', ""), + ] + ) + + def ssh(instance, command, **kwargs): + calls.append((instance, command, kwargs)) + return next(replies) + + monkeypatch.setattr(item, "_ssh", ssh) + + with pytest.raises(gcp.Gate13CloudError, match="captured output"): + item.run_client(platform, artifact(platform)) + + captured = json.loads((tmp_path / f"{platform}-host-job-failure-output.json").read_text()) + assert "lifecycle_failed" in captured["terminal"]["stdout"] + assert captured["stderr"]["stdout"] == "actual lifecycle error\n" + assert '"phase":"launch"' in captured["evidence"]["stdout"] + assert any("actual lifecycle error" in message for message in messages) + observed = json.loads((tmp_path / f"{item.clients[platform]}-host-job-command-output.json").read_text()) + assert observed["stdout"] == '{"job_state":"failed"}\n' + for call, filename in zip(calls[2:], ("terminal.json", "stderr.log", "evidence.json")): + assert call[0] == item.clients[platform] + if platform == "windows": + assert call[2]["user"] == "Gate13Admin" + script = base64.b64decode(call[1].split()[-1]).decode("utf-16-le") + assert f"'C:\\Gate13Run\\{filename}'" in script + assert "[IO.File]::ReadAllText" in script + else: + assert call[1] == f"sudo cat /qualification/{filename}" + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +def test_failure_capture_keeps_earlier_files_when_a_later_read_times_out(tmp_path, monkeypatch, platform): + item = provider(tmp_path, LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None)) + output_path = tmp_path / f"{platform}-host-job-failure-output.json" + calls = [] + + def ssh(_instance, _command, **_kwargs): + calls.append(_command) + if len(calls) == 1: + return subprocess.CompletedProcess([], 0, '{"failure_code":"lifecycle_failed"}\n', "") + assert "lifecycle_failed" in json.loads(output_path.read_text())["terminal"]["stdout"] + if len(calls) == 2: + raise gcp.CommandError("transport timed out") + return subprocess.CompletedProcess([], 0, '{"failed_step":"initial_session"}\n', "") + + def fail(*_args): + raise gcp.Gate13CloudError(f"{platform} host job ended in state failed") + + monkeypatch.setattr(item, "_run_client", fail) + monkeypatch.setattr(item, "_ssh", ssh) + with pytest.raises(gcp.Gate13CloudError, match=f"{platform} host job ended in state failed"): + item.run_client(platform, artifact(platform)) + + captured = json.loads(output_path.read_text()) + assert captured["stderr"] == {"capture_error": "CommandError"} + assert "initial_session" in captured["evidence"]["stdout"] + + +def test_failure_capture_disk_error_does_not_replace_original_failure(tmp_path, monkeypatch): + item = provider(tmp_path, LoggedRunner(tmp_path / "journal.jsonl", progress=lambda _message: None)) + + def fail(*_args): + raise gcp.Gate13CloudError("windows host job ended in state failed") + + def capture(*_args): + raise OSError("disk full") + + monkeypatch.setattr(item, "_run_client", fail) + monkeypatch.setattr(item, "_capture_host_failure", capture) + with pytest.raises(gcp.Gate13CloudError, match="windows host job ended in state failed") as error: + item.run_client("windows", artifact("windows")) + assert "collection failed (OSError)" in str(error.value) + + +def test_cleanup_instance_inspection_retries_instead_of_claiming_absence(tmp_path): + calls = [] + waits = [] + + class Runner: + def run(self, argv, **_kwargs): + calls.append(argv) + if len(calls) == 1: + return subprocess.CompletedProcess(argv, 1, "", "temporary DNS failure") + name = str(argv[argv.index("describe") + 1]) + value = { + "name": name, + "labels": {"communityai_run": RUN_ID}, + "deletionProtection": False, + "disks": [{"autoDelete": True, "source": f"https://example.invalid/disks/{name}"}], + } + return subprocess.CompletedProcess(argv, 0, json.dumps(value), "") + + item = provider(tmp_path, Runner()) + item.sleeper = waits.append + + value = item._describe_instance(item.clients["windows"], check=False) + + assert value["name"] == item.clients["windows"] + assert len(calls) == 2 + assert waits == [15] + + +def test_cleanup_instance_inspection_raises_when_gcp_never_answers(tmp_path): + calls = [] + + class Runner: + def run(self, argv, **_kwargs): + calls.append(argv) + return subprocess.CompletedProcess(argv, 1, "", "temporary DNS failure") + + item = provider(tmp_path, Runner()) + item.sleeper = lambda _seconds: None + + with pytest.raises(gcp.CommandError, match="failed after 5 attempts"): + item._describe_instance(item.clients["windows"], check=False) + + assert len(calls) == 5 + + +def test_cleanup_instance_inspection_accepts_an_explicit_not_found_response(tmp_path): + calls = [] + + class Runner: + def run(self, argv, **_kwargs): + calls.append(argv) + return subprocess.CompletedProcess(argv, 1, "", "The resource was not found") + + item = provider(tmp_path, Runner()) + item.sleeper = lambda _seconds: pytest.fail("an explicit not-found response must not be retried") + + assert item._describe_instance(item.clients["windows"], check=False) is None + assert len(calls) == 1 diff --git a/tests/test_gate13_host_job.py b/tests/test_gate13_host_job.py new file mode 100644 index 000000000..90a66837c --- /dev/null +++ b/tests/test_gate13_host_job.py @@ -0,0 +1,657 @@ +import hashlib +import io +import json +import subprocess +import sys +import threading +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_host_job as host_job # noqa: E402 + + +def sha256(path): + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture +def config_factory(tmp_path, monkeypatch): + def make(platform="linux"): + root = tmp_path / platform + root.mkdir() + adapter = root / "gate13_host_job.py" + adapter.write_bytes(host_job.ADAPTER_PATH.read_bytes()) + entrypoint = root / ( + "gate13_windows_packaged_lifecycle.ps1" if platform == "windows" else "gate13_linux_packaged_lifecycle.py" + ) + entrypoint.write_text("# bound lifecycle\n", encoding="utf-8") + lifecycle_config = root / ("gate13-windows-run.json" if platform == "windows" else "gate13-linux-run.json") + lifecycle_config.write_text('{"bound":true}\n', encoding="utf-8") + python = Path(sys.executable).resolve() + + monkeypatch.setitem(host_job.HOST_ROOTS, platform, root) + monkeypatch.setitem(host_job.HOST_PYTHON, platform, python) + monkeypatch.setattr(host_job, "ADAPTER_PATH", adapter.resolve()) + + run_id = "gate13-test-a" + raw = { + "schema_version": 1, + "run_id": run_id, + "lifecycle_run_id": f"{run_id}-{platform}", + "platform": platform, + "attempt_ordinal": 1, + "source_commit": "a" * 40, + "job_name": f"communityai-gate13-{run_id}-{platform}", + "host_user": "gate13", + "adapter_path": str(adapter.resolve()), + "adapter_sha256": sha256(adapter), + "config_path": str((root / "host-job.json").resolve()), + "entrypoint_path": str(entrypoint.resolve()), + "entrypoint_sha256": sha256(entrypoint), + "lifecycle_config_path": str(lifecycle_config.resolve()), + "lifecycle_config_sha256": sha256(lifecycle_config), + "evidence_path": str((root / "evidence.json").resolve()), + "stderr_path": str((root / "stderr.log").resolve()), + "status_path": str((root / "status.json").resolve()), + "terminal_path": str((root / "terminal.json").resolve()), + "working_directory": str(root.resolve()), + "python_executable": str(python), + "max_run_seconds": 3600, + } + path = root / "host-job.json" + path.write_text(json.dumps(raw), encoding="utf-8") + return path, raw + + return make + + +def test_load_config_binds_exact_files_paths_and_single_attempt(config_factory): + path, raw = config_factory() + + config = host_job.load_config(path) + + assert config.attempt_ordinal == 1 + assert config.job_name == "communityai-gate13-gate13-test-a-linux" + assert config.adapter_sha256 == raw["adapter_sha256"] + assert config.entrypoint_sha256 == raw["entrypoint_sha256"] + assert config.lifecycle_config_sha256 == raw["lifecycle_config_sha256"] + assert config.host_user == "gate13" + + +def test_windows_environment_keeps_standard_user_runtime_and_drops_secrets(config_factory, monkeypatch): + path, _raw = config_factory("windows") + config = host_job.load_config(path) + expected = { + "APPDATA": r"C:\\Users\\M\\AppData\\Roaming", + "LOCALAPPDATA": r"C:\\Users\\M\\AppData\\Local", + "PATH": r"C:\\Windows\\System32", + "USERPROFILE": r"C:\\Users\\M", + } + for key, value in expected.items(): + monkeypatch.setenv(key, value) + monkeypatch.setenv("GH_TOKEN", "must-not-cross-the-host-boundary") + monkeypatch.setenv("COMMUNITYAI_CONTROL_TOKEN", "must-not-cross-the-host-boundary") + + environment = host_job._bounded_environment(config) + + assert all(environment[key] == value for key, value in expected.items()) + assert set(environment).issubset(set(host_job.WINDOWS_RUNTIME_ENVIRONMENT)) + assert "GH_TOKEN" not in environment + assert "COMMUNITYAI_CONTROL_TOKEN" not in environment + + +def test_linux_environment_keeps_display_and_secret_service_session(config_factory, monkeypatch): + path, _raw = config_factory("linux") + config = host_job.load_config(path) + expected = { + "DISPLAY": ":99", + "DBUS_SESSION_BUS_ADDRESS": "unix:path=/run/user/1000/bus", + "GNOME_KEYRING_CONTROL": "/run/user/1000/keyring", + "QT_QPA_PLATFORM": "offscreen", + } + for key, value in expected.items(): + monkeypatch.setenv(key, value) + monkeypatch.setenv("UNRELATED_SECRET", "must-not-cross-the-host-boundary") + + environment = host_job._bounded_environment(config) + + assert all(environment[key] == value for key, value in expected.items()) + assert set(environment).issubset(set(host_job.LINUX_RUNTIME_ENVIRONMENT)) + assert "UNRELATED_SECRET" not in environment + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("attempt_ordinal", 2), + ("job_name", "communityai-gate13-foreign-linux"), + ("lifecycle_run_id", "foreign-linux"), + ("max_run_seconds", 86_400), + ("host_user", "root"), + ], +) +def test_changed_execution_binding_fails_closed(config_factory, field, value): + path, raw = config_factory() + raw[field] = value + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(host_job.HostJobError): + host_job.load_config(path) + + +def test_path_escape_and_entrypoint_tampering_fail_closed(config_factory, tmp_path): + path, raw = config_factory() + raw["evidence_path"] = str((tmp_path / "escaped.json").resolve()) + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(host_job.HostJobError, match="escapes"): + host_job.load_config(path) + + raw["evidence_path"] = str((path.parent / "evidence.json").resolve()) + Path(raw["entrypoint_path"]).write_text("# changed\n", encoding="utf-8") + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(host_job.HostJobError, match="entrypoint digest changed"): + host_job.load_config(path) + + +def test_lifecycle_config_tampering_fails_closed(config_factory): + path, raw = config_factory() + Path(raw["lifecycle_config_path"]).write_text('{"changed":true}\n', encoding="utf-8") + + with pytest.raises(host_job.HostJobError, match="lifecycle config digest changed"): + host_job.load_config(path) + + +def test_windows_lifecycle_config_must_be_beside_entrypoint(config_factory): + path, raw = config_factory("windows") + nested = path.parent / "nested" + nested.mkdir() + nominated = nested / "gate13-windows-run.json" + nominated.write_text('{"bound":true}\n', encoding="utf-8") + raw["lifecycle_config_path"] = str(nominated.resolve()) + raw["lifecycle_config_sha256"] = sha256(nominated) + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(host_job.HostJobError, match="not beside"): + host_job.load_config(path) + + +def test_windows_task_is_bounded_interactive_ordinary_user_single_instance(config_factory): + path, _raw = config_factory("windows") + config = host_job.load_config(path) + + script = host_job._windows_register_script(config) + + assert "New-ScheduledTaskPrincipal -UserId $targetAccount.Value" in script + assert "-LogonType Interactive -RunLevel Limited" in script + assert "privileged task registration required" in script + assert "Get-LocalUser -Name 'gate13'" in script + assert "'SYSTEM'" not in script + assert "-MultipleInstances IgnoreNew" in script + assert "-ExecutionTimeLimit" in script + assert str(config.adapter_path) in script + assert str(config.config_path) in script + assert "password" not in script.lower() + assert "token" not in script.lower() + + snapshot = host_job._windows_snapshot_script(config) + assert "MultipleInstances -eq 'IgnoreNew'" in snapshot + assert "ExecutionTimeLimit -eq $expectedLimit" in snapshot + assert "LogonType -eq 'Interactive'" in snapshot + assert "$taskSid -eq $targetSid" in snapshot + assert "NTAccount]::new([string]$task.Principal.UserId)" in snapshot + assert "RunLevel -eq 'Limited'" in snapshot + + +@pytest.mark.skipif(sys.platform != "win32", reason="requires Windows PowerShell parser") +def test_windows_task_scripts_parse_natively(config_factory): + import base64 + + path, _raw = config_factory("windows") + config = host_job.load_config(path) + for source in ( + host_job._windows_register_script(config), + host_job._windows_snapshot_script(config), + ): + encoded = base64.b64encode(source.encode("utf-16le")).decode("ascii") + probe = ( + "$source=[Text.Encoding]::Unicode.GetString(" + f"[Convert]::FromBase64String('{encoded}'));" + "$tokens=$null;$errors=$null;" + "[Management.Automation.Language.Parser]::ParseInput(" + "$source,[ref]$tokens,[ref]$errors)|Out-Null;" + "if($errors.Count -ne 0){exit 2}" + ) + result = subprocess.run( + host_job._powershell_argv(probe), + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +def test_linux_unit_is_bounded_non_root_and_non_restarting(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + + argv = host_job._linux_start_argv(config) + + assert argv[:4] == ["sudo", "-n", "/usr/bin/systemd-run", "--quiet"] + assert f"--unit" in argv + assert config.job_name in argv + assert "--property=User=gate13" in argv + assert "--property=Restart=no" in argv + assert "--property=KillMode=control-group" in argv + assert "--property=NoNewPrivileges=no" in argv + assert "--property=PrivateTmp=no" in argv + assert "--property=TimeoutStartSec=120" in argv + assert f"--property=RuntimeMaxSec={config.max_run_seconds + 2 * host_job.SUPERVISOR_GRACE_SECONDS}" in argv + assert "--setenv=DISPLAY=:99" in argv + assert "--setenv=HOME=/home/gate13" in argv + assert "--setenv=XDG_RUNTIME_DIR=/qualification/runtime" in argv + assert "/usr/bin/dbus-run-session" in argv + assert "execute-linux-desktop-session" in argv + assert "--wait" not in argv + assert host_job._entrypoint_argv(config)[-2:] == [ + "--config", + str(config.lifecycle_config_path), + ] + + +def test_linux_desktop_session_starts_secret_service_before_execute(tmp_path, monkeypatch): + config_path = tmp_path / "host-job.json" + observed = [] + monkeypatch.setattr(host_job.sys, "platform", "linux") + monkeypatch.setenv("DISPLAY", ":99") + monkeypatch.setenv("HOME", "/home/gate13") + monkeypatch.setenv("XDG_RUNTIME_DIR", "/qualification/runtime") + monkeypatch.setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path=/qualification/runtime/bus") + + def run(argv, **kwargs): + observed.append((argv, kwargs)) + return subprocess.CompletedProcess( + argv, 0, stdout="GNOME_KEYRING_CONTROL=/qualification/runtime/keyring\n", stderr="" + ) + + monkeypatch.setattr(host_job.subprocess, "run", run) + monkeypatch.setattr(host_job, "execute", lambda path: {"result": "passed", "path": str(path)}) + + assert host_job._execute_linux_desktop_session(config_path)["result"] == "passed" + assert observed[0][0] == ["/usr/bin/gnome-keyring-daemon", "--unlock", "--components=secrets"] + assert observed[0][1]["input"] == "\n" + assert host_job.os.environ["GNOME_KEYRING_CONTROL"] == "/qualification/runtime/keyring" + + +def test_windows_automated_python_replay_uses_the_bound_python(config_factory): + path, _raw = config_factory("windows") + config = host_job.load_config(path) + automated = replace(config, entrypoint_path=config.entrypoint_path.with_suffix(".py")) + + assert host_job._entrypoint_argv(automated) == [ + str(config.python_executable), + str(automated.entrypoint_path), + "--config", + str(config.lifecycle_config_path), + ] + + +def test_bounded_copy_caps_private_diagnostics(tmp_path): + destination = tmp_path / "stderr.log" + overflow = threading.Event() + errors = [] + + host_job._bounded_copy( + io.BytesIO(b"x" * 257), + destination, + 256, + overflow, + errors, + ) + + assert overflow.is_set() + assert errors == [] + assert destination.stat().st_size == 256 + + +def test_real_entrypoint_output_is_capped(config_factory): + path, raw = config_factory() + entrypoint = Path(raw["entrypoint_path"]) + entrypoint.write_text( + "import sys\nsys.stdout.buffer.write(b'x' * 1048577)\n", + encoding="utf-8", + ) + raw["entrypoint_sha256"] = sha256(entrypoint) + path.write_text(json.dumps(raw), encoding="utf-8") + config = host_job.load_config(path) + + assert host_job._run_entrypoint(config) == 126 + assert config.evidence_path.stat().st_size == host_job.MAX_EVIDENCE_BYTES + assert config.stderr_path.stat().st_size == 0 + + +def test_linux_tree_shutdown_escalates_to_process_group(config_factory, monkeypatch): + path, _raw = config_factory() + config = host_job.load_config(path) + signals = [] + + class Process: + pid = 4321 + + def __init__(self): + self.waits = 0 + + def wait(self, timeout): + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired("entrypoint", timeout) + return -9 + + monkeypatch.setattr( + host_job.os, + "killpg", + lambda pid, requested: signals.append((pid, requested)), + raising=False, + ) + host_job._stop_process_tree(config, Process()) + + assert signals == [ + (4321, host_job.POSIX_SIGTERM), + (4321, host_job.POSIX_SIGKILL), + ] + + +def test_execute_persists_status_validates_evidence_and_never_relaunches(config_factory, monkeypatch): + path, _raw = config_factory() + calls = [] + + monkeypatch.setattr(host_job.lifecycle, "load_lifecycle_json", lambda _payload: {}) + monkeypatch.setattr( + host_job.lifecycle, + "validate_lifecycle_document", + lambda _document: { + "run_id": "gate13-test-a-linux", + "platform": "linux", + "source_commit": "a" * 40, + }, + ) + + def runner(config): + calls.append(config.job_name) + config.evidence_path.write_text('{"canonical":true}', encoding="utf-8") + config.stderr_path.write_bytes(b"") + return 0 + + terminal = host_job.execute(path, clock=lambda: 2_000_000_000, entrypoint_runner=runner) + repeated = host_job.execute(path, clock=lambda: 2_000_000_001, entrypoint_runner=runner) + + assert terminal["result"] == "passed" + assert terminal["failure_code"] is None + assert terminal["evidence_digest"].startswith("sha256:") + assert repeated == terminal + assert calls == ["communityai-gate13-gate13-test-a-linux"] + status = json.loads((path.parent / "status.json").read_text(encoding="utf-8")) + assert status["state"] == "running" + assert status["attempt_ordinal"] == 1 + + +def test_started_attempt_without_terminal_is_never_relaunched(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + host_job._atomic_json( + config.status_path, + { + "schema_version": 1, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": 1, + "state": "running", + "started_at_unix": 2_000_000_000, + }, + exclusive=True, + ) + called = False + + def runner(_config): + nonlocal called + called = True + return 0 + + with pytest.raises(host_job.HostJobError, match="already started"): + host_job.execute(path, entrypoint_runner=runner) + assert called is False + + +def test_observation_distinguishes_pristine_active_terminal_and_ambiguous(config_factory, monkeypatch): + path, _raw = config_factory() + config = host_job.load_config(path) + + assert host_job.observe_job(config, {"native_state": "absent", "binding_ok": False}) == { + "job_state": "absent", + "attempt_ordinal": 0, + "evidence_digest": None, + } + assert host_job.observe_job(config, {"native_state": "running", "binding_ok": True}) == { + "job_state": "starting", + "attempt_ordinal": 1, + "evidence_digest": None, + } + assert host_job.observe_job(config, {"native_state": "running", "binding_ok": False}) == { + "job_state": "ambiguous", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + monkeypatch.setattr(host_job.lifecycle, "load_lifecycle_json", lambda _payload: {}) + monkeypatch.setattr( + host_job.lifecycle, + "validate_lifecycle_document", + lambda _document: { + "run_id": "gate13-test-a-linux", + "platform": "linux", + "source_commit": "a" * 40, + }, + ) + + def runner(bound): + bound.evidence_path.write_text("{}", encoding="utf-8") + bound.stderr_path.write_bytes(b"") + return 0 + + terminal = host_job.execute(path, clock=lambda: 2_000_000_000, entrypoint_runner=runner) + observed = host_job.observe_job(config, {"native_state": "absent", "binding_ok": False}) + assert observed["job_state"] == "passed" + assert observed["evidence_digest"] == terminal["evidence_digest"] + assert host_job.observe_job(config, {"native_state": "running", "binding_ok": False}) == { + "job_state": "ambiguous", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + +def test_inactive_after_persisted_start_is_ambiguous(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + host_job._atomic_json( + config.status_path, + { + "schema_version": 1, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": 1, + "state": "running", + "started_at_unix": 2_000_000_000, + }, + exclusive=True, + ) + + assert host_job.observe_job(config, {"native_state": "inactive", "binding_ok": True}) == { + "job_state": "ambiguous", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + +def test_collect_revalidates_terminal_digest_and_lifecycle_binding(config_factory, monkeypatch): + path, _raw = config_factory() + monkeypatch.setattr(host_job.lifecycle, "load_lifecycle_json", lambda _payload: {}) + monkeypatch.setattr( + host_job.lifecycle, + "validate_lifecycle_document", + lambda _document: { + "run_id": "gate13-test-a-linux", + "platform": "linux", + "source_commit": "a" * 40, + }, + ) + + def runner(config): + config.evidence_path.write_text('{"ok":true}', encoding="utf-8") + config.stderr_path.write_bytes(b"") + return 0 + + host_job.execute(path, clock=lambda: 2_000_000_000, entrypoint_runner=runner) + assert host_job.collect(path) == b'{"ok":true}' + + (path.parent / "evidence.json").write_text('{"ok":false}', encoding="utf-8") + with pytest.raises(host_job.HostJobError, match="digest changed"): + host_job.collect(path) + + +def test_start_reattaches_to_bound_native_job_without_mutation(config_factory, monkeypatch): + path, _raw = config_factory() + monkeypatch.setattr( + host_job, + "native_snapshot", + lambda _config, _runner: {"native_state": "running", "binding_ok": True}, + ) + + def forbidden(*_args, **_kwargs): + raise AssertionError("native start must not run") + + observed = host_job.start(path, runner=forbidden) + + assert observed == { + "job_state": "starting", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + +def test_linux_snapshot_binds_exact_service_command(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + stdout = "\n".join( + [ + "LoadState=loaded", + "ActiveState=active", + "SubState=running", + "User=gate13", + "Group=gate13", + ( + "ExecStart={ path=/usr/bin/dbus-run-session ; argv[]=/usr/bin/dbus-run-session " + f"{config.python_executable} {config.adapter_path} " + f"execute-linux-desktop-session --config {config.config_path} ; " + "ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; " + "pid=0 ; code=(null) ; status=0/0 }" + ), + f"WorkingDirectory={config.working_directory}", + "Restart=no", + "KillMode=control-group", + "UMask=0077", + "NoNewPrivileges=no", + "PrivateTmp=no", + 'Environment="DISPLAY=:99" "HOME=/home/gate13" "XDG_RUNTIME_DIR=/qualification/runtime"', + "TimeoutStartUSec=2min", + "RuntimeMaxUSec=1h 2min", + ] + ) + + def runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=stdout, stderr="") + + assert host_job._linux_snapshot(config, runner) == { + "native_state": "running", + "binding_ok": True, + } + + foreign_stdout = stdout.replace( + f"execute-linux-desktop-session --config {config.config_path} ;", + f"execute-linux-desktop-session --config {config.config_path} --extra ;", + ) + + def foreign_runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=foreign_stdout, stderr="") + + assert host_job._linux_snapshot(config, foreign_runner) == { + "native_state": "running", + "binding_ok": False, + } + + for foreign_stdout in ( + stdout.replace("ignore_errors=no", "ignore_errors=yes"), + stdout.replace("status=0/0 }", "status=0/0 ; arbitrary=value }"), + ): + + def foreign_metadata_runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=foreign_stdout, stderr="") + + assert host_job._linux_snapshot(config, foreign_metadata_runner) == { + "native_state": "running", + "binding_ok": False, + } + + +def test_linux_snapshot_accepts_fresh_systemd_inventory_without_exec_start(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + stdout = "\n".join( + [ + "Restart=no", + "TimeoutStartUSec=1min 30s", + "RuntimeMaxUSec=infinity", + "Environment=", + "UMask=0022", + "WorkingDirectory=", + "User=", + "Group=", + "PrivateTmp=no", + "NoNewPrivileges=no", + "KillMode=control-group", + "LoadState=not-found", + "ActiveState=inactive", + "SubState=dead", + ] + ) + + def runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=stdout, stderr="") + + assert host_job._linux_snapshot(config, runner) == { + "native_state": "absent", + "binding_ok": False, + } + + +def test_public_cli_failure_is_bounded_and_path_free(capsys, tmp_path): + missing = tmp_path / "secret-token-config.json" + + exit_code = host_job.main(["status", "--config", str(missing)]) + + assert exit_code == 2 + output = capsys.readouterr().out + assert json.loads(output) == { + "failure_code": "host_job_rejected", + "result": "failed", + "schema_version": 1, + } + assert str(missing) not in output diff --git a/tests/test_gate13_linux_packaged_lifecycle.py b/tests/test_gate13_linux_packaged_lifecycle.py index 45e60baf7..b4a5b4315 100644 --- a/tests/test_gate13_linux_packaged_lifecycle.py +++ b/tests/test_gate13_linux_packaged_lifecycle.py @@ -742,6 +742,11 @@ def run(command, **kwargs): assert owner.owned == [] +def test_termination_signal_enters_lifecycle_cleanup_path(): + with pytest.raises(linux_lifecycle.LifecycleRunError, match="termination"): + linux_lifecycle._termination_requested(15, None) + + def test_main_failure_is_generic_and_does_not_echo_config(monkeypatch, capsys): marker = "/private/path/must-not-escape" monkeypatch.setattr(linux_lifecycle, "_disable_core_dumps", lambda: None) diff --git a/tests/test_gate13_packaged_lifecycle.py b/tests/test_gate13_packaged_lifecycle.py index ffad861dc..86fefcc7f 100644 --- a/tests/test_gate13_packaged_lifecycle.py +++ b/tests/test_gate13_packaged_lifecycle.py @@ -480,3 +480,50 @@ def test_public_summary_does_not_retain_raw_phase_only_fields(): "recovery_action_count", ): assert forbidden_value not in rendered + + +def test_current_gate13_automated_replay_is_accepted_by_the_host_evidence_boundary(): + document = { + "schema_version": 2, + "scope": "gate13-automated-desktop-replay", + "run_id": "gate13-automated-a", + "platform": "windows", + "result": "passed", + "source_commit": SOURCE_COMMIT, + "package": { + "sha256": "sha256:" + PACKAGE_DIGEST, + "bytes": 123_456_789, + "verified_before_run": True, + "self_test_count": 4, + }, + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:" + MANIFEST_DIGEST, + "real_window_sessions": 2, + "localhost_inference_count": 1, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": False, + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": "gate13-manual-cpu-v1", + "sequence_profile": "gate13-manual-windows-v1", + "start_observation_seconds": 25.0, + "session_duration_seconds": {"initial": 100.0, "restart": 80.0}, + "privacy_safe": True, + "qualification_temporaries_removed": True, + } + + evidence = lifecycle.validate_lifecycle_document(document) + + assert evidence["result"] == "passed" + assert evidence["source_commit"] == SOURCE_COMMIT + assert evidence["package_sha256"] == PACKAGE_DIGEST + assert evidence["manifest_digest"] == MANIFEST_DIGEST + assert evidence["lifecycle"]["real_window_sessions"] == 2 + assert evidence["lifecycle"]["restart_resume_observed"] is False + assert evidence["lifecycle"]["policy_profile"] == "gate13-manual-cpu-v1" + + document["pause_clicked"] = False + with pytest.raises(lifecycle.LifecycleEvidenceError): + lifecycle.validate_lifecycle_document(document) diff --git a/tests/test_gate13_route_fence.py b/tests/test_gate13_route_fence.py new file mode 100644 index 000000000..6c8f06424 --- /dev/null +++ b/tests/test_gate13_route_fence.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_route_fence as fence + + +class Response: + status = 200 + + class Headers: + @staticmethod + def get_content_type(): + return "application/json" + + headers = Headers() + + def __init__(self, document): + self.payload = json.dumps(document).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _maximum): + return self.payload + + +def profile(tmp_path: Path) -> fence.Profile: + local = tmp_path / "local-api.key" + control = tmp_path / "control-api.key" + local.write_text("local-secret\n", encoding="ascii") + control.write_text("control-secret\n", encoding="ascii") + return fence.Profile( + target="windows", + service="communityai-qwen.service", + other_service="communityai-gemma.service", + origin="http://127.0.0.1:8081", + local_key=local, + control_key=control, + model_id="Qwen3.5 2B", + manifest_digest="sha256:" + "a" * 64, + total_blocks=24, + ) + + +def ready_opener(item: fence.Profile): + models = { + "data": [ + { + "id": item.model_id, + "availability": "complete", + "manifest_digest": item.manifest_digest, + } + ] + } + status = { + "auto_selection": { + "status": "selected", + "model": item.model_id, + "manifest_digest": item.manifest_digest, + "covered_blocks": item.total_blocks, + "total_blocks": item.total_blocks, + "peer_count": 1, + } + } + opener = MagicMock() + opener.open.side_effect = [Response(models), Response(status), Response(models), Response(status)] + return opener + + +def test_fence_restarts_only_target_and_rechecks_exact_route_after_settle(tmp_path): + item = profile(tmp_path) + calls = [] + timeouts = [] + + def runner(argv, **kwargs): + assert kwargs["stdin"] is subprocess.DEVNULL + assert kwargs["stdout"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.DEVNULL + calls.append(tuple(argv[1:])) + timeouts.append(kwargs["timeout"]) + inactive_probe = argv[1:3] == ["is-active", "--quiet"] and argv[3] == item.other_service + return subprocess.CompletedProcess(argv, 3 if inactive_probe else 0) + + sleeps = [] + result = fence.fence_route( + item, + timeout_seconds=60, + settle_seconds=30, + runner=runner, + opener=ready_opener(item), + sleeper=sleeps.append, + ) + + assert calls[:2] == [("stop", item.other_service), ("restart", item.service)] + assert timeouts[:2] == [fence.SERVICE_ACTION_TIMEOUT_SECONDS] * 2 + assert ("is-active", "--quiet", item.other_service) in calls + assert sleeps == [30] + assert result == { + "schema_version": 1, + "scope": "gate13-route-client-fence", + "result": "passed", + "target": "windows", + "model_id": item.model_id, + "manifest_digest": item.manifest_digest, + "covered_blocks": 24, + "total_blocks": 24, + "peer_count_minimum": 1, + "target_service_restarted": True, + "standby_service_stopped": True, + "stable_rechecks": 2, + "settle_seconds": 30, + "privacy_safe": True, + } + + +def test_fence_fails_if_standby_is_still_active(tmp_path): + item = profile(tmp_path) + + def runner(argv, **_kwargs): + return subprocess.CompletedProcess(argv, 0) + + with pytest.raises(fence.FenceError, match="remain stable"): + fence.fence_route( + item, + timeout_seconds=60, + settle_seconds=30, + runner=runner, + opener=ready_opener(item), + sleeper=lambda _seconds: None, + ) + + +def test_fence_retries_when_stale_advertisement_expires_during_settle(tmp_path): + item = profile(tmp_path) + opener = ready_opener(item) + ready_responses = list(opener.open.side_effect) + incomplete_models = { + "data": [ + { + "id": item.model_id, + "availability": "incomplete", + "manifest_digest": item.manifest_digest, + } + ] + } + opener.open.side_effect = [ + *ready_responses[:2], + Response(incomplete_models), + ready_responses[3], + *ready_responses, + ] + + def runner(argv, **_kwargs): + inactive_probe = argv[1:3] == ["is-active", "--quiet"] and argv[3] == item.other_service + return subprocess.CompletedProcess(argv, 3 if inactive_probe else 0) + + sleeps = [] + result = fence.fence_route( + item, + timeout_seconds=60, + settle_seconds=30, + runner=runner, + opener=opener, + sleeper=sleeps.append, + ) + + assert result["result"] == "passed" + assert sleeps == [30, 5.0, 30] + + +def test_snapshot_rejects_wrong_model_manifest_even_when_control_coverage_is_complete(tmp_path): + item = profile(tmp_path) + opener = ready_opener(item) + wrong_manifest_models = { + "data": [ + { + "id": item.model_id, + "availability": "complete", + "manifest_digest": "sha256:" + "b" * 64, + } + ] + } + responses = list(opener.open.side_effect) + responses[0] = Response(wrong_manifest_models) + opener.open.side_effect = responses + + assert fence._snapshot(item, opener) is False + + +@pytest.mark.parametrize("target", ["windows", "linux"]) +def test_cli_defaults_to_1200_second_readiness_timeout(monkeypatch, target): + run_fence = MagicMock(return_value={"result": "passed"}) + monkeypatch.setattr(fence, "fence_route", run_fence) + monkeypatch.setattr(fence.os, "geteuid", lambda: 0, raising=False) + + assert fence.main(["--target", target]) == 0 + run_fence.assert_called_once_with(fence.PROFILES[target], timeout_seconds=1_200.0, settle_seconds=30.0) + + +def test_secret_rejects_links(tmp_path): + target = tmp_path / "target" + target.write_text("secret", encoding="ascii") + link = tmp_path / "link" + try: + link.symlink_to(target) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + with pytest.raises(fence.FenceError, match="unsafe"): + fence._secret(link) diff --git a/tests/test_gate13_route_setup.py b/tests/test_gate13_route_setup.py new file mode 100644 index 000000000..a6c4036d4 --- /dev/null +++ b/tests/test_gate13_route_setup.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_route_setup_pins_configured_runtime_and_preserves_the_route_helpers(): + source = (ROOT / "scripts" / "gate13_route_setup.sh").read_text(encoding="utf-8") + config = json.loads((ROOT / "config" / "gate13_gcp.json").read_text(encoding="utf-8")) + + assert f'test "$(stat -c %s "$wheel")" = "{config["route_wheel_bytes"]}"' in source + assert config["route_wheel_sha256"] in source + assert "fc385f74e02ca955203b1fc5e8ae493c7f4ccd31bd7383c2ae0a1c461c91363e" in source + assert "bdcc9f499a7cd6b727c0e33a0c4c2b0e71e76e28f3f21cb99804a8f39edfa0d2" in source + assert "metadata.google.internal" in source + assert "communityai-qwen.service" in source + assert "communityai-gemma.service" in source + assert 'rm -rf "$root"' in source diff --git a/tests/test_gate13_run_controller.py b/tests/test_gate13_run_controller.py new file mode 100644 index 000000000..d80d52119 --- /dev/null +++ b/tests/test_gate13_run_controller.py @@ -0,0 +1,625 @@ +import hashlib +import json +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_run_controller as controller # noqa: E402 + +AUTHORIZATION = ROOT / "docs" / "evidence" / "gate13-20260831-a-cost-authorization.json" +LEDGER = ROOT / "docs" / "RELEASE_READINESS.md" +NOW = 2_000_000_000 +ROUTE_DIGEST = "sha256:" + "a" * 64 +WINDOWS_DIGEST = "sha256:" + "b" * 64 +LINUX_DIGEST = "sha256:" + "c" * 64 + + +def reserved_ledger_text(*, old_digest, new_digest): + lines = LEDGER.read_text(encoding="utf-8").splitlines(keepends=True) + matches = [index for index, line in enumerate(lines) if line.startswith("| gate13-20260831-a |")] + assert len(matches) == 1 + index = matches[0] + assert old_digest in lines[index] + assert lines[index].rstrip().endswith("| CLEANED-COMMITTED |") + lines[index] = lines[index].replace(old_digest, new_digest, 1).replace("| CLEANED-COMMITTED |", "| RESERVED |", 1) + return "".join(lines) + + +@pytest.fixture +def plan(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + raw["provider_plan"]["sequencing"]["clients_may_run_concurrently"] = False + old_digest = raw["provider_plan_digest"] + new_digest = controller._provider_digest(raw["provider_plan"]) + raw["provider_plan_digest"] = new_digest + authorization = tmp_path / "authorization.json" + authorization.write_text(json.dumps(raw), encoding="utf-8") + + ledger = tmp_path / "ledger.md" + ledger.write_text( + reserved_ledger_text(old_digest=old_digest, new_digest=new_digest), + encoding="utf-8", + ) + return controller.load_plan(authorization, ledger) + + +def observation( + plan, *, route=False, windows=False, linux=False, route_job="absent", windows_job="absent", linux_job="absent" +): + present = { + plan.route_instance: (route, plan.route_source_commit), + plan.windows_instance: (windows, plan.windows_source_commit), + plan.linux_instance: (linux, plan.linux_source_commit), + } + return { + "schema_version": 1, + "run_id": plan.run_id, + "observed_at_unix": NOW, + "instances": { + name: { + "present": exists, + "run_id": plan.run_id if exists else None, + "source_commit": source if exists else None, + "termination_unix": NOW + 20_000 if exists else None, + } + for name, (exists, source) in present.items() + }, + "disks": { + plan.route_disk: route, + plan.windows_disk: windows, + plan.linux_disk: linux, + }, + "firewalls": { + plan.route_firewalls[0]: route, + plan.route_firewalls[1]: route, + }, + "protected_bootstrap_running": True, + "route_acceptance": { + "job_state": route_job, + "evidence_digest": ROUTE_DIGEST if route_job == "passed" else None, + }, + "clients": { + "windows": { + "job_state": windows_job, + "attempt_ordinal": 1 if windows_job != "absent" else 0, + "evidence_digest": WINDOWS_DIGEST if windows_job == "passed" else None, + }, + "linux": { + "job_state": linux_job, + "attempt_ordinal": 1 if linux_job != "absent" else 0, + "evidence_digest": LINUX_DIGEST if linux_job == "passed" else None, + }, + }, + } + + +def test_load_plan_binds_exact_cost_and_resources(plan): + assert plan.run_id == "gate13-20260831-a" + assert plan.provider_plan_digest.startswith("sha256:") + assert plan.ledger_state == "RESERVED" + assert plan.instance_names == ( + "route-20260831-a-node", + "gate13-20260831-a-win", + "gate13-20260831-a-linux", + ) + assert controller.PROTECTED_INSTANCE not in plan.instance_names + assert plan.clients_may_run_concurrently is False + + +def test_load_plan_accepts_the_automated_replay_instead_of_legacy_16_phases(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + sequencing = raw["provider_plan"]["sequencing"] + sequencing["clients_may_run_concurrently"] = False + sequencing["all_16_phases_required_per_platform"] = False + sequencing["automated_gate13_replay_required"] = True + old_digest = raw["provider_plan_digest"] + new_digest = controller._provider_digest(raw["provider_plan"]) + raw["provider_plan_digest"] = new_digest + authorization = tmp_path / "authorization.json" + authorization.write_text(json.dumps(raw), encoding="utf-8") + ledger = tmp_path / "ledger.md" + ledger.write_text( + reserved_ledger_text(old_digest=old_digest, new_digest=new_digest), + encoding="utf-8", + ) + + replay_plan = controller.load_plan(authorization, ledger) + + assert replay_plan.clients_may_run_concurrently is False + + +def test_load_plan_accepts_only_documented_owner_ceiling(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + raw["provider_plan"]["sequencing"]["clients_may_run_concurrently"] = False + old_digest = raw["provider_plan_digest"] + new_digest = controller._provider_digest(raw["provider_plan"]) + raw["provider_plan_digest"] = new_digest + raw["authorization"].update( + { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "52.00", + "maximum_estimate_usd": "56.00", + "remaining_after_run_maximum_usd": "392.00", + } + ) + authorization = tmp_path / "authorization.json" + authorization.write_text(json.dumps(raw), encoding="utf-8") + ledger = tmp_path / "ledger.md" + ledger.write_text( + reserved_ledger_text(old_digest=old_digest, new_digest=new_digest), + encoding="utf-8", + ) + + raised_plan = controller.load_plan(authorization, ledger) + assert raised_plan.ledger_state == "RESERVED" + assert controller.initial_state(raised_plan)["next_action"] == "start_route" + + raw["authorization"]["combined_cloud_ceiling_usd"] = "499.00" + raw["authorization"]["remaining_after_run_maximum_usd"] = "391.00" + authorization.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(controller.RunControllerError, match="inconsistent"): + controller.load_plan(authorization, ledger) + + +def test_cleaned_committed_ledger_cannot_start_a_new_run(): + historical_plan = controller.load_plan(AUTHORIZATION, LEDGER) + + assert historical_plan.ledger_state == "CLEANED-COMMITTED" + with pytest.raises(controller.RunControllerError, match="not reserved"): + controller.initial_state(historical_plan) + + +def test_non_reserved_ledger_allows_cleanup_only(): + historical_plan = controller.load_plan(AUTHORIZATION, LEDGER) + reserved_plan = replace( + historical_plan, + ledger_state="RESERVED", + clients_may_run_concurrently=False, + ) + reserved_state = controller.initial_state(reserved_plan) + + forward_actions = controller.ACTION_STATES - controller.CLEANUP_ACTIONS - {"none"} + for action in forward_actions: + with pytest.raises(controller.RunControllerError, match="not reserved"): + controller.begin_action(reserved_state, historical_plan, action=action) + + cleanup_state = dict(reserved_state) + cleanup_state.update( + { + "phase": "CLEANING_FAILED", + "failure_code": "operator_cleanup", + "next_action": "cleanup_failure", + } + ) + cleaning = controller.begin_action( + cleanup_state, + historical_plan, + action="cleanup_failure", + ) + assert cleaning["phase"] == "CLEANING_FAILED" + assert cleaning["next_action"] == "none" + + +def test_reserved_parallel_client_plan_cannot_start(tmp_path): + ledger = tmp_path / "ledger.md" + digest = json.loads(AUTHORIZATION.read_text(encoding="utf-8"))["provider_plan_digest"] + ledger.write_text( + reserved_ledger_text(old_digest=digest, new_digest=digest), + encoding="utf-8", + ) + parallel = controller.load_plan(AUTHORIZATION, ledger) + + with pytest.raises(controller.RunControllerError, match="concurrent clients"): + controller.initial_state(parallel) + + +def test_changed_authorization_fails_closed(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + raw["provider_plan"]["route"]["machine_type"] = "e2-micro" + changed = tmp_path / "authorization.json" + changed.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(controller.RunControllerError, match="digest changed"): + controller.load_plan(changed, LEDGER) + + +def test_inventory_precedes_route_and_route_acceptance_precedes_clients(plan): + state = controller.initial_state(plan) + + absent = controller.reconcile(state, observation(plan), plan, now_unix=NOW) + assert absent["phase"] == "ABSENT" + assert absent["next_action"] == "start_route" + + accepting = controller.reconcile( + absent, + observation(plan, route=True, route_job="running"), + plan, + now_unix=NOW, + ) + assert accepting["phase"] == "ROUTE_ACCEPTING" + assert accepting["next_action"] == "accept_route" + + invalid = controller.reconcile( + accepting, + observation(plan, route=True, windows=True, route_job="running", windows_job="starting"), + plan, + now_unix=NOW, + ) + assert invalid["phase"] == "CLEANING_FAILED" + assert invalid["failure_code"] == "client_started_before_route_acceptance" + + +def test_exact_name_with_foreign_identity_fails_closed(plan): + raw = observation(plan, route=True, route_job="running") + raw["instances"][plan.route_instance]["run_id"] = "foreign-run" + + with pytest.raises(controller.RunControllerError, match="foreign exact-name"): + controller.reconcile(controller.initial_state(plan), raw, plan, now_unix=NOW) + + +def test_route_acceptance_starts_windows_before_linux(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + assert state["phase"] == "ROUTE_ACCEPTED" + assert state["next_action"] == "start_windows" + + invalid = controller.reconcile( + state, + observation(plan, route=True, linux=True, route_job="passed", linux_job="running"), + plan, + now_unix=NOW, + ) + assert invalid["phase"] == "CLEANING_FAILED" + assert invalid["failure_code"] == "linux_started_before_windows_evidence" + + +@pytest.mark.parametrize("job_state", ["failed", "ambiguous"]) +def test_failed_or_ambiguous_windows_is_consumed_and_never_resumed(plan, job_state): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job=job_state), + plan, + now_unix=NOW, + ) + + assert state["phase"] == "CLEANING_FAILED" + assert state["windows_consumed"] is True + assert state["next_action"] == "cleanup_failure" + + +def test_action_intent_is_persisted_before_mutation_and_cannot_relaunch(plan): + state = controller.initial_state(plan) + started = controller.begin_action(state, plan, action="start_route") + + assert started["phase"] == "ROUTE_STARTING" + assert started["next_action"] == "none" + missing = controller.reconcile(started, observation(plan), plan, now_unix=NOW) + assert missing["phase"] == "CLEANED_FAILURE" + assert missing["failure_code"] == "resources_disappeared_before_completion" + with pytest.raises(controller.RunControllerError, match="out of order"): + controller.begin_action(started, plan, action="start_route") + + +def test_route_acceptance_intent_cannot_rearm_after_dispatch(plan): + ready = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="absent"), + plan, + now_unix=NOW, + ) + dispatched = controller.begin_action(ready, plan, action="accept_route") + + running = controller.reconcile( + dispatched, + observation(plan, route=True, route_job="running"), + plan, + now_unix=NOW, + ) + assert running["phase"] == "ROUTE_ACCEPTING" + assert running["next_action"] == "none" + + missing = controller.reconcile( + dispatched, + observation(plan, route=True, route_job="absent"), + plan, + now_unix=NOW, + ) + + assert missing["phase"] == "CLEANING_FAILED" + assert missing["failure_code"] == "route_acceptance_disappeared" + assert missing["next_action"] == "cleanup_failure" + + +def test_client_start_intent_cannot_rearm_after_dispatch(plan): + route_ready = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + windows_dispatched = controller.begin_action(route_ready, plan, action="start_windows") + + provisioning = controller.reconcile( + windows_dispatched, + observation(plan, route=True, windows=True, route_job="passed", windows_job="absent"), + plan, + now_unix=NOW, + ) + assert provisioning["phase"] == "WINDOWS_RUNNING" + assert provisioning["next_action"] == "none" + + missing = controller.reconcile( + windows_dispatched, + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + assert missing["phase"] == "CLEANING_FAILED" + assert missing["failure_code"] == "windows_disappeared_after_start_intent" + + linux_ready = dict(route_ready) + linux_ready.update( + { + "phase": "WINDOWS_COLLECTED", + "windows_evidence_digest": WINDOWS_DIGEST, + "windows_consumed": True, + "next_action": "start_linux", + } + ) + linux_dispatched = controller.begin_action(linux_ready, plan, action="start_linux") + linux_provisioning = controller.reconcile( + linux_dispatched, + observation(plan, route=True, linux=True, route_job="passed", linux_job="absent"), + plan, + now_unix=NOW, + ) + assert linux_provisioning["phase"] == "LINUX_RUNNING" + assert linux_provisioning["next_action"] == "none" + + linux_missing = controller.reconcile( + linux_dispatched, + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + assert linux_missing["phase"] == "CLEANING_FAILED" + assert linux_missing["failure_code"] == "linux_disappeared_after_start_intent" + + +def test_observed_attempt_cannot_disappear_and_relaunch(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + disappeared = observation(plan, route=True, route_job="passed") + disappeared["clients"]["windows"]["attempt_ordinal"] = 1 + + failed = controller.reconcile(state, disappeared, plan, now_unix=NOW) + assert failed["phase"] == "CLEANING_FAILED" + assert failed["failure_code"] == "windows_attempt_disappeared" + + +def test_active_host_job_is_observed_not_relaunched(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="running"), + plan, + now_unix=NOW, + ) + + assert state["phase"] == "WINDOWS_RUNNING" + assert state["next_action"] == "none" + + +def test_collect_binds_canonical_evidence_then_deletes_windows(monkeypatch, plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="passed"), + plan, + now_unix=NOW, + ) + assert state["phase"] == "WINDOWS_COLLECTING" + + payload = b'{"bounded":true}' + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + monkeypatch.setattr(controller.lifecycle, "load_lifecycle_json", lambda _payload: {"validated": True}) + monkeypatch.setattr( + controller.lifecycle, + "validate_lifecycle_document", + lambda _raw: { + "source_commit": plan.windows_source_commit, + "package_sha256": plan.windows_package_sha256, + "package_bytes": plan.windows_package_bytes, + "model_id": "Qwen3.5 2B", + "manifest_digest": plan.qwen_manifest.removeprefix("sha256:"), + }, + ) + + collected = controller.collect_platform( + state, + plan, + platform="windows", + evidence_payload=payload, + observed_digest=digest, + ) + assert collected["phase"] == "WINDOWS_DELETING" + assert collected["next_action"] == "delete_windows" + assert collected["windows_consumed"] is True + + deleting = observation(plan, route=True, route_job="passed") + deleting["clients"]["windows"]["attempt_ordinal"] = 1 + deleting["disks"][plan.windows_disk] = True + still_present = controller.reconcile(collected, deleting, plan, now_unix=NOW) + assert still_present["phase"] == "WINDOWS_DELETING" + assert still_present["next_action"] == "delete_windows" + with pytest.raises(controller.RunControllerError, match="absence is not proved"): + controller.mark_client_absent( + collected, + plan, + platform="windows", + observation=deleting, + now_unix=NOW, + ) + + absent = observation(plan, route=True, route_job="passed") + absent["clients"]["windows"]["attempt_ordinal"] = 1 + after_delete = controller.mark_client_absent( + collected, + plan, + platform="windows", + observation=absent, + now_unix=NOW, + ) + assert after_delete["phase"] == "WINDOWS_COLLECTED" + assert after_delete["next_action"] == "start_linux" + + +def test_collect_accepts_current_automated_desktop_replay(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="passed"), + plan, + now_unix=NOW, + ) + evidence = { + "schema_version": 2, + "scope": "gate13-automated-desktop-replay", + "run_id": f"{plan.run_id}-windows", + "platform": "windows", + "result": "passed", + "source_commit": plan.windows_source_commit, + "package": { + "sha256": "sha256:" + plan.windows_package_sha256.removeprefix("sha256:"), + "bytes": plan.windows_package_bytes, + "verified_before_run": True, + "self_test_count": 4, + }, + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:" + plan.qwen_manifest.removeprefix("sha256:"), + "real_window_sessions": 2, + "localhost_inference_count": 1, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": False, + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": "gate13-manual-cpu-v1", + "sequence_profile": "gate13-manual-windows-v1", + "start_observation_seconds": 25.0, + "session_duration_seconds": {"initial": 120.0, "restart": 90.0}, + "privacy_safe": True, + "qualification_temporaries_removed": True, + } + payload = (json.dumps(evidence, sort_keys=True, separators=(",", ":")) + "\n").encode() + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + + collected = controller.collect_platform( + state, + plan, + platform="windows", + evidence_payload=payload, + observed_digest=digest, + ) + + assert collected["phase"] == "WINDOWS_DELETING" + assert collected["windows_consumed"] is True + + +def test_partial_or_wrong_digest_evidence_cannot_advance(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="passed"), + plan, + now_unix=NOW, + ) + + with pytest.raises(controller.RunControllerError, match="digest changed"): + controller.collect_platform( + state, + plan, + platform="windows", + evidence_payload=b"{}", + observed_digest=WINDOWS_DIGEST, + ) + + +def test_success_requires_both_records_and_exact_absence(plan): + state = controller.initial_state(plan) + state.update( + { + "phase": "LINUX_COLLECTED", + "route_acceptance_digest": ROUTE_DIGEST, + "windows_evidence_digest": WINDOWS_DIGEST, + "linux_evidence_digest": LINUX_DIGEST, + "windows_consumed": True, + "linux_consumed": True, + "next_action": "delete_route", + } + ) + + complete = controller.reconcile(state, observation(plan), plan, now_unix=NOW) + assert complete["phase"] == "CLEANED_PASS" + assert complete["cleanup_verified"] is True + assert complete["next_action"] == "none" + + +def test_failure_cleanup_is_idempotent_and_never_becomes_pass(plan): + state = controller.initial_state(plan) + state.update( + { + "phase": "CLEANING_FAILED", + "failure_code": "windows_failed_or_ambiguous", + "windows_consumed": True, + "next_action": "cleanup_failure", + } + ) + + cleaned = controller.reconcile(state, observation(plan), plan, now_unix=NOW) + assert cleaned["phase"] == "CLEANED_FAILURE" + assert controller.reconcile(cleaned, observation(plan), plan, now_unix=NOW) == cleaned + + +def test_stale_observation_and_expired_deadline_fail_closed(plan): + stale = observation(plan) + stale["observed_at_unix"] = NOW - 301 + with pytest.raises(controller.RunControllerError, match="stale"): + controller.reconcile(controller.initial_state(plan), stale, plan, now_unix=NOW) + + expired = observation(plan, route=True, route_job="running") + expired["instances"][plan.route_instance]["termination_unix"] = NOW + with pytest.raises(controller.RunControllerError, match="deadline expired"): + controller.reconcile(controller.initial_state(plan), expired, plan, now_unix=NOW) + + +def test_atomic_state_round_trip_and_public_status_are_bounded(tmp_path, plan): + state_path = tmp_path / "state.json" + state = controller.initial_state(plan) + controller.persist(state_path, state, plan) + + assert controller.load_state(state_path, plan) == state + public = controller.public_status(state, plan) + assert set(public) == { + "schema_version", + "run_id", + "phase", + "next_action", + "failure_code", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + } + rendered = json.dumps(public) + for forbidden in ("token", "password", "prompt", "endpoint", str(tmp_path)): + assert forbidden not in rendered.lower() diff --git a/tests/test_gate13_windows_packaged_lifecycle.py b/tests/test_gate13_windows_packaged_lifecycle.py index b850aade3..997bae485 100644 --- a/tests/test_gate13_windows_packaged_lifecycle.py +++ b/tests/test_gate13_windows_packaged_lifecycle.py @@ -81,6 +81,22 @@ def test_json_input_bound_accepts_production_scale_and_rejects_above_limit(tmp_p } +@pytest.mark.skipif(not POWERSHELL.is_file(), reason="native Windows PowerShell is required") +def test_sha256_does_not_depend_on_powershell_module_autoload(tmp_path): + target = tmp_path / "payload.bin" + target.write_bytes(b"clean-host-hash") + source = f""" +. {_ps_literal(LIFECYCLE)} +Remove-Module Microsoft.PowerShell.Utility -Force -ErrorAction Stop +$PSModuleAutoLoadingPreference = 'None' +[Console]::Out.WriteLine((Get-Gate13Sha256 -Path {_ps_literal(target)})) +""" + + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == hashlib.sha256(target.read_bytes()).hexdigest() + + @pytest.mark.skipif(not POWERSHELL.is_file(), reason="native Windows PowerShell is required") def test_windows_build_platform_accepts_production_runner_and_rejects_spoofs(tmp_path): source = f""" @@ -588,6 +604,12 @@ def test_adapter_contains_exact_safety_and_lifecycle_contracts(): positions = [lifecycle.index(f'-Name "{phase}"') for phase in phases] assert positions == sorted(positions) assert lifecycle.count('-Name "') >= len(phases) + assert "$script:LifecycleFailurePhase = $Name" in lifecycle + assert "$script:LifecycleFailureOperation = $Name" in lifecycle + assert "failure_phase = $failurePhase" in lifecycle + assert "failure_operation = $failureOperation" in lifecycle + assert '"product_readiness"' in lifecycle + assert "ConvertTo-Json -Compress" in lifecycle for required in ( "CreateSuspended", diff --git a/tests/test_gate14_cache_materializer.py b/tests/test_gate14_cache_materializer.py new file mode 100644 index 000000000..60606a56f --- /dev/null +++ b/tests/test_gate14_cache_materializer.py @@ -0,0 +1,1317 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import os +import shutil +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_cache_materializer as materializer # noqa: E402 +import gate14_packaged_lifecycle as lifecycle # noqa: E402 + +from drift.model_manifest import ModelManifest # noqa: E402 + +MANIFESTS = { + "windows": ROOT / "manifests" / "candidates" / "qwen3.5-2b-bfloat16-eager.json", + "linux": ROOT / "manifests" / "candidates" / "gemma-4-e2b-it-bfloat16-eager.json", +} +SOURCE = "a" * 40 + + +class NoopLease: + def __init__(self): + self.stability_checks = 0 + self.closed = False + + def assert_stable(self): + self.stability_checks += 1 + + def close(self): + self.closed = True + + +def _record(platform_name: str) -> dict: + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[platform_name] + profile = lifecycle.acceptance.MODEL_PROFILES[model_id] + expected = lifecycle._GATE9_WARM_CACHE[platform_name] + artifacts = [ + { + "path": path, + "role": role, + "sha256": digest.removeprefix("sha256:"), + "size_bytes": size, + "materialization_attempts": 1, + "resumptions": 0, + "resumed_from_bytes": [], + "elapsed_seconds": 0.1, + } + for path, role, digest, size in expected["artifacts"] + ] + startup_roles = { + "chat_template", + "config", + "tokenizer", + "weight_index", + } + return { + "schema_version": 1, + "acquired_at_unix": 1_800_000_000, + "runtime": { + "python": "3.12", + "platform": ("Windows-Server-2022-test" if platform_name == "windows" else "Linux-Ubuntu-24.04-test"), + "drift": "test", + }, + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "repository": lifecycle._MODEL_SOURCE[model_id][0], + "revision": profile["revision_commit"], + "dtype": lifecycle._MODEL_SOURCE[model_id][1], + }, + "selection": { + "startup_artifact_paths": sorted(item["path"] for item in artifacts if item["role"] in startup_roles), + "weight_artifact_paths": sorted(item["path"] for item in artifacts if item["role"] == "weight"), + "artifact_count": len(artifacts), + "artifact_bytes": sum(item["size_bytes"] for item in artifacts), + "weight_artifact_bytes": sum(item["size_bytes"] for item in artifacts if item["role"] == "weight"), + }, + "artifacts": artifacts, + "transfer": { + "direct_upstream_transfer": True, + "mirror_used": False, + "source_class_verified": True, + "transport_override_present": False, + "elapsed_seconds": 1.0, + "max_resumptions": 3, + "resumptions": 0, + "completed": True, + }, + "storage": { + "cold_start": True, + "cache_bytes_before": 0, + "cache_bytes_after": profile["selected_artifact_bytes"], + "cache_growth_bytes": profile["selected_artifact_bytes"], + "verified": True, + }, + "privacy": { + "credentials_retained": False, + "local_paths_retained": False, + "response_bodies_retained": False, + "urls_retained": False, + }, + } + + +def _template(platform_name: str, work: Path, staging: Path) -> dict: + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[platform_name] + value = {field: None for field in lifecycle._CONFIG_FIELDS} + value.update( + { + "schema_version": 1, + "scope": lifecycle.SCOPE, + "run_id": "gate14-cache-test-a", + "platform": platform_name, + "attempt_ordinal": 1, + "source_commit": SOURCE, + "warm_cache": None, + "model_id": model_id, + "manifest_digest": lifecycle.acceptance.MODEL_PROFILES[model_id]["manifest_digest"], + "staging_root": str(staging.resolve()), + "work_root": str(work.resolve()), + } + ) + return value + + +def _write_plan( + tmp_path: Path, + monkeypatch, + platform_name: str, + *, + manifest_path: Path | None = None, + template: dict | None = None, +): + monkeypatch.setattr(materializer, "getproxies", lambda: {}) + for name in materializer._OVERRIDE_NAMES: + monkeypatch.delenv(name, raising=False) + base = tmp_path / platform_name + work = base / "work" + staging = base / "staging" + work.mkdir(parents=True) + staging.mkdir() + template_value = _template(platform_name, work, staging) if template is None else template + template_payload = materializer._canonical(template_value) + template_path = staging / materializer.TEMPLATE_NAME + template_path.write_bytes(template_payload) + source_manifest = MANIFESTS[platform_name] if manifest_path is None else manifest_path + staged_manifest = staging / materializer._MANIFEST_NAMES[platform_name] + staged_manifest.write_bytes(source_manifest.read_bytes()) + acquirer_root = staging.joinpath(*materializer._ACQUIRER_RUNTIME_PARTS) + acquirer_root.mkdir(parents=True) + acquirer_path = acquirer_root / materializer._ACQUIRER_NAMES[platform_name] + acquirer_path.write_bytes(b"packaged-node") + plan = { + "schema_version": 1, + "scope": materializer.PLAN_SCOPE, + "platform": platform_name, + "source_commit": SOURCE, + "manifest_path": str(staged_manifest.resolve()), + "manifest_sha256": lifecycle._digest(staged_manifest.read_bytes()), + "acquirer_path": str(acquirer_path.resolve()), + "acquirer_sha256": lifecycle._digest(acquirer_path.read_bytes()), + "acquirer_bytes": acquirer_path.stat().st_size, + "work_root": str(work.resolve()), + "staging_root": str(staging.resolve()), + "lifecycle_template_sha256": lifecycle._digest(template_payload), + "sources": dict(materializer.current_source_bindings()), + } + plan_path = staging / materializer.PLAN_NAME + plan_path.write_bytes(materializer._canonical(plan)) + return plan_path, plan, work, staging + + +def _allow_owned(_path, *, directory): + assert isinstance(directory, bool) + + +def _allow_protected(_path): + return None + + +def _fake_cache_verifier(_cache, _binding): + return NoopLease() + + +def _binding_kwargs(platform_name: str) -> dict: + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[platform_name] + return { + "platform": platform_name, + "source_commit": SOURCE, + "materialization_plan_sha256": "sha256:" + "1" * 64, + "materializer_sources_sha256": "sha256:" + "2" * 64, + "model_id": model_id, + "manifest_digest": lifecycle.acceptance.MODEL_PROFILES[model_id]["manifest_digest"], + } + + +@pytest.mark.parametrize("platform_name", ["windows", "linux"]) +def test_materializer_emits_private_handoff_for_both_exact_profiles( + tmp_path, + monkeypatch, + platform_name, +): + plan_path, _plan, work, staging = _write_plan( + tmp_path, + monkeypatch, + platform_name, + ) + calls = [] + + def acquire(manifest, **kwargs): + calls.append((manifest.name, kwargs)) + return _record(platform_name) + + result = materializer.materialize( + plan_path=plan_path, + acquirer=acquire, + ownership_verifier=_allow_owned, + cache_verifier=_fake_cache_verifier, + native_platform=platform_name, + ) + + assert calls[0][1] == { + "cache_dir": work / materializer.CACHE_NAME, + "token": False, + "max_resumptions": 3, + "require_direct_upstream": True, + "manifest_path": staging / materializer._MANIFEST_NAMES[platform_name], + "manifest_sha256": lifecycle._digest((staging / materializer._MANIFEST_NAMES[platform_name]).read_bytes()), + "acquirer_path": staging.joinpath( + *materializer._ACQUIRER_RUNTIME_PARTS, + materializer._ACQUIRER_NAMES[platform_name], + ), + "acquirer_sha256": lifecycle._digest(b"packaged-node"), + "acquirer_bytes": len(b"packaged-node"), + } + assert result["phase"] == "materialized" + assert result["platform"] == platform_name + assert result["source_commit"] == SOURCE + assert result["warm_cache_binding_sha256"].startswith("sha256:") + assert (work / materializer.RECORD_NAME).is_file() + assert (work / materializer.BINDING_NAME).is_file() + assert (work / materializer.HANDOFF_NAME).is_file() + assert not (staging / materializer.RECORD_NAME).exists() + assert not (staging / materializer.CONFIG_NAME).exists() + rendered = json.dumps(result) + assert str(work) not in rendered + assert str(staging) not in rendered + assert "huggingface.co" not in rendered + + +@pytest.mark.parametrize( + ("field", "value"), + [ + (("transfer", "direct_upstream_transfer"), False), + (("transfer", "mirror_used"), True), + (("transfer", "source_class_verified"), False), + (("transfer", "transport_override_present"), True), + (("transfer", "completed"), False), + (("transfer", "max_resumptions"), 2), + (("storage", "cold_start"), False), + (("storage", "cache_bytes_before"), 1), + (("privacy", "credentials_retained"), True), + (("model", "repository"), "mirror.invalid/model"), + ], +) +def test_binding_rejects_materialization_lies(field, value): + record = _record("windows") + record[field[0]][field[1]] = value + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="materialization", + ): + lifecycle.build_warm_cache_binding( + materializer._canonical(record), + **_binding_kwargs("windows"), + ) + + +def test_runtime_platform_is_bound_to_requested_platform(): + record = _record("linux") + record["runtime"]["platform"] = "Windows-Server-2022" + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="materialization", + ): + lifecycle.build_warm_cache_binding( + materializer._canonical(record), + **_binding_kwargs("linux"), + ) + + +@pytest.mark.parametrize("override", materializer._OVERRIDE_NAMES) +def test_transport_override_fails_before_cache_creation( + tmp_path, + monkeypatch, + override, +): + plan_path, _plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + monkeypatch.setenv(override, "set") + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="overridden", + ): + materializer.materialize( + plan_path=plan_path, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert not (work / materializer.CACHE_NAME).exists() + + +def test_system_proxy_fails_before_cache_creation(tmp_path, monkeypatch): + plan_path, _plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + monkeypatch.setattr( + materializer, + "getproxies", + lambda: {"https": "http://system-proxy.invalid"}, + ) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="overridden", + ): + materializer.materialize( + plan_path=plan_path, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert not (work / materializer.CACHE_NAME).exists() + + +def test_changed_source_binding_fails_before_cache_creation( + tmp_path, + monkeypatch, +): + plan_path, plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + plan["sources"]["gate14_cache_materializer.py"] = "sha256:" + "0" * 64 + plan_path.write_bytes(materializer._canonical(plan)) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="source binding", + ): + materializer.materialize( + plan_path=plan_path, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert not (work / materializer.CACHE_NAME).exists() + + +@pytest.mark.parametrize("target", ["manifest", "acquirer"]) +def test_changed_bound_acquisition_input_fails_before_cache_creation( + tmp_path, + monkeypatch, + target, +): + plan_path, plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + Path(plan[f"{target}_path"]).write_bytes(b"substituted") + + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match=f"{target}.*identity changed", + ): + materializer.materialize( + plan_path=plan_path, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert not (work / materializer.CACHE_NAME).exists() + + +def test_changed_exact_name_manifest_fails_before_cache_creation( + tmp_path, + monkeypatch, +): + manifest_path = tmp_path / MANIFESTS["windows"].name + manifest = json.loads(MANIFESTS["windows"].read_text(encoding="utf-8")) + manifest["source"]["repository"] = "mirror.invalid/model" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + plan_path, _plan, work, _staging = _write_plan( + tmp_path / "run", + monkeypatch, + "windows", + manifest_path=manifest_path, + ) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="manifest profile", + ): + materializer.materialize( + plan_path=plan_path, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert not (work / materializer.CACHE_NAME).exists() + + +def test_template_source_commit_substitution_fails_before_acquisition( + tmp_path, + monkeypatch, +): + work = tmp_path / "windows" / "work" + staging = tmp_path / "windows" / "staging" + template = _template("windows", work, staging) + template["source_commit"] = "b" * 40 + plan_path, _plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + template=template, + ) + acquisitions = 0 + + def acquire(_manifest, **_kwargs): + nonlocal acquisitions + acquisitions += 1 + return _record("windows") + + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="lifecycle template binding changed", + ): + materializer.materialize( + plan_path=plan_path, + acquirer=acquire, + ownership_verifier=_allow_owned, + cache_verifier=_fake_cache_verifier, + native_platform="windows", + ) + assert acquisitions == 0 + assert not (work / materializer.CACHE_NAME).exists() + + +def _install_tiny_profile(monkeypatch, tmp_path, platform_name: str): + payloads = { + "config.json": b"c", + "model.safetensors": b"weights", + "tokenizer.json": b"tokenizer", + } + source = json.loads(MANIFESTS[platform_name].read_text(encoding="utf-8")) + source["artifacts"] = [ + { + "path": path, + "role": ("config" if path == "config.json" else "tokenizer" if path == "tokenizer.json" else "weight"), + "sha256": hashlib.sha256(payload).hexdigest(), + "size": len(payload), + } + for path, payload in sorted(payloads.items()) + ] + manifest = ModelManifest.from_dict(source) + manifest_path = tmp_path / MANIFESTS[platform_name].name + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(source), encoding="utf-8") + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[platform_name] + + profiles = copy.deepcopy(lifecycle.acceptance.MODEL_PROFILES) + profiles[model_id]["manifest_digest"] = manifest.digest_id + profiles[model_id]["selected_artifact_count"] = len(payloads) + profiles[model_id]["selected_artifact_bytes"] = sum(len(payload) for payload in payloads.values()) + monkeypatch.setattr(lifecycle.acceptance, "MODEL_PROFILES", profiles) + + gate9 = copy.deepcopy(lifecycle._GATE9_WARM_CACHE) + gate9[platform_name]["artifacts"] = tuple( + ( + item.path, + item.role, + "sha256:" + item.sha256, + item.size, + ) + for item in sorted(manifest.artifacts, key=lambda value: value.path) + ) + monkeypatch.setattr(lifecycle, "_GATE9_WARM_CACHE", gate9) + return manifest_path, manifest.digest_id, payloads + + +def _tiny_acquirer(platform_name, manifest_digest, payloads): + def acquire(_manifest, **kwargs): + cache = kwargs["cache_dir"] + root = cache / "manifest-artifacts" / manifest_digest.removeprefix("sha256:") + snapshot = root / "snapshot" + partial = root / "partial" + locks = root / "locks" + snapshot.mkdir(parents=True) + partial.mkdir() + locks.mkdir() + for path, payload in payloads.items(): + destination = snapshot / path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(payload) + lock = hashlib.sha256(path.encode("utf-8")).hexdigest() + ".lock" + (locks / lock).write_bytes(b"") + return _record(platform_name) + + return acquire + + +def _fake_lifecycle_loader(path: Path): + payload = path.read_bytes() + raw = json.loads(payload) + warm = raw["warm_cache"] + return SimpleNamespace( + platform=raw["platform"], + source_commit=raw["source_commit"], + config_sha256=lifecycle._digest(payload), + warm_cache=SimpleNamespace( + binding_sha256=lifecycle._digest(lifecycle._canonical(warm)), + materialization_plan_sha256=warm["materialization_plan_sha256"], + materializer_sources_sha256=warm["materializer_sources_sha256"], + materialization_record_sha256=warm["materialization_record_sha256"], + materialization_record_bytes=warm["materialization_record_bytes"], + ), + ) + + +def test_tiny_physical_cache_round_trip_through_protected_promotion( + tmp_path, + monkeypatch, +): + platform_name = materializer._native_platform() + manifest_path, digest, payloads = _install_tiny_profile( + monkeypatch, + tmp_path / "manifest", + platform_name, + ) + plan_path, _plan, work, staging = _write_plan( + tmp_path / "run", + monkeypatch, + platform_name, + manifest_path=manifest_path, + ) + ownership_checks = [] + protected_outputs = [] + + def owned(path, *, directory): + ownership_checks.append((Path(path), directory)) + + def protect(path): + protected_outputs.append(Path(path)) + + materialized = materializer.materialize( + plan_path=plan_path, + acquirer=_tiny_acquirer(platform_name, digest, payloads), + ownership_verifier=owned, + native_platform=platform_name, + ) + cache = work / materializer.CACHE_NAME + cache_identity = cache.stat() + promoted = materializer.promote( + plan_path=plan_path, + ownership_verifier=owned, + lifecycle_loader=_fake_lifecycle_loader, + output_protector=protect, + ) + + assert materialized["warm_cache_binding_sha256"] == promoted["warm_cache_binding_sha256"] + assert (staging / materializer.RECORD_NAME).is_file() + assert (staging / materializer.CONFIG_NAME).is_file() + assert cache.is_dir() + after = cache.stat() + assert (after.st_dev, after.st_ino, after.st_uid) == ( + cache_identity.st_dev, + cache_identity.st_ino, + cache_identity.st_uid, + ) + assert not (work / materializer.RECORD_NAME).exists() + assert not (work / materializer.BINDING_NAME).exists() + assert not (work / materializer.HANDOFF_NAME).exists() + assert (staging.parent, True) in ownership_checks + assert (staging, True) in ownership_checks + assert (plan_path, False) in ownership_checks + assert (staging / materializer.TEMPLATE_NAME, False) in ownership_checks + assert protected_outputs == [ + staging / materializer.RECORD_NAME, + staging / materializer.CONFIG_NAME, + ] + config = json.loads((staging / materializer.CONFIG_NAME).read_text(encoding="utf-8")) + assert config["source_commit"] == SOURCE + assert config["warm_cache"]["materialization_plan_sha256"] == promoted["plan_sha256"] + assert config["warm_cache"]["materializer_sources_sha256"] == promoted["materializer_sources_sha256"] + + +def test_promoter_defaults_are_controller_writable_and_structurally_verified( + monkeypatch, + tmp_path, +): + assert materializer.promote.__kwdefaults__["ownership_verifier"] is lifecycle._assert_controller_managed + assert materializer.promote.__kwdefaults__["output_protector"] is materializer._protect_promoted_output + assert materializer._WINDOWS_CONTROLLER_SDDL == ("O:BAG:BAD:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;GR;;;AU)") + assert materializer._POSIX_CONTROLLER_FILE_MODE == 0o644 + observed = [] + + def load(_path, *, ownership_verifier): + observed.append(ownership_verifier) + return object() + + monkeypatch.setattr(lifecycle, "load_config", load) + materializer._load_promoted_config(tmp_path / "gate14-lifecycle.json") + assert observed == [lifecycle._assert_controller_managed] + + +def test_posix_promoted_output_is_qualification_readable_but_not_writable(monkeypatch): + events = [] + + class Output: + def chmod(self, mode): + events.append(("chmod", mode)) + + output = Output() + monkeypatch.setattr( + lifecycle, + "_assert_controller_managed", + lambda candidate, *, directory: events.append(("validated", candidate, directory)), + ) + + materializer._protect_promoted_output(output, os_name="posix") + + assert materializer._POSIX_CONTROLLER_FILE_MODE & 0o444 == 0o444 + assert materializer._POSIX_CONTROLLER_FILE_MODE & 0o022 == 0 + assert events == [ + ("chmod", 0o644), + ("validated", output, False), + ] + + +def test_windows_promoted_output_installs_descriptor_before_validation( + tmp_path, + monkeypatch, +): + path = tmp_path / materializer.RECORD_NAME + events = [] + + monkeypatch.setattr( + materializer, + "_windows_protect_controller_output", + lambda candidate: events.append(("installed", candidate)), + ) + monkeypatch.setattr( + lifecycle, + "_assert_controller_managed", + lambda candidate, *, directory: events.append(("validated", candidate, directory)), + ) + + materializer._protect_promoted_output(path, os_name="nt") + + assert events == [ + ("installed", path), + ("validated", path, False), + ] + + +def test_output_protection_failure_rolls_back_staged_file( + tmp_path, + monkeypatch, +): + plan_path, _plan, work, staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + materializer.materialize( + plan_path=plan_path, + acquirer=lambda _manifest, **_kwargs: _record("windows"), + ownership_verifier=_allow_owned, + cache_verifier=_fake_cache_verifier, + native_platform="windows", + ) + + def fail_protection(_path): + raise materializer.Gate14CacheMaterializationError("protection failed") + + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="protection failed", + ): + materializer.promote( + plan_path=plan_path, + ownership_verifier=_allow_owned, + cache_verifier=_fake_cache_verifier, + lifecycle_loader=_fake_lifecycle_loader, + output_protector=fail_protection, + ) + + assert not (staging / materializer.RECORD_NAME).exists() + assert not (staging / materializer.CONFIG_NAME).exists() + assert (work / materializer.HANDOFF_NAME).is_file() + assert (work / materializer.BINDING_NAME).is_file() + assert (work / materializer.RECORD_NAME).is_file() + + +def test_committed_promotion_retries_partial_handoff_cleanup( + tmp_path, + monkeypatch, +): + platform_name = materializer._native_platform() + manifest_path, digest, payloads = _install_tiny_profile( + monkeypatch, + tmp_path / "manifest", + platform_name, + ) + plan_path, _plan, work, staging = _write_plan( + tmp_path / "run", + monkeypatch, + platform_name, + manifest_path=manifest_path, + ) + materializer.materialize( + plan_path=plan_path, + acquirer=_tiny_acquirer(platform_name, digest, payloads), + ownership_verifier=_allow_owned, + native_platform=platform_name, + ) + + blocked = work / materializer.BINDING_NAME + unlink = Path.unlink + attempts = 0 + + def fail_middle_unlink(self, *args, **kwargs): + nonlocal attempts + if self == blocked: + attempts += 1 + raise OSError("locked") + return unlink(self, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_middle_unlink) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="retry promotion cleanup", + ): + materializer.promote( + plan_path=plan_path, + ownership_verifier=_allow_owned, + lifecycle_loader=_fake_lifecycle_loader, + output_protector=_allow_protected, + ) + + assert attempts == 2 + assert (staging / materializer.RECORD_NAME).is_file() + assert (staging / materializer.CONFIG_NAME).is_file() + assert not (work / materializer.HANDOFF_NAME).exists() + assert blocked.is_file() + assert not (work / materializer.RECORD_NAME).exists() + + monkeypatch.setattr(Path, "unlink", unlink) + result = materializer.promote( + plan_path=plan_path, + ownership_verifier=_allow_owned, + lifecycle_loader=_fake_lifecycle_loader, + output_protector=_allow_protected, + ) + assert result["phase"] == "promoted" + assert not blocked.exists() + assert (staging / materializer.RECORD_NAME).is_file() + assert (staging / materializer.CONFIG_NAME).is_file() + + +def test_cache_lease_detects_or_prevents_aba_swap(tmp_path, monkeypatch): + platform_name = materializer._native_platform() + _manifest, digest, payloads = _install_tiny_profile( + monkeypatch, + tmp_path / "manifest", + platform_name, + ) + model_id = lifecycle.acceptance.EXPECTED_PLATFORM_MODELS[platform_name] + record_payload = materializer._canonical(_record(platform_name)) + binding = lifecycle.build_warm_cache_binding( + record_payload, + platform=platform_name, + source_commit=SOURCE, + materialization_plan_sha256="sha256:" + "1" * 64, + materializer_sources_sha256="sha256:" + "2" * 64, + model_id=model_id, + manifest_digest=digest, + ) + cache = tmp_path / "cache" + snapshot = cache / "manifest-artifacts" / digest.removeprefix("sha256:") / "snapshot" + snapshot.mkdir(parents=True) + for path, payload in payloads.items(): + (snapshot / path).write_bytes(payload) + lease = materializer.verify_exact_cache(cache, binding) + target = snapshot / "config.json" + replacement = snapshot / "replacement" + replacement.write_bytes(payloads["config.json"]) + try: + try: + os.replace(replacement, target) + except PermissionError: + assert os.name == "nt" + else: + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="changed", + ): + lease.assert_stable() + finally: + lease.close() + + +def test_failed_download_removes_partial_cache_and_handoff( + tmp_path, + monkeypatch, +): + plan_path, _plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + + def fail(_manifest, **kwargs): + (kwargs["cache_dir"] / "partial.bin").write_bytes(b"partial") + raise RuntimeError("interrupted") + + with pytest.raises(RuntimeError, match="interrupted"): + materializer.materialize( + plan_path=plan_path, + acquirer=fail, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert not (work / materializer.CACHE_NAME).exists() + assert not (work / materializer.RECORD_NAME).exists() + assert not (work / materializer.BINDING_NAME).exists() + assert not (work / materializer.HANDOFF_NAME).exists() + + +def test_cleanup_retries_one_shot_tree_removal( + tmp_path, + monkeypatch, +): + plan_path, _plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + "windows", + ) + remove_tree = materializer.shutil.rmtree + attempts = 0 + + def flaky(path): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError("scanner busy") + return remove_tree(path) + + monkeypatch.setattr(materializer.shutil, "rmtree", flaky) + + def fail(_manifest, **kwargs): + (kwargs["cache_dir"] / "partial.bin").write_bytes(b"partial") + raise RuntimeError("interrupted") + + with pytest.raises(RuntimeError, match="interrupted"): + materializer.materialize( + plan_path=plan_path, + acquirer=fail, + ownership_verifier=_allow_owned, + native_platform="windows", + ) + assert attempts == 2 + assert not (work / materializer.CACHE_NAME).exists() + + +def test_write_failure_retries_unlink_and_preserves_original( + tmp_path, + monkeypatch, +): + path = tmp_path / "record.json" + fsync = materializer.os.fsync + unlink = Path.unlink + unlink_attempts = 0 + + def fail_fsync(_descriptor): + raise OSError("flush interrupted") + + def flaky_unlink(self, *args, **kwargs): + nonlocal unlink_attempts + if self == path: + unlink_attempts += 1 + if unlink_attempts == 1: + raise OSError("scanner busy") + return unlink(self, *args, **kwargs) + + monkeypatch.setattr(materializer.os, "fsync", fail_fsync) + monkeypatch.setattr(Path, "unlink", flaky_unlink) + with pytest.raises(OSError, match="flush interrupted"): + materializer._write_new(path, b"payload") + monkeypatch.setattr(materializer.os, "fsync", fsync) + assert unlink_attempts == 2 + assert not path.exists() + + +def test_permanent_write_cleanup_failure_is_bounded( + tmp_path, + monkeypatch, +): + path = tmp_path / "private-secret-record.json" + unlink = Path.unlink + + def fail_fsync(_descriptor): + raise OSError("flush interrupted") + + def fail_unlink(self, *args, **kwargs): + if self == path: + raise OSError("locked") + return unlink(self, *args, **kwargs) + + monkeypatch.setattr(materializer.os, "fsync", fail_fsync) + monkeypatch.setattr(Path, "unlink", fail_unlink) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="incomplete materialization output", + ) as captured: + materializer._write_new(path, b"payload") + assert str(path) not in str(captured.value) + monkeypatch.setattr(Path, "unlink", unlink) + path.unlink() + + +def test_locked_read_detects_path_identity_swap(tmp_path, monkeypatch): + path = tmp_path / "input.json" + path.write_bytes(b"{}\n") + real_fstat = materializer.os.fstat + calls = 0 + + def changed_fstat(descriptor): + nonlocal calls + value = real_fstat(descriptor) + calls += 1 + if calls == 2: + return SimpleNamespace( + st_dev=value.st_dev, + st_ino=value.st_ino + 1, + st_size=value.st_size, + st_mtime_ns=value.st_mtime_ns, + st_mode=value.st_mode, + st_file_attributes=getattr(value, "st_file_attributes", 0), + ) + return value + + monkeypatch.setattr(materializer.os, "fstat", changed_fstat) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="changed", + ): + materializer._read_locked_regular(path, 1024) + + +def test_internal_metadata_is_exact_and_removed(tmp_path): + cache = tmp_path / "cache" + digest = "b" * 64 + manifest_root = cache / "manifest-artifacts" / digest + partial = manifest_root / "partial" + locks = manifest_root / "locks" + partial.mkdir(parents=True) + locks.mkdir() + paths = ["a.bin", "nested/b.bin"] + for path in paths: + name = hashlib.sha256(path.encode("utf-8")).hexdigest() + ".lock" + (locks / name).write_bytes(b"") + materializer._clear_acquisition_metadata( + cache, + "sha256:" + digest, + paths, + ) + assert not partial.exists() + assert not locks.exists() + + +def test_packaged_acquirer_runs_source_bound_binary_without_credentials( + tmp_path, + monkeypatch, +): + acquirer_path = tmp_path / "CommunityAI-Node.exe" + acquirer_path.write_bytes(b"source-bound-node") + manifest_path = tmp_path / "manifest.json" + manifest_path.write_bytes(b"{}") + cache = tmp_path / "cache" + cache.mkdir() + observed = {} + + for name in ( + "HOME", + "LOCALAPPDATA", + "USERPROFILE", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HTTPS_PROXY", + ): + monkeypatch.setenv(name, "must-not-cross-process-boundary") + + def run(argv, **kwargs): + observed["argv"] = argv + observed["kwargs"] = kwargs + Path(argv[argv.index("--output") + 1]).write_bytes(b'{"ok":true}\n') + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(materializer.subprocess, "run", run) + payload = acquirer_path.read_bytes() + manifest_payload = manifest_path.read_bytes() + result = materializer._packaged_acquirer( + materializer.ManifestProfile(name="test"), + cache_dir=cache, + token=False, + max_resumptions=3, + require_direct_upstream=True, + manifest_path=manifest_path, + manifest_sha256=lifecycle._digest(manifest_payload), + acquirer_path=acquirer_path, + acquirer_sha256=lifecycle._digest(payload), + acquirer_bytes=len(payload), + ) + + result_path = cache / materializer.ACQUIRER_RESULT_NAME + assert result == {"ok": True} + argv = observed["argv"] + assert argv[0] == str(acquirer_path) + assert argv[1:] == [ + "edge-acquire", + "--manifest_stdin_sha256", + lifecycle._digest(manifest_payload), + "--cache_dir", + str(cache), + "--max_resumptions", + "3", + "--require_direct_upstream", + "--no_token", + "--output", + str(result_path), + ] + kwargs = observed["kwargs"] + assert kwargs["check"] is False + assert kwargs["input"] == manifest_payload + assert kwargs["stdout"] == materializer.subprocess.DEVNULL + assert kwargs["stderr"] == materializer.subprocess.DEVNULL + assert kwargs["timeout"] == materializer.ACQUIRER_TIMEOUT_SECONDS + assert kwargs["shell"] is False + assert kwargs["cwd"] == str(acquirer_path.parent) + if sys.platform == "win32": + assert "executable" not in kwargs + assert "pass_fds" not in kwargs + else: + assert len(kwargs["pass_fds"]) == 1 + assert kwargs["executable"] == f"/proc/self/fd/{kwargs['pass_fds'][0]}" + assert kwargs["close_fds"] is True + assert kwargs["env"]["HF_HUB_DISABLE_IMPLICIT_TOKEN"] == "1" + assert not { + "HOME", + "LOCALAPPDATA", + "USERPROFILE", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HTTPS_PROXY", + } & set(kwargs["env"]) + assert not result_path.exists() + + +@pytest.mark.parametrize( + ("returncode", "result_payload"), + [ + (1, b"{}"), + (0, b"not-json"), + (0, b"x" * (materializer.MAX_ACQUIRER_OUTPUT_BYTES + 1)), + ], + ids=["nonzero", "malformed", "oversized"], +) +def test_packaged_acquirer_rejects_failed_or_unbounded_results( + tmp_path, + monkeypatch, + returncode, + result_payload, +): + acquirer_path = tmp_path / "CommunityAI-Node" + acquirer_path.write_bytes(b"source-bound-node") + manifest_path = tmp_path / "manifest.json" + manifest_path.write_bytes(b"{}") + cache = tmp_path / "cache" + cache.mkdir() + + def run(argv, **_kwargs): + Path(argv[argv.index("--output") + 1]).write_bytes(result_payload) + return SimpleNamespace(returncode=returncode) + + payload = acquirer_path.read_bytes() + monkeypatch.setattr(materializer.subprocess, "run", run) + + with pytest.raises(materializer.Gate14CacheMaterializationError): + materializer._packaged_acquirer( + materializer.ManifestProfile(name="test"), + cache_dir=cache, + token=False, + max_resumptions=3, + require_direct_upstream=True, + manifest_path=manifest_path, + manifest_sha256=lifecycle._digest(manifest_path.read_bytes()), + acquirer_path=acquirer_path, + acquirer_sha256=lifecycle._digest(payload), + acquirer_bytes=len(payload), + ) + assert not (cache / materializer.ACQUIRER_RESULT_NAME).exists() + + +def test_packaged_acquirer_mutation_removes_partial_materialization( + tmp_path, + monkeypatch, +): + platform_name = materializer._native_platform() + plan_path, plan, work, _staging = _write_plan( + tmp_path, + monkeypatch, + platform_name, + ) + acquirer_path = Path(plan["acquirer_path"]) + captured = {} + bind = materializer._bound_acquirer_execution + + def capture_bind(path, lease): + captured["lease"] = lease + return bind(path, lease) + + def run(argv, **_kwargs): + cache = Path(argv[argv.index("--cache_dir") + 1]) + (cache / "partial.bin").write_bytes(b"partial") + Path(argv[argv.index("--output") + 1]).write_bytes(b"{}\n") + captured["lease"].close() + acquirer_path.write_bytes(b"mutated-node") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(materializer, "_bound_acquirer_execution", capture_bind) + monkeypatch.setattr(materializer.subprocess, "run", run) + with pytest.raises( + materializer.Gate14CacheMaterializationError, + match="identity changed", + ): + materializer.materialize( + plan_path=plan_path, + ownership_verifier=_allow_owned, + native_platform=platform_name, + ) + + assert not (work / materializer.CACHE_NAME).exists() + assert not (work / materializer.RECORD_NAME).exists() + assert not (work / materializer.BINDING_NAME).exists() + assert not (work / materializer.HANDOFF_NAME).exists() + + +@pytest.mark.skipif(sys.platform != "win32", reason="requires native Windows sharing semantics") +def test_windows_packaged_acquirer_locks_inputs_across_launch_boundary(tmp_path, monkeypatch): + acquirer_path = tmp_path / "CommunityAI-Node.exe" + acquirer_path.write_bytes(b"source-bound-node") + manifest_path = tmp_path / "manifest.json" + manifest_path.write_bytes(b"{}\n") + cache = tmp_path / "cache" + cache.mkdir() + blocked = [] + + def run(argv, **_kwargs): + for target in (manifest_path, acquirer_path): + replacement = target.with_suffix(target.suffix + ".replacement") + replacement.write_bytes(b"substituted") + for operation in ( + lambda: target.write_bytes(b"substituted"), + target.unlink, + lambda: os.replace(replacement, target), + ): + with pytest.raises(OSError): + operation() + blocked.append((target.name, operation)) + replacement.unlink() + Path(argv[argv.index("--output") + 1]).write_bytes(b'{"ok":true}\n') + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(materializer.subprocess, "run", run) + acquirer_payload = acquirer_path.read_bytes() + manifest_payload = manifest_path.read_bytes() + result = materializer._packaged_acquirer( + materializer.ManifestProfile(name="test"), + cache_dir=cache, + token=False, + max_resumptions=3, + require_direct_upstream=True, + manifest_path=manifest_path, + manifest_sha256=lifecycle._digest(manifest_payload), + acquirer_path=acquirer_path, + acquirer_sha256=lifecycle._digest(acquirer_payload), + acquirer_bytes=len(acquirer_payload), + ) + + assert result == {"ok": True} + assert len(blocked) == 6 + manifest_path.write_bytes(b"released") + acquirer_path.write_bytes(b"released") + assert manifest_path.read_bytes() == b"released" + assert acquirer_path.read_bytes() == b"released" + + +@pytest.mark.skipif(sys.platform != "win32", reason="requires native Windows CreateProcess") +def test_windows_locked_executable_still_launches_original_image(tmp_path): + acquirer_path = tmp_path / "CommunityAI-Node.exe" + shutil.copyfile(os.environ["COMSPEC"], acquirer_path) + payload = acquirer_path.read_bytes() + lease = materializer._lock_verified_acquirer( + acquirer_path, + materializer.MAX_ACQUIRER_BYTES, + len(payload), + lifecycle._digest(payload), + ) + replacement = tmp_path / "replacement.exe" + replacement.write_bytes(b"substituted") + + try: + executable, options = materializer._bound_acquirer_execution(acquirer_path, lease) + with pytest.raises(OSError): + os.replace(replacement, acquirer_path) + result = materializer.subprocess.run( + [executable, "/d", "/c", "ver"], + check=False, + stdout=materializer.subprocess.DEVNULL, + stderr=materializer.subprocess.DEVNULL, + **options, + ) + assert result.returncode == 0 + lease.assert_stable("packaged acquirer") + finally: + lease.close() + + os.replace(replacement, acquirer_path) + assert acquirer_path.read_bytes() == b"substituted" + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux fd execution") +def test_linux_packaged_acquirer_executes_verified_fd_after_path_replacement(tmp_path): + acquirer_path = tmp_path / "CommunityAI-Node" + shutil.copyfile(sys.executable, acquirer_path) + acquirer_path.chmod(0o700) + payload = acquirer_path.read_bytes() + lease = materializer._lock_verified_acquirer( + acquirer_path, + materializer.MAX_ACQUIRER_BYTES, + len(payload), + lifecycle._digest(payload), + ) + verified_marker = tmp_path / "verified" + substituted_marker = tmp_path / "substituted" + moved_path = tmp_path / "verified-original" + acquirer_path.rename(moved_path) + acquirer_path.write_text( + f'#!/bin/sh\nprintf substituted > "{substituted_marker}"\n', + encoding="utf-8", + ) + acquirer_path.chmod(0o700) + + try: + executable, options = materializer._bound_acquirer_execution(acquirer_path, lease) + result = materializer.subprocess.run( + [ + executable, + "-c", + f'from pathlib import Path; Path(r"{verified_marker}").write_text("verified")', + ], + check=False, + **options, + ) + assert result.returncode == 0 + assert verified_marker.read_text(encoding="utf-8") == "verified" + assert not substituted_marker.exists() + with pytest.raises(materializer.Gate14CacheMaterializationError, match="identity changed"): + lease.assert_stable("packaged acquirer") + finally: + lease.close() + + +def test_source_bindings_execute_without_site_packages(): + result = materializer.subprocess.run( + [ + sys.executable, + "-S", + str(ROOT / "scripts" / "gate14_cache_materializer.py"), + "source-bindings", + ], + check=False, + capture_output=True, + cwd=ROOT, + text=True, + ) + + assert result.returncode == 0 + payload = json.loads(result.stdout) + assert payload["scope"] == "gate14-cache-materializer-sources" + assert set(payload["sources"]) == set(materializer._SOURCE_NAMES) + + +def test_materializer_cli_uses_two_explicit_phases(): + parser = materializer.build_parser() + assert parser.parse_args(["materialize", "--plan", str(ROOT / "plan.json")]).command == "materialize" + assert parser.parse_args(["promote", "--plan", str(ROOT / "plan.json")]).command == "promote" + assert parser.parse_args(["source-bindings"]).command == "source-bindings" diff --git a/tests/test_gate14_gcp_executor.py b/tests/test_gate14_gcp_executor.py new file mode 100644 index 000000000..35a84baca --- /dev/null +++ b/tests/test_gate14_gcp_executor.py @@ -0,0 +1,326 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_gcp_executor as executor # noqa: E402 +import gate14_run_controller as controller # noqa: E402 + +RUN_ID = "gate14-20260902-a" +SOURCE = "1" * 40 +NOW = 2_000_000_000 + + +def client(platform: str) -> controller.ClientPlan: + return controller.ClientPlan( + platform=platform, + instance=f"{RUN_ID}-{platform}", + disk=f"{RUN_ID}-{platform}-disk", + source_commit=SOURCE, + termination_unix=NOW + 7_200, + package_sha256="sha256:" + ("a" if platform == "windows" else "b") * 64, + model_id="Qwen3.5 2B" if platform == "windows" else "Gemma 4 E2B IT", + manifest_digest=( + "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33" + if platform == "windows" + else "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd" + ), + machine_type="g2-standard-8", + image_project="windows-cloud" if platform == "windows" else "ubuntu-os-cloud", + image=("windows-server-2022-dc-v20260814" if platform == "windows" else "ubuntu-2404-noble-amd64-v20260826"), + boot_disk_gib=100, + boot_disk_type="pd-balanced", + max_run_seconds=7_200, + ) + + +def plan() -> controller.RunPlan: + return controller.RunPlan( + run_id=RUN_ID, + authorization_sha256="sha256:" + "c" * 64, + provider_plan_digest="sha256:" + "d" * 64, + source_commit=SOURCE, + ledger_state="RESERVED", + project="community-ai-506321", + zone="us-central1-a", + windows=client("windows"), + linux=client("linux"), + ) + + +def jobs(**states) -> dict: + return { + "schema_version": 1, + "run_id": RUN_ID, + "clients": { + platform: { + "job_state": states.get(platform, "absent"), + "attempt_ordinal": 0 if states.get(platform, "absent") == "absent" else 1, + "evidence_digest": None, + } + for platform in ("windows", "linux") + }, + } + + +def action_state(run_plan: controller.RunPlan, action: str) -> dict: + state = controller.initial_state(run_plan) + if action == "start_windows": + state["next_action"] = action + return state + if action == "delete_windows": + state.update( + revision=1, + phase="WINDOWS_DELETING", + windows_consumed=True, + windows_evidence_digest="sha256:" + "e" * 64, + windows_challenge_sha256="sha256:" + "f" * 64, + windows_challenge_consumed=True, + next_action=action, + ) + return state + raise AssertionError(f"unsupported test action: {action}") + + +class FakeGcloud: + def __init__(self, run_plan: controller.RunPlan): + self.plan = run_plan + self.instances: dict[str, dict] = { + controller.PROTECTED_INSTANCE: { + "name": controller.PROTECTED_INSTANCE, + "status": "RUNNING", + } + } + self.disks: dict[str, dict] = {} + self.calls: list[tuple[str, ...]] = [] + + @staticmethod + def result(value=None, *, returncode=0, stderr=b""): + stdout = b"" if value is None else json.dumps(value).encode("utf-8") + return executor.CommandResult(returncode, stdout, stderr) + + @staticmethod + def option(arguments: list[str], name: str) -> str: + prefix = name + "=" + return next(item[len(prefix) :] for item in arguments if item.startswith(prefix)) + + def disk_value(self, item: controller.ClientPlan) -> dict: + return { + "name": item.disk, + "status": "READY", + "labels": executor._labels(self.plan, item), + "type": f"zones/{self.plan.zone}/diskTypes/{item.boot_disk_type}", + "sourceImage": f"projects/{item.image_project}/global/images/{item.image}", + "sizeGb": str(item.boot_disk_gib), + } + + def instance_value(self, item: controller.ClientPlan) -> dict: + return { + "name": item.instance, + "status": "RUNNING", + "labels": executor._labels(self.plan, item), + "machineType": f"zones/{self.plan.zone}/machineTypes/{item.machine_type}", + "disks": [{"source": f"zones/{self.plan.zone}/disks/{item.disk}"}], + "guestAccelerators": [ + { + "acceleratorType": f"zones/{self.plan.zone}/acceleratorTypes/nvidia-l4", + "acceleratorCount": 1, + } + ], + "metadata": { + "items": [ + {"key": "communityai-run-id", "value": self.plan.run_id}, + {"key": "communityai-source-commit", "value": item.source_commit}, + {"key": "communityai-termination-unix", "value": str(item.termination_unix)}, + ] + }, + } + + def __call__(self, argv, timeout): + command = tuple(argv) + self.calls.append(command) + arguments = list(command[1:]) + assert arguments.pop() == "--quiet" + + if arguments[:2] == ["auth", "list"]: + return executor.CommandResult(0, b"operator@example.invalid\n", b"") + if arguments[:2] == ["projects", "describe"]: + return self.result({"lifecycleState": "ACTIVE"}) + if arguments[:3] == ["compute", "accelerator-types", "describe"]: + return self.result({"name": "nvidia-l4"}) + if arguments[:3] == ["compute", "images", "describe"]: + return self.result({"name": arguments[3], "status": "READY"}) + if arguments[:3] == ["compute", "firewall-rules", "list"]: + return self.result([]) + if arguments[:3] == ["compute", "project-info", "describe"]: + return self.result({"quotas": [{"metric": "GPUS_ALL_REGIONS", "limit": 1, "usage": 0}]}) + if arguments[:3] == ["compute", "instances", "list"]: + values = [ + value + for name, value in self.instances.items() + if name != controller.PROTECTED_INSTANCE and value.get("status") == "RUNNING" + ] + return self.result(values) + + if arguments[:2] == ["compute", "instances"] and arguments[2] == "describe": + name = arguments[3] + if name not in self.instances: + return self.result(returncode=1, stderr=b"resource was not found") + return self.result(self.instances[name]) + if arguments[:2] == ["compute", "disks"] and arguments[2] == "describe": + name = arguments[3] + if name not in self.disks: + return self.result(returncode=1, stderr=b"resource was not found") + return self.result(self.disks[name]) + + if arguments[:3] == ["compute", "disks", "create"]: + name = arguments[3] + item = next(value for value in (self.plan.windows, self.plan.linux) if value.disk == name) + self.disks[name] = self.disk_value(item) + return self.result() + if arguments[:3] == ["compute", "instances", "create"]: + name = arguments[3] + item = next(value for value in (self.plan.windows, self.plan.linux) if value.instance == name) + self.instances[name] = self.instance_value(item) + return self.result() + if arguments[:3] == ["compute", "instances", "delete"]: + name = arguments[3] + item = next(value for value in (self.plan.windows, self.plan.linux) if value.instance == name) + self.instances.pop(name, None) + self.disks.pop(item.disk, None) + return self.result() + if arguments[:3] == ["compute", "disks", "delete"]: + self.disks.pop(arguments[3], None) + return self.result() + + raise AssertionError(f"unexpected command: {command}") + + +def test_clean_preflight_revalidates_auth_images_l4_and_bootstrap(): + run_plan = plan() + fake = FakeGcloud(run_plan) + provider = executor.GcpExecutor(run_plan, runner=fake, clock=lambda: NOW) + + result = provider.preflight(jobs()) + + assert result["result"] == "passed" + assert result["maximum_estimate_usd"] == "44.00" + assert result["planned_resources_absent"] is True + assert any(call[1:3] == ("auth", "list") for call in fake.calls) + assert len([call for call in fake.calls if call[1:4] == ("compute", "images", "describe")]) == 2 + + +def test_exact_start_and_delete_use_bound_disk_image_and_no_service_account(): + run_plan = plan() + fake = FakeGcloud(run_plan) + provider = executor.GcpExecutor(run_plan, runner=fake, clock=lambda: NOW) + + provider.execute( + "start_windows", + state=action_state(run_plan, "start_windows"), + jobs=jobs(), + ) + + assert run_plan.windows.instance in fake.instances + assert run_plan.windows.disk in fake.disks + create = next(call for call in fake.calls if call[1:4] == ("compute", "instances", "create")) + assert "--no-service-account" in create + assert "--no-address" in create + assert "--max-run-duration=7200s" in create + observation = provider.inventory(jobs(windows="starting")) + assert observation["l4_usage"] == 1 + assert observation["instances"][run_plan.windows.instance]["present"] is True + + completed_jobs = jobs(windows="passed") + completed_jobs["clients"]["windows"]["evidence_digest"] = "sha256:" + "e" * 64 + provider.execute( + "delete_windows", + state=action_state(run_plan, "delete_windows"), + jobs=completed_jobs, + ) + + assert run_plan.windows.instance not in fake.instances + assert run_plan.windows.disk not in fake.disks + + +def test_foreign_exact_name_instance_fails_closed_before_mutation(): + run_plan = plan() + fake = FakeGcloud(run_plan) + fake.instances[run_plan.windows.instance] = fake.instance_value(run_plan.windows) + fake.instances[run_plan.windows.instance]["labels"]["communityai-run"] = "foreign-run" + fake.disks[run_plan.windows.disk] = fake.disk_value(run_plan.windows) + provider = executor.GcpExecutor(run_plan, runner=fake, clock=lambda: NOW) + + with pytest.raises(executor.Gate14GcpError, match="ownership"): + provider.inventory(jobs(windows="starting")) + + assert not any(call[1:4] == ("compute", "instances", "delete") for call in fake.calls) + + +def test_same_image_name_from_foreign_project_fails_closed(): + run_plan = plan() + fake = FakeGcloud(run_plan) + fake.disks[run_plan.windows.disk] = fake.disk_value(run_plan.windows) + fake.disks[run_plan.windows.disk][ + "sourceImage" + ] = f"projects/foreign-project/global/images/{run_plan.windows.image}" + provider = executor.GcpExecutor(run_plan, runner=fake, clock=lambda: NOW) + + with pytest.raises(executor.Gate14GcpError, match="shape"): + provider.inventory(jobs()) + + assert not any(call[1:4] == ("compute", "disks", "delete") for call in fake.calls) + + +def test_attached_service_account_fails_closed(): + run_plan = plan() + fake = FakeGcloud(run_plan) + fake.instances[run_plan.windows.instance] = fake.instance_value(run_plan.windows) + fake.instances[run_plan.windows.instance]["serviceAccounts"] = [ + { + "email": "unexpected@example.invalid", + "scopes": ["https://www.googleapis.com/auth/cloud-platform"], + } + ] + fake.disks[run_plan.windows.disk] = fake.disk_value(run_plan.windows) + provider = executor.GcpExecutor(run_plan, runner=fake, clock=lambda: NOW) + + with pytest.raises(executor.Gate14GcpError, match="shape"): + provider.inventory(jobs(windows="starting")) + + assert not any(call[1:4] == ("compute", "instances", "delete") for call in fake.calls) + + +def test_execute_requires_fresh_inventory_and_bound_controller_action(): + run_plan = plan() + fake = FakeGcloud(run_plan) + provider = executor.GcpExecutor(run_plan, runner=fake, clock=lambda: NOW) + + with pytest.raises(executor.Gate14GcpError, match="stale or unbound"): + provider.execute( + "start_windows", + state=controller.initial_state(run_plan), + jobs=jobs(), + ) + + assert any(call[1:3] == ("auth", "list") for call in fake.calls) + assert not any(call[1:4] == ("compute", "instances", "create") for call in fake.calls) + + +def test_jobs_default_to_absent_and_reject_wrong_run(tmp_path): + run_plan = plan() + missing = executor.load_jobs(tmp_path / "missing.json", run_plan) + assert missing["clients"]["windows"]["job_state"] == "absent" + + path = tmp_path / "jobs.json" + value = jobs() + value["run_id"] = "gate14-20260902-b" + path.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises(executor.Gate14GcpError, match="run changed"): + executor.load_jobs(path, run_plan) diff --git a/tests/test_gate14_hardware_acceptance.py b/tests/test_gate14_hardware_acceptance.py new file mode 100644 index 000000000..e15cd0ec7 --- /dev/null +++ b/tests/test_gate14_hardware_acceptance.py @@ -0,0 +1,564 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_hardware_acceptance as acceptance # noqa: E402 + +RUN_ID = "gate14-20260902-a" +CONTROLLER_SOURCE = "2" * 40 +SOURCE = CONTROLLER_SOURCE +DIGEST = "sha256:" + "a" * 64 +PROJECT = "community-ai-506321" +ZONE = "us-central1-a" +INSTANCES = ("gate14-20260902-a-windows", "gate14-20260902-a-linux") +DISKS = ("gate14-20260902-a-windows-disk", "gate14-20260902-a-linux-disk") +CHALLENGE_SHA256 = "sha256:" + "c" * 64 +CHALLENGE_ISSUED = 2_000_000_000 +CHALLENGE_EXPIRES = CHALLENGE_ISSUED + 900 + + +def provider_plan_document() -> dict: + return { + "project": PROJECT, + "zone": ZONE, + "clients": [ + { + "platform": platform, + "instance": INSTANCES[index], + "disk": DISKS[index], + "source_commit": SOURCE, + "termination_unix": 2_000_010_000 + index, + "package_sha256": DIGEST, + "model_id": acceptance.EXPECTED_PLATFORM_MODELS[platform], + "manifest_digest": acceptance.MODEL_PROFILES[acceptance.EXPECTED_PLATFORM_MODELS[platform]][ + "manifest_digest" + ], + "machine_type": "g2-standard-8", + "image_project": "windows-cloud" if platform == "windows" else "ubuntu-os-cloud", + "image": ( + "windows-server-2022-dc-v20260814" if platform == "windows" else "ubuntu-2404-noble-amd64-v20260826" + ), + "boot_disk_gib": 100, + "boot_disk_type": "pd-balanced", + "service_account_disabled": True, + "max_run_seconds": 7_200, + "termination_action": "DELETE", + } + for index, platform in enumerate(("windows", "linux")) + ], + "sequencing": { + "clients_may_run_concurrently": False, + "windows_first": True, + "fresh_host_per_platform": True, + }, + } + + +PROVIDER_PLAN = provider_plan_document() +PLAN_DIGEST = ( + "sha256:" + + hashlib.sha256(json.dumps(PROVIDER_PLAN, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() +) + + +def authorization_document() -> dict: + return { + "schema_version": 1, + "gate": 14, + "result": "authorized", + "run_id": RUN_ID, + "source_commit": CONTROLLER_SOURCE, + "provider_plan_digest": PLAN_DIGEST, + "provider_plan": PROVIDER_PLAN, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "44.00", + "remaining_after_run_maximum_usd": "0.00", + "reservation_recorded": True, + "native_auth_revalidated": True, + "provisioning_authorized_after_fail_closed_preflight": True, + }, + "prohibited": {"credits": 0, "macos": 0, "fly_gpu": 0}, + } + + +def platform_document(platform: str) -> dict: + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + return { + "schema_version": 1, + "scope": acceptance.PLATFORM_SCOPE, + "run_id": RUN_ID, + "platform": platform, + "result": "passed", + "source_commit": SOURCE, + "gate13_evidence_sha256": acceptance.EXPECTED_GATE13_EVIDENCE_SHA256, + "package": { + "source_commit": SOURCE, + "archive_sha256": DIGEST, + "archive_bytes": 1024, + "release_metadata_sha256": "sha256:" + "b" * 64, + }, + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "revision_commit": profile["revision_commit"], + "gate9_envelope_sha256": acceptance.EXPECTED_GATE9_ENVELOPES[platform], + "selected_artifact_count": profile["selected_artifact_count"], + "selected_artifact_bytes": profile["selected_artifact_bytes"], + "total_blocks": profile["total_blocks"], + }, + "hardware": { + "os_name": "Windows Server 2022" if platform == "windows" else "Ubuntu 24.04", + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": 24 * 1024**3, + }, + "cache": { + "verified_bytes_before": profile["selected_artifact_bytes"], + "verified_bytes_after": profile["selected_artifact_bytes"], + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": min(4, profile["total_blocks"]), + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "calibration_challenge": { + "challenge_sha256": CHALLENGE_SHA256, + "controller_state_revision": 2, + "issued_at_unix": CHALLENGE_ISSUED, + "expires_at_unix": CHALLENGE_EXPIRES, + }, + "suspensions": [ + { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": 2.5, + "calibration": { + "measurement_source": { + "bandwidth": "host-network-counters", + "power": "nvidia-nvml-device-power", + "schedule": "utc-policy-clock", + }[kind], + "measurement_scope": { + "bandwidth": "aggregate-host-network", + "power": "selected-nvidia-l4-device", + "schedule": "utc-schedule-policy", + }[kind], + "sample_count": 4, + "sample_interval_seconds": 0.5, + "baseline_value": 1.0 if kind == "schedule" else 10.0, + "configured_limit": { + "bandwidth": 100.0, + "power": 250.0, + "schedule": 0.5, + }[kind], + "trigger_value": 0.0 if kind == "schedule" else (120.0 if kind == "bandwidth" else 275.0), + "resume_value": 1.0 if kind == "schedule" else 10.0, + "challenge_sha256": CHALLENGE_SHA256, + "sample_started_at_unix": CHALLENGE_ISSUED + 10, + "sample_ended_at_unix": CHALLENGE_ISSUED + 12, + }, + } + for kind in ("bandwidth", "power", "schedule") + ], + "recovery": { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 4.5, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + "pause": { + "requested": True, + "completed": True, + "duration_seconds": 1.5, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + "restart": { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 8.0, + "cache_reused": True, + }, + "unsupported_telemetry": { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + "privacy": { + "prompt_retained": False, + "response_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + "provider_output_retained": False, + }, + "qualification_temporaries_removed": True, + } + + +def cleanup_document(terminal_state_sha256: str) -> dict: + return { + "schema_version": 1, + "scope": acceptance.CLEANUP_SCOPE, + "run_id": RUN_ID, + "result": "passed", + "provider": "GCP", + "controller_source_commit": CONTROLLER_SOURCE, + "provider_plan_digest": PLAN_DIGEST, + "project": PROJECT, + "zone": ZONE, + "deleted_instances": list(INSTANCES), + "deleted_disks": list(DISKS), + "controller_terminal_state_sha256": terminal_state_sha256, + "native_auth_revalidated": True, + "expected_instances": 2, + "remaining_instances": 0, + "expected_disks": 2, + "remaining_disks": 0, + "remaining_firewalls": 0, + "l4_usage": 0, + "protected_bootstrap_running": True, + "product_processes_remaining": 0, + "temporary_credentials_remaining": 0, + } + + +def write_documents(tmp_path: Path) -> tuple[Path, Path, Path, Path, Path]: + windows_path = tmp_path / "windows.json" + linux_path = tmp_path / "linux.json" + cleanup_path = tmp_path / "cleanup.json" + terminal_state_path = tmp_path / "state.json" + authorization_path = tmp_path / "authorization.json" + windows_path.write_text(json.dumps(platform_document("windows")), encoding="utf-8") + linux_path.write_text(json.dumps(platform_document("linux")), encoding="utf-8") + authorization_path.write_text(json.dumps(authorization_document()), encoding="utf-8") + windows_digest = "sha256:" + hashlib.sha256(windows_path.read_bytes()).hexdigest() + linux_digest = "sha256:" + hashlib.sha256(linux_path.read_bytes()).hexdigest() + authorization_digest = "sha256:" + hashlib.sha256(authorization_path.read_bytes()).hexdigest() + terminal_state = { + "schema_version": 1, + "run_id": RUN_ID, + "authorization_sha256": authorization_digest, + "provider_plan_digest": PLAN_DIGEST, + "revision": 10, + "phase": "CLEANED_PASS", + "failure_code": None, + "windows_evidence_digest": windows_digest, + "linux_evidence_digest": linux_digest, + "windows_challenge_sha256": CHALLENGE_SHA256, + "linux_challenge_sha256": CHALLENGE_SHA256, + "windows_challenge_consumed": True, + "linux_challenge_consumed": True, + "windows_consumed": True, + "linux_consumed": True, + "cleanup_verified": True, + "next_action": "none", + } + terminal_state_path.write_text(json.dumps(terminal_state), encoding="utf-8") + terminal_digest = "sha256:" + hashlib.sha256(terminal_state_path.read_bytes()).hexdigest() + cleanup_path.write_text( + json.dumps(cleanup_document(terminal_digest)), + encoding="utf-8", + ) + return windows_path, linux_path, cleanup_path, terminal_state_path, authorization_path + + +def validate_documents(paths: tuple[Path, Path, Path, Path, Path]) -> dict: + windows, linux, cleanup, terminal_state, authorization = paths + return acceptance.validate_files( + windows, + linux, + cleanup, + CONTROLLER_SOURCE, + provider_plan_digest=PLAN_DIGEST, + project=PROJECT, + zone=ZONE, + expected_instances=INSTANCES, + expected_disks=DISKS, + terminal_state_path=terminal_state, + authorization_path=authorization, + ) + + +def test_validate_platform_documents_cover_both_models_and_hardware_contract(): + windows = acceptance.validate_platform_document(platform_document("windows")) + linux = acceptance.validate_platform_document(platform_document("linux")) + + assert windows["model_id"] == "Qwen3.5 2B" + assert linux["model_id"] == "Gemma 4 E2B IT" + assert windows["accelerator"] == linux["accelerator"] == "NVIDIA L4" + assert windows["block_start"] == 0 + assert windows["block_end"] == 4 + + +def test_validate_files_emits_digest_bound_privacy_safe_aggregate(tmp_path): + paths = write_documents(tmp_path) + + result = validate_documents(paths) + + assert result["scope"] == acceptance.AGGREGATE_SCOPE + assert result["result"] == "passed" + assert result["controller_source_commit"] == CONTROLLER_SOURCE + assert result["package_source_commit"] == SOURCE + assert [item["platform"] for item in result["platforms"]] == ["windows", "linux"] + assert all(item["evidence_sha256"].startswith("sha256:") for item in result["platforms"]) + assert result["cleanup"]["resource_absence_proved"] is True + assert result["credits_in_scope"] is False + assert result["macos_in_scope"] is False + assert result["privacy_safe"] is True + + +@pytest.mark.parametrize( + "mutator", + [ + lambda value: value.update(gate13_evidence_sha256="sha256:" + "0" * 64), + lambda value: value["model"].update(gate9_envelope_sha256="sha256:" + "0" * 64), + lambda value: value["hardware"].update(os_name="Ubuntu 24.04"), + lambda value: value["cache"].update(transfer_bytes_during_gate=1), + lambda value: value["placement"].update(remote_acknowledged=False), + lambda value: value["limits"].update(low_vram_rejected=False), + lambda value: value["limits"].update(power_watts=None), + lambda value: value["suspensions"].pop(), + lambda value: value["suspensions"][0].update(resumed=False), + lambda value: value["suspensions"][0]["calibration"].update(trigger_value=50.0), + lambda value: value["suspensions"][1]["calibration"].update(measurement_scope="aggregate-host-network"), + lambda value: value["recovery"].update(worker_restarted=False), + lambda value: value["pause"].update(descendant_count_after=1), + lambda value: value["restart"].update(policy_persisted=False), + lambda value: value["unsupported_telemetry"].update(start_rejected=False), + lambda value: value["privacy"].update(paths_retained=True), + lambda value: value.update(qualification_temporaries_removed=False), + ], +) +def test_platform_evidence_fails_closed(mutator): + value = platform_document("windows") + mutator(value) + + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(value) + + +def test_wrong_model_and_unsafe_block_range_fail_closed(): + value = platform_document("windows") + value["model"] = platform_document("linux")["model"] + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(value) + + value = platform_document("windows") + value["placement"]["block_end"] = value["placement"]["block_start"] + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(value) + + +def test_calibration_requires_challenge_bound_bounded_sample_windows(): + missing_timestamp = platform_document("windows") + missing_timestamp["suspensions"][0]["calibration"].pop("sample_started_at_unix") + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(missing_timestamp) + + wrong_challenge = platform_document("windows") + wrong_challenge["suspensions"][0]["calibration"]["challenge_sha256"] = "sha256:" + "d" * 64 + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(wrong_challenge) + + stale = platform_document("windows") + stale["suspensions"][0]["calibration"]["sample_ended_at_unix"] = CHALLENGE_EXPIRES + 1 + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(stale) + + oversized = platform_document("windows") + oversized["suspensions"][0]["calibration"]["sample_ended_at_unix"] = CHALLENGE_ISSUED + 131 + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(oversized) + + +def test_aggregate_rejects_mismatched_run_source_and_incomplete_cleanup(tmp_path): + paths = write_documents(tmp_path) + linux = paths[1] + linux_value = json.loads(linux.read_text(encoding="utf-8")) + linux_value["run_id"] = "gate14-20260902-b" + linux.write_text(json.dumps(linux_value), encoding="utf-8") + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + paths = write_documents(tmp_path) + cleanup = paths[2] + cleanup_value = json.loads(cleanup.read_text(encoding="utf-8")) + cleanup_value["remaining_disks"] = 1 + cleanup.write_text(json.dumps(cleanup_value), encoding="utf-8") + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +@pytest.mark.parametrize( + ("field", "replacement"), + [ + ("controller_source_commit", "3" * 40), + ("provider_plan_digest", "sha256:" + "3" * 64), + ("project", "different-project"), + ("zone", "us-east1-b"), + ("deleted_instances", list(reversed(INSTANCES))), + ("deleted_disks", list(reversed(DISKS))), + ("controller_terminal_state_sha256", "sha256:" + "3" * 64), + ], +) +def test_cleanup_must_bind_exact_plan_resources_and_terminal_state( + tmp_path, + field, + replacement, +): + paths = write_documents(tmp_path) + cleanup = paths[2] + value = json.loads(cleanup.read_text(encoding="utf-8")) + value[field] = replacement + cleanup.write_text(json.dumps(value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +def test_terminal_state_must_be_a_real_digest_bound_pass(tmp_path): + paths = write_documents(tmp_path) + cleanup = paths[2] + terminal_state = paths[3] + state_value = json.loads(terminal_state.read_text(encoding="utf-8")) + state_value["cleanup_verified"] = False + terminal_state.write_text(json.dumps(state_value), encoding="utf-8") + terminal_digest = "sha256:" + hashlib.sha256(terminal_state.read_bytes()).hexdigest() + cleanup_value = json.loads(cleanup.read_text(encoding="utf-8")) + cleanup_value["controller_terminal_state_sha256"] = terminal_digest + cleanup.write_text(json.dumps(cleanup_value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +def test_protected_bootstrap_cannot_enter_cleanup_inventory(tmp_path): + windows, linux, cleanup, terminal_state, authorization = write_documents(tmp_path) + + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_files( + windows, + linux, + cleanup, + CONTROLLER_SOURCE, + provider_plan_digest=PLAN_DIGEST, + project=PROJECT, + zone=ZONE, + expected_instances=(acceptance.PROTECTED_INSTANCE, INSTANCES[1]), + expected_disks=DISKS, + terminal_state_path=terminal_state, + authorization_path=authorization, + ) + + +def test_terminal_state_binds_exact_semantic_authorization_file(tmp_path): + paths = write_documents(tmp_path) + cleanup = paths[2] + terminal_state = paths[3] + authorization = paths[4] + authorization_value = json.loads(authorization.read_text(encoding="utf-8")) + authorization_value["source_commit"] = "3" * 40 + authorization.write_text(json.dumps(authorization_value), encoding="utf-8") + authorization_digest = "sha256:" + hashlib.sha256(authorization.read_bytes()).hexdigest() + + state_value = json.loads(terminal_state.read_text(encoding="utf-8")) + state_value["authorization_sha256"] = authorization_digest + terminal_state.write_text(json.dumps(state_value), encoding="utf-8") + terminal_digest = "sha256:" + hashlib.sha256(terminal_state.read_bytes()).hexdigest() + cleanup_value = json.loads(cleanup.read_text(encoding="utf-8")) + cleanup_value["controller_terminal_state_sha256"] = terminal_digest + cleanup.write_text(json.dumps(cleanup_value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +def test_duplicate_and_non_finite_json_fail_closed(): + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance._strict_json(b'{"schema_version":1,"schema_version":1}') + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance._strict_json(b'{"value":NaN}') + + +def test_cli_prints_canonical_aggregate(tmp_path): + windows, linux, cleanup, terminal_state, authorization = write_documents(tmp_path) + + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "gate14_hardware_acceptance.py"), + "--windows", + str(windows), + "--linux", + str(linux), + "--cleanup", + str(cleanup), + "--controller-state", + str(terminal_state), + "--authorization", + str(authorization), + "--controller-source-commit", + CONTROLLER_SOURCE, + "--provider-plan-digest", + PLAN_DIGEST, + "--project", + PROJECT, + "--zone", + ZONE, + "--instances", + *INSTANCES, + "--disks", + *DISKS, + ], + check=True, + capture_output=True, + text=True, + ) + + result = json.loads(completed.stdout) + assert result["result"] == "passed" + assert completed.stdout.strip() == json.dumps( + result, + sort_keys=True, + separators=(",", ":"), + ) diff --git a/tests/test_gate14_host_job.py b/tests/test_gate14_host_job.py new file mode 100644 index 000000000..528827b31 --- /dev/null +++ b/tests/test_gate14_host_job.py @@ -0,0 +1,259 @@ +import hashlib +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_host_job as shared # noqa: E402 +import gate14_host_job as host_job # noqa: E402 + + +def sha256(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture +def config_factory(tmp_path, monkeypatch): + def make(platform="linux"): + root = tmp_path / platform + root.mkdir() + adapter = root / "gate14_host_job.py" + adapter.write_bytes(host_job.ADAPTER_PATH.read_bytes()) + entrypoint = root / "gate14_host_lifecycle.py" + entrypoint.write_text("# bound Gate 14 lifecycle\n", encoding="utf-8") + lifecycle_config = root / "gate14-lifecycle.json" + lifecycle_config.write_text('{"bound":true}\n', encoding="utf-8") + python = Path(sys.executable).resolve() + + monkeypatch.setitem(host_job.HOST_ROOTS, platform, root) + monkeypatch.setitem(host_job.HOST_PYTHON, platform, python) + monkeypatch.setattr(host_job, "ADAPTER_PATH", adapter.resolve()) + monkeypatch.setattr(host_job, "LINUX_HOME", "/home/gate14-test") + monkeypatch.setattr(host_job, "LINUX_RUNTIME_DIR", "/qualification/gate14-test/runtime") + + run_id = "gate14-test-a" + raw = { + "schema_version": 1, + "run_id": run_id, + "lifecycle_run_id": run_id, + "platform": platform, + "attempt_ordinal": 1, + "source_commit": "a" * 40, + "job_name": f"communityai-gate14-{run_id}-{platform}", + "host_user": "Gate14Admin" if platform == "windows" else "gate14", + "adapter_path": str(adapter.resolve()), + "adapter_sha256": sha256(adapter), + "config_path": str((root / "host-job.json").resolve()), + "entrypoint_path": str(entrypoint.resolve()), + "entrypoint_sha256": sha256(entrypoint), + "lifecycle_config_path": str(lifecycle_config.resolve()), + "lifecycle_config_sha256": sha256(lifecycle_config), + "evidence_path": str((root / "evidence.json").resolve()), + "stderr_path": str((root / "stderr.log").resolve()), + "status_path": str((root / "status.json").resolve()), + "terminal_path": str((root / "terminal.json").resolve()), + "working_directory": str(root.resolve()), + "python_executable": str(python), + "max_run_seconds": 3600, + } + path = root / "host-job.json" + path.write_text(json.dumps(raw), encoding="utf-8") + return path, raw + + return make + + +def test_tampered_shared_core_is_rejected_before_import(tmp_path): + wrapper = tmp_path / "gate14_host_job.py" + shared_core = tmp_path / "gate13_host_job.py" + wrapper.write_bytes(host_job.ADAPTER_PATH.read_bytes()) + shared_core.write_bytes(host_job._SHARED_CORE_PATH.read_bytes() + b"\nTAMPERED = True\n") + spec = importlib.util.spec_from_file_location("_tampered_gate14_host_job", wrapper) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + + with pytest.raises(ImportError, match="core digest changed"): + spec.loader.exec_module(module) + + +def test_crlf_shared_core_has_the_same_bound_source_digest(tmp_path): + wrapper = tmp_path / "gate14_host_job.py" + shared_core = tmp_path / "gate13_host_job.py" + wrapper.write_bytes(host_job.ADAPTER_PATH.read_bytes()) + shared_core.write_bytes(host_job._SHARED_CORE_PATH.read_bytes().replace(b"\n", b"\r\n")) + spec = importlib.util.spec_from_file_location("_crlf_gate14_host_job", wrapper) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + + spec.loader.exec_module(module) + + assert module._SHARED_CORE_PATH == shared_core.resolve() + + +def test_gate14_config_uses_separate_namespace_root_and_run_binding(config_factory): + path, raw = config_factory() + + config = host_job.load_config(path) + + assert config.run_id == "gate14-test-a" + assert config.lifecycle_run_id == config.run_id + assert config.job_name == "communityai-gate14-gate14-test-a-linux" + assert config.host_user == "gate14" + assert config.adapter_sha256 == raw["adapter_sha256"] + assert config.lifecycle_config_path.name == "gate14-lifecycle.json" + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("job_name", "communityai-gate13-gate14-test-a-linux"), + ("lifecycle_run_id", "gate14-test-a-linux"), + ("host_user", "gate13"), + ], +) +def test_gate13_or_foreign_execution_bindings_are_rejected(config_factory, field, value): + path, raw = config_factory() + raw[field] = value + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(host_job.HostJobError): + host_job.load_config(path) + + +def test_gate14_uses_an_isolated_core_and_never_mutates_gate13_defaults(config_factory): + path, _raw = config_factory() + expected = { + "HOST_ROOTS": shared.HOST_ROOTS, + "HOST_PYTHON": shared.HOST_PYTHON, + "ADAPTER_PATH": shared.ADAPTER_PATH, + "GATE_NAME": shared.GATE_NAME, + "LINUX_HOST_USER": shared.LINUX_HOST_USER, + "LINUX_HOME": shared.LINUX_HOME, + "LINUX_RUNTIME_DIR": shared.LINUX_RUNTIME_DIR, + "LIFECYCLE_CONFIG_NAMES": shared.LIFECYCLE_CONFIG_NAMES, + "LIFECYCLE_RUN_ID_BUILDER": shared.LIFECYCLE_RUN_ID_BUILDER, + "_JOB_RE": shared._JOB_RE, + "MAX_EVIDENCE_BYTES": shared.MAX_EVIDENCE_BYTES, + "EVIDENCE_VALIDATOR": shared.EVIDENCE_VALIDATOR, + } + + host_job.load_config(path) + + assert host_job.core is not shared + assert all(getattr(shared, name) is value for name, value in expected.items()) + + +def test_platform_evidence_validator_calls_strict_gate14_contract(monkeypatch): + document = { + "run_id": "gate14-test-a", + "platform": "linux", + "source_commit": "a" * 40, + } + calls = [] + + def strict(payload): + assert payload == b'{"gate":14}\n' + return document + + def validate(value): + calls.append(value) + + monkeypatch.setattr(host_job.acceptance, "_strict_json", strict) + monkeypatch.setattr(host_job.acceptance, "validate_platform_document", validate) + + assert host_job._validate_platform_evidence(b'{"gate":14}\n') == document + assert calls == [document] + + +def test_execute_is_exactly_once_and_collects_digest_bound_platform_evidence(config_factory, monkeypatch): + path, raw = config_factory() + payload = b'{"gate":14}\n' + calls = [] + + def validate(value): + assert value == payload + return { + "run_id": raw["run_id"], + "platform": raw["platform"], + "source_commit": raw["source_commit"], + } + + def entrypoint(config): + calls.append(config.run_id) + config.evidence_path.write_bytes(payload) + config.stderr_path.write_bytes(b"") + return 0 + + monkeypatch.setattr(host_job, "_validate_platform_evidence", validate) + + first = host_job.execute(path, clock=lambda: 100, entrypoint_runner=entrypoint) + second = host_job.execute(path, clock=lambda: 200, entrypoint_runner=lambda _config: 99) + + assert first == second + assert first["result"] == "passed" + assert first["evidence_digest"] == "sha256:" + hashlib.sha256(payload).hexdigest() + assert calls == [raw["run_id"]] + assert host_job.collect(path) == payload + + +def test_invalid_platform_evidence_fails_terminally(config_factory, monkeypatch): + path, _raw = config_factory() + + def reject(_payload): + raise ValueError("not Gate 14 platform evidence") + + def entrypoint(config): + config.evidence_path.write_text('{"gate":13}\n', encoding="utf-8") + config.stderr_path.write_bytes(b"") + return 0 + + monkeypatch.setattr(host_job, "_validate_platform_evidence", reject) + + terminal = host_job.execute(path, clock=lambda: 100, entrypoint_runner=entrypoint) + + assert terminal["result"] == "failed" + assert terminal["failure_code"] == "invalid_lifecycle_evidence" + assert terminal["evidence_digest"] is None + with pytest.raises(host_job.HostJobError, match="successful terminal"): + host_job.collect(path) + + +def test_linux_native_command_is_gate14_bound(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + + host_job._configure_core() + argv = host_job.core._linux_start_argv(config) + + assert "--unit" in argv + assert argv[argv.index("--unit") + 1] == config.job_name + assert f"--setenv=HOME={host_job.LINUX_HOME}" in argv + assert f"--setenv=XDG_RUNTIME_DIR={host_job.LINUX_RUNTIME_DIR}" in argv + assert str(host_job.ADAPTER_PATH) in argv + + +def test_linux_desktop_session_accepts_gate14_home_and_runtime(config_factory, monkeypatch): + path, _raw = config_factory() + host_job._configure_core() + monkeypatch.setattr(host_job.core.sys, "platform", "linux") + monkeypatch.setenv("DISPLAY", ":99") + monkeypatch.setenv("HOME", host_job.LINUX_HOME) + monkeypatch.setenv("XDG_RUNTIME_DIR", host_job.LINUX_RUNTIME_DIR) + monkeypatch.setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/gate14/bus") + monkeypatch.setattr( + host_job.core.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args[0], 0, "", ""), + ) + monkeypatch.setattr(host_job.core, "execute", lambda value: {"result": "passed", "config": value}) + + assert host_job.core._execute_linux_desktop_session(path) == { + "result": "passed", + "config": path, + } diff --git a/tests/test_gate14_host_probe.py b/tests/test_gate14_host_probe.py new file mode 100644 index 000000000..504f80878 --- /dev/null +++ b/tests/test_gate14_host_probe.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_calibration_challenge as challenge_contract # noqa: E402 +import gate14_hardware_acceptance as acceptance # noqa: E402 +import gate14_host_probe as probe # noqa: E402 + +SOURCE = "1" * 40 +RUN_ID = "gate14-20260902-a" +CHALLENGE_ISSUED = 2_000_000_000 + + +def calibration_challenge(platform: str, package_payload: bytes) -> dict: + return dict( + challenge_contract.create( + run_id=RUN_ID, + platform=platform, + source_commit=SOURCE, + package_sha256="sha256:" + hashlib.sha256(package_payload).hexdigest(), + checkpoint_sha256="sha256:" + "c" * 64, + controller_state_revision=2, + issued_at_unix=CHALLENGE_ISSUED, + nonce="a" * 64, + ) + ) + + +def calibration(kind: str, challenge_sha256: str) -> dict: + return { + "measurement_source": { + "bandwidth": "host-network-counters", + "power": "nvidia-nvml-device-power", + "schedule": "utc-policy-clock", + }[kind], + "measurement_scope": { + "bandwidth": "aggregate-host-network", + "power": "selected-nvidia-l4-device", + "schedule": "utc-schedule-policy", + }[kind], + "sample_count": 4, + "sample_interval_seconds": 0.5, + "baseline_value": 1.0 if kind == "schedule" else 10.0, + "configured_limit": {"bandwidth": 100.0, "power": 250.0, "schedule": 0.5}[kind], + "trigger_value": 0.0 if kind == "schedule" else (120.0 if kind == "bandwidth" else 275.0), + "resume_value": 1.0 if kind == "schedule" else 10.0, + "challenge_sha256": challenge_sha256, + "sample_started_at_unix": CHALLENGE_ISSUED + 10, + "sample_ended_at_unix": CHALLENGE_ISSUED + 12, + } + + +def facts(platform: str, package_payload: bytes) -> dict: + challenge_sha256 = challenge_contract.digest(calibration_challenge(platform, package_payload)) + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + selected = profile["selected_artifact_bytes"] + return { + "schema_version": 1, + "scope": probe.FACT_SCOPE, + "run_id": RUN_ID, + "platform": platform, + "source_commit": SOURCE, + "gate13_evidence_sha256": acceptance.EXPECTED_GATE13_EVIDENCE_SHA256, + "expected_package_sha256": "sha256:" + hashlib.sha256(package_payload).hexdigest(), + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "revision_commit": profile["revision_commit"], + "gate9_envelope_sha256": acceptance.EXPECTED_GATE9_ENVELOPES[platform], + "selected_artifact_count": profile["selected_artifact_count"], + "selected_artifact_bytes": selected, + "total_blocks": profile["total_blocks"], + }, + "cache": { + "verified_bytes_before": selected, + "verified_bytes_after": selected, + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": 4, + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "suspensions": [ + { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": 2.0, + "calibration": calibration(kind, challenge_sha256), + } + for kind in ("bandwidth", "power", "schedule") + ], + "recovery": { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 3.0, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + "pause": { + "requested": True, + "completed": True, + "duration_seconds": 2.0, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + "restart": { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 5.0, + "cache_reused": True, + }, + "unsupported_telemetry": { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + "qualification_temporaries_removed": True, + } + + +def hardware(platform: str) -> dict: + return { + "os_name": "Windows Server 2022" if platform == "windows" else "Ubuntu 24.04", + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": 24 * 1024**3, + } + + +def write_inputs(tmp_path: Path, platform: str = "windows"): + package_payload = b"source-bound-production-package" + package = tmp_path / "package.zip" + metadata = tmp_path / "release-metadata.json" + facts_path = tmp_path / "facts.json" + challenge_path = tmp_path / "challenge.json" + output = tmp_path / "evidence.json" + package.write_bytes(package_payload) + metadata.write_text('{"schema_version":1}', encoding="utf-8") + facts_path.write_text(json.dumps(facts(platform, package_payload)), encoding="utf-8") + challenge_path.write_text( + json.dumps(calibration_challenge(platform, package_payload)), + encoding="utf-8", + ) + return facts_path, challenge_path, package, metadata, output + + +def test_probe_hashes_inputs_measures_hardware_and_emits_only_safe_evidence(tmp_path): + facts_path, challenge_path, package, metadata, output = write_inputs(tmp_path) + + document = probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED + 20, + ) + + assert acceptance.validate_platform_document(document)["platform"] == "windows" + assert document["package"]["archive_bytes"] == package.stat().st_size + assert document["hardware"]["accelerator"] == "NVIDIA L4" + assert all(value is False for value in document["privacy"].values()) + assert json.loads(output.read_text(encoding="utf-8")) == document + + +def test_probe_rejects_package_drift_and_does_not_publish(tmp_path): + facts_path, challenge_path, package, metadata, output = write_inputs(tmp_path) + package.write_bytes(b"changed") + + with pytest.raises(probe.Gate14ProbeError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED + 20, + ) + + assert not output.exists() + + +def test_probe_rejects_uncalibrated_physical_trigger(tmp_path): + facts_path, challenge_path, package, metadata, output = write_inputs(tmp_path) + value = json.loads(facts_path.read_text(encoding="utf-8")) + value["suspensions"][0]["calibration"]["trigger_value"] = 50.0 + facts_path.write_text(json.dumps(value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED + 20, + ) + + +def test_probe_rejects_expired_or_mismatched_challenge(tmp_path): + facts_path, challenge_path, package, metadata, output = write_inputs(tmp_path) + + with pytest.raises(challenge_contract.Gate14ChallengeError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED - 1, + ) + assert not output.exists() + + with pytest.raises(probe.Gate14ProbeError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED, + ) + assert not output.exists() + + with pytest.raises(challenge_contract.Gate14ChallengeError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED + challenge_contract.MAX_LIFETIME_SECONDS + 1, + ) + assert not output.exists() + + value = json.loads(facts_path.read_text(encoding="utf-8")) + value["suspensions"][0]["calibration"]["sample_started_at_unix"] = CHALLENGE_ISSUED + 49 + value["suspensions"][0]["calibration"]["sample_ended_at_unix"] = CHALLENGE_ISSUED + 51 + facts_path.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises(probe.Gate14ProbeError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED + 20, + ) + assert not output.exists() + + package_payload = package.read_bytes() + value = facts("windows", package_payload) + value["suspensions"][0]["calibration"]["challenge_sha256"] = "sha256:" + "f" * 64 + facts_path.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises(acceptance.Gate14EvidenceError): + probe.run_probe( + platform_name="windows", + facts_path=facts_path, + challenge_path=challenge_path, + package_path=package, + release_metadata_path=metadata, + output_path=output, + hardware_probe=hardware, + now_unix=CHALLENGE_ISSUED + 20, + ) + assert not output.exists() + + +def test_probe_hardware_requires_one_l4(monkeypatch): + monkeypatch.setattr(probe, "_operating_system", lambda platform: "Ubuntu 24.04") + + def runner(argv, timeout): + assert tuple(argv) == ( + "nvidia-smi", + "--query-gpu=name,memory.total", + "--format=csv,noheader,nounits", + ) + assert timeout == 30 + return subprocess.CompletedProcess(argv, 0, "NVIDIA L4, 23034\n", "") + + result = probe.probe_hardware("linux", runner=runner) + + assert result["accelerator_count"] == 1 + assert result["accelerator_memory_bytes"] == 23034 * 1024**2 + + +def test_platform_wrappers_fix_their_platform(): + windows = (ROOT / "scripts" / "gate14_windows_probe.ps1").read_text(encoding="utf-8") + linux = (ROOT / "scripts" / "gate14_linux_probe.sh").read_text(encoding="utf-8") + + assert "--platform windows" in windows + assert "--platform linux" in linux + assert "macos" not in windows.casefold() + linux.casefold() diff --git a/tests/test_gate14_linux_action_transport.py b/tests/test_gate14_linux_action_transport.py new file mode 100644 index 000000000..2df2890ae --- /dev/null +++ b/tests/test_gate14_linux_action_transport.py @@ -0,0 +1,289 @@ +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_calibration_challenge as challenge_contract # noqa: E402 +import gate14_linux_action_transport as transport # noqa: E402 + + +def config_fixture(tmp_path): + path = tmp_path / "gate14-lifecycle.json" + path.write_text('{"bound":true}\n', encoding="utf-8") + return SimpleNamespace( + attempt_ordinal=1, + config_path=path, + config_sha256="sha256:" + hashlib.sha256(path.read_bytes()).hexdigest(), + package_sha256="sha256:" + "b" * 64, + platform="linux", + run_id="gate14-linux-rpc-a", + source_commit="a" * 40, + ) + + +def challenge(): + return challenge_contract.create( + run_id="gate14-linux-rpc-a", + platform="linux", + source_commit="a" * 40, + package_sha256="sha256:" + "b" * 64, + checkpoint_sha256="sha256:" + "c" * 64, + controller_state_revision=7, + issued_at_unix=1_000, + lifetime_seconds=900, + nonce="d" * 64, + ) + + +def test_challenge_payload_contains_the_full_safe_time_binding(): + value = challenge() + assert transport._challenge_payload(value) == { + "challenge_sha256": challenge_contract.digest(value), + "controller_state_revision": 7, + "issued_at_unix": 1_000, + "expires_at_unix": 1_900, + } + + +@pytest.mark.parametrize( + "timeouts", + [ + {}, + {"prepare": 1, "calibrate": 1, "cleanup": 1, "extra": 1}, + {"prepare": True, "calibrate": 1, "cleanup": 1}, + {"prepare": 7_201, "calibrate": 1, "cleanup": 1}, + {"prepare": 1, "calibrate": 3_601, "cleanup": 1}, + {"prepare": 1, "calibrate": 1, "cleanup": 601}, + ], +) +def test_operation_timeout_contract_fails_closed(timeouts): + with pytest.raises(transport.Gate14ActionTransportError, match="timeout"): + transport._timeouts(timeouts) + + +def test_duplicate_nonfinite_and_private_response_material_fail_closed(): + with pytest.raises(transport.Gate14ActionTransportError, match="duplicate"): + transport._strict_json(b'{"result":"passed","result":"failed"}') + with pytest.raises(transport.Gate14ActionTransportError, match="non-finite"): + transport._strict_json(b'{"sample":NaN}') + with pytest.raises(transport.Gate14ActionTransportError, match="private"): + transport._assert_safe_payload({"control_token": "not-serialized"}) + with pytest.raises(transport.Gate14ActionTransportError, match="private"): + transport._assert_safe_payload({"value": "drift_control_never-serialized"}) + + +def test_native_host_preserves_one_process_and_state_across_operations(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + + action_host = transport.LinuxActionTransport( + config, + python=sys.executable, + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + prepared = action_host.request("prepare", {}) + calibrated = action_host.request( + "calibrate", + transport._challenge_payload(challenge()), + ) + cleaned = action_host.request("cleanup", {}) + + assert prepared["helpers_verified"] is True + assert prepared["host_process_id"] == calibrated["host_process_id"] + assert prepared["state_nonce"] == calibrated["state_nonce"] + assert calibrated["challenge_sha256"] == challenge_contract.digest(challenge()) + assert cleaned == { + "action_temporaries_removed": True, + "attempt_ordinal": 1, + "credentials_removed": True, + "platform": "linux", + "processes_absent": True, + "run_id": "gate14-linux-rpc-a", + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + } + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +def test_native_host_runs_cleanup_on_eof(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.LinuxActionTransport( + config, + python=sys.executable, + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + action_host.request("prepare", {}) + action_host.close() + assert marker.read_text(encoding="utf-8") == "cleaned" + + +def test_native_host_rejects_out_of_order_operation_and_cleans(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.LinuxActionTransport( + config, + python=sys.executable, + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + with pytest.raises( + transport.Gate14ActionTransportError, + match="response binding is invalid", + ): + action_host.request( + "calibrate", + transport._challenge_payload(challenge()), + ) + assert marker.read_text(encoding="utf-8") == "cleaned" + + +def test_production_prepare_fails_closed_when_required_inputs_are_absent(tmp_path): + config = config_fixture(tmp_path) + with transport.LinuxActionTransport(config, python=sys.executable) as action_host: + with pytest.raises( + transport.Gate14ActionTransportError, + match="product-prepare-failed", + ): + action_host.prepare(config) + + +@pytest.mark.parametrize( + "attack", + [ + {"challenge_sha256": "sha256:" + "c" * 64}, + { + "challenge_sha256": "sha256:" + "c" * 64, + "controller_state_revision": True, + "issued_at_unix": 1_000, + "expires_at_unix": 1_900, + }, + { + "challenge_sha256": "sha256:" + "c" * 64, + "controller_state_revision": 1, + "issued_at_unix": 1_900, + "expires_at_unix": 1_000, + }, + ], +) +def test_native_host_rejects_incomplete_or_coerced_challenge(tmp_path, attack): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.LinuxActionTransport( + config, + python=sys.executable, + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + action_host.request("prepare", {}) + with pytest.raises(transport.Gate14ActionTransportError): + action_host.request("calibrate", attack) + assert marker.read_text(encoding="utf-8") == "cleaned" + + +def test_python_transport_rejects_boolean_integer_response_fields(tmp_path): + config = config_fixture(tmp_path) + child = """ +import json +import sys +request = json.loads(sys.stdin.readline()) +response = { + "failure_code": None, + "operation": request["operation"], + "payload": {}, + "request_id": True, + "result": "passed", + "schema_version": 1, + "scope": request["scope"], + "session_id": request["session_id"], +} +print(json.dumps(response, separators=(",", ":"), sort_keys=True), flush=True) +""" + + def process_factory(_arguments, **kwargs): + return subprocess.Popen( + [sys.executable, "-u", "-c", child], + stdin=kwargs["stdin"], + stdout=kwargs["stdout"], + stderr=kwargs["stderr"], + bufsize=kwargs["bufsize"], + start_new_session=kwargs["start_new_session"], + ) + + action_host = transport.LinuxActionTransport( + config, + python=sys.executable, + process_factory=process_factory, + ) + try: + with pytest.raises( + transport.Gate14ActionTransportError, + match="response binding is invalid", + ): + action_host.request("prepare", {}) + finally: + action_host.close() + + +def test_python_transport_rejects_noncanonical_response_frame(tmp_path): + config = config_fixture(tmp_path) + child = """ +import json +import sys + +request = json.loads(sys.stdin.readline()) +response = { + "failure_code": None, + "operation": request["operation"], + "payload": {}, + "request_id": request["request_id"], + "result": "passed", + "schema_version": 1, + "scope": request["scope"], + "session_id": request["session_id"], +} +print(json.dumps(response), flush=True) +""" + + def process_factory(_arguments, **kwargs): + return subprocess.Popen([sys.executable, "-u", "-c", child], **kwargs) + + action_host = transport.LinuxActionTransport( + config, + python=sys.executable, + process_factory=process_factory, + ) + try: + with pytest.raises( + transport.Gate14ActionTransportError, + match="response is invalid", + ): + action_host.request("prepare", {}) + finally: + action_host.close() + + +def test_normalized_helper_binding_accepts_crlf_and_rejects_mutation(tmp_path): + original = ROOT / "scripts" / "gate13_linux_packaged_lifecycle.py" + payload = original.read_bytes().replace(b"\r\n", b"\n") + expected = hashlib.sha256(payload).hexdigest() + crlf = tmp_path / original.name + crlf.write_bytes(payload.replace(b"\n", b"\r\n")) + + assert transport._normalized_source(crlf, expected) == crlf.resolve() + + crlf.write_bytes(crlf.read_bytes() + b"# mutation\r\n") + with pytest.raises(transport.Gate14ActionTransportError, match="digest changed"): + transport._normalized_source(crlf, expected) diff --git a/tests/test_gate14_linux_product_actions.py b/tests/test_gate14_linux_product_actions.py new file mode 100644 index 000000000..73898ecd2 --- /dev/null +++ b/tests/test_gate14_linux_product_actions.py @@ -0,0 +1,369 @@ +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_linux_product_actions as actions # noqa: E402 + +RUN_ID = "gate14-linux-product-a" +SOURCE_COMMIT = "a" * 40 +PACKAGE_SHA256 = "sha256:" + "b" * 64 +MODEL_ID = "Qwen3.5 2B" +PROFILE = actions.MODEL_PROFILES[MODEL_ID] + + +class FakeOwner: + instances = [] + + def __init__(self, name): + self.name = name + self.stopped = False + self.product = SimpleNamespace(pid=4100) + self.__class__.instances.append(self) + + def process_ids(self, _unit): + return set() if self.stopped else {4100, 4200} + + def process_count(self): + return 0 if self.stopped else 2 + + def stop_all(self): + self.stopped = True + + +def _write_inputs(tmp_path): + work_root = tmp_path / "work" + work_root.mkdir() + warm_cache = work_root / actions.WARM_CACHE_NAME + warm_cache.mkdir() + (warm_cache / "weights.bin").write_bytes(b"verified") + + package = tmp_path / "communityai-linux.tar.gz" + package.write_bytes(b"archive") + staging_root = tmp_path / "staging" + audit_root = staging_root / "release-audit" + audit_root.mkdir(parents=True) + for name in ( + "SHA256SUMS", + "desktop-metrics.json", + "provenance.json", + "release-metadata.json", + ): + (audit_root / name).write_text("{}\n", encoding="utf-8") + + config = { + "run_id": RUN_ID, + "attempt_ordinal": 1, + "source_commit": SOURCE_COMMIT, + "package_sha256": PACKAGE_SHA256, + "platform": "linux", + "model_id": MODEL_ID, + "manifest_digest": PROFILE["manifest_digest"], + "work_root": str(work_root), + "package_path": str(package), + "package_bytes": package.stat().st_size, + "staging_root": str(staging_root), + "disk_bytes": 20_000_000_000, + "vram_bytes": 16_000_000_000, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "pause_timeout_seconds": 30.0, + "sample_interval_seconds": 1.0, + "warm_cache": { + "artifacts": [ + { + "path": "weights.bin", + "role": "model", + "sha256": "sha256:" + "c" * 64, + "size_bytes": PROFILE["selected_artifact_bytes"], + } + ] + }, + } + config_path = tmp_path / "gate14-lifecycle.json" + config_path.write_text( + json.dumps(config, separators=(",", ":"), sort_keys=True), + encoding="utf-8", + ) + return config_path, work_root + + +def _fake_gate13(monkeypatch, *, audit_failure=None): + credential = {"present": False} + FakeOwner.instances = [] + + def strict_json(payload, maximum=None): + if maximum is not None and len(payload) > maximum: + raise ValueError("oversized") + return json.loads(payload) + + def audit_package(_release_root, _digest, _size): + if audit_failure is not None: + raise audit_failure + return SimpleNamespace( + source_commit=SOURCE_COMMIT, + package_version="1.0.0-alpha", + ) + + def extract_package(_audit, install_root): + product_root = install_root / "CommunityAI" + (product_root / "node").mkdir(parents=True) + + def bootstrap(_owner, _product_root, persistent_root, *_args): + manifest = persistent_root / "manifest.json" + manifest.write_text("{}\n", encoding="utf-8") + return persistent_root / "bootstrap.json", manifest + + def verify_cache(_cache, _manifest_digest, _artifacts): + return PROFILE["selected_artifact_bytes"], {"weights.bin": (1, 2, 3, 4, 5)} + + def store_control_token(_token): + credential["present"] = True + + def clear_control_token(): + credential["present"] = False + + fake = SimpleNamespace( + ARCHIVE_NAME="communityai-linux.tar.gz", + ProxyHandler=lambda _mapping: object(), + SystemdUnitOwner=FakeOwner, + _RejectRedirects=lambda: object(), + _assert_cache_unchanged=lambda *_args: PROFILE["selected_artifact_bytes"], + _audit_package=audit_package, + _bootstrap=bootstrap, + _clear_control_token=clear_control_token, + _control_request=lambda *_args, **_kwargs: {}, + _credential_count=lambda: int(credential["present"]), + _extract_package=extract_package, + _run_self_tests=lambda *_args: None, + _start_products=lambda owner, *_args: (owner.product, object()), + _status_identity=lambda *_args: None, + _stop_products=lambda owner, *_args: owner.stop_all(), + _store_control_token=store_control_token, + _strict_json=strict_json, + _verify_cache=verify_cache, + build_opener=lambda *_args: object(), + ) + monkeypatch.setattr(actions, "gate13", fake) + return credential + + +def _product(tmp_path, monkeypatch, *, audit_failure=None): + config_path, work_root = _write_inputs(tmp_path) + credential = _fake_gate13(monkeypatch, audit_failure=audit_failure) + product = actions.LinuxProductActions( + config_path=config_path, + run_id=RUN_ID, + attempt_ordinal=1, + source_commit=SOURCE_COMMIT, + package_sha256=PACKAGE_SHA256, + clock=lambda: 1_100.0, + ) + return product, work_root, credential + + +def _running_worker(): + return { + "automatic": True, + "block_indices": "0:24", + "desired_running": True, + "intent_published": True, + "model": MODEL_ID, + "pid": 4200, + "remote_acknowledged": True, + "state": "running", + } + + +def test_prepare_calibrate_and_cleanup_with_controlled_product_boundaries(tmp_path, monkeypatch): + product, work_root, credential = _product(tmp_path, monkeypatch) + worker = _running_worker() + api_calls = [] + + def request(method, path, payload=None): + api_calls.append((method, path)) + if method == "GET" and path == "/control/v1/contribution-policy": + return { + "schema_version": 1, + "config_revision": "revision-a", + "policy": product.expected_policy, + } + if method == "PUT" and path == "/control/v1/contribution-policy": + return { + "schema_version": 1, + "config_revision": "revision-b", + "policy": payload["policy"], + } + return {} + + monkeypatch.setattr(product, "_request", request) + monkeypatch.setattr(product, "_wait_running", lambda timeout=300.0: worker) + monkeypatch.setattr( + product, + "_status_worker", + lambda: { + "id": "automatic", + "placement": {"automatic": True, "block_indices": "0:24"}, + "resources": { + "limits": { + "disk_bytes": product.config["disk_bytes"], + "vram_bytes": product.config["vram_bytes"], + "bandwidth_mbps": product.config["bandwidth_mbps"], + "power_watts": product.config["power_watts"], + } + }, + }, + ) + monkeypatch.setattr(product, "_low_vram_probe", lambda: None) + monkeypatch.setattr( + product, + "_cpu_power_probe", + lambda: { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + ) + monkeypatch.setattr( + product, + "_crash_recovery", + lambda: { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 1.0, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + ) + monkeypatch.setattr( + product, + "_pause", + lambda: { + "requested": True, + "completed": True, + "duration_seconds": 1.0, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + ) + monkeypatch.setattr( + product, + "_restart", + lambda: { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 1.0, + "cache_reused": True, + }, + ) + + prepared = product.prepare() + + assert prepared["scope"] == "gate14-prepared-host-observations" + assert prepared["placement"] == { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": 24, + "intent_published": True, + "remote_acknowledged": True, + } + assert prepared["cache"]["verified_bytes_before"] == PROFILE["selected_artifact_bytes"] + assert prepared["cache"]["verified_bytes_after"] == PROFILE["selected_artifact_bytes"] + assert prepared["limits"]["resource_limit_count"] == 5 + assert credential["present"] is True + assert ("POST", "/control/v1/workers/automatic/start") in api_calls + + calibration_calls = [] + + def record(kind): + calibration_calls.append(kind) + return {"kind": kind, "suspended": True, "resumed": True} + + monkeypatch.setattr(product, "_calibrate_bandwidth", lambda _challenge: record("bandwidth")) + monkeypatch.setattr(product, "_calibrate_power", lambda _challenge: record("power")) + monkeypatch.setattr(product, "_calibrate_schedule", lambda _challenge: record("schedule")) + challenge = { + "challenge_sha256": "sha256:" + "d" * 64, + "controller_state_revision": 7, + "issued_at_unix": 1_000, + "expires_at_unix": 1_200, + } + + calibrated = product.calibrate(challenge) + + assert [item["kind"] for item in calibrated] == [ + "bandwidth", + "power", + "schedule", + ] + assert calibration_calls == ["bandwidth", "power", "schedule"] + + cleanup = product.cleanup() + + assert cleanup == { + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + "run_id": RUN_ID, + "platform": "linux", + "attempt_ordinal": 1, + "processes_absent": True, + "credentials_removed": True, + "action_temporaries_removed": True, + } + assert credential["present"] is False + assert not (work_root / actions.ACTION_ROOT_NAME).exists() + assert not (work_root / actions.WARM_CACHE_NAME).exists() + assert FakeOwner.instances[0].stopped is True + + +def test_prepare_failure_runs_exact_cleanup_and_preserves_original_error(tmp_path, monkeypatch): + product, work_root, credential = _product( + tmp_path, + monkeypatch, + audit_failure=actions.Gate14LinuxProductError("audit rejected"), + ) + + with pytest.raises(actions.Gate14LinuxProductError, match="audit rejected"): + product.prepare() + + assert credential["present"] is False + assert not (work_root / actions.ACTION_ROOT_NAME).exists() + assert not (work_root / actions.WARM_CACHE_NAME).exists() + assert product.cleaned is True + + +def test_prepare_start_failure_after_credential_runs_exact_cleanup(tmp_path, monkeypatch): + product, work_root, credential = _product(tmp_path, monkeypatch) + startup_error = RuntimeError("product startup failed") + + def fail_start(owner, *_args): + assert credential["present"] is True + assert owner is FakeOwner.instances[0] + raise startup_error + + monkeypatch.setattr(actions.gate13, "_start_products", fail_start) + + with pytest.raises( + actions.Gate14LinuxProductError, + match="packaged product prepare failed", + ) as caught: + product.prepare() + + assert caught.value.__cause__ is startup_error + assert FakeOwner.instances[0].stopped is True + assert credential["present"] is False + assert not (work_root / actions.ACTION_ROOT_NAME).exists() + assert not (work_root / actions.WARM_CACHE_NAME).exists() + assert product.cleaned is True diff --git a/tests/test_gate14_packaged_lifecycle.py b/tests/test_gate14_packaged_lifecycle.py new file mode 100644 index 000000000..6ae3a4e35 --- /dev/null +++ b/tests/test_gate14_packaged_lifecycle.py @@ -0,0 +1,1337 @@ +from __future__ import annotations + +import hashlib +import json +import stat +import sys +import zipfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_calibration_challenge as challenge_contract # noqa: E402 +import gate14_hardware_acceptance as acceptance # noqa: E402 +import gate14_packaged_lifecycle as lifecycle # noqa: E402 + +SOURCE = "1" * 40 +RUN_ID = "gate14-lifecycle-test-a" +NOW = 2_000_000_000 +PACKAGE_PAYLOAD = b"source-bound-production-package" +PACKAGE_SHA256 = "sha256:" + hashlib.sha256(PACKAGE_PAYLOAD).hexdigest() +REAL_CONTROLLER_GUARD = lifecycle._assert_controller_owned + + +class Clock: + def __init__(self): + self.wall = float(NOW) + self.steady = 0.0 + + def time(self): + return self.wall + + def monotonic(self): + return self.steady + + def sleep(self, seconds): + self.steady += seconds + + +def model(platform="windows"): + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + return { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "revision_commit": profile["revision_commit"], + "gate9_envelope_sha256": acceptance.EXPECTED_GATE9_ENVELOPES[platform], + "selected_artifact_count": profile["selected_artifact_count"], + "selected_artifact_bytes": profile["selected_artifact_bytes"], + "total_blocks": profile["total_blocks"], + } + + +def prepared(platform="windows"): + selected = model(platform)["selected_artifact_bytes"] + return { + "schema_version": 1, + "scope": lifecycle.PREPARED_SCOPE, + "run_id": RUN_ID, + "platform": platform, + "attempt_ordinal": 1, + "source_commit": SOURCE, + "package_sha256": PACKAGE_SHA256, + "model": model(platform), + "cache": { + "verified_bytes_before": selected, + "verified_bytes_after": selected, + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": 4, + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "recovery": { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 3.0, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + "pause": { + "requested": True, + "completed": True, + "duration_seconds": 2.0, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + "restart": { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 5.0, + "cache_reused": True, + }, + "unsupported_telemetry": { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + } + + +def calibrations(challenge): + challenge_sha256 = challenge_contract.digest(challenge) + + def item(kind): + return { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": 2.0, + "calibration": { + "measurement_source": { + "bandwidth": "host-network-counters", + "power": "nvidia-nvml-device-power", + "schedule": "utc-policy-clock", + }[kind], + "measurement_scope": { + "bandwidth": "aggregate-host-network", + "power": "selected-nvidia-l4-device", + "schedule": "utc-schedule-policy", + }[kind], + "sample_count": 4, + "sample_interval_seconds": 0.5, + "baseline_value": 1.0 if kind == "schedule" else 10.0, + "configured_limit": { + "bandwidth": 100.0, + "power": 250.0, + "schedule": 0.5, + }[kind], + "trigger_value": (0.0 if kind == "schedule" else (120.0 if kind == "bandwidth" else 275.0)), + "resume_value": 1.0 if kind == "schedule" else 10.0, + "challenge_sha256": challenge_sha256, + "sample_started_at_unix": challenge["issued_at_unix"] + 1, + "sample_ended_at_unix": challenge["issued_at_unix"] + 3, + }, + } + + return [item(kind) for kind in ("bandwidth", "power", "schedule")] + + +def cleanup(config, **overrides): + value = { + "schema_version": 1, + "scope": lifecycle.CLEANUP_SCOPE, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": config.attempt_ordinal, + "processes_absent": True, + "credentials_removed": True, + "action_temporaries_removed": True, + } + value.update(overrides) + return value + + +def _json_bytes(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + + +def _release_audit(staging, platform, source_commit, package): + audit_root = staging / lifecycle._RELEASE_AUDIT_DIRECTORY_NAME + audit_root.mkdir() + title = platform.title() + package_name = lifecycle._PACKAGE_NAMES[platform] + package_digest = hashlib.sha256(package.read_bytes()).hexdigest() + archive_record = { + "schema_version": 1, + "path": package_name, + "format": "zip" if platform == "windows" else "tar.gz", + "platform": title, + "artifact_root": "CommunityAI", + "sha256": package_digest, + "size_bytes": package.stat().st_size, + "entry_count": 1, + "preserves_executable_modes": platform == "linux", + "preserves_internal_file_symlinks": platform == "linux", + } + publication = {"scope": "test-publication-binding"} + release_artifacts = { + "schema_version": 1, + "artifact_count": 1, + "artifact_bytes": 1, + "checksums_sha256": hashlib.sha256(b"0" * 64 + b" CommunityAI/test.bin\n").hexdigest(), + "source_commit": source_commit, + "source_tree": "2" * 40, + "unsigned": True, + "complete_release_qualification": False, + "install_archive": archive_record, + } + metrics = { + "schema_version": 1, + "application": "CommunityAI", + "package": "communityai-desktop", + "platform": f"{title}-test", + "signed": False, + "catalog_bootstrap_bundled": True, + "catalog_publication_bundle": publication, + "node_sidecar": { + "self_test_passed": True, + "node_entrypoint_smoke_passed": True, + "worker_entrypoint_smoke_passed": True, + "worker_self_test_passed": True, + }, + "release_artifacts": release_artifacts, + } + metrics_payload = _json_bytes(metrics) + artifact = { + "path": "CommunityAI/test.bin", + "kind": "file", + "sha256": "0" * 64, + "size_bytes": 1, + "mode": 0o755, + } + provenance = { + "schema_version": 1, + "product": "CommunityAI", + "package": "communityai-desktop", + "release_channel": "public-alpha", + "source_commit": source_commit, + "source_tree": "2" * 40, + "build_workflow": "test", + "build_platform": f"{title}-test", + "build_python": "3.12", + "build_pyinstaller": "test", + "artifact_root": "CommunityAI", + "checksum_manifest": "SHA256SUMS", + "artifacts": [artifact], + "install_archive": archive_record, + "desktop_metrics": { + "schema_version": 1, + "path": "desktop-metrics.json", + "sha256": hashlib.sha256(metrics_payload).hexdigest(), + "size_bytes": len(metrics_payload), + }, + "catalog_publication_bundle": publication, + "unsigned": True, + "publisher_signature": False, + "automatic_updates": False, + "complete_release_qualification": False, + } + member_payloads = { + "SHA256SUMS": b"0" * 64 + b" CommunityAI/test.bin\n", + "desktop-metrics.json": metrics_payload, + "provenance.json": _json_bytes(provenance), + "release-metadata.json": _json_bytes(lifecycle._RELEASE_METADATA), + } + for name, payload in member_payloads.items(): + (audit_root / name).write_bytes(payload) + + archive_path = staging / lifecycle._RELEASE_AUDIT_ARCHIVE_NAME + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_STORED) as archive: + for name in lifecycle._RELEASE_AUDIT_MEMBERS: + archive.writestr(name, member_payloads[name]) + archive_payload = archive_path.read_bytes() + binding = { + "schema_version": 1, + "artifact_name": f"communityai-desktop-audit-{platform}", + "artifact_sha256": lifecycle._digest(archive_payload), + "artifact_bytes": len(archive_payload), + "members": [ + { + "name": name, + "sha256": lifecycle._digest(member_payloads[name]), + "size_bytes": len(member_payloads[name]), + } + for name in lifecycle._RELEASE_AUDIT_MEMBERS + ], + } + return binding, audit_root / "release-metadata.json" + + +def _warm_cache(staging, platform): + expected = lifecycle._GATE9_WARM_CACHE[platform] + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + artifacts = [ + { + "path": path, + "role": role, + "sha256": digest, + "size_bytes": size, + } + for path, role, digest, size in expected["artifacts"] + ] + materialized = [ + { + "path": item["path"], + "role": item["role"], + "sha256": item["sha256"].removeprefix("sha256:"), + "size_bytes": item["size_bytes"], + "materialization_attempts": 1, + "resumptions": 0, + "resumed_from_bytes": [], + "elapsed_seconds": 0.1, + } + for item in artifacts + ] + record = { + "schema_version": 1, + "acquired_at_unix": NOW - 60, + "runtime": {"python": "3.12", "platform": platform, "drift": "test"}, + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "repository": lifecycle._MODEL_SOURCE[model_id][0], + "revision": profile["revision_commit"], + "dtype": lifecycle._MODEL_SOURCE[model_id][1], + }, + "selection": { + "startup_artifact_paths": sorted( + item["path"] + for item in artifacts + if item["role"] in {"chat_template", "config", "tokenizer", "weight_index"} + ), + "weight_artifact_paths": sorted(item["path"] for item in artifacts if item["role"] == "weight"), + "artifact_count": len(artifacts), + "artifact_bytes": sum(item["size_bytes"] for item in artifacts), + "weight_artifact_bytes": sum(item["size_bytes"] for item in artifacts if item["role"] == "weight"), + }, + "artifacts": materialized, + "transfer": { + "direct_upstream_transfer": True, + "mirror_used": False, + "source_class_verified": True, + "transport_override_present": False, + "elapsed_seconds": 1.0, + "max_resumptions": 3, + "resumptions": 0, + "completed": True, + }, + "storage": { + "cold_start": True, + "cache_bytes_before": 0, + "cache_bytes_after": profile["selected_artifact_bytes"], + "cache_growth_bytes": profile["selected_artifact_bytes"], + "verified": True, + }, + "privacy": { + "credentials_retained": False, + "local_paths_retained": False, + "response_bodies_retained": False, + "urls_retained": False, + }, + } + payload = _json_bytes(record) + record_path = staging / lifecycle._MATERIALIZATION_RECORD_NAME + record_path.write_bytes(payload) + return { + "schema_version": 1, + "layout": "manifest-artifacts-v1", + "gate9_acquisition_record_sha256": expected["gate9_acquisition_record_sha256"], + "gate9_resource_envelope_sha256": expected["gate9_resource_envelope_sha256"], + "source_commit": SOURCE, + "materialization_plan_sha256": "sha256:" + "3" * 64, + "materializer_sources_sha256": "sha256:" + "4" * 64, + "materialization_record_sha256": lifecycle._digest(payload), + "materialization_record_bytes": len(payload), + "artifact_count": profile["selected_artifact_count"], + "artifact_bytes": profile["selected_artifact_bytes"], + "artifacts": artifacts, + } + + +@pytest.fixture +def config_factory(tmp_path, monkeypatch): + monkeypatch.setattr( + lifecycle, + "_assert_controller_owned", + lambda _path, *, directory: None, + ) + + def make(platform="windows", **overrides): + base = tmp_path / platform + staging = base / "staging" + root = base / "work" + staging.mkdir(parents=True) + root.mkdir() + package = staging / ( + "communityai-desktop-windows.zip" if platform == "windows" else "communityai-desktop-linux.tar.gz" + ) + package.write_bytes(PACKAGE_PAYLOAD) + release_audit, metadata = _release_audit( + staging, + platform, + SOURCE, + package, + ) + warm_cache = _warm_cache(staging, platform) + metadata_payload = metadata.read_bytes() + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + raw = { + "schema_version": 1, + "scope": lifecycle.SCOPE, + "run_id": RUN_ID, + "platform": platform, + "attempt_ordinal": 1, + "source_commit": SOURCE, + "package_path": str(package.resolve()), + "package_sha256": PACKAGE_SHA256, + "package_bytes": package.stat().st_size, + "release_metadata_path": str(metadata.resolve()), + "release_metadata_sha256": lifecycle._digest(metadata_payload), + "release_audit": release_audit, + "warm_cache": warm_cache, + "model_id": model_id, + "manifest_digest": acceptance.MODEL_PROFILES[model_id]["manifest_digest"], + "gate13_evidence_sha256": acceptance.EXPECTED_GATE13_EVIDENCE_SHA256, + "staging_root": str(staging.resolve()), + "work_root": str(root.resolve()), + "challenge_path": str((staging / "gate14-challenge.json").resolve()), + "checkpoint_path": str((root / "gate14-checkpoint.json").resolve()), + "facts_path": str((root / "gate14-facts.json").resolve()), + "evidence_path": str((root / "gate14-platform-evidence.json").resolve()), + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "pause_timeout_seconds": 120.0, + "sample_interval_seconds": 0.5, + "max_challenge_wait_seconds": 10.0, + } + raw.update(overrides) + path = staging / "gate14-lifecycle.json" + path.write_text(json.dumps(raw), encoding="utf-8") + return path, raw + + return make + + +class FakeActions: + def __init__(self, clock, *, prepared_value=None, calibration_mutator=None): + self.clock = clock + self.prepared_value = prepared() if prepared_value is None else prepared_value + self.calibration_mutator = calibration_mutator + self.events = [] + self.cleanup_calls = 0 + + def prepare(self, config): + self.events.append("prepare") + return self.prepared_value + + def calibrate(self, config, challenge): + self.events.append("calibrate") + value = calibrations(challenge) + if self.calibration_mutator is not None: + self.calibration_mutator(value) + self.clock.wall = challenge["issued_at_unix"] + 5 + return value + + def cleanup(self, config): + self.events.append("cleanup") + self.cleanup_calls += 1 + return cleanup(config) + + +def challenge(config, *, issued_at=NOW, checkpoint_sha256=None): + if checkpoint_sha256 is None: + if config.checkpoint_path.exists(): + checkpoint_sha256 = lifecycle.checkpoint_digest( + json.loads(config.checkpoint_path.read_text(encoding="utf-8")) + ) + else: + checkpoint_sha256 = "sha256:" + "f" * 64 + return challenge_contract.create( + run_id=config.run_id, + platform=config.platform, + source_commit=config.source_commit, + package_sha256=config.package_sha256, + checkpoint_sha256=checkpoint_sha256, + controller_state_revision=2, + issued_at_unix=issued_at, + nonce="a" * 64, + ) + + +def hardware(platform): + return { + "os_name": ("Windows Server 2022" if platform == "windows" else "Ubuntu 24.04"), + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": 24 * 1024**3, + } + + +def test_config_has_no_claim_fields_and_binds_exact_inputs(config_factory): + path, raw = config_factory() + + config = lifecycle.load_config(path) + + assert config.run_id == RUN_ID + assert config.model_id == "Qwen3.5 2B" + assert config.package_bytes == len(PACKAGE_PAYLOAD) + assert config.config_sha256.startswith("sha256:") + + raw["suspended"] = True + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="configuration schema", + ): + lifecycle.load_config(path) + + +def test_load_config_threads_selected_controller_ownership_policy( + config_factory, +): + path, raw = config_factory() + observed = set() + + def verify(candidate, *, directory): + observed.add((Path(candidate), directory)) + + lifecycle.load_config(path, ownership_verifier=verify) + + staging = path.parent + audit = staging / lifecycle._RELEASE_AUDIT_DIRECTORY_NAME + expected = { + (staging.parent, True), + (staging, True), + (path, False), + (Path(raw["package_path"]), False), + (Path(raw["release_metadata_path"]), False), + (staging / lifecycle._MATERIALIZATION_RECORD_NAME, False), + (audit, True), + (staging / lifecycle._RELEASE_AUDIT_ARCHIVE_NAME, False), + } + expected.update((audit / name, False) for name in lifecycle._RELEASE_AUDIT_MEMBERS) + assert expected <= observed + + +@pytest.mark.parametrize( + ("platform", "cell"), + [("windows", "qwen_windows"), ("linux", "gemma_linux")], +) +def test_gate9_warm_cache_constants_match_committed_evidence( + config_factory, + platform, + cell, +): + evidence = json.loads( + (ROOT / "docs" / "evidence" / "gate9-20260830-e-edge-resource-envelopes.json").read_text(encoding="utf-8") + ) + source = evidence["client_results"][cell] + expected = lifecycle._GATE9_WARM_CACHE[platform] + projected = tuple( + ( + item["path"], + item["role"], + "sha256:" + item["sha256"], + item["size_bytes"], + ) + for item in source["acquisition_record"]["artifacts"] + ) + + assert source["acquisition_record_sha256"] == expected["gate9_acquisition_record_sha256"] + assert source["resource_envelope_sha256"] == expected["gate9_resource_envelope_sha256"] + assert projected == expected["artifacts"] + + path, _raw = config_factory(platform=platform) + config = lifecycle.load_config(path) + assert config.warm_cache.artifacts == tuple(lifecycle.CacheArtifactBinding(*item) for item in expected["artifacts"]) + + +def test_config_binds_full_release_audit_and_fresh_warm_cache(config_factory): + path, _raw = config_factory() + + config = lifecycle.load_config(path) + checkpoint = lifecycle.write_or_load_checkpoint( + config, + prepared(), + now_unix=NOW, + ) + + assert config.release_audit.artifact_name == "communityai-desktop-audit-windows" + assert [item.name for item in config.release_audit.members] == list(lifecycle._RELEASE_AUDIT_MEMBERS) + assert ( + config.warm_cache.gate9_acquisition_record_sha256 + == lifecycle._GATE9_WARM_CACHE["windows"]["gate9_acquisition_record_sha256"] + ) + assert checkpoint["release_audit_sha256"] == config.release_audit.binding_sha256 + assert checkpoint["warm_cache_sha256"] == config.warm_cache.binding_sha256 + assert checkpoint["materialization_record_sha256"] == config.warm_cache.materialization_record_sha256 + + +@pytest.mark.parametrize( + ("mutator", "message"), + [ + ( + lambda raw: raw.update({"schema_version": True}), + "lifecycle configuration schema", + ), + ( + lambda raw: raw["release_audit"].update({"schema_version": True}), + "release audit identity", + ), + ( + lambda raw: raw["warm_cache"].update({"schema_version": True}), + "warm-cache Gate 9 identity", + ), + ( + lambda raw: raw["release_audit"]["members"].reverse(), + "members are not exact and sorted", + ), + ( + lambda raw: raw["release_audit"].update({"passed": True}), + "release audit binding schema", + ), + ( + lambda raw: raw["warm_cache"].update({"gate9_acquisition_record_sha256": "sha256:" + "f" * 64}), + "Gate 9 identity", + ), + ( + lambda raw: raw["warm_cache"]["artifacts"][0].update({"path": "../chat_template.jinja"}), + "artifact path", + ), + ( + lambda raw: raw["warm_cache"]["artifacts"][0].update( + {"size_bytes": raw["warm_cache"]["artifacts"][0]["size_bytes"] + 1} + ), + "artifact identity", + ), + ( + lambda raw: raw["warm_cache"].update({"verified": True}), + "warm-cache binding schema", + ), + ], +) +def test_public_input_binding_mutations_fail_closed( + config_factory, + mutator, + message, +): + path, raw = config_factory() + mutator(raw) + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(lifecycle.Gate14LifecycleError, match=message): + lifecycle.load_config(path) + + +@pytest.mark.parametrize("attack", ["reverse", "symlink"]) +def test_release_audit_zip_members_are_exact_regular_files(config_factory, attack): + path, raw = config_factory() + archive_path = path.parent / lifecycle._RELEASE_AUDIT_ARCHIVE_NAME + audit_root = path.parent / lifecycle._RELEASE_AUDIT_DIRECTORY_NAME + names = list(lifecycle._RELEASE_AUDIT_MEMBERS) + if attack == "reverse": + names.reverse() + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_STORED) as archive: + for index, name in enumerate(names): + payload = (audit_root / name).read_bytes() + if attack == "symlink" and index == 0: + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + archive.writestr(info, payload) + else: + archive.writestr(name, payload) + archive_payload = archive_path.read_bytes() + raw["release_audit"]["artifact_sha256"] = lifecycle._digest(archive_payload) + raw["release_audit"]["artifact_bytes"] = len(archive_payload) + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="archive members are invalid", + ): + lifecycle.load_config(path) + + +@pytest.mark.parametrize( + "target", + [ + "provenance", + "metrics", + "release_metadata", + "archive", + "metrics_record", + "release_artifacts", + "release_totals", + ], +) +def test_release_audit_schema_versions_reject_boolean(config_factory, target): + path, raw = config_factory() + audit_root = path.parent / lifecycle._RELEASE_AUDIT_DIRECTORY_NAME + provenance = json.loads((audit_root / "provenance.json").read_text(encoding="utf-8")) + metrics = json.loads((audit_root / "desktop-metrics.json").read_text(encoding="utf-8")) + metadata = json.loads((audit_root / "release-metadata.json").read_text(encoding="utf-8")) + + if target == "provenance": + provenance["schema_version"] = True + elif target == "metrics": + metrics["schema_version"] = True + elif target == "release_metadata": + metadata["schema_version"] = True + elif target == "archive": + provenance["install_archive"]["schema_version"] = True + metrics["release_artifacts"]["install_archive"]["schema_version"] = True + elif target == "metrics_record": + provenance["desktop_metrics"]["schema_version"] = True + elif target == "release_artifacts": + metrics["release_artifacts"]["schema_version"] = True + else: + metrics["release_artifacts"]["artifact_count"] = 999 + metrics["release_artifacts"]["artifact_bytes"] = -1 + metrics["release_artifacts"]["checksums_sha256"] = "f" * 64 + + metrics_payload = _json_bytes(metrics) + provenance["desktop_metrics"]["sha256"] = hashlib.sha256(metrics_payload).hexdigest() + provenance["desktop_metrics"]["size_bytes"] = len(metrics_payload) + payloads = { + "SHA256SUMS": (audit_root / "SHA256SUMS").read_bytes(), + "desktop-metrics.json": metrics_payload, + "provenance.json": _json_bytes(provenance), + "release-metadata.json": _json_bytes(metadata), + } + + with pytest.raises(lifecycle.Gate14LifecycleError): + lifecycle._validate_release_semantics( + payloads, + platform=raw["platform"], + source_commit=raw["source_commit"], + package_sha256=raw["package_sha256"], + package_bytes=raw["package_bytes"], + ) + + +@pytest.mark.parametrize( + ("target", "message"), + [ + ("release-audit.zip", "release audit artifact"), + ("release-audit/SHA256SUMS", "release audit extracted member"), + ( + lifecycle._MATERIALIZATION_RECORD_NAME, + "cache materialization record identity", + ), + ], +) +def test_staged_public_input_drift_fails_before_prepare_and_cleans( + config_factory, + target, + message, +): + path, _raw = config_factory() + config = lifecycle.load_config(path) + candidate = config.staging_root / Path(target) + payload = candidate.read_bytes() + candidate.write_bytes(b" " + payload[1:]) + actions = FakeActions(Clock()) + + with pytest.raises(lifecycle.Gate14LifecycleError, match=message): + lifecycle.run_lifecycle(config, actions, hardware_probe=hardware) + + assert actions.events == ["cleanup"] + assert not config.checkpoint_path.exists() + + +def _rewrite_materialization(path, raw, mutator): + record_path = path.parent / lifecycle._MATERIALIZATION_RECORD_NAME + record = json.loads(record_path.read_text(encoding="utf-8")) + mutator(record) + payload = _json_bytes(record) + record_path.write_bytes(payload) + raw["warm_cache"]["materialization_record_sha256"] = lifecycle._digest(payload) + raw["warm_cache"]["materialization_record_bytes"] = len(payload) + path.write_text(json.dumps(raw), encoding="utf-8") + + +def test_materialization_mirror_or_transport_override_is_rejected(config_factory): + path, raw = config_factory() + + def mutate(record): + record["transfer"]["direct_upstream_transfer"] = False + record["transfer"]["mirror_used"] = True + + _rewrite_materialization(path, raw, mutate) + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="materialization proof", + ): + lifecycle.load_config(path) + + +@pytest.mark.parametrize( + "mutator", + [ + lambda record: record.update({"schema_version": True}), + lambda record: record["runtime"].update({"private_path": "C:/secret"}), + lambda record: record["model"].update({"repository": "attacker/model"}), + lambda record: record["selection"].update({"verified": True}), + lambda record: record["transfer"].update({"elapsed_seconds": True}), + lambda record: record["transfer"].update({"elapsed_seconds": float("inf")}), + lambda record: record["transfer"].update({"resumptions": False}), + lambda record: record["storage"].update({"cache_bytes_before": False}), + lambda record: record["privacy"].update({"credentials_retained": 0}), + ], +) +def test_materialization_nested_schema_and_source_fail_closed( + config_factory, + mutator, +): + path, raw = config_factory() + _rewrite_materialization(path, raw, mutator) + + with pytest.raises(lifecycle.Gate14LifecycleError): + lifecycle.load_config(path) + + +def test_config_rejects_wrong_model_and_escaped_private_path( + config_factory, + tmp_path, +): + path, raw = config_factory() + raw["model_id"] = "Gemma 4 E2B IT" + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(lifecycle.Gate14LifecycleError, match="model binding"): + lifecycle.load_config(path) + + raw["model_id"] = "Qwen3.5 2B" + raw["facts_path"] = str((tmp_path / "gate14-facts.json").resolve()) + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(lifecycle.Gate14LifecycleError, match="escaped"): + lifecycle.load_config(path) + + +def test_checkpoint_is_immutable_and_prepared_digest_bound(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + observed = prepared() + + first = lifecycle.write_or_load_checkpoint( + config, + observed, + now_unix=NOW, + ) + second = lifecycle.write_or_load_checkpoint( + config, + observed, + now_unix=NOW + 1, + ) + + assert first == second + assert first["phase"] == "challenge-ready" + assert first["prepared_facts_sha256"] == lifecycle._digest(lifecycle._canonical(observed)) + + changed = prepared() + changed["placement"]["block_end"] = 5 + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="checkpoint binding", + ): + lifecycle.write_or_load_checkpoint( + config, + changed, + now_unix=NOW + 1, + ) + + +@pytest.mark.parametrize("field", ["schema_version", "attempt_ordinal"]) +def test_boolean_checkpoint_identity_fails_direct_and_persisted(config_factory, field): + path, _raw = config_factory() + config = lifecycle.load_config(path) + observed = prepared() + checkpoint = lifecycle._checkpoint_value(config, observed, NOW) + checkpoint[field] = True + + with pytest.raises(lifecycle.Gate14LifecycleError, match="checkpoint schema"): + lifecycle.validate_checkpoint( + checkpoint, + config, + observed, + now_unix=NOW, + ) + with pytest.raises(lifecycle.Gate14LifecycleError, match="checkpoint schema"): + lifecycle.checkpoint_digest(checkpoint) + + config.checkpoint_path.write_text(json.dumps(checkpoint), encoding="utf-8") + with pytest.raises(lifecycle.Gate14LifecycleError, match="checkpoint schema"): + lifecycle.write_or_load_checkpoint( + config, + observed, + now_unix=NOW, + ) + + +def test_boolean_attempt_ordinal_fails_prepared_and_cleanup(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + observed = prepared() + observed["attempt_ordinal"] = True + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="prepared observation schema or binding", + ): + lifecycle.validate_prepared(observed, config) + + with pytest.raises(lifecycle.Gate14LifecycleError, match="cleanup is incomplete"): + lifecycle.validate_cleanup( + cleanup(config, attempt_ordinal=True), + config, + ) + + +def test_full_sequence_waits_for_challenge_cleans_then_probes(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + actions = FakeActions(clock) + challenge_written = False + + def sleeper(seconds): + nonlocal challenge_written + assert config.checkpoint_path.is_file() + clock.sleep(seconds) + if not challenge_written: + challenge_contract.write_new( + config.challenge_path, + challenge(config, issued_at=NOW), + ) + challenge_written = True + actions.events.append("challenge") + + document = lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.events == [ + "prepare", + "challenge", + "calibrate", + "cleanup", + ] + assert acceptance.validate_platform_document(document)["platform"] == ("windows") + assert config.checkpoint_path.is_file() + assert config.challenge_path.is_file() + assert config.evidence_path.is_file() + assert not config.facts_path.exists() + assert document["qualification_temporaries_removed"] is True + + +def test_early_challenge_is_rejected_without_running_actions(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + challenge_contract.write_new(config.challenge_path, challenge(config)) + actions = FakeActions(Clock()) + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="fresh lifecycle outputs", + ): + lifecycle.run_lifecycle(config, actions, hardware_probe=hardware) + + assert actions.events == ["cleanup"] + assert actions.cleanup_calls == 1 + + +def test_challenge_that_predates_checkpoint_fails_and_cleans(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + actions = FakeActions(clock) + written = False + + def sleeper(seconds): + nonlocal written + clock.sleep(seconds) + if not written: + challenge_contract.write_new( + config.challenge_path, + challenge(config, issued_at=NOW - 1), + ) + written = True + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="predates readiness", + ): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.cleanup_calls == 1 + assert not config.evidence_path.exists() + + +def test_expired_challenge_fails_before_calibration_and_cleans(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + clock.wall = NOW + 901 + actions = FakeActions(clock) + written = False + + def sleeper(seconds): + nonlocal written + clock.sleep(seconds) + if not written: + challenge_contract.write_new( + config.challenge_path, + challenge(config, issued_at=NOW), + ) + written = True + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="challenge is invalid", + ): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.events == ["prepare", "cleanup"] + assert not config.evidence_path.exists() + + +def test_non_crossing_native_calibration_never_reaches_probe(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + + def no_crossing(value): + value[0]["calibration"]["trigger_value"] = 99.0 + + actions = FakeActions(clock, calibration_mutator=no_crossing) + written = False + + def sleeper(seconds): + nonlocal written + clock.sleep(seconds) + if not written: + challenge_contract.write_new( + config.challenge_path, + challenge(config), + ) + written = True + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="calibration observations", + ): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.cleanup_calls == 1 + assert not config.evidence_path.exists() + + +def test_timeout_runs_cleanup_and_emits_no_evidence(config_factory): + path, raw = config_factory(max_challenge_wait_seconds=1.0) + config = lifecycle.load_config(path) + clock = Clock() + actions = FakeActions(clock) + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="challenge timed out", + ): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=clock.sleep, + ) + + assert raw["max_challenge_wait_seconds"] == 1.0 + assert actions.events == ["prepare", "cleanup"] + assert not config.evidence_path.exists() + + +def test_package_drift_fails_before_prepare_and_still_cleans(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + config.package_path.write_bytes(b"same-sized-mutated-package-bytes!!") + actions = FakeActions(Clock()) + + with pytest.raises(lifecycle.Gate14LifecycleError, match="package"): + lifecycle.run_lifecycle(config, actions, hardware_probe=hardware) + + assert actions.events == ["cleanup"] + assert not config.checkpoint_path.exists() + assert not config.evidence_path.exists() + + +def test_incomplete_cleanup_prevents_platform_evidence(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + actions = FakeActions(clock) + written = False + + def bad_cleanup(value): + actions.events.append("cleanup") + actions.cleanup_calls += 1 + return cleanup(value, credentials_removed=False) + + actions.cleanup = bad_cleanup + + def sleeper(seconds): + nonlocal written + clock.sleep(seconds) + if not written: + challenge_contract.write_new( + config.challenge_path, + challenge(config), + ) + written = True + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="cleanup did not complete", + ): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.cleanup_calls == 2 + assert not config.evidence_path.exists() + + +def test_run_rejects_retained_checkpoint_replay_and_cleans(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + lifecycle.write_or_load_checkpoint(config, prepared(), now_unix=NOW) + actions = FakeActions(Clock()) + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="fresh lifecycle outputs", + ): + lifecycle.run_lifecycle(config, actions, hardware_probe=hardware) + + assert actions.events == ["cleanup"] + assert actions.cleanup_calls == 1 + + +def test_same_second_challenge_requires_exact_checkpoint_digest(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + actions = FakeActions(clock) + written = False + + def sleeper(seconds): + nonlocal written + clock.sleep(seconds) + if not written: + challenge_contract.write_new( + config.challenge_path, + challenge( + config, + issued_at=NOW, + checkpoint_sha256="sha256:" + "0" * 64, + ), + ) + written = True + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="challenge is invalid", + ): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.events == ["prepare", "cleanup"] + assert not config.evidence_path.exists() + + +def test_release_metadata_drift_fails_before_prepare_and_cleans(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + payload = config.release_metadata_path.read_bytes() + config.release_metadata_path.write_bytes(b" " + payload[1:]) + actions = FakeActions(Clock()) + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="release metadata", + ): + lifecycle.run_lifecycle(config, actions, hardware_probe=hardware) + + assert actions.events == ["cleanup"] + assert not config.checkpoint_path.exists() + + +def test_failed_final_probe_removes_private_and_pass_shaped_outputs(config_factory): + path, _raw = config_factory() + config = lifecycle.load_config(path) + clock = Clock() + actions = FakeActions(clock) + written = False + + def sleeper(seconds): + nonlocal written + clock.sleep(seconds) + if not written: + challenge_contract.write_new( + config.challenge_path, + challenge(config), + ) + written = True + + def invalid_hardware(_platform): + value = hardware(config.platform) + value["accelerator"] = "foreign-device" + return value + + with pytest.raises(Exception): + lifecycle.run_lifecycle( + config, + actions, + hardware_probe=invalid_hardware, + clock=clock.time, + monotonic=clock.monotonic, + sleeper=sleeper, + ) + + assert actions.cleanup_calls == 2 + assert not config.facts_path.exists() + assert not config.evidence_path.exists() + assert not (config.work_root / lifecycle._PENDING_EVIDENCE_NAME).exists() + + +def test_real_controller_guard_rejects_qualification_user_owned_staging(tmp_path): + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="controller staging", + ): + REAL_CONTROLLER_GUARD(tmp_path, directory=True) + + +def test_windows_acl_probes_each_dangerous_right_independently(): + invalid = -1 + opened = [] + closed = [] + + def opener(mask): + opened.append(mask) + return 7 if mask == 0x40000000 else invalid + + with pytest.raises( + lifecycle.Gate14LifecycleError, + match="writable by the qualification process", + ): + lifecycle._assert_windows_access_denied( + directory=True, + opener=opener, + closer=closed.append, + invalid_handle=invalid, + get_last_error=lambda: 5, + ) + + assert opened == [0x00010000, 0x00040000, 0x00080000, 0x40000000] + assert closed == [7] + + all_denied = [] + + def denied(mask): + all_denied.append(mask) + return invalid + + lifecycle._assert_windows_access_denied( + directory=True, + opener=denied, + closer=closed.append, + invalid_handle=invalid, + get_last_error=lambda: 5, + ) + assert all_denied == [ + 0x00010000, + 0x00040000, + 0x00080000, + 0x40000000, + 0x00000002, + 0x00000004, + 0x00000040, + ] diff --git a/tests/test_gate14_run_controller.py b/tests/test_gate14_run_controller.py new file mode 100644 index 000000000..a88a77bee --- /dev/null +++ b/tests/test_gate14_run_controller.py @@ -0,0 +1,1042 @@ +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_calibration_challenge as challenge_contract # noqa: E402 +import gate14_hardware_acceptance as acceptance # noqa: E402 +import gate14_packaged_lifecycle as lifecycle # noqa: E402 +import gate14_run_controller as controller # noqa: E402 + +RUN_ID = "gate14-20260902-a" +SOURCE = "1" * 40 +NOW = 2_000_000_000 +PACKAGE_DIGESTS = { + "windows": "sha256:" + "a" * 64, + "linux": "sha256:" + "b" * 64, +} + + +def challenge_document(platform: str, revision: int = 2) -> dict: + return dict( + challenge_contract.create( + run_id=RUN_ID, + platform=platform, + source_commit=SOURCE, + package_sha256=PACKAGE_DIGESTS[platform], + checkpoint_sha256="sha256:" + "c" * 64, + controller_state_revision=revision, + issued_at_unix=NOW, + nonce=("a" if platform == "windows" else "b") * 64, + ) + ) + + +def write_challenge(tmp_path: Path, platform: str, value: dict | None = None) -> Path: + path = tmp_path / f"{platform}-challenge.json" + path.write_text(json.dumps(value or challenge_document(platform)), encoding="utf-8") + return path + + +def write_checkpoint( + tmp_path: Path, + plan: controller.RunPlan, + platform: str, + *, + created_at: int = NOW, +) -> Path: + client = plan.windows if platform == "windows" else plan.linux + value = { + "schema_version": lifecycle.SCHEMA_VERSION, + "scope": lifecycle.CHECKPOINT_SCOPE, + "run_id": plan.run_id, + "platform": platform, + "attempt_ordinal": 1, + "source_commit": client.source_commit, + "lifecycle_config_sha256": "sha256:" + "d" * 64, + "package_sha256": client.package_sha256, + "release_metadata_sha256": "sha256:" + "e" * 64, + "release_audit_sha256": "sha256:" + "a" * 64, + "warm_cache_sha256": "sha256:" + "b" * 64, + "materialization_record_sha256": "sha256:" + "c" * 64, + "prepared_facts_sha256": "sha256:" + "f" * 64, + "phase": "challenge-ready", + "created_at_unix": created_at, + } + path = tmp_path / f"{platform}-checkpoint.json" + path.write_text(json.dumps(value), encoding="utf-8") + return path + + +def provider_plan() -> dict: + clients = [] + for index, platform in enumerate(("windows", "linux"), start=1): + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + clients.append( + { + "platform": platform, + "instance": f"gate14-20260902-a-{platform}", + "disk": f"gate14-20260902-a-{platform}-disk", + "source_commit": SOURCE, + "termination_unix": NOW + index * 10_000, + "package_sha256": PACKAGE_DIGESTS[platform], + "model_id": model_id, + "manifest_digest": acceptance.MODEL_PROFILES[model_id]["manifest_digest"], + "machine_type": "g2-standard-8", + "image_project": "windows-cloud" if platform == "windows" else "ubuntu-os-cloud", + "image": ( + "windows-server-2022-dc-v20260814" if platform == "windows" else "ubuntu-2404-noble-amd64-v20260826" + ), + "boot_disk_gib": 100, + "boot_disk_type": "pd-balanced", + "service_account_disabled": True, + "max_run_seconds": 7_200, + "termination_action": "DELETE", + } + ) + return { + "project": "community-ai-506321", + "zone": "us-central1-a", + "clients": clients, + "sequencing": { + "clients_may_run_concurrently": False, + "windows_first": True, + "fresh_host_per_platform": True, + }, + } + + +def write_plan( + tmp_path: Path, + *, + additional_current_maximum: str | None = None, + hide_additional_below_anchor: bool = False, +) -> controller.RunPlan: + provider = provider_plan() + digest = controller._canonical_digest(provider) + authorization = { + "schema_version": 1, + "gate": 14, + "result": "authorized", + "run_id": RUN_ID, + "source_commit": SOURCE, + "provider_plan_digest": digest, + "provider_plan": provider, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "44.00", + "remaining_after_run_maximum_usd": "0.00", + "reservation_recorded": True, + "native_auth_revalidated": True, + "provisioning_authorized_after_fail_closed_preflight": True, + }, + "prohibited": {"credits": 0, "macos": 0, "fly_gpu": 0}, + } + authorization_path = tmp_path / "authorization.json" + authorization_path.write_text(json.dumps(authorization), encoding="utf-8") + ledger_path = tmp_path / "ledger.md" + ledger_lines = [ + "## Cloud authorization and spend ledger", + "", + "| Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State |", + "| --- | --- | --- | ---: | ---: | --- | --- |", + f"| {RUN_ID} | GCP | Gate 14 packaged hardware [plan {digest}] | USD 44.00 | — | — | RESERVED |", + ] + additional_row = ( + "| gate14-prior-run | GCP | Unexpected same-epoch reservation | " + f"USD {additional_current_maximum} | — | — | RESERVED |" + if additional_current_maximum is not None + else None + ) + if additional_row is not None and not hide_additional_below_anchor: + ledger_lines.append(additional_row) + ledger_lines.append( + "| gate13-20260901-a | GCP | Current epoch anchor | " + "USD 56.00 | — | Existing cleanup proof | CLEANED-COMMITTED |" + ) + if additional_row is not None and hide_additional_below_anchor: + ledger_lines.append(additional_row) + ledger_lines.append("") + ledger_path.write_text("\n".join(ledger_lines), encoding="utf-8") + return controller.load_plan(authorization_path, ledger_path) + + +def observation( + plan: controller.RunPlan, + *, + windows: bool = False, + linux: bool = False, + windows_job: str = "absent", + linux_job: str = "absent", + windows_digest: str | None = None, + linux_digest: str | None = None, + windows_disk: bool | None = None, + linux_disk: bool | None = None, +) -> dict: + present = {"windows": windows, "linux": linux} + jobs = {"windows": windows_job, "linux": linux_job} + digests = {"windows": windows_digest, "linux": linux_digest} + disks = { + "windows": windows if windows_disk is None else windows_disk, + "linux": linux if linux_disk is None else linux_disk, + } + return { + "schema_version": 1, + "run_id": plan.run_id, + "observed_at_unix": NOW, + "instances": { + client.instance: { + "present": present[client.platform], + "run_id": plan.run_id if present[client.platform] else None, + "source_commit": client.source_commit if present[client.platform] else None, + "termination_unix": client.termination_unix if present[client.platform] else None, + } + for client in (plan.windows, plan.linux) + }, + "disks": {client.disk: disks[client.platform] for client in (plan.windows, plan.linux)}, + "clients": { + platform: { + "job_state": jobs[platform], + "attempt_ordinal": 1 if jobs[platform] != "absent" else 0, + "evidence_digest": digests[platform] if jobs[platform] == "passed" else None, + } + for platform in ("windows", "linux") + }, + "l4_usage": int(windows or linux), + "protected_bootstrap_running": True, + } + + +def platform_evidence(platform: str, challenge_value: dict | None = None) -> dict: + challenge = challenge_value or challenge_document(platform) + challenge_sha256 = challenge_contract.digest(challenge) + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + selected = profile["selected_artifact_bytes"] + return { + "schema_version": 1, + "scope": acceptance.PLATFORM_SCOPE, + "run_id": RUN_ID, + "platform": platform, + "result": "passed", + "source_commit": SOURCE, + "gate13_evidence_sha256": acceptance.EXPECTED_GATE13_EVIDENCE_SHA256, + "package": { + "source_commit": SOURCE, + "archive_sha256": PACKAGE_DIGESTS[platform], + "archive_bytes": 1024, + "release_metadata_sha256": "sha256:" + "e" * 64, + }, + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "revision_commit": profile["revision_commit"], + "gate9_envelope_sha256": acceptance.EXPECTED_GATE9_ENVELOPES[platform], + "selected_artifact_count": profile["selected_artifact_count"], + "selected_artifact_bytes": selected, + "total_blocks": profile["total_blocks"], + }, + "hardware": { + "os_name": "Windows Server 2022" if platform == "windows" else "Ubuntu 24.04", + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": 24 * 1024**3, + }, + "cache": { + "verified_bytes_before": selected, + "verified_bytes_after": selected, + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": 4, + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "calibration_challenge": { + "challenge_sha256": challenge_sha256, + "controller_state_revision": challenge["controller_state_revision"], + "issued_at_unix": challenge["issued_at_unix"], + "expires_at_unix": challenge["expires_at_unix"], + }, + "suspensions": [ + { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": 3.0, + "calibration": { + "measurement_source": { + "bandwidth": "host-network-counters", + "power": "nvidia-nvml-device-power", + "schedule": "utc-policy-clock", + }[kind], + "measurement_scope": { + "bandwidth": "aggregate-host-network", + "power": "selected-nvidia-l4-device", + "schedule": "utc-schedule-policy", + }[kind], + "sample_count": 4, + "sample_interval_seconds": 0.5, + "baseline_value": 1.0 if kind == "schedule" else 10.0, + "configured_limit": { + "bandwidth": 100.0, + "power": 250.0, + "schedule": 0.5, + }[kind], + "trigger_value": 0.0 if kind == "schedule" else (120.0 if kind == "bandwidth" else 275.0), + "resume_value": 1.0 if kind == "schedule" else 10.0, + "challenge_sha256": challenge_sha256, + "sample_started_at_unix": NOW + 10, + "sample_ended_at_unix": NOW + 12, + }, + } + for kind in ("bandwidth", "power", "schedule") + ], + "recovery": { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 3.0, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + "pause": { + "requested": True, + "completed": True, + "duration_seconds": 3.0, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + "restart": { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 3.0, + "cache_reused": True, + }, + "unsupported_telemetry": { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + "privacy": { + "prompt_retained": False, + "response_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + "provider_output_retained": False, + }, + "qualification_temporaries_removed": True, + } + + +@pytest.fixture +def plan(tmp_path): + return write_plan(tmp_path) + + +def test_load_plan_binds_budget_sequence_models_and_exact_resources(plan): + assert plan.run_id == RUN_ID + assert plan.ledger_state == "RESERVED" + assert plan.instances == ( + "gate14-20260902-a-windows", + "gate14-20260902-a-linux", + ) + assert plan.windows.model_id == "Qwen3.5 2B" + assert plan.linux.model_id == "Gemma 4 E2B IT" + assert controller.PROTECTED_INSTANCE not in plan.instances + + +def test_controller_issues_one_time_source_bound_calibration_challenge(tmp_path, plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, windows=True, windows_job="running"), + plan, + ) + path = tmp_path / "controller-challenge.json" + checkpoint_path = write_checkpoint(tmp_path, plan, "windows") + + next_state, value = controller.issue_calibration_challenge( + state, + plan, + "windows", + path, + checkpoint_path, + issued_at_unix=NOW + 1, + nonce="f" * 64, + ) + + assert path.exists() + assert value["run_id"] == plan.run_id + assert value["platform"] == "windows" + assert value["source_commit"] == plan.windows.source_commit + assert value["package_sha256"] == plan.windows.package_sha256 + assert value["checkpoint_sha256"] == lifecycle.checkpoint_digest( + json.loads(checkpoint_path.read_text(encoding="utf-8")) + ) + assert value["controller_state_revision"] == state["revision"] + assert next_state["windows_challenge_sha256"] == challenge_contract.digest(value) + recovered_state, recovered = controller.issue_calibration_challenge( + state, + plan, + "windows", + path, + checkpoint_path, + issued_at_unix=NOW + 2, + nonce="e" * 64, + ) + assert recovered == value + assert recovered_state["windows_challenge_sha256"] == next_state["windows_challenge_sha256"] + + path.unlink() + with pytest.raises(controller.Gate14ControllerError): + controller.issue_calibration_challenge( + next_state, + plan, + "windows", + path, + checkpoint_path, + issued_at_unix=NOW + 2, + nonce="e" * 64, + ) + + +def test_load_plan_rejects_spend_above_remaining_ceiling(tmp_path): + provider = provider_plan() + digest = controller._canonical_digest(provider) + authorization = { + "schema_version": 1, + "gate": 14, + "result": "authorized", + "run_id": RUN_ID, + "source_commit": SOURCE, + "provider_plan_digest": digest, + "provider_plan": provider, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "45.00", + "remaining_after_run_maximum_usd": "-1.00", + "reservation_recorded": True, + "native_auth_revalidated": True, + "provisioning_authorized_after_fail_closed_preflight": True, + }, + "prohibited": {"credits": 0, "macos": 0, "fly_gpu": 0}, + } + authorization_path = tmp_path / "bad.json" + authorization_path.write_text(json.dumps(authorization), encoding="utf-8") + ledger_path = tmp_path / "ledger.md" + ledger_path.write_text( + "\n".join( + ( + "## Cloud authorization and spend ledger", + "| Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State |", + "| --- | --- | --- | ---: | ---: | --- | --- |", + f"| {RUN_ID} | GCP | Gate 14 [plan {digest}] | USD 45.00 | — | — | RESERVED |", + ) + ), + encoding="utf-8", + ) + with pytest.raises(controller.Gate14ControllerError): + controller.load_plan(authorization_path, ledger_path) + + +def test_lifecycle_reattaches_collects_serially_and_cleanup_passes( + tmp_path, + plan, +): + state = controller.initial_state(plan) + assert state["next_action"] == "none" + + state = controller.reconcile(state, observation(plan), plan) + assert state["phase"] == "ABSENT" + assert state["next_action"] == "start_windows" + + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + assert state["phase"] == "WINDOWS_RUNNING" + assert state["windows_consumed"] is True + assert state["next_action"] == "none" + + windows_challenge_path = tmp_path / "windows-challenge.json" + windows_checkpoint_path = write_checkpoint(tmp_path, plan, "windows") + state, windows_challenge = controller.issue_calibration_challenge( + state, + plan, + "windows", + windows_challenge_path, + windows_checkpoint_path, + issued_at_unix=NOW, + nonce="a" * 64, + ) + windows_path = tmp_path / "windows.json" + windows_path.write_text( + json.dumps(platform_evidence("windows", windows_challenge)), + encoding="utf-8", + ) + windows_digest = "sha256:" + hashlib.sha256(windows_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=windows_digest, + ), + plan, + ) + assert state["next_action"] == "collect_windows" + state = controller.collect_platform( + state, + plan, + "windows", + windows_path, + windows_challenge_path, + ) + assert state["phase"] == "WINDOWS_DELETING" + assert state["next_action"] == "delete_windows" + + state = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + ), + plan, + ) + assert state["phase"] == "WINDOWS_DELETING" + assert state["next_action"] == "start_linux" + + state = controller.reconcile( + state, + observation( + plan, + linux=True, + windows_job="passed", + windows_digest=windows_digest, + linux_job="running", + ), + plan, + ) + assert state["phase"] == "LINUX_RUNNING" + assert state["linux_consumed"] is True + + linux_challenge_path = tmp_path / "linux-challenge.json" + linux_checkpoint_path = write_checkpoint(tmp_path, plan, "linux") + state, linux_challenge = controller.issue_calibration_challenge( + state, + plan, + "linux", + linux_challenge_path, + linux_checkpoint_path, + issued_at_unix=NOW, + nonce="b" * 64, + ) + linux_path = tmp_path / "linux.json" + linux_path.write_text( + json.dumps(platform_evidence("linux", linux_challenge)), + encoding="utf-8", + ) + linux_digest = "sha256:" + hashlib.sha256(linux_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + linux=True, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest=linux_digest, + ), + plan, + ) + assert state["next_action"] == "collect_linux" + state = controller.collect_platform( + state, + plan, + "linux", + linux_path, + linux_challenge_path, + ) + assert state["phase"] == "LINUX_DELETING" + assert state["next_action"] == "delete_linux" + + state = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest=linux_digest, + ), + plan, + ) + assert state["phase"] == "CLEANED_PASS" + assert state["cleanup_verified"] is True + assert state["next_action"] == "none" + + +def test_failed_job_goes_directly_to_exact_cleanup(plan): + state = controller.initial_state(plan) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="failed"), + plan, + ) + assert state["phase"] == "CLEANING_FAILED" + assert state["next_action"] == "cleanup_failure" + + state = controller.reconcile(state, observation(plan), plan) + assert state["phase"] == "CLEANED_FAILURE" + assert state["cleanup_verified"] is True + + +def test_foreign_or_overlapping_resource_observations_fail_closed(plan): + state = controller.initial_state(plan) + value = observation(plan, windows=True, windows_job="running") + value["instances"][plan.windows.instance]["source_commit"] = "9" * 40 + with pytest.raises(controller.Gate14ControllerError): + controller.reconcile(state, value, plan) + + value = observation( + plan, + windows=True, + linux=True, + windows_job="running", + linux_job="running", + ) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + state = { + **state, + "phase": "WINDOWS_DELETING", + "next_action": "delete_windows", + } + with pytest.raises(controller.Gate14ControllerError): + controller.validate_observation( + { + **value, + "l4_usage": 2, + }, + plan, + ) + + +def test_collect_rejects_wrong_package_and_save_round_trips(tmp_path, plan): + state = controller.initial_state(plan) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + challenge_path = tmp_path / "wrong-package-challenge.json" + checkpoint_path = write_checkpoint(tmp_path, plan, "windows") + state, challenge = controller.issue_calibration_challenge( + state, + plan, + "windows", + challenge_path, + checkpoint_path, + issued_at_unix=NOW, + nonce="a" * 64, + ) + evidence = platform_evidence("windows", challenge) + evidence["package"]["archive_sha256"] = "sha256:" + "9" * 64 + evidence_path = tmp_path / "wrong.json" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + evidence_digest = "sha256:" + hashlib.sha256(evidence_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=evidence_digest, + ), + plan, + ) + with pytest.raises(controller.Gate14ControllerError): + controller.collect_platform( + state, + plan, + "windows", + evidence_path, + challenge_path, + ) + + state_path = tmp_path / "state.json" + controller.save_state(state_path, state, plan) + assert controller.load_state(state_path, plan) == state + + +def test_collect_rejects_evidence_from_a_different_challenge(tmp_path, plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, windows=True, windows_job="running"), + plan, + ) + issued_path = tmp_path / "issued-challenge.json" + checkpoint_path = write_checkpoint(tmp_path, plan, "windows") + state, issued = controller.issue_calibration_challenge( + state, + plan, + "windows", + issued_path, + checkpoint_path, + issued_at_unix=NOW, + nonce="a" * 64, + ) + evidence_path = tmp_path / "evidence.json" + evidence_path.write_text( + json.dumps(platform_evidence("windows", issued)), + encoding="utf-8", + ) + evidence_digest = "sha256:" + hashlib.sha256(evidence_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=evidence_digest, + ), + plan, + ) + different = dict(issued) + different["nonce"] = "f" * 64 + challenge_path = tmp_path / "different-challenge.json" + challenge_path.write_text(json.dumps(different), encoding="utf-8") + + with pytest.raises(controller.Gate14ControllerError): + controller.collect_platform( + state, + plan, + "windows", + evidence_path, + challenge_path, + ) + + +def test_begin_cleanup_is_idempotent_after_terminal_state(plan): + state = controller.initial_state(plan) + state = controller.begin_cleanup(state, plan, "manual-stop") + assert state["phase"] == "CLEANING_FAILED" + state = controller.reconcile(state, observation(plan), plan) + assert state["phase"] == "CLEANED_FAILURE" + assert controller.begin_cleanup(state, plan, "manual-stop") == state + + +def test_forged_success_and_deletion_states_fail_closed(plan): + initial = controller.initial_state(plan) + forged_pass = { + **initial, + "phase": "CLEANED_PASS", + "cleanup_verified": True, + "next_action": "none", + } + with pytest.raises(controller.Gate14ControllerError): + controller.validate_state(forged_pass, plan) + + forged_windows_deleting = { + **initial, + "phase": "WINDOWS_DELETING", + "windows_consumed": True, + "next_action": "start_linux", + } + with pytest.raises(controller.Gate14ControllerError): + controller.validate_state(forged_windows_deleting, plan) + + forged_linux_deleting = { + **initial, + "phase": "LINUX_DELETING", + "windows_consumed": True, + "linux_consumed": True, + "windows_evidence_digest": "sha256:" + "c" * 64, + "linux_evidence_digest": "sha256:" + "d" * 64, + "next_action": "delete_linux", + } + with pytest.raises(controller.Gate14ControllerError): + controller.reconcile(forged_linux_deleting, observation(plan), plan) + + +def test_expired_run_never_returns_a_start_action(plan): + value = observation(plan) + value["observed_at_unix"] = plan.windows.termination_unix + + state = controller.reconcile(controller.initial_state(plan), value, plan) + + assert state["phase"] == "CLEANED_FAILURE" + assert state["failure_code"] == "run-expired" + assert state["cleanup_verified"] is True + assert state["next_action"] == "none" + + +def test_passed_job_requires_and_binds_exact_evidence_digest(tmp_path, plan): + missing = observation(plan, windows=True, windows_job="passed") + with pytest.raises(controller.Gate14ControllerError): + controller.validate_observation(missing, plan) + + stale = controller.reconcile( + controller.initial_state(plan), + observation( + plan, + windows_job="passed", + windows_digest="sha256:" + "c" * 64, + ), + plan, + ) + assert stale["phase"] == "CLEANED_FAILURE" + assert stale["failure_code"] == "stale-windows-job" + assert stale["next_action"] == "none" + + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, windows=True, windows_job="running"), + plan, + ) + challenge_path = tmp_path / "reported-challenge.json" + checkpoint_path = write_checkpoint(tmp_path, plan, "windows") + state, challenge = controller.issue_calibration_challenge( + state, + plan, + "windows", + challenge_path, + checkpoint_path, + issued_at_unix=NOW, + nonce="a" * 64, + ) + reported_digest = "sha256:" + "c" * 64 + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=reported_digest, + ), + plan, + ) + evidence_path = tmp_path / "different.json" + evidence_path.write_text( + json.dumps(platform_evidence("windows", challenge)), + encoding="utf-8", + ) + assert "sha256:" + hashlib.sha256(evidence_path.read_bytes()).hexdigest() != reported_digest + with pytest.raises(controller.Gate14ControllerError): + controller.collect_platform( + state, + plan, + "windows", + evidence_path, + challenge_path, + ) + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +def test_initial_state_requires_all_planned_disks_absent_before_start(plan, platform): + value = observation( + plan, + **{f"{platform}_disk": True}, + ) + + state = controller.reconcile(controller.initial_state(plan), value, plan) + + assert state["phase"] == "CLEANING_FAILED" + assert state["failure_code"] == "orphaned-planned-disk" + assert state["next_action"] == "cleanup_failure" + + +def test_stale_passed_job_with_orphan_disk_cannot_claim_terminal_cleanup(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation( + plan, + windows_job="passed", + windows_digest="sha256:" + "c" * 64, + windows_disk=True, + ), + plan, + ) + + assert state["phase"] == "CLEANING_FAILED" + assert state["cleanup_verified"] is False + assert state["next_action"] == "cleanup_failure" + + +def test_linux_start_requires_absent_disk_and_absent_stale_job(plan): + windows_digest = "sha256:" + "c" * 64 + state = { + **controller.initial_state(plan), + "revision": 2, + "phase": "WINDOWS_DELETING", + "windows_evidence_digest": windows_digest, + "windows_challenge_sha256": "sha256:" + "e" * 64, + "windows_challenge_consumed": True, + "windows_consumed": True, + "next_action": "delete_windows", + } + orphan = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_disk=True, + ), + plan, + ) + assert orphan["phase"] == "CLEANING_FAILED" + assert orphan["failure_code"] == "orphaned-linux-disk" + assert orphan["next_action"] == "cleanup_failure" + + stale_job = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest="sha256:" + "d" * 64, + ), + plan, + ) + assert stale_job["phase"] == "CLEANED_FAILURE" + assert stale_job["failure_code"] == "stale-linux-job" + assert stale_job["cleanup_verified"] is True + assert stale_job["next_action"] == "none" + + +@pytest.mark.parametrize("returned_resource", [{"windows": True}, {"windows_disk": True}]) +def test_linux_deletion_escalates_returned_windows_resources(plan, returned_resource): + windows_digest = "sha256:" + "c" * 64 + linux_digest = "sha256:" + "d" * 64 + state = { + **controller.initial_state(plan), + "revision": 5, + "phase": "LINUX_DELETING", + "windows_evidence_digest": windows_digest, + "linux_evidence_digest": linux_digest, + "windows_challenge_sha256": "sha256:" + "e" * 64, + "linux_challenge_sha256": "sha256:" + "f" * 64, + "windows_challenge_consumed": True, + "linux_challenge_consumed": True, + "windows_consumed": True, + "linux_consumed": True, + "next_action": "delete_linux", + } + + state = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest=linux_digest, + **returned_resource, + ), + plan, + ) + + assert state["phase"] == "CLEANING_FAILED" + assert state["failure_code"] == "windows-resources-returned" + assert state["next_action"] == "cleanup_failure" + + +def test_load_plan_recomputes_total_ledger_commitment(tmp_path): + with pytest.raises(controller.Gate14ControllerError): + write_plan(tmp_path, additional_current_maximum="99.00") + + +def test_load_plan_rejects_hidden_active_reservation_below_epoch_anchor(tmp_path): + with pytest.raises(controller.Gate14ControllerError): + write_plan( + tmp_path, + additional_current_maximum="1.00", + hide_additional_below_anchor=True, + ) + + +def test_challenge_rejects_missing_or_foreign_ready_checkpoint(tmp_path, plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, windows=True, windows_job="running"), + plan, + ) + challenge_path = tmp_path / "challenge.json" + missing = tmp_path / "missing-checkpoint.json" + with pytest.raises( + controller.Gate14ControllerError, + match="checkpoint is invalid", + ): + controller.issue_calibration_challenge( + state, + plan, + "windows", + challenge_path, + missing, + issued_at_unix=NOW, + nonce="a" * 64, + ) + + checkpoint_path = write_checkpoint(tmp_path, plan, "windows") + value = json.loads(checkpoint_path.read_text(encoding="utf-8")) + value["source_commit"] = "9" * 40 + checkpoint_path.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises( + controller.Gate14ControllerError, + match="checkpoint is invalid", + ): + controller.issue_calibration_challenge( + state, + plan, + "windows", + challenge_path, + checkpoint_path, + issued_at_unix=NOW, + nonce="a" * 64, + ) diff --git a/tests/test_gate14_run_packaged_lifecycle.py b/tests/test_gate14_run_packaged_lifecycle.py new file mode 100644 index 000000000..de2e4e614 --- /dev/null +++ b/tests/test_gate14_run_packaged_lifecycle.py @@ -0,0 +1,144 @@ +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_run_packaged_lifecycle as entrypoint # noqa: E402 + + +class FakeActions: + def __init__(self): + self.closed = False + + def prepare(self, _config): + raise AssertionError("mocked sequencer should own prepare") + + def calibrate(self, _config, _challenge): + raise AssertionError("mocked sequencer should own calibrate") + + def cleanup(self, _config): + raise AssertionError("mocked sequencer should own cleanup") + + def close(self): + self.closed = True + + +def test_factory_selects_only_the_native_platform_adapters(): + assert entrypoint._factory("windows") is entrypoint.windows_transport.WindowsActionTransport + assert entrypoint._factory("linux") is entrypoint.linux_transport.LinuxActionTransport + with pytest.raises(entrypoint.Gate14LifecycleEntrypointError, match="platform"): + entrypoint._factory("macos") + + +def test_run_from_config_binds_platform_and_closes_the_adapter(monkeypatch, tmp_path): + config = SimpleNamespace(platform="linux") + actions = FakeActions() + expected = { + "run_id": "gate14-a", + "platform": "linux", + "source_commit": "a" * 40, + } + seen = [] + + monkeypatch.setattr(entrypoint.lifecycle, "load_config", lambda path: config) + + def run_lifecycle(observed_config, observed_actions): + seen.append((observed_config, observed_actions)) + return expected + + monkeypatch.setattr(entrypoint.lifecycle, "run_lifecycle", run_lifecycle) + + assert ( + entrypoint.run_from_config( + tmp_path / "gate14-lifecycle.json", + action_factory=lambda observed: actions if observed is config else None, + native_platform="linux", + ) + == expected + ) + assert seen == [(config, actions)] + assert actions.closed is True + + +def test_run_from_config_rejects_cross_platform_execution_before_actions( + monkeypatch, + tmp_path, +): + monkeypatch.setattr( + entrypoint.lifecycle, + "load_config", + lambda path: SimpleNamespace(platform="windows"), + ) + called = False + + def factory(_config): + nonlocal called + called = True + return FakeActions() + + with pytest.raises( + entrypoint.Gate14LifecycleEntrypointError, + match="platform binding", + ): + entrypoint.run_from_config( + tmp_path / "gate14-lifecycle.json", + action_factory=factory, + native_platform="linux", + ) + assert called is False + + +def test_run_from_config_closes_actions_when_the_sequencer_fails( + monkeypatch, + tmp_path, +): + config = SimpleNamespace(platform="linux") + actions = FakeActions() + monkeypatch.setattr(entrypoint.lifecycle, "load_config", lambda path: config) + + def fail(_config, _actions): + raise RuntimeError("private diagnostic") + + monkeypatch.setattr(entrypoint.lifecycle, "run_lifecycle", fail) + with pytest.raises(RuntimeError, match="private diagnostic"): + entrypoint.run_from_config( + tmp_path / "gate14-lifecycle.json", + action_factory=lambda _config: actions, + native_platform="linux", + ) + assert actions.closed is True + + +def test_invalid_adapter_is_rejected(monkeypatch, tmp_path): + monkeypatch.setattr( + entrypoint.lifecycle, + "load_config", + lambda path: SimpleNamespace(platform="linux"), + ) + with pytest.raises( + entrypoint.Gate14LifecycleEntrypointError, + match="adapter", + ): + entrypoint.run_from_config( + tmp_path / "gate14-lifecycle.json", + action_factory=lambda _config: object(), + native_platform="linux", + ) + + +def test_main_emits_only_a_bounded_failure_code(monkeypatch, capsys): + def fail(_path): + raise RuntimeError("secret private detail") + + monkeypatch.setattr(entrypoint, "run_from_config", fail) + assert entrypoint.main(["--config", "/qualification/gate14/gate14-lifecycle.json"]) == 2 + assert json.loads(capsys.readouterr().out) == { + "failure_code": "gate14_lifecycle_failed", + "result": "failed", + "schema_version": 1, + } diff --git a/tests/test_gate14_windows_action_transport.py b/tests/test_gate14_windows_action_transport.py new file mode 100644 index 000000000..d7c108abd --- /dev/null +++ b/tests/test_gate14_windows_action_transport.py @@ -0,0 +1,628 @@ +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_calibration_challenge as challenge_contract # noqa: E402 +import gate14_windows_action_transport as transport # noqa: E402 + + +def config_fixture(tmp_path): + path = tmp_path / "gate14-lifecycle.json" + path.write_text('{"bound":true}\n', encoding="utf-8") + return SimpleNamespace( + attempt_ordinal=1, + config_path=path, + config_sha256="sha256:" + hashlib.sha256(path.read_bytes()).hexdigest(), + package_sha256="sha256:" + "b" * 64, + platform="windows", + run_id="gate14-rpc-a", + source_commit="a" * 40, + ) + + +def challenge(): + return challenge_contract.create( + run_id="gate14-rpc-a", + platform="windows", + source_commit="a" * 40, + package_sha256="sha256:" + "b" * 64, + checkpoint_sha256="sha256:" + "c" * 64, + controller_state_revision=7, + issued_at_unix=1_000, + lifetime_seconds=900, + nonce="d" * 64, + ) + + +def test_challenge_payload_contains_the_full_safe_time_binding(): + value = challenge() + assert transport._challenge_payload(value) == { + "challenge_sha256": challenge_contract.digest(value), + "controller_state_revision": 7, + "issued_at_unix": 1_000, + "expires_at_unix": 1_900, + } + + +@pytest.mark.parametrize( + "timeouts", + [ + {}, + {"prepare": 1, "calibrate": 1, "cleanup": 1, "extra": 1}, + {"prepare": True, "calibrate": 1, "cleanup": 1}, + {"prepare": 7_201, "calibrate": 1, "cleanup": 1}, + {"prepare": 1, "calibrate": 3_601, "cleanup": 1}, + {"prepare": 1, "calibrate": 1, "cleanup": 601}, + ], +) +def test_operation_timeout_contract_fails_closed(timeouts): + with pytest.raises(transport.Gate14ActionTransportError, match="timeout"): + transport._operation_timeouts(timeouts) + + +def test_host_job_and_lifecycle_use_the_same_config_filename(): + import gate14_host_job as host_job + import gate14_packaged_lifecycle as lifecycle + + assert set(host_job.LIFECYCLE_CONFIG_NAMES.values()) == {"gate14-lifecycle.json"} + assert lifecycle._OUTPUT_NAMES["evidence_path"] == "gate14-platform-evidence.json" + + +def test_duplicate_nonfinite_and_private_response_material_fail_closed(): + with pytest.raises(transport.Gate14ActionTransportError, match="duplicate"): + transport._strict_json(b'{"result":"passed","result":"failed"}') + with pytest.raises(transport.Gate14ActionTransportError, match="non-finite"): + transport._strict_json(b'{"sample":NaN}') + with pytest.raises(transport.Gate14ActionTransportError, match="private"): + transport._assert_safe_payload({"control_token": "not-serialized"}) + with pytest.raises(transport.Gate14ActionTransportError, match="private"): + transport._assert_safe_payload({"value": "drift_control_never-serialized"}) + + +@pytest.mark.parametrize("boolean_field", ["schema_version", "request_id"]) +def test_python_transport_rejects_boolean_integer_response_fields( + tmp_path, + boolean_field, +): + config = config_fixture(tmp_path) + child = """ +import json +import sys + +request = json.loads(sys.stdin.readline()) +response = { + "failure_code": None, + "operation": request["operation"], + "payload": {}, + "request_id": request["request_id"], + "result": "passed", + "schema_version": 1, + "scope": request["scope"], + "session_id": request["session_id"], +} +response[sys.argv[1]] = True +print(json.dumps(response, allow_nan=False, separators=(",", ":"), sort_keys=True), flush=True) +""" + + def process_factory(_arguments, **kwargs): + return subprocess.Popen( + [sys.executable, "-u", "-c", child, boolean_field], + **kwargs, + ) + + action_host = transport.WindowsActionTransport( + config, + powershell="test-powershell", + process_factory=process_factory, + ) + try: + with pytest.raises( + transport.Gate14ActionTransportError, + match="response binding is invalid", + ): + action_host.request("prepare", {}) + finally: + action_host.close() + + +def test_python_transport_rejects_noncanonical_response_frame(tmp_path): + config = config_fixture(tmp_path) + child = """ +import json +import sys + +request = json.loads(sys.stdin.readline()) +response = { + "failure_code": None, + "operation": request["operation"], + "payload": {}, + "request_id": request["request_id"], + "result": "passed", + "schema_version": 1, + "scope": request["scope"], + "session_id": request["session_id"], +} +print(json.dumps(response), flush=True) +""" + + def process_factory(_arguments, **kwargs): + return subprocess.Popen([sys.executable, "-u", "-c", child], **kwargs) + + action_host = transport.WindowsActionTransport( + config, + powershell="test-powershell", + process_factory=process_factory, + ) + try: + with pytest.raises( + transport.Gate14ActionTransportError, + match="response is invalid", + ): + action_host.request("prepare", {}) + finally: + action_host.close() + + +def test_normalized_helper_binding_accepts_crlf_and_rejects_mutation(tmp_path): + original = ROOT / "scripts" / "gate13_windows_packaged_lifecycle.ps1" + payload = original.read_bytes().replace(b"\r\n", b"\n") + expected = hashlib.sha256(payload).hexdigest() + crlf = tmp_path / original.name + crlf.write_bytes(payload.replace(b"\n", b"\r\n")) + + assert transport._normalized_source(crlf, expected) == crlf.resolve() + + crlf.write_bytes(crlf.read_bytes() + b"# mutation\r\n") + with pytest.raises(transport.Gate14ActionTransportError, match="digest changed"): + transport._normalized_source(crlf, expected) + + +@pytest.mark.skipif(os.name != "nt", reason="Windows file sharing is required") +def test_verified_source_lock_denies_write_and_replace_until_release(tmp_path): + source = tmp_path / "source.ps1" + replacement = tmp_path / "replacement.ps1" + payload = b"Write-Output 'bound'\n" + source.write_bytes(payload) + expected = hashlib.sha256(payload).hexdigest() + + verified, handle = transport._open_verified_source(source, expected) + assert verified == source.resolve() + try: + with pytest.raises(OSError): + source.write_bytes(b"changed\n") + replacement.write_bytes(b"replacement\n") + with pytest.raises(OSError): + replacement.replace(source) + finally: + handle.close() + + source.write_bytes(b"released\n") + assert source.read_bytes() == b"released\n" + + config = tmp_path / "gate14-lifecycle.json" + config_payload = b'{"bound":true}\n' + config.write_bytes(config_payload) + config_digest = "sha256:" + hashlib.sha256(config_payload).hexdigest() + verified_config, config_handle = transport._open_verified_config( + config, + config_digest, + 65_536, + ) + assert verified_config == config.resolve() + try: + with pytest.raises(OSError): + config.write_bytes(b'{"bound":false}\n') + finally: + config_handle.close() + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_preserves_one_process_and_state_across_operations(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + prepared = action_host.request("prepare", {}) + calibrated = action_host.request( + "calibrate", + transport._challenge_payload(challenge()), + ) + cleaned = action_host.request("cleanup", {}) + + assert prepared["helpers_loaded"] is True + assert prepared["host_process_id"] == calibrated["host_process_id"] + assert prepared["state_nonce"] == calibrated["state_nonce"] + assert cleaned == { + "action_temporaries_removed": True, + "attempt_ordinal": 1, + "credentials_removed": True, + "platform": "windows", + "processes_absent": True, + "run_id": "gate14-rpc-a", + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + } + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_runs_cleanup_on_eof(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + + action_host.request("prepare", {}) + action_host.close() + + assert marker.read_text(encoding="utf-8") == "cleaned" + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_out_of_order_operation_and_cleans(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + + with pytest.raises( + transport.Gate14ActionTransportError, + match="response binding is invalid", + ): + action_host.request( + "calibrate", + transport._challenge_payload(challenge()), + ) + + assert marker.read_text(encoding="utf-8") == "cleaned" + + +@pytest.mark.parametrize( + "attack", + [ + {"challenge_sha256": "sha256:" + "c" * 64}, + { + "challenge_sha256": "sha256:" + "c" * 64, + "controller_state_revision": True, + "issued_at_unix": 1_000, + "expires_at_unix": 1_900, + }, + { + "challenge_sha256": "sha256:" + "c" * 64, + "controller_state_revision": 1, + "issued_at_unix": 1_000, + "expires_at_unix": 1_059, + }, + ], +) +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_incomplete_or_coerced_challenge(tmp_path, attack): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + action_host.request("prepare", {}) + with pytest.raises(transport.Gate14ActionTransportError): + action_host.request("calibrate", attack) + assert marker.read_text(encoding="utf-8") == "cleaned" + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_duplicate_keys_before_dispatch(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + frame = { + "binding": action_host._binding, + "operation": "prepare", + "payload": {}, + "request_id": 1, + "schema_version": 1, + "scope": transport.SCOPE, + "session_id": action_host._session_id, + } + rendered = transport._canonical(frame).replace( + b'"operation":"prepare"', + b'"operation":"prepare","operation":"cleanup"', + ) + action_host._process.stdin.write(rendered + b"\n") + action_host._process.stdin.flush() + response = action_host._responses.get(timeout=10) + + assert isinstance(response, bytes) + assert transport._strict_json(response)["failure_code"] == "invalid-action-frame" + action_host._process.wait(timeout=10) + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +@pytest.mark.parametrize("boolean_field", ["schema_version", "request_id"]) +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_boolean_integer_frame_fields_and_cleans( + tmp_path, + boolean_field, +): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + frame = { + "binding": action_host._binding, + "operation": "prepare", + "payload": {}, + "request_id": 1, + "schema_version": 1, + "scope": transport.SCOPE, + "session_id": action_host._session_id, + } + frame[boolean_field] = True + action_host._process.stdin.write(transport._canonical(frame) + b"\n") + action_host._process.stdin.flush() + response = action_host._responses.get(timeout=10) + + assert isinstance(response, bytes) + assert transport._strict_json(response)["failure_code"] == "invalid-action-frame" + action_host._process.wait(timeout=10) + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +@pytest.mark.parametrize( + "attack", + [ + "binding-run-id-number", + "binding-run-id-array", + "binding-platform-array", + "binding-source-array", + "binding-package-array", + "binding-config-digest-array", + "binding-run-id-changed", + "binding-attempt-changed", + "binding-source-changed", + "binding-package-changed", + "binding-config-digest-changed", + "frame-scope-array", + "frame-session-array", + "frame-operation-array", + ], +) +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_coerced_or_changed_controller_binding( + tmp_path, + attack, +): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + frame = { + "binding": dict(action_host._binding), + "operation": "prepare", + "payload": {}, + "request_id": 1, + "schema_version": 1, + "scope": transport.SCOPE, + "session_id": action_host._session_id, + } + mutations = { + "binding-run-id-number": ("binding", "run_id", 1), + "binding-run-id-array": ("binding", "run_id", [config.run_id]), + "binding-platform-array": ("binding", "platform", ["windows"]), + "binding-source-array": ("binding", "source_commit", [config.source_commit]), + "binding-package-array": ("binding", "package_sha256", [config.package_sha256]), + "binding-config-digest-array": ( + "binding", + "lifecycle_config_sha256", + [config.config_sha256], + ), + "binding-run-id-changed": ("binding", "run_id", "gate14-rpc-b"), + "binding-attempt-changed": ("binding", "attempt_ordinal", 2), + "binding-source-changed": ("binding", "source_commit", "c" * 40), + "binding-package-changed": ( + "binding", + "package_sha256", + "sha256:" + "c" * 64, + ), + "binding-config-digest-changed": ( + "binding", + "lifecycle_config_sha256", + "sha256:" + "c" * 64, + ), + "frame-scope-array": ("frame", "scope", [transport.SCOPE]), + "frame-session-array": ("frame", "session_id", [action_host._session_id]), + "frame-operation-array": ("frame", "operation", ["prepare"]), + } + target, field, value = mutations[attack] + if target == "binding": + frame["binding"][field] = value + else: + frame[field] = value + + action_host._process.stdin.write(transport._canonical(frame) + b"\n") + action_host._process.stdin.flush() + response = action_host._responses.get(timeout=10) + + assert isinstance(response, bytes) + assert transport._strict_json(response)["failure_code"] == "invalid-action-frame" + action_host._process.wait(timeout=10) + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_array_calibration_digest_and_cleans(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + action_host.request("prepare", {}) + frame = { + "binding": action_host._binding, + "operation": "calibrate", + "payload": {"challenge_sha256": ["sha256:" + "c" * 64]}, + "request_id": 2, + "schema_version": 1, + "scope": transport.SCOPE, + "session_id": action_host._session_id, + } + action_host._process.stdin.write(transport._canonical(frame) + b"\n") + action_host._process.stdin.flush() + response = action_host._responses.get(timeout=10) + + assert isinstance(response, bytes) + assert transport._strict_json(response)["failure_code"] == "invalid-action-frame" + action_host._process.wait(timeout=10) + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_native_host_rejects_replayed_request_id_and_cleans(tmp_path): + config = config_fixture(tmp_path) + marker = tmp_path / "cleanup.marker" + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + transport_self_test=True, + self_test_cleanup_marker=marker, + ) + try: + action_host.request("prepare", {}) + replay = { + "binding": action_host._binding, + "operation": "cleanup", + "payload": {}, + "request_id": 1, + "schema_version": 1, + "scope": transport.SCOPE, + "session_id": action_host._session_id, + } + action_host._process.stdin.write(transport._canonical(replay) + b"\n") + action_host._process.stdin.flush() + response = action_host._responses.get(timeout=10) + + assert isinstance(response, bytes) + assert transport._strict_json(response)["failure_code"] == "invalid-action-frame" + action_host._process.wait(timeout=10) + assert marker.read_text(encoding="utf-8") == "cleaned" + finally: + action_host.close() + + +@pytest.mark.skipif( + shutil.which("powershell.exe") is None, + reason="native Windows PowerShell is unavailable", +) +def test_production_rejects_incomplete_lifecycle_config_before_dispatch(tmp_path): + config = config_fixture(tmp_path) + action_host = transport.WindowsActionTransport( + config, + powershell=shutil.which("powershell.exe"), + ) + try: + with pytest.raises( + transport.Gate14ActionTransportError, + match="action host ended before response", + ): + action_host.prepare(config) + + assert action_host._process.poll() not in (None, 0) + finally: + action_host.close() + + +def test_binding_rejects_wrong_platform_or_changed_config(tmp_path): + config = config_fixture(tmp_path) + config.platform = "linux" + with pytest.raises(transport.Gate14ActionTransportError, match="binding"): + transport._binding(config) + + config.platform = "windows" + config.config_path.write_text(json.dumps({"bound": False}), encoding="utf-8") + with pytest.raises( + transport.Gate14ActionTransportError, + match="configuration binding changed", + ): + transport.WindowsActionTransport(config, powershell="powershell.exe") diff --git a/tests/test_gate14_windows_product_actions.py b/tests/test_gate14_windows_product_actions.py new file mode 100644 index 000000000..d6137be6e --- /dev/null +++ b/tests/test_gate14_windows_product_actions.py @@ -0,0 +1,758 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / "scripts" / "gate13_windows_packaged_lifecycle.ps1" +INFERENCE = ROOT / "scripts" / "gate13_windows_localhost_inference.ps1" +PRODUCT = ROOT / "scripts" / "gate14_windows_product_actions.ps1" +HOST = ROOT / "scripts" / "gate14_windows_lifecycle_actions.ps1" +TRANSPORT = ROOT / "scripts" / "gate14_windows_action_transport.py" +POWERSHELL = Path(r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe") +MODEL_ID = "Qwen3.5 2B" +MANIFEST = "3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33" + + +def _ps_literal(value: Path | str) -> str: + return "'" + str(value).replace("'", "''") + "'" + + +def _run_powershell( + source: str, + tmp_path: Path, + *, + timeout: int = 120, +) -> subprocess.CompletedProcess[str]: + driver = tmp_path / "driver.ps1" + driver.write_text( + "$ErrorActionPreference = 'Stop'\n$global:LASTEXITCODE = 0\n" + source, + encoding="utf-8", + newline="\n", + ) + return subprocess.run( + [ + str(POWERSHELL), + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(driver), + ], + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + + +def test_sources_bind_real_windows_product_actions(): + product = PRODUCT.read_text(encoding="utf-8") + host = HOST.read_text(encoding="utf-8") + transport = TRANSPORT.read_text(encoding="utf-8") + lifecycle = LIFECYCLE.read_text(encoding="utf-8") + + for required in ( + "function Initialize-Gate14WindowsProductActions", + "function Invoke-Gate14WindowsProductPrepare", + "function Invoke-Gate14WindowsProductCalibrate", + "function Invoke-Gate14WindowsProductCleanup", + "Test-Gate13PackageAudit", + "Install-Gate13VerifiedPackage", + "Invoke-Gate13Bootstrap", + "Move-Gate14WindowsWarmCache", + "Start-Gate14WindowsProduct", + "Get-Gate14WindowsExactCacheInventory", + "Invoke-Gate14WindowsLowVramProbe", + "Invoke-Gate14WindowsCpuPowerProbe", + "Invoke-Gate14WindowsCrashRecovery", + "Invoke-Gate14WindowsPause", + "Invoke-Gate14WindowsRestart", + "Gate14.LoopbackLoad", + "Start-Gate14WindowsPowerBurn", + "Get-Gate14WindowsClosedSchedule", + "challenge expired during calibration", + ): + assert required in product + + assert ". $inferencePath" in host + assert ". $productActionsPath" in host + assert "product-prepare-failed" in host + assert "product-calibration-failed" in host + assert "Invoke-Gate14WindowsProductCleanup" in host + assert "_PRODUCT_ACTIONS_SHA256" in transport + assert '"-ProductActionsSha256"' in transport + assert "_open_verified_source" in transport + assert "Remove-Item -LiteralPath $full -Recurse" not in product + assert "Stop-Gate13Product\n Start-Gate14WindowsProduct" in product + assert "public bool ContainsProcessId" in lifecycle + assert "public void KillMemberProcess" in lifecycle + + +@pytest.mark.skipif( + not POWERSHELL.is_file(), + reason="native Windows PowerShell is required", +) +def test_job_membership_and_exact_member_termination_are_native(tmp_path): + source = f""" +. {_ps_literal(LIFECYCLE)} +Initialize-Gate13NativeHost +$owner = [Gate13.NativeHost]::Start( + {_ps_literal(POWERSHELL)}, + [string[]]@( + '-NoLogo', + '-NoProfile', + '-NonInteractive', + '-Command', + 'Start-Sleep -Seconds 30' + ), + {_ps_literal(tmp_path)} +) +try {{ + $field = $owner.GetType().GetField( + 'processId', + [Reflection.BindingFlags]'NonPublic,Instance' + ) + $memberPid = [int]$field.GetValue($owner) + $containedBefore = $owner.ContainsProcessId($memberPid) + $owner.KillMemberProcess($memberPid, 30000) + $emptyAfter = ($owner.ActiveProcessCount -eq 0) + [Console]::Out.WriteLine((@{{ + contained_before = $containedBefore + empty_after = $emptyAfter + }} | ConvertTo-Json -Compress)) +}} +finally {{ + $owner.ForceAndVerify(30000) + $owner.Dispose() +}} +""" + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == { + "contained_before": True, + "empty_after": True, + } + + +@pytest.mark.skipif( + not POWERSHELL.is_file(), + reason="native Windows PowerShell is required", +) +def test_product_start_uses_only_action_specific_node_paths_on_initial_and_restart( + tmp_path, +): + product_root = tmp_path / "CommunityAI" + persistent_root = tmp_path / "persistent" + node_config = persistent_root / "node-config.json" + bootstrap = product_root / "_internal" / "bootstrap" / "catalog-bootstrap.json" + desktop = product_root / "CommunityAI.exe" + fake_profile = tmp_path / "profile" + persistent_root.mkdir() + bootstrap.parent.mkdir(parents=True) + node_config.write_text("{}\n", encoding="utf-8") + bootstrap.write_text("{}\n", encoding="utf-8") + desktop.write_bytes(b"desktop") + + source = f""" +. {_ps_literal(LIFECYCLE)} +. {_ps_literal(INFERENCE)} +. {_ps_literal(PRODUCT)} + +$env:USERPROFILE = {_ps_literal(fake_profile)} +$script:LifecycleProcess = $null +$script:LifecycleDesktopExe = {_ps_literal(desktop)} +$script:LifecycleProductRoot = {_ps_literal(product_root)} +$script:LifecycleNodeConfig = {_ps_literal(node_config)} +$script:LifecyclePersistentRoot = {_ps_literal(persistent_root)} +$script:LifecycleBootstrap = {_ps_literal(bootstrap)} +$script:capturedStarts = New-Object Collections.ArrayList + +function Initialize-Gate13NativeHost {{ }} +function New-Gate14WindowsContainedProductProcess {{ + param($Executable, $Arguments, $WorkingDirectory) + [void]$script:capturedStarts.Add([pscustomobject]@{{ + executable = $Executable + arguments = @($Arguments) + working_directory = $WorkingDirectory + }}) + return [pscustomobject]@{{ ActiveProcessCount = 1 }} +}} + +Start-Gate14WindowsProduct +$script:LifecycleProcess = $null +Start-Gate14WindowsProduct +$defaultNodeRoot = Join-Path $env:USERPROFILE '.drift\\node' +[Console]::Out.WriteLine((@{{ + starts = @($script:capturedStarts) + default_root_absent = -not (Test-Path -LiteralPath $defaultNodeRoot) +}} | ConvertTo-Json -Compress -Depth 8)) +""" + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["default_root_absent"] is True + assert len(payload["starts"]) == 2 + for start in payload["starts"]: + assert start == { + "executable": str(desktop), + "arguments": [ + "--node-config", + str(node_config), + "--node-data-dir", + str(persistent_root), + "--bootstrap-config", + str(bootstrap), + ], + "working_directory": str(product_root), + } + + +@pytest.mark.skipif( + not POWERSHELL.is_file(), + reason="native Windows PowerShell is required", +) +def test_post_credential_start_failure_runs_exact_cleanup(tmp_path): + work_root = tmp_path / "work" + staging_root = tmp_path / "staging" + package = tmp_path / "CommunityAI-windows.zip" + warm_cache = work_root / "gate14-warm-cache" + config_path = tmp_path / "gate14-lifecycle.json" + work_root.mkdir() + staging_root.mkdir() + warm_cache.mkdir() + package.write_bytes(b"package") + artifacts = [ + { + "path": f"blobs/{index:02d}.bin", + "role": "weight", + "sha256": "sha256:" + f"{index + 1:064x}", + "size_bytes": index + 1, + } + for index in range(8) + ] + config_path.write_text( + json.dumps( + { + "run_id": "gate14-windows-product-a", + "attempt_ordinal": 1, + "source_commit": "a" * 40, + "package_sha256": "sha256:" + "b" * 64, + "platform": "windows", + "model_id": MODEL_ID, + "manifest_digest": "sha256:" + MANIFEST, + "work_root": str(work_root), + "staging_root": str(staging_root), + "package_path": str(package), + "package_bytes": package.stat().st_size, + "disk_bytes": 8_000_000_000, + "vram_bytes": 20_000_000_000, + "bandwidth_mbps": 250.0, + "power_watts": 200.0, + "pause_timeout_seconds": 30.0, + "sample_interval_seconds": 1.0, + "warm_cache": {"artifacts": artifacts}, + }, + separators=(",", ":"), + sort_keys=True, + ) + + "\n", + encoding="utf-8", + newline="\n", + ) + + source = f""" +. {_ps_literal(LIFECYCLE)} +. {_ps_literal(INFERENCE)} +. {_ps_literal(PRODUCT)} + +$script:testCredentialCount = 0 +$script:testCleanupCalls = 0 +function Initialize-Gate13CredentialInterop {{ }} +function New-Gate14WindowsRunInput {{ }} +function Test-Gate13PackageAudit {{ + return [pscustomobject]@{{ + SourceCommit = ('a' * 40) + PackageDigest = ('b' * 64) + }} +}} +function Install-Gate13VerifiedPackage {{ + param($Audit) + New-Item -ItemType Directory -Path $script:LifecycleProductRoot -Force | Out-Null + [IO.File]::WriteAllText( + $script:LifecycleDesktopExe, + 'desktop', + (New-Object Text.UTF8Encoding($false)) + ) +}} +function Test-Gate13PackagedSelfTests {{ + return [pscustomobject]@{{ result = 'passed' }} +}} +function Invoke-Gate13Bootstrap {{ + if ($script:testCredentialCount -ne 0) {{ + throw 'bootstrap observed an unexpected credential' + }} + return [pscustomobject]@{{ result = 'passed' }} +}} +function Get-Gate13CredentialCount {{ + return [int]$script:testCredentialCount +}} +function Get-Gate13ProductProcessCount {{ return 0 }} +function Get-Gate13SelectedManifestContext {{ + param($Profile) + return [pscustomobject]@{{ + CacheDir = (Join-Path $script:Gate14ProductActionRoot 'cache') + ManifestPath = (Join-Path $script:Gate14ProductActionRoot 'manifest.json') + }} +}} +function Move-Gate14WindowsWarmCache {{ }} +function Get-Gate14WindowsExactCacheInventory {{ + param($Root, $Context) + return [pscustomobject]@{{ Entries = @(); Count = 8; Bytes = 4571197320 }} +}} +function Assert-Gate14WindowsSameCacheInventory {{ }} +function Start-Gate14WindowsProduct {{ + if ($script:testCredentialCount -ne 0) {{ + throw 'start observed an unexpected credential' + }} + $script:testCredentialCount = 1 +}} +function Wait-Gate13ProductStatus {{ + param($TimeoutSeconds) + return [pscustomobject]@{{ + Profile = [pscustomobject]@{{ + ModelId = "Qwen3.5 2B" + ManifestDigest = "3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33" + }} + ControlToken = ('drift_control_' + ('A' * 43)) + }} +}} +function Set-Gate14WindowsContributionPolicy {{ + param($ControlToken, $Schedule) + if ( + $script:testCredentialCount -ne 1 -or + $ControlToken -cne ('drift_control_' + ('A' * 43)) + ) {{ + throw 'post-start state was not established' + }} + throw 'post-start-policy-failure' +}} +function Force-Gate13ProductCleanup {{ + $script:testCleanupCalls += 1 +}} +function Invoke-Gate13Contained {{ + param($Executable, $Arguments, $WorkingDirectory, $TimeoutSeconds) + if ($Arguments -ccontains '--delete-control-key') {{ + $script:testCredentialCount = 0 + }} + return '{{"result":"passed"}}' +}} + +Initialize-Gate14WindowsProductActions ` + -LifecycleConfig {_ps_literal(config_path)} ` + -RunId 'gate14-windows-product-a' ` + -AttemptOrdinal 1 ` + -SourceCommit ('a' * 40) ` + -PackageSha256 ('sha256:' + ('b' * 64)) ` + -ProductActionsPath {_ps_literal(PRODUCT)} + +$actionRoot = $script:Gate14ProductActionRoot +$warmRoot = $script:Gate14ProductWarmCache +$message = $null +try {{ + [void](Invoke-Gate14WindowsProductPrepare) +}} +catch {{ + $message = $_.Exception.Message +}} +$cleanup = Invoke-Gate14WindowsProductCleanup +[Console]::Out.WriteLine((@{{ + error = $message + cleanup_calls = $script:testCleanupCalls + credential_count = $script:testCredentialCount + action_root_absent = -not (Test-Path -LiteralPath $actionRoot) + warm_root_absent = -not (Test-Path -LiteralPath $warmRoot) + cleaned = $script:Gate14ProductCleaned + cleanup = $cleanup +}} | ConvertTo-Json -Compress -Depth 8)) +""" + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["error"] == "post-start-policy-failure" + assert payload["cleanup_calls"] == 1 + assert payload["credential_count"] == 0 + assert payload["action_root_absent"] is True + assert payload["warm_root_absent"] is True + assert payload["cleaned"] is True + assert payload["cleanup"] == { + "schema_version": 1, + "scope": "gate14-host-lifecycle-cleanup", + "run_id": "gate14-windows-product-a", + "platform": "windows", + "attempt_ordinal": 1, + "processes_absent": True, + "credentials_removed": True, + "action_temporaries_removed": True, + } + + +@pytest.mark.parametrize("failure_mode", ["process", "burn", "credential"]) +@pytest.mark.skipif( + not POWERSHELL.is_file(), + reason="native Windows PowerShell is required", +) +def test_cleanup_preserves_deletion_tool_and_retries_transient_failures( + tmp_path, + failure_mode, +): + action_root = tmp_path / "action" + warm_root = tmp_path / "warm" + product_root = action_root / "install" / "CommunityAI" + desktop = product_root / "CommunityAI.exe" + product_root.mkdir(parents=True) + warm_root.mkdir() + desktop.write_bytes(b"desktop") + + source = f""" +. {_ps_literal(LIFECYCLE)} +. {_ps_literal(INFERENCE)} +. {_ps_literal(PRODUCT)} + +$script:Gate14ProductInitialized = $true +$script:Gate14ProductCleaned = $false +$script:Gate14ProductConfig = [pscustomobject]@{{ + run_id = 'gate14-cleanup-retry-a' + attempt_ordinal = 1 +}} +$script:Gate14ProductActionRoot = [IO.Path]::GetFullPath({_ps_literal(action_root)}) +$script:Gate14ProductWarmCache = [IO.Path]::GetFullPath({_ps_literal(warm_root)}) +$script:LifecycleProductRoot = {_ps_literal(product_root)} +$script:LifecycleDesktopExe = {_ps_literal(desktop)} +$script:Gate14ProductBurns = New-Object Collections.ArrayList +$script:Gate14ProductCacheLocks = New-Object Collections.ArrayList +$script:testFailureMode = '{failure_mode}' +$script:testForceCalls = 0 +$script:testDeleteCalls = 0 +$script:testBurnCalls = 0 +$script:testProcessCount = 1 +$script:testCredentialCount = 1 +if ($script:testFailureMode -ceq 'burn') {{ + $burn = [pscustomobject]@{{ ActiveProcessCount = 1 }} + $burn | Add-Member -MemberType ScriptMethod -Name ForceAndVerify -Value {{ + param([int]$TimeoutMilliseconds) + $script:testBurnCalls += 1 + if ($script:testBurnCalls -eq 1) {{ + throw 'one-shot burn cleanup failure' + }} + $this.ActiveProcessCount = 0 + }} + [void]$script:Gate14ProductBurns.Add($burn) +}} + +function Force-Gate13ProductCleanup {{ + $script:testForceCalls += 1 + if ( + $script:testFailureMode -ceq 'process' -and + $script:testForceCalls -eq 1 + ) {{ + throw 'one-shot process cleanup failure' + }} + $script:testProcessCount = 0 +}} +function Get-Gate13ProductProcessCount {{ + return [int]$script:testProcessCount +}} +function Get-Gate13CredentialCount {{ + return [int]$script:testCredentialCount +}} +function Invoke-Gate13Contained {{ + param($Executable, $Arguments, $WorkingDirectory, $TimeoutSeconds) + if (-not ($Arguments -ccontains '--delete-control-key')) {{ + throw 'unexpected cleanup command' + }} + $script:testDeleteCalls += 1 + if ( + $script:testFailureMode -ceq 'credential' -and + $script:testDeleteCalls -eq 1 + ) {{ + throw 'one-shot credential cleanup failure' + }} + $script:testCredentialCount = 0 + return '{{"result":"passed"}}' +}} + +$firstError = $null +try {{ + [void](Invoke-Gate14WindowsProductCleanup) +}} +catch {{ + $firstError = $_.Exception.Message +}} +$preserved = ( + (Test-Path -LiteralPath $script:Gate14ProductActionRoot) -and + (Test-Path -LiteralPath $script:Gate14ProductWarmCache) -and + (Test-Path -LiteralPath $script:LifecycleDesktopExe) +) +$firstCredential = $script:testCredentialCount +$firstProcess = $script:testProcessCount +$firstBurnCount = $script:Gate14ProductBurns.Count +$cleanup = Invoke-Gate14WindowsProductCleanup +[Console]::Out.WriteLine((@{{ + first_error = $firstError + preserved = $preserved + first_credential = $firstCredential + first_process = $firstProcess + first_burn_count = $firstBurnCount + force_calls = $script:testForceCalls + burn_calls = $script:testBurnCalls + delete_calls = $script:testDeleteCalls + final_credential = $script:testCredentialCount + final_process = $script:testProcessCount + final_burn_count = $script:Gate14ProductBurns.Count + action_absent = -not (Test-Path -LiteralPath $script:Gate14ProductActionRoot) + warm_absent = -not (Test-Path -LiteralPath $script:Gate14ProductWarmCache) + cleaned = $script:Gate14ProductCleaned + cleanup = $cleanup +}} | ConvertTo-Json -Compress -Depth 8)) +""" + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + expected_phase = "credential" if failure_mode == "credential" else "process" + assert payload["first_error"] == (f"Windows packaged product {expected_phase} cleanup was not proved") + assert payload["preserved"] is True + assert payload["first_credential"] == 1 + assert payload["first_process"] == (1 if failure_mode == "process" else 0) + assert payload["first_burn_count"] == (1 if failure_mode == "burn" else 0) + assert payload["force_calls"] == 2 + assert payload["burn_calls"] == (2 if failure_mode == "burn" else 0) + assert payload["delete_calls"] == (2 if failure_mode == "credential" else 1) + assert payload["final_credential"] == 0 + assert payload["final_process"] == 0 + assert payload["final_burn_count"] == 0 + assert payload["action_absent"] is True + assert payload["warm_absent"] is True + assert payload["cleaned"] is True + assert payload["cleanup"]["scope"] == "gate14-host-lifecycle-cleanup" + + +@pytest.mark.skipif( + not POWERSHELL.is_file(), + reason="native Windows PowerShell is required", +) +def test_exact_cache_inventory_and_cleanup_reject_unexpected_or_reparse_entries( + tmp_path, +): + cache_root = tmp_path / "cache" + outside = tmp_path / "outside" + artifact_path = cache_root / "manifest-artifacts" / MANIFEST / "snapshot" / "blobs" / "a.bin" + artifact_path.parent.mkdir(parents=True) + artifact_path.write_bytes(b"abc") + outside.mkdir() + (outside / "sentinel.txt").write_text("outside", encoding="utf-8") + artifact_digest = hashlib.sha256(b"abc").hexdigest() + + source = f""" +. {_ps_literal(LIFECYCLE)} +. {_ps_literal(INFERENCE)} +. {_ps_literal(PRODUCT)} + +$script:Gate14ProductProfile = [pscustomobject]@{{ + SelectedCount = 1 + SelectedBytes = 3 +}} +$script:Gate14ProductArtifacts = @([pscustomobject]@{{ + path = 'blobs/a.bin' + role = 'weight' + sha256 = '{artifact_digest}' + size_bytes = [int64]3 +}}) +$context = [pscustomobject]@{{ ManifestDigest = '{MANIFEST}' }} +$valid = Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context + +$unexpectedRejected = $false +[IO.File]::WriteAllText( + (Join-Path {_ps_literal(cache_root)} 'unexpected.bin'), + 'x', + (New-Object Text.UTF8Encoding($false)) +) +try {{ + [void](Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context) +}} +catch {{ + $unexpectedRejected = $_.Exception.Message -like '*unexpected file*' +}} +[IO.File]::Delete((Join-Path {_ps_literal(cache_root)} 'unexpected.bin')) + +$digestRejected = $false +[IO.File]::WriteAllBytes({_ps_literal(artifact_path)}, [byte[]](97, 98, 100)) +try {{ + [void](Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context) +}} +catch {{ + $digestRejected = $_.Exception.Message -like '*artifact verification failed*' +}} +[IO.File]::WriteAllBytes({_ps_literal(artifact_path)}, [byte[]](97, 98, 99)) + +$missingRejected = $false +[IO.File]::Delete({_ps_literal(artifact_path)}) +try {{ + [void](Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context) +}} +catch {{ + $missingRejected = $_.Exception.Message -like '*inventory is incomplete*' +}} +[IO.File]::WriteAllBytes({_ps_literal(artifact_path)}, [byte[]](97, 98, 99)) +$restored = Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context + +$junction = Join-Path {_ps_literal(cache_root)} 'outside-junction' +New-Item -ItemType Junction -Path $junction -Target {_ps_literal(outside)} | Out-Null +$reparseRejected = $false +try {{ + [void](Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context) +}} +catch {{ + $reparseRejected = $_.Exception.Message -like '*reparse point*' +}} + +$script:Gate14ProductActionRoot = [IO.Path]::GetFullPath({_ps_literal(cache_root)}) +$script:Gate14ProductWarmCache = [IO.Path]::GetFullPath( + (Join-Path {_ps_literal(tmp_path)} 'unused-warm-cache') +) +$cleanupRejected = $false +try {{ + Remove-Gate14WindowsExactTree -Path $script:Gate14ProductActionRoot +}} +catch {{ + $cleanupRejected = $_.Exception.Message -like '*descendant is unsafe*' +}} +$outsidePreserved = Test-Path -LiteralPath (Join-Path {_ps_literal(outside)} 'sentinel.txt') +[IO.Directory]::Delete($junction, $false) +$locked = Get-Gate14WindowsExactCacheInventory -Root {_ps_literal(cache_root)} -Context $context -HoldLocks +$identityMatches = ( + $locked.Entries[0].FileIdentity -ceq $restored.Entries[0].FileIdentity +) +$mutationRejected = $false +try {{ + [IO.File]::WriteAllBytes({_ps_literal(artifact_path)}, [byte[]](120, 121, 122)) +}} +catch {{ + $mutationRejected = $true +}} +Close-Gate14WindowsCacheLocks +Remove-Gate14WindowsExactTree -Path $script:Gate14ProductActionRoot + +[Console]::Out.WriteLine((@{{ + valid_count = $valid.Count + valid_bytes = $valid.Bytes + unexpected_rejected = $unexpectedRejected + digest_rejected = $digestRejected + missing_rejected = $missingRejected + reparse_rejected = $reparseRejected + cleanup_rejected = $cleanupRejected + outside_preserved = $outsidePreserved + identity_matches = $identityMatches + mutation_rejected = $mutationRejected + cache_removed = -not (Test-Path -LiteralPath {_ps_literal(cache_root)}) +}} | ConvertTo-Json -Compress)) +""" + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == { + "valid_count": 1, + "valid_bytes": 3, + "unexpected_rejected": True, + "digest_rejected": True, + "missing_rejected": True, + "reparse_rejected": True, + "cleanup_rejected": True, + "outside_preserved": True, + "identity_matches": True, + "mutation_rejected": True, + "cache_removed": True, + } + + +@pytest.mark.skipif( + not POWERSHELL.is_file(), + reason="native Windows PowerShell is required", +) +def test_calibration_record_and_stale_challenge_fail_closed(tmp_path): + source = f""" +. {_ps_literal(LIFECYCLE)} +. {_ps_literal(INFERENCE)} +. {_ps_literal(PRODUCT)} +$script:Gate14ProductConfig = [pscustomobject]@{{ + sample_interval_seconds = 1.0 +}} +$challenge = [pscustomobject]@{{ + challenge_sha256 = ('sha256:' + ('c' * 64)) +}} +$record = New-Gate14WindowsCalibrationRecord ` + -Kind 'bandwidth' ` + -Challenge $challenge ` + -StartedAt 100.0 ` + -EndedAt 102.0 ` + -Baseline 10.0 ` + -Trigger 300.0 ` + -Resume 9.0 ` + -Source 'host-network-counters' ` + -Scope 'aggregate-host-network' ` + -Configured 250.0 ` + -Duration 2.0 +$badWindowRejected = $false +try {{ + [void](New-Gate14WindowsCalibrationRecord ` + -Kind 'bandwidth' ` + -Challenge $challenge ` + -StartedAt 100.0 ` + -EndedAt 101.0 ` + -Baseline 10.0 ` + -Trigger 300.0 ` + -Resume 9.0 ` + -Source 'host-network-counters' ` + -Scope 'aggregate-host-network' ` + -Configured 250.0 ` + -Duration 1.0) +}} +catch {{ + $badWindowRejected = $true +}} +$script:Gate14ProductPrepared = $true +$script:Gate14ProductCleaned = $false +$staleRejected = $false +try {{ + [void](Invoke-Gate14WindowsProductCalibrate -Challenge ([pscustomobject]@{{ + challenge_sha256 = ('sha256:' + ('d' * 64)) + controller_state_revision = 1 + issued_at_unix = 0 + expires_at_unix = 60 + }})) +}} +catch {{ + $staleRejected = ($_.Exception.Message -ceq 'controller challenge is invalid or stale') +}} +[Console]::Out.WriteLine((@{{ + kind = $record.kind + sample_count = $record.calibration.sample_count + challenge_sha256 = $record.calibration.challenge_sha256 + bad_window_rejected = $badWindowRejected + stale_rejected = $staleRejected +}} | ConvertTo-Json -Compress)) +""" + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == { + "kind": "bandwidth", + "sample_count": 3, + "challenge_sha256": "sha256:" + "c" * 64, + "bad_window_rejected": True, + "stale_rejected": True, + } diff --git a/tests/test_gate16_catalog_channel.py b/tests/test_gate16_catalog_channel.py new file mode 100644 index 000000000..1180f3f29 --- /dev/null +++ b/tests/test_gate16_catalog_channel.py @@ -0,0 +1,196 @@ +"""Isolated signed-channel checks with in-memory consumer/status adapters.""" + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import gate16_catalog_channel as channel + +from drift.model_catalog import SignedModelCatalog +from drift.node.catalog_bootstrap import CatalogBootstrapInstaller + +ROOT = Path(__file__).resolve().parents[1] +RELEASE = ROOT / "public-alpha/catalog-qwen-v2" +PEER = "/ip4/8.8.8.8/tcp/31337/p2p/QmZhGcSVR6qPLZTq3TJPZEi734GbMkouv3kPxQLdDY2qUo" + + +@pytest.fixture +def bundle(tmp_path, monkeypatch): + original = SignedModelCatalog.from_json((RELEASE / "catalog.signed.json").read_text()) + now = original.signed.issued_at_ms / 1000 + 1 + monkeypatch.setattr(channel.time, "time", lambda: now) + args = SimpleNamespace( + release=RELEASE, + base_url="https://canary.example.com/gate16/", + initial_peer=PEER, + run_id="test-channel", + output=tmp_path / "bundle", + ) + result = channel.prepare(args, now=now) + assert result["published"] is False + return args.output + + +def test_prepare_uses_separate_root_presigns_all_phases_and_never_retains_key(bundle): + index, bootstrap = channel.load_bundle(bundle) + production = json.loads((RELEASE / "catalog-bootstrap.json").read_text()) + assert bootstrap.trust_root.catalog_id.startswith("communityai-canary-") + assert bootstrap.trust_root.to_dict() != production["trust_root"] + assert [index["phases"][phase]["sequence"] for phase in channel.PHASES] == [1, 2, 3] + assert index["private_signing_key_retained"] is False + assert not list(bundle.rglob("*.pem")) and not list(bundle.rglob("*.key")) + config = json.loads((bundle / "private-node/node-config.json").read_text()) + assert config["inference_mode"] == "local_only" + assert config["contribution_policy"]["sharing_enabled"] is False + assert not list((bundle / "private-node").rglob("*.safetensors")) + + +def test_phase_advance_rejects_skip_replay_and_tamper(bundle): + with pytest.raises(channel.ChannelError, match="exactly_once"): + channel.advance(SimpleNamespace(bundle=bundle, phase="restore")) + assert channel.advance(SimpleNamespace(bundle=bundle, phase="withdrawal"))["sequence"] == 2 + with pytest.raises(channel.ChannelError, match="exactly_once"): + channel.advance(SimpleNamespace(bundle=bundle, phase="withdrawal")) + active = (bundle / "channel/catalog.signed.json").read_bytes() + path = bundle / "phases/restore.signed.json" + path.write_bytes(path.read_bytes() + b" ") + with pytest.raises(channel.ChannelError, match="phase_digest_mismatch"): + channel.advance(SimpleNamespace(bundle=bundle, phase="restore")) + assert (bundle / "channel/catalog.signed.json").read_bytes() == active + + +def status(started_at, config_path): + return { + "status": "running", + "started_at": started_at, + "inference_mode": "local_only", + "contribution": { + "policy": { + "config_revision": "sha256:" + channel.sha(config_path.read_bytes()), + "policy": {"sharing_enabled": False}, + } + }, + "workers": [{"state": "paused"}], + "runtime_budget": {"resident_models": 0}, + } + + +def observe_args(bundle, tmp_path, phase, previous=None): + return SimpleNamespace( + bundle=bundle, + phase=phase, + previous=previous, + node_config=bundle / "private-node/node-config.json", + node_url="http://127.0.0.1:18116", + credential_service="unused-test", + credential_account="unused-test", + timeout=10, + output=tmp_path / ("observe-" + phase), + ) + + +def refresh(bundle): + _, bootstrap = channel.load_bundle(bundle) + + def fetch(url, _limit): + suffix = url.removeprefix("https://canary.example.com/gate16/") + return (bundle / "channel" / suffix).read_text() + + return CatalogBootstrapInstaller( + bootstrap, + data_dir=bundle / "private-node", + config_path=bundle / "private-node/node-config.json", + fetch_text=fetch, + ).refresh() + + +def test_observer_requires_restart_and_preserves_preferences_through_forward_restore(bundle, tmp_path): + baseline_args = observe_args(bundle, tmp_path, "baseline") + baseline = channel.observe(baseline_args, get_status=lambda: status(100, baseline_args.node_config)) + assert baseline["result"] == "passed" + previous = baseline_args.output / "result.json" + for phase, started_at in (("withdrawal", 101), ("restore", 102)): + channel.advance(SimpleNamespace(bundle=bundle, phase=phase)) + assert refresh(bundle).created + samples = iter( + ( + httpx.ConnectError("restart in progress"), + status(started_at - 1, baseline_args.node_config), + status(started_at, baseline_args.node_config), + ) + ) + + def read_status(): + value = next(samples) + if isinstance(value, Exception): + raise value + return value + + args = observe_args(bundle, tmp_path, phase, previous) + result = channel.observe(args, get_status=read_status, sleep=lambda _: None) + assert result["result"] == "passed" and result["node_restarted"] is True + assert result["local_preferences_sha256"] == baseline["local_preferences_sha256"] + assert result["cleanup"] == {"created_processes": 0, "created_credentials": 0, "owned_http_client_closed": True} + previous = args.output / "result.json" + + +def test_file_update_without_active_node_restart_cannot_pass(bundle, tmp_path): + args = observe_args(bundle, tmp_path, "baseline") + assert channel.observe(args, get_status=lambda: status(100, args.node_config))["result"] == "passed" + channel.advance(SimpleNamespace(bundle=bundle, phase="withdrawal")) + assert refresh(bundle).created + clock = iter(range(30)) + waiting = observe_args(bundle, tmp_path, "withdrawal", args.output / "result.json") + result = channel.observe( + waiting, get_status=lambda: status(100, args.node_config), monotonic=lambda: next(clock), sleep=lambda _: None + ) + assert result["result"] == "failed" and result["error_code"] == "catalog_observation_deadline" + + +@pytest.mark.parametrize("problem", ["stale_active_revision", "stopping"]) +def test_saved_new_catalog_with_unrelated_restart_or_stopping_node_cannot_pass(bundle, tmp_path, problem): + args = observe_args(bundle, tmp_path, "baseline") + before = status(100, args.node_config) + assert channel.observe(args, get_status=lambda: before)["result"] == "passed" + channel.advance(SimpleNamespace(bundle=bundle, phase="withdrawal")) + assert refresh(bundle).created + sample = status(101, args.node_config) + if problem == "stale_active_revision": + sample["contribution"]["policy"]["config_revision"] = before["contribution"]["policy"]["config_revision"] + else: + sample["status"] = "stopping" + clock = iter(range(30)) + waiting = observe_args(bundle, tmp_path, "withdrawal", args.output / "result.json") + result = channel.observe(waiting, get_status=lambda: sample, monotonic=lambda: next(clock), sleep=lambda _: None) + assert result["result"] == "failed" and result["error_code"] == "catalog_observation_deadline" + assert json.loads((waiting.output / "result.json").read_text())["result"] == "failed" + + +def test_http_cleanup_failure_preserves_failed_result_json(bundle, tmp_path, monkeypatch): + monkeypatch.syspath_prepend(str(ROOT / "desktop/src")) + from communityai_desktop.credentials import NativeCredentialStore + + args = observe_args(bundle, tmp_path, "baseline") + monkeypatch.setattr(NativeCredentialStore, "get", lambda self: "never-transmitted-test-value") + + class Client: + def __init__(self, **kwargs): + pass + + def get(self, path): + return SimpleNamespace(raise_for_status=lambda: None, json=lambda: status(100, args.node_config)) + + def close(self): + raise OSError("simulated HTTP cleanup failure") + + monkeypatch.setattr(channel.httpx, "Client", Client) + result = channel.observe(args) + assert result["result"] == "failed" and result["cleanup_error_type"] == "OSError" + assert result["cleanup"]["owned_http_client_closed"] is False + assert json.loads((args.output / "result.json").read_text())["result"] == "failed" diff --git a/tests/test_gate16_catalog_drill.py b/tests/test_gate16_catalog_drill.py new file mode 100644 index 000000000..83a9a269b --- /dev/null +++ b/tests/test_gate16_catalog_drill.py @@ -0,0 +1,97 @@ +"""Offline signed withdrawal and forward-restore drill for the real alpha catalog. + +No publication key, HTTPS server, DHT, or model weights are used. A real packaged +consumer still needs to repeat this sequence in the monitored public canary. +""" + +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from drift.model_catalog import CatalogSigningKey, SignedModelCatalog +from drift.node.catalog_bootstrap import CatalogBootstrapConfig, CatalogBootstrapError, CatalogBootstrapInstaller +from drift.node.catalog_refresh import load_configured_catalog +from drift.node.config import NodeConfig + + +def test_qwen_catalog_withdrawal_and_forward_restore_preserve_local_settings_and_cache(tmp_path): + release = Path(__file__).resolve().parents[1] / "public-alpha" / "catalog-qwen-v2" + original = SignedModelCatalog.from_json((release / "catalog.signed.json").read_text()).signed + now = original.issued_at_ms / 1000 + 1 + local = next(model for model in original.models if model.execution == "local") + remote = next(model for model in original.models if model.execution == "distributed") + key = CatalogSigningKey.generate() + bootstrap_source = json.loads((release / "catalog-bootstrap.json").read_text()) + bootstrap_source["trust_root"]["keys"] = [key.trusted_key.to_dict()] + bootstrap_source["trust_root"]["threshold"] = 1 + bootstrap_source.pop("replaces_trust_roots", None) + bootstrap = CatalogBootstrapConfig.from_dict(bootstrap_source) + active = [SignedModelCatalog(1, original, ()).add_signature(key)] + manifests = { + url: (release / "manifests" / (model.manifest_digest.removeprefix("sha256:") + ".json")).read_text() + for model in original.models + for url in model.manifest_urls + } + + def fetch(url, _maximum_bytes): + return json.dumps(active[0].to_dict()) if url in bootstrap.catalog_mirrors else manifests[url] + + path = tmp_path / "node-config.json" + installer = CatalogBootstrapInstaller(bootstrap, data_dir=tmp_path, config_path=path, fetch_text=fetch, now=now) + installer.install() + document = json.loads(path.read_text()) + cache = tmp_path / "custom-local-cache" + cache.mkdir() + (cache / "retained-sentinel").write_bytes(b"cache stays across withdrawal") + local_config = next(item for item in document["models"] if item.get("execution") == "local") + local_config.update(cache_dir=str(cache), local_device="cpu", request_timeout=17) + document["contribution_policy"].update(sharing_enabled=False, max_vram="75%", max_processing_percent=50) + path.write_text(json.dumps(document)) + retained_policy = NodeConfig.load(path).contribution_policy + + withdrawal = replace( + original, + sequence=original.sequence + 1, + models=(local,), + rungs=tuple(rung for rung in original.rungs if rung.rung_id == local.rung_id), + ) + active[0] = SignedModelCatalog(1, withdrawal, ()).add_signature(key) + assert installer.refresh().created + withdrawn = NodeConfig.load(path) + # Older exact models remain manual choices by design. Withdrawal removes + # catalog approval and automatic routing, not operator-owned model history. + assert len(withdrawn.models) == 2 + withdrawn_local = next(model for model in withdrawn.models if model.execution == "local") + assert withdrawn.auto_model_priority == (local.manifest_digest,) + assert remote.manifest_digest not in {model.manifest_digest for model in load_configured_catalog(withdrawn).models} + assert withdrawn_local.cache_dir == cache + assert withdrawn_local.local_device == "cpu" + assert withdrawn_local.request_timeout == 17 + assert withdrawn.contribution_policy == retained_policy + accepted_bytes = path.read_bytes() + accepted_rollback = installer.rollback_path.read_bytes() + + # An operator rollback must be a newer signed release of known-good content, + # never a lower sequence or removal of the persistent rollback guard. + active[0] = SignedModelCatalog(1, original, ()).add_signature(key) + with pytest.raises(CatalogBootstrapError): + installer.refresh() + assert path.read_bytes() == accepted_bytes + assert installer.rollback_path.read_bytes() == accepted_rollback + + restored = replace(original, sequence=original.sequence + 2) + active[0] = SignedModelCatalog(1, restored, ()).add_signature(key) + assert installer.refresh().created + resumed = NodeConfig.load(path) + assert {model.manifest_digest for model in load_configured_catalog(resumed).models} == { + local.manifest_digest, + remote.manifest_digest, + } + resumed_local = next(model for model in resumed.models if model.execution == "local") + assert resumed_local.cache_dir == cache + assert resumed_local.local_device == "cpu" + assert resumed_local.request_timeout == 17 + assert resumed.contribution_policy == retained_policy + assert (cache / "retained-sentinel").read_bytes() == b"cache stays across withdrawal" diff --git a/tests/test_gate16_live_rpc.py b/tests/test_gate16_live_rpc.py new file mode 100644 index 000000000..f91e45092 --- /dev/null +++ b/tests/test_gate16_live_rpc.py @@ -0,0 +1,401 @@ +"""Small canary-driver checks; no model, GUI, GPU, or external route is used.""" + +import asyncio +import json +import sys +from dataclasses import asdict, replace +from pathlib import Path +from types import SimpleNamespace + +import pytest +from hivemind.p2p import P2PHandlerError, PeerID +from hivemind.utils.serializer import MSGPackSerializer + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import gate16_live_rpc as canary + +from drift.model_manifest import ModelManifest +from drift.server.admission import AdmissionPolicy, AdmissionState +from drift.server.handler import TransformerConnectionHandler +from drift.server.health import build_public_worker_health + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = ( + ROOT + / "public-alpha/catalog-qwen-v2/manifests/c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4.json" +) +PEER = PeerID.from_base58("QmZhGcSVR6qPLZTq3TJPZEi734GbMkouv3kPxQLdDY2qUo") + + +def policy_file(tmp_path, **changes): + source = {"schema_version": 1, "admission": asdict(AdmissionPolicy()), "step_timeout": 30, "session_timeout": 60} + source.update(changes) + path = tmp_path / "policy.json" + path.write_text(json.dumps(source)) + return path + + +@pytest.mark.parametrize( + "address", + ["/ip4/127.0.0.1/udp/31337/p2p/" + str(PEER), "/ip4/127.0.0.1/tcp/31337/p2p/" + str(PEER) + "/p2p-circuit"], +) +def test_exact_peer_rejects_non_tcp_and_relay_targets(address): + with pytest.raises(canary.CanaryError): + canary.exact_peer(address) + + +def test_exact_peer_accepts_one_direct_authenticated_target(): + assert canary.exact_peer("/ip4/127.0.0.1/tcp/31337/p2p/" + str(PEER)) == PEER + + +@pytest.mark.parametrize( + "changes", + [ + {"step_timeout": float("nan")}, + {"session_timeout": 61}, + {"step_timeout": True}, + {"step_timeout": 31, "session_timeout": 30}, + {"step_timeout": 5}, + {"admission": asdict(AdmissionPolicy(max_active_sessions=1))}, + ], +) +def test_policy_rejects_unbounded_or_inconsistent_deadlines(tmp_path, changes): + with pytest.raises(canary.CanaryError): + canary.load_policy(policy_file(tmp_path, **changes)) + + +def test_payloads_are_bounded_and_cannot_execute_model_tensors(): + manifest = ModelManifest.load(MANIFEST) + cases = canary.malformed_cases(manifest.dht_prefix + ".0", manifest.digest) + assert len(cases) == 5 + assert sum(request.ByteSize() for _, request, _ in cases) < canary.MAX_INPUT_BYTES + assert all(not request.tensors for _, request, _ in cases) + assert len(cases[0][1].metadata) == canary.MAX_INFERENCE_METADATA_BYTES + 1 + assert MSGPackSerializer.loads(cases[-1][1].metadata)["max_length"] == -1 + + +def test_dry_preflight_never_opens_network(tmp_path, monkeypatch): + manifest = ModelManifest.load(MANIFEST) + state = AdmissionState.local(AdmissionPolicy()) + health = tmp_path / "health.json" + health.write_text( + json.dumps( + build_public_worker_health( + manifest_digest=manifest.digest_id, + start_block=0, + end_block=1, + admission_snapshot=state.snapshot(), + ready=True, + announcer_alive=True, + handlers_alive=True, + pools_alive=True, + ) + ) + ) + + async def forbidden(**kwargs): + raise AssertionError("dry preflight must not create a transport") + + monkeypatch.setattr(canary.P2P, "create", forbidden) + args = SimpleNamespace( + manifest=MANIFEST, + expected_manifest_digest=manifest.digest_id, + block=0, + worker_multiaddr="/ip4/127.0.0.1/tcp/31337/p2p/" + str(PEER), + worker_label="owned-worker-a", + policy=policy_file(tmp_path), + health=health, + output=tmp_path / "result", + execute=False, + ) + result = asyncio.run(canary.run(args)) + assert result["result"] == "preflight-passed" + assert result["network_connections"] == 0 and result["executed"] is False + assert str(PEER) not in json.dumps(result) + + +def test_health_rejects_stale_or_wrong_manifest(tmp_path): + manifest = ModelManifest.load(MANIFEST) + payload = build_public_worker_health( + manifest_digest=manifest.digest_id, + start_block=0, + end_block=1, + admission_snapshot=AdmissionState.local(AdmissionPolicy()).snapshot(), + ready=True, + announcer_alive=True, + handlers_alive=True, + pools_alive=True, + observed_at="2026-01-01T00:00:00Z", + ) + path = tmp_path / "health.json" + path.write_text(json.dumps(payload)) + with pytest.raises(canary.CanaryError, match="health_stale"): + canary.read_health(path, manifest.digest_id, 0) + with pytest.raises(canary.CanaryError, match="health_manifest_mismatch"): + canary.read_health(path, "sha256:" + "0" * 64, 0) + + +def test_child_inspection_failure_preserves_failed_result_json(tmp_path, monkeypatch): + manifest = ModelManifest.load(MANIFEST) + snapshots = iter(([], canary.psutil.AccessDenied())) + + def children(**kwargs): + value = next(snapshots) + if isinstance(value, Exception): + raise value + return value + + async def create(**kwargs): + return SimpleNamespace() + + async def passed(self): + return {"checks": [], "rpc_calls": 0, "input_bytes": 0} + + monkeypatch.setattr(canary.psutil, "Process", lambda: SimpleNamespace(children=children)) + monkeypatch.setattr(canary.P2P, "create", create) + monkeypatch.setattr(canary.TransformerConnectionHandler, "get_stub", lambda *args: None) + monkeypatch.setattr(canary.Probe, "run", passed) + monkeypatch.setattr(canary, "read_health", lambda *args: {"admission": {"active_sessions": 0, "pending_pushes": 0}}) + args = SimpleNamespace( + manifest=MANIFEST, + expected_manifest_digest=manifest.digest_id, + block=0, + worker_multiaddr="/ip4/127.0.0.1/tcp/31337/p2p/" + str(PEER), + worker_label="owned-worker-a", + policy=policy_file(tmp_path), + health=tmp_path / "unused-health.json", + output=tmp_path / "result", + execute=True, + ) + result = asyncio.run(canary.run(args)) + assert result["result"] == "failed" and result["client_cleanup_error_type"] == "AccessDenied" + assert result["cleanup"] == {"owned_client_stopped": False, "worker_sessions_released": True} + assert json.loads((args.output / "result.json").read_text())["result"] == "failed" + + +@pytest.mark.parametrize( + "real_transport,enforce_peer_cap", + [(False, True), (True, True), (False, False)], + ids=["local-handler", "loopback-tls", "missing-peer-cap"], +) +def test_probe_uses_real_handler_admission_and_rejects_without_cache_allocation(real_transport, enforce_peer_cap): + """The real handler validates every case, including over actual loopback TLS.""" + manifest = ModelManifest.load(MANIFEST) + policy = AdmissionPolicy(global_session_rate=1000, global_session_burst=100, peer_session_rate=1000) + state = AdmissionState.local(policy if enforce_peer_cap else replace(policy, max_active_sessions_per_peer=2)) + handler = object.__new__(TransformerConnectionHandler) + handler._admission_state = state + handler.step_timeout, handler.session_timeout = 1.0, 3.0 + handler.manifest_digest = manifest.digest + handler.identity_key_id = "sha256:" + "a" * 64 + handler.inference_max_length = 512 + handler._log_request = lambda *args, **kwargs: None + handler.dht_prefix = manifest.dht_prefix + handler.dht = SimpleNamespace(peer_id=PEER, client_mode=False) + handler.module_backends = { + manifest.dht_prefix + + ".0": SimpleNamespace( + memory_cache=SimpleNamespace(bytes_left=4096), cache_bytes_per_token={"test": 16}, get_info=lambda: {} + ) + } + handler._allocate_cache = lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("invalid probe allocated cache") + ) + context = SimpleNamespace(remote_id=PEER) + + class Stub: + _peer = PEER + + async def rpc_info(self, request): + return await handler.rpc_info(request, context) + + async def rpc_forward(self, request): + try: + return await handler.rpc_forward(request, context) + except Exception as exc: + raise P2PHandlerError(str(exc)) from None + + async def rpc_inference(self, requests): + async def responses(): + try: + async for response in handler.rpc_inference(requests, context): + yield response + except Exception as exc: + raise P2PHandlerError(str(exc)) from None + + return responses() + + def health(): + return build_public_worker_health( + manifest_digest=manifest.digest_id, + start_block=0, + end_block=1, + admission_snapshot=state.snapshot(), + ready=True, + announcer_alive=True, + handlers_alive=True, + pools_alive=True, + ) + + async def exercise(): + if not real_transport: + return await canary.Probe( + Stub(), health, manifest, 0, policy, refill=0, step_timeout=handler.step_timeout + ).run() + server = client = None + options = dict( + host_maddrs=["/ip4/127.0.0.1/tcp/0"], + auto_nat=False, + conn_manager=False, + nat_port_map=False, + use_relay=False, + tls=True, + startup_timeout=10, + ) + try: + server = await canary.P2P.create(initial_peers=[], **options) + handler.dht.peer_id = server.peer_id + for method, request, stream in ( + ("rpc_info", canary.runtime_pb2.ExpertUID, False), + ("rpc_forward", canary.runtime_pb2.ExpertRequest, False), + ("rpc_inference", canary.runtime_pb2.ExpertRequest, True), + ): + await server.add_protobuf_handler( + TransformerConnectionHandler._get_handle_name(None, method), + getattr(handler, method), + request, + stream_input=stream, + stream_output=stream, + ) + client = await canary.P2P.create(initial_peers=await server.get_visible_maddrs(), **options) + stub = TransformerConnectionHandler.get_stub(client, server.peer_id) + return await asyncio.wait_for( + canary.Probe(stub, health, manifest, 0, policy, refill=0, step_timeout=handler.step_timeout).run(), 20 + ) + finally: + for transport in (client, server): + if transport is not None: + await asyncio.wait_for(transport.shutdown(), 5) + assert transport._child.returncode is not None + + if not enforce_peer_cap: + with pytest.raises(canary.CanaryError, match="unexpected_rpc_rejection"): + asyncio.run(exercise()) + assert state.snapshot()["active_sessions"] == 0 + return + result = asyncio.run(exercise()) + assert len(result["checks"]) == 7 + assert result["rpc_calls"] == 20 + assert result["input_bytes"] < canary.MAX_INPUT_BYTES + assert result["after"]["admission"]["active_sessions"] == 0 + assert result["after"]["admission"]["rejected_sessions"] == 1 + + +def test_overload_does_not_count_as_malformed_rejection(): + class Stub: + async def rpc_inference(self, requests): + async def responses(): + raise P2PHandlerError(canary.PUBLIC_OVERLOAD_MESSAGE) + yield + + return responses() + + probe = canary.Probe(Stub(), None, None, 0, None, refill=0, step_timeout=1) + with pytest.raises(canary.CanaryError, match="unexpected_rpc_rejection"): + asyncio.run(probe.reject(canary.runtime_pb2.ExpertRequest(), "metadata is invalid")) + + +@pytest.mark.parametrize( + "elapsed,rejected,closure,error", + [ + (1.0, 1, ConnectionResetError, None), + (1.0, 1, StopAsyncIteration, None), + (0.5, 1, ConnectionResetError, "idle_lease_released_before_timeout"), + (0.5, 1, StopAsyncIteration, "idle_lease_released_before_timeout"), + (1.0, 2, ConnectionResetError, "admission_counters_contaminated_or_missing"), + (1.0, 1, "already_reset", "idle_stream_closed_before_producer_release"), + (1.0, 1, RuntimeError, RuntimeError), + ], +) +def test_idle_closure_requires_independent_timeout_and_counter_proof(elapsed, rejected, closure, error, monkeypatch): + """A deterministic health clock isolates the proof required before closing the producer.""" + manifest = ModelManifest.load(MANIFEST) + now = [0.0] + released = [] + disconnected = asyncio.Event() + + class Stub: + async def rpc_inference(self, requests): + async def responses(): + if closure == "already_reset": + await disconnected.wait() + raise ConnectionResetError("reset before producer release") + async for _ in requests: + raise AssertionError("idle producer sent a request") + released.append(True) + if closure is not StopAsyncIteration: + raise closure("synthetic transport closure") + if False: + yield + + return responses() + + class Probe(canary.Probe): + observations = 0 + + async def info(self): + return 256 + + async def reject(self, request, expected, **kwargs): + return + + async def wait_health(self, predicate, timeout=15): + self.observations += 1 + active = self.observations in (2, 3) + if self.observations >= 4: + now[0] = elapsed + disconnected.set() + await asyncio.sleep(0) + value = { + "admission": { + "active_sessions": int(active), + "pending_pushes": 0, + "accepted_sessions": int(self.observations > 1), + "rejected_sessions": rejected if self.observations >= 4 else 0, + } + } + assert predicate(value) + return value + + monkeypatch.setattr(canary, "malformed_cases", lambda *args: []) + probe = Probe(Stub(), None, manifest, 0, AdmissionPolicy(), refill=0, step_timeout=1, clock=lambda: now[0]) + if error is None: + result = asyncio.run(probe.run()) + assert result["checks"][0]["transport_closure"] == ( + "end_of_stream" if closure is StopAsyncIteration else "connection_reset_after_idle_release" + ) + assert released == [True] + elif isinstance(error, str): + with pytest.raises(canary.CanaryError, match=error): + asyncio.run(probe.run()) + assert not released and not probe.checks + else: + with pytest.raises(error, match="synthetic transport closure"): + asyncio.run(probe.run()) + assert not probe.checks + + +@pytest.mark.parametrize("expected", [canary.PUBLIC_OVERLOAD_MESSAGE, "metadata is invalid"]) +def test_reset_during_admission_or_malformed_probe_is_not_accepted(expected): + class Stub: + async def rpc_inference(self, requests): + async def responses(): + raise ConnectionResetError("unexpected reset") + yield + + return responses() + + probe = canary.Probe(Stub(), None, None, 0, None, refill=0, step_timeout=1) + with pytest.raises(ConnectionResetError, match="unexpected reset"): + asyncio.run(probe.reject(canary.runtime_pb2.ExpertRequest(), expected)) diff --git a/tests/test_gateq38_gcp_adapter.py b/tests/test_gateq38_gcp_adapter.py new file mode 100644 index 000000000..2ce989018 --- /dev/null +++ b/tests/test_gateq38_gcp_adapter.py @@ -0,0 +1,1340 @@ +from __future__ import annotations + +import copy +import json +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +import test_gateq38_route_controller as route_test # noqa: E402 + +from scripts import ( # noqa: E402 + gateq38_gcp_adapter as adapter, + gateq38_linux_host_transport as transport, + gateq38_route_controller as route, +) + +NOW = 1_900_000_000 + + +def _revalidation(plan: route.RoutePlan, verified_at_unix: int) -> dict[str, object]: + verifier = next( + item["sha256"] for item in plan.source_bindings if item["relative_path"] == route.VERIFIER_SOURCE_PATH + ) + return { + "verified_at_unix": verified_at_unix, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "model_revision": plan.model_revision, + "index_digest": route.EXPECTED_INDEX_DIGEST, + "block_prefix": route.EXPECTED_BLOCK_PREFIX, + "worker_plan_digest": plan.worker_plan_digest, + "verifier_source_sha256": verifier, + } + + +@pytest.fixture(autouse=True) +def _trusted_sources(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(adapter, "_assert_source_bound", lambda *args, **kwargs: None) + monkeypatch.setattr(route, "_assert_protected_path", lambda *args, **kwargs: None) + monkeypatch.setattr(route, "revalidate_authorization_evidence", lambda *args, **kwargs: None) + monkeypatch.setattr( + route, + "revalidate_production_artifact_plan", + lambda plan, manifest, artifacts, source, *, verified_at_unix: _revalidation(plan, verified_at_unix), + ) + + +class FakeGcloud: + def __init__(self, plan: route.RoutePlan) -> None: + self.plan = plan + self.resources: dict[str, dict[str, object]] = {} + self.calls: list[tuple[str, ...]] = [] + self.extra: dict[str, set[str]] = { + "instances": set(), + "disks": set(), + "firewall": set(), + } + self.fail_delete_once: set[str] = set() + self.fail_create_once: set[str] = set() + self.guest_attributes: dict[str, bytes] = {} + self.guest_response_overrides: dict[str, object] = {} + self.recreate_after_guest_read: str | None = None + + @staticmethod + def result(value: object | None = None, *, returncode: int = 0, stderr: bytes = b""): + stdout = b"" if value is None else json.dumps(value).encode("utf-8") + return adapter.CommandResult(returncode, stdout, stderr) + + def resource(self, name: str) -> route.ResourcePlan: + return self.plan.resource_by_name[name] + + def disk_value(self, resource: route.ResourcePlan) -> dict[str, object]: + image_project, image = route.EXPECTED_SOURCE_IMAGE.split("/", 1) + return { + "name": resource.name, + "status": "READY", + "labels": adapter._labels(self.plan), + "type": f"zones/{route.EXPECTED_ZONE}/diskTypes/{route.EXPECTED_DISK_TYPE}", + "sizeGb": str(route.EXPECTED_DISK_SIZE_GB), + "sourceImage": f"projects/{image_project}/global/images/{image}", + "description": adapter._description(self.plan, resource), + } + + def instance_value(self, resource: route.ResourcePlan) -> dict[str, object]: + spec = route._expected_resource_spec(resource) + disk = next( + item + for item in self.plan.resources + if item.kind == ("worker_disk" if resource.worker_id else "bootstrap_disk") + and item.worker_id == resource.worker_id + ) + value: dict[str, object] = { + "name": resource.name, + "id": str(20_000_000 + list(self.plan.resource_by_name).index(resource.name)), + "creationTimestamp": "2026-09-03T01:20:00+00:00", + "status": "RUNNING", + "labels": adapter._labels(self.plan), + "machineType": f"zones/{route.EXPECTED_ZONE}/machineTypes/{spec['machine_type']}", + "disks": [ + { + "source": f"zones/{route.EXPECTED_ZONE}/disks/{disk.name}", + "boot": True, + "autoDelete": True, + } + ], + "networkInterfaces": [ + { + "network": f"global/networks/{route.EXPECTED_NETWORK}", + "subnetwork": f"regions/{route.EXPECTED_REGION}/subnetworks/{route.EXPECTED_SUBNET}", + "accessConfigs": [], + "ipv6AccessConfigs": [], + "stackType": "IPV4_ONLY", + } + ], + "tags": {"items": [adapter._route_tag(self.plan)]}, + "metadata": { + "items": [ + {"key": key, "value": field} + for key, field in adapter._instance_metadata(self.plan, resource).items() + ] + }, + "guestAccelerators": [], + "canIpForward": False, + "deletionProtection": False, + "scheduling": { + "automaticRestart": True, + "provisioningModel": "STANDARD", + "onHostMaintenance": "TERMINATE", + "instanceTerminationAction": "DELETE", + "maxRunDuration": { + "seconds": str(route.EXPECTED_MAX_LIFETIME_SECONDS), + "nanos": 0, + }, + }, + } + if resource.kind == "worker_instance": + value["guestAccelerators"] = [ + { + "acceleratorType": ( + f"zones/{route.EXPECTED_ZONE}/acceleratorTypes/" f"{route.EXPECTED_ACCELERATOR_TYPE}" + ), + "acceleratorCount": 1, + } + ] + return value + + def firewall_value(self, resource: route.ResourcePlan) -> dict[str, object]: + if resource.kind == "iap_firewall": + return { + "name": resource.name, + "network": f"global/networks/{route.EXPECTED_NETWORK}", + "direction": "INGRESS", + "sourceRanges": [adapter.IAP_SOURCE_RANGE], + "targetTags": [adapter._route_tag(self.plan)], + "allowed": [{"IPProtocol": "tcp", "ports": ["22"]}], + "disabled": False, + "description": adapter._description(self.plan, resource), + } + return { + "name": resource.name, + "network": f"global/networks/{route.EXPECTED_NETWORK}", + "direction": "INGRESS", + "sourceTags": [adapter._route_tag(self.plan)], + "targetTags": [adapter._route_tag(self.plan)], + "allowed": [{"IPProtocol": "tcp", "ports": ["31330-31339"]}], + "disabled": False, + "description": adapter._description(self.plan, resource), + } + + def value(self, resource: route.ResourcePlan) -> dict[str, object]: + if resource.kind.endswith("disk"): + return self.disk_value(resource) + if resource.kind.endswith("instance"): + return self.instance_value(resource) + return self.firewall_value(resource) + + def populate_all(self) -> None: + self.resources = {item.name: self.value(item) for item in self.plan.resources} + + def _listed(self, kind: str) -> list[dict[str, str]]: + if kind == "instances": + names = { + item.name + for item in self.plan.resources + if item.kind.endswith("instance") and item.name in self.resources + } + elif kind == "disks": + names = { + item.name for item in self.plan.resources if item.kind.endswith("disk") and item.name in self.resources + } + else: + names = { + item.name + for item in self.plan.resources + if item.kind.endswith("firewall") and item.name in self.resources + } + return [{"name": name} for name in sorted(names | self.extra[kind])] + + def __call__(self, argv, timeout): + del timeout + command = tuple(argv) + self.calls.append(command) + args = list(command[1:]) + assert args.pop() == "--quiet" + if "--format=json" in args: + args.remove("--format=json") + + if args[:2] == ["auth", "list"]: + return adapter.CommandResult(0, b"operator@example.invalid\n", b"") + if args[:2] == ["projects", "describe"]: + return self.result({"lifecycleState": "ACTIVE"}) + if args[:3] == ["compute", "instances", "get-guest-attributes"]: + name = args[3] + if name not in self.resources: + return self.result(returncode=1, stderr=b"resource was not found") + if name in self.guest_response_overrides: + response = self.guest_response_overrides[name] + else: + items = [] + payload = self.guest_attributes.get(name) + if payload is not None: + items.append( + { + "namespace": adapter.GUEST_ATTRIBUTE_NAMESPACE, + "key": adapter.GUEST_ATTRIBUTE_KEY, + "value": payload.decode("ascii"), + } + ) + response = { + "kind": "compute#guestAttributes", + "queryPath": adapter.GUEST_ATTRIBUTE_QUERY_PATH, + "queryValue": {"items": items}, + } + if self.recreate_after_guest_read == name: + self.resources[name]["id"] = str(int(self.resources[name]["id"]) + 1) + self.recreate_after_guest_read = None + return self.result(response) + if args[:3] == ["compute", "instances", "describe"]: + name = args[3] + if name == route.PROTECTED_INSTANCE: + return self.result({"name": name, "status": "RUNNING"}) + if name not in self.resources: + return self.result(returncode=1, stderr=b"resource was not found") + return self.result(self.resources[name]) + if args[:3] == ["compute", "disks", "describe"]: + name = args[3] + if name not in self.resources: + return self.result(returncode=1, stderr=b"resource was not found") + return self.result(self.resources[name]) + if args[:3] == ["compute", "firewall-rules", "describe"]: + name = args[3] + if name not in self.resources: + return self.result(returncode=1, stderr=b"resource was not found") + return self.result(self.resources[name]) + if args[:3] == ["compute", "instances", "list"]: + return self.result(self._listed("instances")) + if args[:3] == ["compute", "disks", "list"]: + return self.result(self._listed("disks")) + if args[:3] == ["compute", "firewall-rules", "list"]: + return self.result(self._listed("firewall")) + + if args[:3] in ( + ["compute", "disks", "create"], + ["compute", "instances", "create"], + ["compute", "firewall-rules", "create"], + ): + name = args[3] + if name in self.fail_create_once: + self.fail_create_once.remove(name) + return self.result(returncode=1, stderr=b"injected create failure") + resource = self.resource(name) + self.resources[name] = self.value(resource) + return self.result() + if args[:3] in ( + ["compute", "instances", "delete"], + ["compute", "disks", "delete"], + ["compute", "firewall-rules", "delete"], + ): + name = args[3] + if name in self.fail_delete_once: + self.fail_delete_once.remove(name) + return self.result(returncode=1, stderr=b"injected delete failure") + self.resources.pop(name, None) + return self.result() + if args[:2] == ["compute", "ssh"]: + return self.result() + + raise AssertionError(f"unexpected command: {command}") + + +def _make(tmp_path: Path) -> tuple[route.RoutePlan, FakeGcloud, adapter.GcpAdapter]: + plan = route_test._load_plan(tmp_path) + fake = FakeGcloud(plan) + return plan, fake, adapter.GcpAdapter(plan, tmp_path / "source", runner=fake, clock=lambda: NOW) + + +def _start_state(plan: route.RoutePlan) -> dict[str, object]: + return route_test._advance( + "start", + route.initial_state(plan), + route_test._observation(plan, observed_at=NOW), + plan, + ) + + +def _cleanup_state(plan: route.RoutePlan) -> dict[str, object]: + state = route.initial_state(plan) + state.update( + revision=1, + phase="CLEANING", + failure_code="operator-cleanup", + next_action="cleanup_route", + ) + return state + + +def _collect_state(plan: route.RoutePlan) -> dict[str, object]: + observation = route_test._observation( + plan, + resource_state="present", + worker_state="ready", + observed_at=NOW, + ) + state = route.initial_state(plan) + state.update( + revision=3, + phase="COLLECTING", + next_action="collect_route", + instance_generations_digest=observation["instance_generations_digest"], + ) + return state + + +def _ready_status(plan: route.RoutePlan) -> dict[str, object]: + observation = route_test._observation( + plan, + resource_state="present", + worker_state="ready", + observed_at=NOW, + ) + return { + "schema_version": route.SCHEMA_VERSION, + "run_id": plan.run_id, + "workers": observation["workers"], + "route_job": observation["route_job"], + } + + +STATUS_KEY = b"q" * 32 +STATUS_BOOT_ID = "123e4567-e89b-12d3-a456-426614174000" +PREPARED_RECORD_DIGEST = "sha256:" + "a" * 64 + + +def _publish_authenticated_status( + plan: route.RoutePlan, + fake: FakeGcloud, + *, + key: bytes = STATUS_KEY, + revision: int = 1, +) -> None: + status = _ready_status(plan) + for resource in plan.resources: + if not resource.kind.endswith("instance"): + continue + provider_value = fake.resources[resource.name] + context = transport.build_instance_context( + plan, + resource.name, + provider_value["id"], + provider_value["creationTimestamp"], + issued_at_unix=NOW - 60, + expires_at_unix=min(plan.deadline_unix, NOW + 600), + key=key, + ) + payload = status["workers"][resource.worker_id] if resource.kind == "worker_instance" else status["route_job"] + envelope = transport.build_status_envelope( + context, + payload, + plan, + key=key, + boot_id=STATUS_BOOT_ID, + revision=revision, + published_at_unix=NOW, + prepared_record_digest=PREPARED_RECORD_DIGEST, + ) + fake.guest_attributes[resource.name] = transport.encode_status_envelope(envelope) + + +def _authenticated_provider( + plan: route.RoutePlan, + fake: FakeGcloud, + tmp_path: Path, + *, + key: bytes = STATUS_KEY, + checkpoint: tuple[str | None, int] = (None, 0), +) -> adapter.GcpAdapter: + return adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + clock=lambda: NOW, + status_key_resolver=lambda _resource, _generation: key, + status_checkpoint_resolver=lambda _resource, _generation: checkpoint, + ) + + +def _execute( + provider: adapter.GcpAdapter, + plan: route.RoutePlan, + state: dict[str, object], + status: dict[str, object], + tmp_path: Path, +) -> dict[str, object]: + return provider.execute( + state, + route.action_record(state, plan), + status, + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_absent_inventory_is_exact_and_read_only(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + + observation = provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert set(observation["resources"]) == set(plan.resource_by_name) + assert all(not item["present"] for item in observation["resources"].values()) + assert all(item["state"] == "absent" for item in observation["workers"].values()) + assert observation["route_job"]["state"] == "absent" + assert not any("create" in call or "delete" in call for call in fake.calls) + + +def test_running_instance_without_host_record_is_only_starting(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + + observation = provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert {item["state"] for item in observation["workers"].values()} == {"starting"} + assert all(item["peer_id"] is None for item in observation["workers"].values()) + + +def test_compiled_start_is_exact_private_twelve_resource_inventory( + tmp_path: Path, +) -> None: + plan, fake, provider = _make(tmp_path) + + creates = list(provider.compiled_start_commands()) + + assert len(creates) == 12 + assert fake.calls == [] + assert fake.resources == {} + instance_creates = [call for call in creates if call[1:4] == ("compute", "instances", "create")] + assert len(instance_creates) == 5 + assert all( + "--no-address" in call + and "--no-service-account" in call + and "--stack-type=IPV4_ONLY" in call + and any("boot=yes,auto-delete=yes" in item for item in call) + for call in instance_creates + ) + assert not any(any(item.startswith("--accelerator=") for item in call) for call in instance_creates) + assert all("--maintenance-policy=TERMINATE" in call for call in instance_creates) + assert all("--restart-on-failure" in call for call in instance_creates) + assert all(f"--max-run-duration={route.EXPECTED_MAX_LIFETIME_SECONDS}s" in call for call in instance_creates) + assert all("--instance-termination-action=DELETE" in call for call in instance_creates) + disk_creates = [call for call in creates if call[1:4] == ("compute", "disks", "create")] + assert len(disk_creates) == 5 + assert all("--image=common-cu129-ubuntu-2404-nvidia-580-v20260831" in call for call in disk_creates) + assert all("--image-project=deeplearning-platform-release" in call for call in disk_creates) + firewall_creates = [call for call in creates if call[1:4] == ("compute", "firewall-rules", "create")] + assert len(firewall_creates) == 2 + route_firewall = next( + call for call in firewall_creates if call[4].endswith("-firewall") and not call[4].endswith("-iap-firewall") + ) + iap_firewall = next(call for call in firewall_creates if call[4].endswith("-iap-firewall")) + assert f"--source-tags={adapter._route_tag(plan)}" in route_firewall + assert not any( + "0.0.0.0/0" in item or "source-ranges" in item or item.startswith("--labels=") for item in route_firewall + ) + assert f"--source-ranges={adapter.IAP_SOURCE_RANGE}" in iap_firewall + assert "--rules=tcp:22" in iap_firewall + assert f"--target-tags={adapter._route_tag(plan)}" in iap_firewall + assert not any(item.startswith("--source-tags=") or item.startswith("--labels=") for item in iap_firewall) + assert all(route.PROTECTED_INSTANCE not in call for call in creates) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("sourceRanges", ["0.0.0.0/0"]), + ("sourceTags", ["foreign"]), + ("targetTags", ["foreign"]), + ("allowed", [{"IPProtocol": "tcp", "ports": ["2222"]}]), + ("direction", "EGRESS"), + ("disabled", True), + ("network", "global/networks/default"), + ], +) +def test_iap_firewall_policy_is_exact_and_fail_closed( + tmp_path: Path, + field: str, + value: object, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + resource = next(item for item in plan.resources if item.kind == "iap_firewall") + fake.resources[resource.name][field] = value + + with pytest.raises(adapter.Q38GcpAdapterError, match="IAP firewall policy"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_route_firewall_cannot_substitute_for_iap_firewall(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + route_firewall = next(item for item in plan.resources if item.kind == "firewall") + iap_firewall = next(item for item in plan.resources if item.kind == "iap_firewall") + fake.resources[iap_firewall.name] = fake.firewall_value(route_firewall) + fake.resources[iap_firewall.name]["name"] = iap_firewall.name + fake.resources[iap_firewall.name]["description"] = adapter._description(plan, iap_firewall) + + with pytest.raises(adapter.Q38GcpAdapterError, match="IAP firewall policy"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_route_network_tags_are_plan_scoped_and_rfc1035(tmp_path: Path) -> None: + plan, _fake, _provider = _make(tmp_path) + other = replace(plan, run_id="q38route-002") + + first = adapter._route_tag(plan) + second = adapter._route_tag(other) + + assert first != second + assert route._GCP_RESOURCE_RE.fullmatch(first) + assert route._GCP_RESOURCE_RE.fullmatch(second) + + +def test_stale_decision_rejects_before_provider_inventory(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + state = _start_state(plan) + decision = route.action_record(state, plan) + decision["plan_digest"] = "sha256:" + "0" * 64 + + with pytest.raises(adapter.Q38GcpAdapterError, match="stale or unbound"): + provider.execute( + state, + decision, + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert fake.calls == [] + + +def test_start_execution_is_blocked_before_provider_access(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="host runtime is not plan-bound", + ): + _execute( + provider, + plan, + _start_state(plan), + adapter.blank_host_status(plan), + tmp_path, + ) + + assert fake.calls == [] + assert fake.resources == {} + + +@pytest.mark.parametrize("kind", ["instances", "disks", "firewall"]) +def test_extra_run_scoped_unlabelled_resource_fails_closed( + tmp_path: Path, + kind: str, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.extra[kind].add(f"{plan.run_id}-unplanned") + + with pytest.raises(adapter.Q38GcpAdapterError, match="not exact"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_foreign_exact_name_does_not_strand_other_cleanup(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + foreign = next(item for item in plan.resources if item.kind == "worker_disk") + fake.resources[foreign.name]["labels"] = {"communityai-run": "foreign"} + + with pytest.raises(adapter.Q38GcpAdapterError, match="ownership"): + _execute( + provider, + plan, + _cleanup_state(plan), + adapter.blank_host_status(plan), + tmp_path, + ) + + assert set(fake.resources) == {foreign.name} + deleted = { + call[4] + for call in fake.calls + if call[1:4] + in { + ("compute", "instances", "delete"), + ("compute", "disks", "delete"), + ("compute", "firewall-rules", "delete"), + } + } + assert deleted == set(plan.resource_by_name) - {foreign.name} + assert all(route.PROTECTED_INSTANCE not in call for call in fake.calls) + + +@pytest.mark.parametrize("field", ["serviceAccounts", "networkInterfaces"]) +def test_public_or_privileged_instance_shape_fails_closed( + tmp_path: Path, + field: str, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + resource = next(item for item in plan.resources if item.kind == "worker_instance") + if field == "serviceAccounts": + fake.resources[resource.name][field] = [{"email": "privileged@example.invalid"}] + else: + fake.resources[resource.name][field][0]["accessConfigs"] = [{"natIP": "203.0.113.1"}] + + with pytest.raises(adapter.Q38GcpAdapterError, match="service account|shape"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_public_ipv6_instance_shape_fails_closed(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + resource = next(item for item in plan.resources if item.kind == "worker_instance") + interface = fake.resources[resource.name]["networkInterfaces"][0] + interface["stackType"] = "IPV4_IPV6" + interface["ipv6AccessConfigs"] = [{"externalIpv6": "2001:db8::1"}] + interface["externalIpv6"] = "2001:db8::1" + + with pytest.raises(adapter.Q38GcpAdapterError, match="shape"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_cleanup_continues_after_failure_and_retries_only_remaining( + tmp_path: Path, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + failed = next(item.name for item in plan.resources if item.kind == "worker_disk") + fake.fail_delete_once.add(failed) + state = _cleanup_state(plan) + + with pytest.raises(adapter.Q38GcpAdapterError, match="cleanup is incomplete"): + _execute(provider, plan, state, adapter.blank_host_status(plan), tmp_path) + + assert set(fake.resources) == {failed} + mutations = [call for call in fake.calls if any(action in call for action in ("create", "delete", "ssh"))] + assert all(route.PROTECTED_INSTANCE not in call for call in mutations) + + observation = _execute(provider, plan, state, adapter.blank_host_status(plan), tmp_path) + + assert fake.resources == {} + assert all(not item["present"] for item in observation["resources"].values()) + assert observation["protected_bootstrap_running"] is True + + +def test_terminal_owned_resources_are_deleted_and_retryable( + tmp_path: Path, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + instance = next(item for item in plan.resources if item.kind == "worker_instance") + disk = next(item for item in plan.resources if item.kind == "worker_disk" and item.worker_id == instance.worker_id) + fake.resources[instance.name]["status"] = "TERMINATED" + fake.resources[disk.name]["status"] = "FAILED" + fake.fail_delete_once.update({instance.name, disk.name}) + state = _cleanup_state(plan) + + with pytest.raises(adapter.Q38GcpAdapterError, match="cleanup is incomplete"): + _execute( + provider, + plan, + state, + adapter.blank_host_status(plan), + tmp_path, + ) + + assert set(fake.resources) == {instance.name, disk.name} + + observation = _execute( + provider, + plan, + state, + adapter.blank_host_status(plan), + tmp_path, + ) + + assert fake.resources == {} + assert all(not item["present"] for item in observation["resources"].values()) + + +@pytest.mark.parametrize("field", ["canIpForward", "deletionProtection"]) +def test_forwarding_or_deletion_protection_fails_closed( + tmp_path: Path, + field: str, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + resource = next(item for item in plan.resources if item.kind == "worker_instance") + fake.resources[resource.name][field] = True + + with pytest.raises(adapter.Q38GcpAdapterError, match="shape"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("boot", False), + ("autoDelete", False), + ("automaticRestart", False), + ("provisioningModel", "SPOT"), + ("onHostMaintenance", "MIGRATE"), + ("instanceTerminationAction", "STOP"), + ("maxRunDuration", {"seconds": "1", "nanos": 0}), + ], +) +def test_instance_lifetime_or_disk_binding_change_fails_closed( + tmp_path: Path, + field: str, + value: object, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + resource = next(item for item in plan.resources if item.kind == "worker_instance") + instance = fake.resources[resource.name] + if field in {"boot", "autoDelete"}: + instance["disks"][0][field] = value + else: + instance["scheduling"][field] = value + + with pytest.raises(adapter.Q38GcpAdapterError, match="shape"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_collect_execution_is_blocked_before_provider_access(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="host runtime is not plan-bound", + ): + _execute( + provider, + plan, + _collect_state(plan), + _ready_status(plan), + tmp_path, + ) + + assert fake.calls == [] + + +def test_cleanup_does_not_require_manifest_revalidation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + state = _cleanup_state(plan) + monkeypatch.setattr( + route, + "revalidate_production_artifact_plan", + lambda *args, **kwargs: pytest.fail("cleanup revalidated stale artifact inputs"), + ) + + observation = provider.execute( + state, + route.action_record(state, plan), + adapter.blank_host_status(plan), + manifest_path=None, + artifact_root=None, + ) + + assert all(not item["present"] for item in observation["resources"].values()) + + +def test_nonblank_host_status_is_rejected_without_bound_transport( + tmp_path: Path, +) -> None: + plan, _fake, provider = _make(tmp_path) + status = _ready_status(plan) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="status transport is not plan-bound", + ): + provider.inventory( + status, + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_stopping_instance_cannot_be_promoted_by_static_ready_status( + tmp_path: Path, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + worker = next(item for item in plan.resources if item.kind == "worker_instance") + fake.resources[worker.name]["status"] = "STOPPING" + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="status transport is not plan-bound", + ): + provider.inventory( + _ready_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert fake.calls == [] + + +def test_adapter_never_targets_protected_bootstrap(tmp_path: Path) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + _execute( + provider, + plan, + _cleanup_state(plan), + adapter.blank_host_status(plan), + tmp_path, + ) + + mutation_calls = [call for call in fake.calls if any(action in call for action in ("create", "delete", "ssh"))] + assert mutation_calls + assert all(route.PROTECTED_INSTANCE not in call for call in mutation_calls) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("id", None), + ("id", True), + ("id", "0"), + ("id", "wrong"), + ("id", "18446744073709551616"), + ("creationTimestamp", None), + ("creationTimestamp", True), + ("creationTimestamp", "2026-09-03T01:20:00Z"), + ("creationTimestamp", "2026-13-03T01:20:00+00:00"), + ("creationTimestamp", "2026-09-03T01:20:00+24:00"), + ], +) +def test_provider_instance_generation_is_required( + tmp_path: Path, + field: str, + value: object, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + instance = next(item for item in plan.resources if item.kind.endswith("instance")) + fake.resources[instance.name][field] = value + + with pytest.raises(adapter.Q38GcpAdapterError, match="instance generation"): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_provider_observation_carries_exact_instance_generation_inventory( + tmp_path: Path, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + + observation = provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert observation["instance_generations_digest"] == route.observation_instance_generations_digest( + observation["resources"], + plan, + ) + for resource in plan.resources: + observed = observation["resources"][resource.name] + if resource.kind.endswith("instance"): + provider_value = fake.resources[resource.name] + assert observed["instance_generation_digest"] == route.instance_generation_digest( + resource.name, + provider_value["id"], + provider_value["creationTimestamp"], + ) + else: + assert observed["instance_id"] is None + assert observed["creation_timestamp"] is None + assert observed["instance_generation_digest"] is None + + +def test_authenticated_guest_status_requires_both_protected_resolvers( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="key and checkpoint resolvers", + ): + adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + status_key_resolver=lambda _resource, _generation: STATUS_KEY, + ) + + +def test_authenticated_guest_status_is_consumed_between_stable_inventories( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + _publish_authenticated_status(plan, fake) + provider = _authenticated_provider(plan, fake, tmp_path) + + observation = provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert {item["state"] for item in observation["workers"].values()} == {"ready"} + assert observation["route_job"]["state"] == "absent" + guest_calls = [call for call in fake.calls if call[1:4] == ("compute", "instances", "get-guest-attributes")] + assert len(guest_calls) == 5 + assert all( + f"--query-path={adapter.GUEST_ATTRIBUTE_QUERY_PATH}" in call + and f"--project={route.EXPECTED_PROJECT}" in call + and f"--zone={route.EXPECTED_ZONE}" in call + and "--format=json" in call + for call in guest_calls + ) + worker = next(item for item in plan.resources if item.kind == "worker_instance") + worker_describes = [call for call in fake.calls if call[1:5] == ("compute", "instances", "describe", worker.name)] + assert len(worker_describes) == 2 + + +def test_absent_guest_attributes_do_not_resolve_keys_or_promote_workers( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + resolutions: list[tuple[str, str]] = [] + provider = adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + clock=lambda: NOW, + status_key_resolver=lambda resource, generation: (resolutions.append((resource, generation)) or STATUS_KEY), + status_checkpoint_resolver=lambda _resource, _generation: (None, 0), + ) + + observation = provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + assert resolutions == [] + assert {item["state"] for item in observation["workers"].values()} == {"starting"} + assert observation["route_job"]["state"] == "absent" + + +def test_authenticated_guest_status_rejects_wrong_key( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + _publish_authenticated_status(plan, fake) + provider = _authenticated_provider( + plan, + fake, + tmp_path, + key=b"x" * 32, + ) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="authenticated guest attribute is invalid", + ): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_authenticated_guest_status_rejects_replayed_revision( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + _publish_authenticated_status(plan, fake, revision=1) + provider = _authenticated_provider( + plan, + fake, + tmp_path, + checkpoint=(STATUS_BOOT_ID, 1), + ) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="authenticated guest attribute is invalid", + ): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_guest_attribute_response_rejects_ambiguous_items( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + resource = next(item for item in plan.resources if item.kind.endswith("instance")) + item = { + "namespace": adapter.GUEST_ATTRIBUTE_NAMESPACE, + "key": adapter.GUEST_ATTRIBUTE_KEY, + "value": "{}", + } + fake.guest_response_overrides[resource.name] = { + "kind": "compute#guestAttributes", + "queryPath": adapter.GUEST_ATTRIBUTE_QUERY_PATH, + "queryValue": {"items": [item, item]}, + } + provider = _authenticated_provider(plan, fake, tmp_path) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="guest attribute response is ambiguous", + ): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_authenticated_guest_status_rejects_generation_drift_during_read( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + _publish_authenticated_status(plan, fake) + resource = next(item for item in plan.resources if item.kind.endswith("instance")) + fake.recreate_after_guest_read = resource.name + provider = _authenticated_provider(plan, fake, tmp_path) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="provider generation changed during authenticated host-status read", + ): + provider.inventory( + adapter.blank_host_status(plan), + manifest_path=tmp_path / "manifest.json", + artifact_root=tmp_path / "artifacts", + ) + + +def test_cleanup_never_reads_authenticated_guest_attributes( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + _publish_authenticated_status(plan, fake) + provider = _authenticated_provider(plan, fake, tmp_path) + + _execute( + provider, + plan, + _cleanup_state(plan), + adapter.blank_host_status(plan), + tmp_path, + ) + + assert not any(call[1:4] == ("compute", "instances", "get-guest-attributes") for call in fake.calls) + + +DELIVERY_KEY = b"d" * transport.KEY_BYTES + + +def _delivery( + plan: route.RoutePlan, + fake: FakeGcloud, +) -> transport.InstanceDelivery: + resource = next(item for item in plan.resources if item.kind == "worker_instance") + provider_value = fake.resources[resource.name] + record = route._instance_key_record( + plan, + resource, + provider_value["id"], + provider_value["creationTimestamp"], + key=DELIVERY_KEY, + key_epoch=1, + issued_at_unix=NOW - 10, + previous_record_digest=None, + ) + return transport.build_instance_delivery( + plan, + route.InstanceGenerationKey(record, DELIVERY_KEY), + now_unix=NOW, + ) + + +def test_compiled_delivery_uses_fixed_iap_stdin_command_without_secret( + tmp_path: Path, +) -> None: + plan, fake, provider = _make(tmp_path) + fake.populate_all() + delivery = _delivery(plan, fake) + + command = provider.compiled_instance_delivery_command( + delivery, + now_unix=NOW, + ) + + assert command[:3] == ("gcloud", "compute", "ssh") + assert command[3] == delivery.record["resource_name"] + assert f"--project={route.EXPECTED_PROJECT}" in command + assert f"--zone={route.EXPECTED_ZONE}" in command + assert "--tunnel-through-iap" in command + assert "--ssh-flag=-T" in command + assert "--ssh-flag=-oBatchMode=yes" in command + assert command[-1] == "--quiet" + joined = "\0".join(command).encode() + assert DELIVERY_KEY not in joined + assert DELIVERY_KEY.hex().encode() not in joined + assert "install-delivery" in joined.decode() + assert delivery.record["instance_generation_digest"] in joined.decode() + + +def test_delivery_runs_between_two_stable_exact_provider_inventories( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + delivery = _delivery(plan, fake) + calls: list[tuple[tuple[str, ...], bytes, int]] = [] + + def deliver(argv, payload, timeout): + calls.append((tuple(argv), payload, timeout)) + receipt = transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=NOW, + ) + return adapter.CommandResult( + 0, + json.dumps(receipt, sort_keys=True).encode(), + b"", + ) + + provider = adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + delivery_runner=deliver, + clock=lambda: NOW, + ) + receipt = provider.deliver_instance(delivery, now_unix=NOW) + + assert len(calls) == 1 + assert calls[0][1] == delivery.payload + assert calls[0][2] == adapter.DELIVERY_TIMEOUT_SECONDS + assert receipt["delivery_digest"] == delivery.record["delivery_digest"] + assert sum(call[1:3] == ("auth", "list") for call in fake.calls) == 2 + assert DELIVERY_KEY not in json.dumps(receipt, sort_keys=True).encode() + + +def test_delivery_rejects_generation_drift_after_remote_install( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + delivery = _delivery(plan, fake) + + def deliver(_argv, _payload, _timeout): + resource = delivery.record["resource_name"] + fake.resources[resource]["id"] = str(int(fake.resources[resource]["id"]) + 1) + receipt = transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=NOW, + ) + return adapter.CommandResult(0, json.dumps(receipt).encode(), b"") + + provider = adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + delivery_runner=deliver, + clock=lambda: NOW, + ) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="provider generation changed during instance delivery", + ): + provider.deliver_instance(delivery, now_unix=NOW) + + +def test_invalid_delivery_is_blocked_before_provider_or_delivery_runner( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + delivery = _delivery(plan, fake) + changed = bytearray(delivery.payload) + changed[-1] ^= 1 + forged = transport.InstanceDelivery(delivery.record, bytes(changed)) + delivery_calls: list[object] = [] + provider = adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + delivery_runner=lambda *args: delivery_calls.append(args), + clock=lambda: NOW, + ) + + with pytest.raises(adapter.Q38GcpAdapterError, match="delivery is invalid"): + provider.deliver_instance(forged, now_unix=NOW) + + assert fake.calls == [] + assert delivery_calls == [] + + +def test_delivery_requires_exact_iap_firewall_before_secret_crosses_boundary( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + delivery = _delivery(plan, fake) + iap = next(item for item in plan.resources if item.kind == "iap_firewall") + fake.resources.pop(iap.name) + delivered: list[bytes] = [] + provider = adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + delivery_runner=lambda _argv, payload, _timeout: delivered.append(payload), + clock=lambda: NOW, + ) + + with pytest.raises( + adapter.Q38GcpAdapterError, + match="provider inventory is not ready", + ): + provider.deliver_instance(delivery, now_unix=NOW) + + assert delivered == [] + + +def test_delivery_failure_does_not_expose_key_or_provider_output( + tmp_path: Path, +) -> None: + plan, fake, _provider = _make(tmp_path) + fake.populate_all() + delivery = _delivery(plan, fake) + provider = adapter.GcpAdapter( + plan, + tmp_path / "source", + runner=fake, + delivery_runner=lambda _argv, _payload, _timeout: adapter.CommandResult( + 1, + b"", + DELIVERY_KEY, + ), + clock=lambda: NOW, + ) + + with pytest.raises(adapter.Q38GcpAdapterError) as captured: + provider.deliver_instance(delivery, now_unix=NOW) + + message = str(captured.value).encode() + assert DELIVERY_KEY not in message + assert DELIVERY_KEY.hex().encode() not in message diff --git a/tests/test_gateq38_linux_host_runtime.py b/tests/test_gateq38_linux_host_runtime.py new file mode 100644 index 000000000..132cb43b8 --- /dev/null +++ b/tests/test_gateq38_linux_host_runtime.py @@ -0,0 +1,2397 @@ +from __future__ import annotations + +import copy +import hashlib +import io +import json +import os +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import threading +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path + +import pytest + +from scripts import ( + gateq38_linux_host_runtime as host, + gateq38_linux_host_transport as transport, + gateq38_route_controller as route, +) + +from tests.test_gateq38_route_controller import _plan_value, _source_root, _write_json + +EXECUTABLE_BYTES = b"#!/bin/sh\nexit 0\n" +SIDECAR_BYTES = b"runtime-sidecar\n" +_NATIVE_CHOWN = getattr(os, "chown", None) +_NATIVE_STATE_LOCK = host._prepared_state_lock +NOW = 1_900_000_000 +KEY = bytes(range(transport.KEY_BYTES)) +BOOT_ID = "01234567-89ab-4cde-8fab-0123456789ab" +INSTANCE_ID = "123456789" +CREATED = "2026-09-03T01:20:00+00:00" + + +def _digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _worker_resource(plan: route.RoutePlan) -> route.ResourcePlan: + return next(item for item in plan.resources if item.kind == "worker_instance") + + +def _transport_context(plan: route.RoutePlan) -> dict[str, object]: + resource = _worker_resource(plan) + return transport.build_instance_context( + plan, + resource.name, + INSTANCE_ID, + CREATED, + issued_at_unix=NOW - 10, + expires_at_unix=NOW + 600, + key=KEY, + ) + + +def _transport_kwargs(plan: route.RoutePlan) -> dict[str, str]: + context = _transport_context(plan) + return { + "expected_resource_name": str(context["resource_name"]), + "expected_generation_digest": str(context["instance_generation_digest"]), + } + + +def _prepared_record( + plan: route.RoutePlan, + action: dict[str, object], + identity: host.QualificationIdentity, + result: host.PreflightResult, +) -> dict[str, object]: + return host._prepared_record( + plan, + action, + identity, + result, + _transport_context(plan), + BOOT_ID, + ) + + +def _artifact(path: str, payload: bytes, mode: int) -> dict[str, object]: + return { + "path": path, + "kind": "file", + "mode": mode, + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + } + + +def _write_archive( + path: Path, + artifacts: list[dict[str, object]], + payloads: dict[str, bytes], + *, + mutate=None, +) -> None: + with tarfile.open(path, "w:gz", format=tarfile.PAX_FORMAT) as archive: + for directory in ("CommunityAI", "CommunityAI/node"): + info = tarfile.TarInfo(directory) + info.type = tarfile.DIRTYPE + info.mode = 0o755 + archive.addfile(info) + for raw in artifacts: + info = tarfile.TarInfo(str(raw["path"])) + if raw["kind"] == "file": + payload = payloads[str(raw["path"])] + info.type = tarfile.REGTYPE + info.mode = int(raw["mode"]) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + else: + info.type = tarfile.SYMTYPE + info.mode = 0o777 + target = str(raw["link_target"]) + member_parent = Path(str(raw["path"])).parent + info.linkname = os.path.relpath(target, member_parent.as_posix()).replace("\\", "/") + archive.addfile(info) + if mutate is not None: + mutate(archive) + + +def _release_and_plan( + tmp_path: Path, + *, + large_provenance: bool = False, +) -> tuple[host.HostPaths, route.RoutePlan, dict[str, object]]: + release = tmp_path / "release" + release.mkdir() + executable_path = route.RUNTIME_PACKAGE_NODE_EXECUTABLE + sidecar_path = "CommunityAI/node/_internal/runtime.bin" + artifacts = [ + _artifact(executable_path, EXECUTABLE_BYTES, 0o755), + _artifact(sidecar_path, SIDECAR_BYTES, 0o644), + ] + payloads = { + executable_path: EXECUTABLE_BYTES, + sidecar_path: SIDECAR_BYTES, + } + artifacts.sort(key=lambda item: str(item["path"])) + archive_path = release / route.RUNTIME_PACKAGE_ARCHIVE + _write_archive(archive_path, artifacts, payloads) + + plan_value = _plan_value() + package = plan_value["runtime_package"] + assert isinstance(package, dict) + provenance = { + "source_commit": plan_value["source_commit"], + "source_tree": package["source_tree"], + "artifacts": artifacts, + } + if large_provenance: + provenance["catalog_publication_bundle"] = { + "complete_release_qualification": False, + "member_digests": {f"catalog/member-{index:05d}-{'x' * 100}.json": "a" * 64 for index in range(7_000)}, + } + provenance_payload = (json.dumps(provenance, sort_keys=True) + "\n").encode() + checksums_payload = "".join(f"{item['sha256']} {item['path']}\n" for item in artifacts).encode() + metrics_payload = b'{"schema_version":1}\n' + manifest = { + "schema_version": 1, + "source": {"revision": route.EXPECTED_MODEL_REVISION}, + "model": {"num_blocks": 64}, + } + manifest_payload = (json.dumps(manifest, sort_keys=True) + "\n").encode() + (release / "provenance.json").write_bytes(provenance_payload) + (release / "SHA256SUMS").write_bytes(checksums_payload) + (release / "desktop-metrics.json").write_bytes(metrics_payload) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_bytes(manifest_payload) + + archive_payload = archive_path.read_bytes() + node_artifacts = [ + item + for item in artifacts + if str(item["path"]) == route.RUNTIME_PACKAGE_NODE_EXECUTABLE + or str(item["path"]).startswith(route.RUNTIME_PACKAGE_NODE_ROOT + "/") + ] + package.update( + { + "release_archive_sha256": _digest(archive_payload), + "release_archive_bytes": len(archive_payload), + "checksums_sha256": _digest(checksums_payload), + "checksums_bytes": len(checksums_payload), + "provenance_sha256": _digest(provenance_payload), + "provenance_bytes": len(provenance_payload), + "desktop_metrics_sha256": _digest(metrics_payload), + "desktop_metrics_bytes": len(metrics_payload), + "manifest_sha256": _digest(manifest_payload), + "manifest_bytes": len(manifest_payload), + "node_executable_sha256": _digest(EXECUTABLE_BYTES), + "node_executable_bytes": len(EXECUTABLE_BYTES), + "node_runtime_entry_count": len(node_artifacts), + "node_runtime_bytes": sum(int(item["size_bytes"]) for item in node_artifacts), + "node_runtime_inventory_digest": _digest(host._canonical_bytes(node_artifacts)), + } + ) + package["runtime_package_digest"] = route._runtime_package_digest(package) + + source_root = _source_root(tmp_path) + plan_path = tmp_path / "route-plan.json" + _write_json(plan_path, plan_value) + plan = route.load_plan(plan_path, source_root) + start_action = route.action_record( + {"revision": 3, "next_action": "start_route"}, + plan, + ) + cleanup_action = route.action_record( + {"revision": 7, "next_action": "cleanup_route"}, + plan, + ) + start_path = tmp_path / "start-action.json" + cleanup_path = tmp_path / "cleanup-action.json" + _write_json(start_path, start_action) + _write_json(cleanup_path, cleanup_action) + context_path = tmp_path / "instance-context.json" + key_path = tmp_path / "host-status.key" + boot_id_path = tmp_path / "boot_id" + context_path.write_bytes(transport.encode_instance_context(_transport_context(plan))) + key_path.write_bytes(KEY) + boot_id_path.write_text(BOOT_ID + "\n", encoding="ascii") + paths = host.HostPaths( + plan=plan_path, + start_action=start_path, + cleanup_action=cleanup_path, + source_root=source_root, + release_root=release, + manifest=manifest_path, + runtime_base=tmp_path / "runtime", + work_base=tmp_path / "work", + prepared_record=tmp_path / "state" / "prepared.json", + instance_context=context_path, + transport_key=key_path, + status_envelope=tmp_path / "state" / "host-status.json", + boot_id=boot_id_path, + ) + return paths, plan, plan_value + + +@contextmanager +def _unlocked_state(_parent: Path): + yield + + +@pytest.fixture(autouse=True) +def _structural_protection(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(host, "_assert_root_managed", lambda *args, **kwargs: None) + monkeypatch.setattr(host, "_assert_root_private_file", lambda *args, **kwargs: None) + monkeypatch.setattr(host, "_assert_qualification_traversal", lambda *args, **kwargs: None) + monkeypatch.setattr(host, "_assert_source_bound", lambda *args, **kwargs: None) + monkeypatch.setattr(host, "_prepared_state_lock", _unlocked_state) + monkeypatch.setattr(os, "chown", lambda *_args, **_kwargs: None, raising=False) + + +def _load_inventory( + paths: host.HostPaths, + plan: route.RoutePlan, +) -> tuple[list[host.Artifact], list[host.Artifact]]: + return host._load_release_inventory(plan, paths) + + +def test_host_runtime_source_is_required() -> None: + assert route.LINUX_HOST_RUNTIME_SOURCE_PATH == "scripts/gateq38_linux_host_runtime.py" + assert route.LINUX_HOST_RUNTIME_SOURCE_PATH in route.REQUIRED_SOURCE_PATHS + + +@pytest.mark.parametrize( + "value", + [ + "/CommunityAI/node/a", + "../CommunityAI/node/a", + "CommunityAI/../node/a", + "CommunityAI\\node\\a", + "CommunityAI/node/./a", + "CommunityAI/node/a\x00", + "", + ], +) +def test_safe_member_path_rejects_unsafe_values(value: str) -> None: + with pytest.raises(host.Q38LinuxHostRuntimeError): + host._safe_member_path(value) + + +@pytest.mark.parametrize("payload", [b'{"a":1,"a":2}', b'{"a":NaN}', b"[]", b""]) +def test_strict_json_rejects_ambiguous_values(payload: bytes) -> None: + with pytest.raises(host.Q38LinuxHostRuntimeError): + host._strict_json(payload) + + +def test_load_exact_start_action(tmp_path: Path) -> None: + paths, expected, _ = _release_and_plan(tmp_path) + plan, action = host._load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=1_900_000_000, + ) + assert plan.plan_digest == expected.plan_digest + assert action["action_id"] == route._action_id(plan, "start_route") + + +@pytest.mark.parametrize("mutation", ["wrong_action", "bool_revision", "extra", "expired"]) +def test_load_action_rejects_substitution(tmp_path: Path, mutation: str) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + action = json.loads(paths.start_action.read_text()) + now = 1_900_000_000 + if mutation == "wrong_action": + action = route.action_record( + {"revision": 3, "next_action": "collect_route"}, + plan, + ) + elif mutation == "bool_revision": + action["revision"] = True + elif mutation == "extra": + action["extra"] = "bad" + else: + now = plan.deadline_unix + _write_json(paths.start_action, action) + with pytest.raises(host.Q38LinuxHostRuntimeError): + host._load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=now, + ) + + +def test_start_rejects_incomplete_authorization(tmp_path: Path) -> None: + paths, _plan, value = _release_and_plan(tmp_path) + value["authorization"]["provisioning_authorized"] = False + _write_json(paths.plan, value) + updated = route.load_plan(paths.plan, paths.source_root) + _write_json( + paths.start_action, + route.action_record( + {"revision": 3, "next_action": "start_route"}, + updated, + ), + ) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="fully authorized"): + host._load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=1_900_000_000, + ) + + +def test_release_inventory_accepts_exact_package(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, node = _load_inventory(paths, plan) + assert len(artifacts) == len(node) == 2 + assert [item.path for item in node] == sorted(item.path for item in node) + + +def test_release_inventory_accepts_production_sized_provenance(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path, large_provenance=True) + assert (paths.release_root / "provenance.json").stat().st_size > 1_241_883 + artifacts, node = _load_inventory(paths, plan) + assert len(artifacts) == len(node) == 2 + + +@pytest.mark.parametrize( + "relative", + ["SHA256SUMS", "provenance.json", "desktop-metrics.json"], +) +def test_release_inventory_rejects_companion_mutation( + tmp_path: Path, + relative: str, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + with (paths.release_root / relative).open("ab") as stream: + stream.write(b"x") + with pytest.raises(host.Q38LinuxHostRuntimeError): + _load_inventory(paths, plan) + + +def test_release_inventory_rejects_manifest_mutation(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + paths.manifest.write_text("{}\n", encoding="utf-8") + with pytest.raises(host.Q38LinuxHostRuntimeError): + _load_inventory(paths, plan) + + +def test_release_inventory_rejects_bool_artifact_size(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + provenance = json.loads((paths.release_root / "provenance.json").read_text()) + provenance["artifacts"][0]["size_bytes"] = True + payload = (json.dumps(provenance, sort_keys=True) + "\n").encode() + (paths.release_root / "provenance.json").write_bytes(payload) + package = dict(plan.runtime_package) + package["provenance_sha256"] = _digest(payload) + package["provenance_bytes"] = len(payload) + object.__setattr__(plan, "runtime_package", package) + with pytest.raises(host.Q38LinuxHostRuntimeError): + _load_inventory(paths, plan) + + +def test_extracts_exact_node_inventory(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, node = _load_inventory(paths, plan) + destination = tmp_path / "install" + host._extract_verified_archive( + paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, + plan.runtime_package, + artifacts, + node, + destination, + ) + assert host._verify_runtime_tree(destination, node, protected=False) == ( + 2, + len(EXECUTABLE_BYTES) + len(SIDECAR_BYTES), + ) + + +def _audit_mutated_archive( + tmp_path: Path, + mutation, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, _node = _load_inventory(paths, plan) + path = tmp_path / "mutated.tar.gz" + payloads = {item.path: EXECUTABLE_BYTES if item.mode == 0o755 else SIDECAR_BYTES for item in artifacts} + _write_archive( + path, + [ + { + "path": item.path, + "kind": item.kind, + "mode": item.mode, + "sha256": item.sha256, + "size_bytes": item.size_bytes, + } + for item in artifacts + ], + payloads, + mutate=mutation, + ) + with tarfile.open(path, "r:gz") as archive: + with pytest.raises(host.Q38LinuxHostRuntimeError): + host._audit_members(archive, artifacts) + + +@pytest.mark.parametrize("kind", ["traversal", "hardlink", "fifo", "extra", "case"]) +def test_archive_rejects_unsafe_members(tmp_path: Path, kind: str) -> None: + def mutate(archive: tarfile.TarFile) -> None: + info = tarfile.TarInfo( + "../escape" + if kind == "traversal" + else "CommunityAI/node/CommunityAI-Node" + if kind == "case" + else "CommunityAI/node/unsafe" + ) + if kind == "hardlink": + info.type = tarfile.LNKTYPE + info.linkname = "CommunityAI/node/CommunityAI-Node" + elif kind == "fifo": + info.type = tarfile.FIFOTYPE + else: + info.type = tarfile.REGTYPE + info.size = 1 + info.mode = 0o644 + archive.addfile(info, io.BytesIO(b"x") if info.isfile() else None) + + _audit_mutated_archive(tmp_path, mutate) + + +def _unprotected_verifier( + root: Path, + node: list[host.Artifact] | tuple[host.Artifact, ...], + *, + protected: bool, +) -> tuple[int, int]: + return _ORIGINAL_VERIFY(root, node, protected=False) + + +_ORIGINAL_VERIFY = host._verify_runtime_tree + + +def test_prepare_writes_digest_only_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + monkeypatch.setattr( + host, + "_atomic_prepared", + lambda path, value, _plan: _write_json(path, value), + ) + result = host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + assert host.validate_prepared_record(result, plan) == result + encoded = json.dumps(result, sort_keys=True) + assert not any(value in encoded for value in ("http://", "https://", str(tmp_path), "token")) + assert paths.prepared_record.exists() + assert paths.status_envelope.exists() + envelope = transport.decode_status_envelope(paths.status_envelope.read_bytes()) + context = _transport_context(plan) + assert envelope["prepared_record_digest"] == result["prepared_record_digest"] + assert envelope["boot_id"] == BOOT_ID + assert envelope["payload"]["state"] == "starting" + assert ( + transport.validate_status_envelope( + envelope, + plan, + key=KEY, + now_unix=NOW, + expected_resource_name=str(context["resource_name"]), + expected_generation_digest=str(context["instance_generation_digest"]), + expected_boot_id=BOOT_ID, + ) + == envelope + ) + + +def test_prepare_failure_removes_new_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + with pytest.raises(RuntimeError, match="preflight"): + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: (_ for _ in ()).throw(RuntimeError("preflight")), + ) + assert not host._runtime_destination(plan, paths).exists() + + +def test_prepare_identity_failure_removes_new_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + monkeypatch.setattr( + host, + "_qualification_identity", + lambda: (_ for _ in ()).throw(RuntimeError("identity")), + ) + with pytest.raises(RuntimeError, match="identity"): + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + protector=lambda *_args: None, + ) + assert not host._runtime_destination(plan, paths).exists() + + +def test_prepare_protection_failure_removes_staging_tree( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + with pytest.raises(RuntimeError, match="protect"): + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: (_ for _ in ()).throw(RuntimeError("protect")), + ) + assert paths.runtime_base.exists() + assert not any(paths.runtime_base.iterdir()) + + +def test_prepare_reuses_exact_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + monkeypatch.setattr( + host, + "_atomic_prepared", + lambda path, value, _plan: _write_json(path, value), + ) + calls = [] + for _ in range(2): + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: calls.append("protect"), + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + assert calls == ["protect"] + + +def test_preflight_binds_fd_argv_identity_and_environment( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, node = _load_inventory(paths, plan) + runtime = tmp_path / "install" + host._extract_verified_archive( + paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, + plan.runtime_package, + artifacts, + node, + runtime, + ) + (runtime / route.RUNTIME_PACKAGE_NODE_EXECUTABLE).chmod(0o755) + monkeypatch.setattr(host, "_verify_runtime_tree", lambda *_args, **_kwargs: (2, 1)) + monkeypatch.setattr(host, "_assert_executable_handle", lambda *_args: None) + monkeypatch.setattr( + os, + "killpg", + lambda *_args: (_ for _ in ()).throw(ProcessLookupError()), + raising=False, + ) + + captured: dict[str, object] = {} + + class Process: + pid = 43210 + + def __init__(self, argv, **kwargs): + captured["argv"] = argv + captured.update(kwargs) + kwargs["stdout"].write(b"edge-acquire help\n") + kwargs["stdout"].flush() + + def wait(self, timeout): + captured.setdefault("timeouts", []).append(timeout) + return 0 + + result = host._run_packaged_preflight( + plan, + runtime, + node, + paths, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + popen_factory=Process, + ) + assert result.returncode == 0 + assert captured["argv"][-2:] == ("edge-acquire", "--help") + assert str(captured["executable"]).startswith("/proc/self/fd/") + assert captured["shell"] is False + assert captured["stdin"] == subprocess.DEVNULL + environment = captured["env"] + assert environment["HF_HUB_OFFLINE"] == "1" + assert environment["TRANSFORMERS_OFFLINE"] == "1" + assert not any("TOKEN" in key or "PROXY" in key and key != "NO_PROXY" for key in environment) + + +def test_preflight_timeout_kills_process_group( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, node = _load_inventory(paths, plan) + runtime = tmp_path / "install" + host._extract_verified_archive( + paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, + plan.runtime_package, + artifacts, + node, + runtime, + ) + (runtime / route.RUNTIME_PACKAGE_NODE_EXECUTABLE).chmod(0o755) + monkeypatch.setattr(host, "_verify_runtime_tree", lambda *_args, **_kwargs: (2, 1)) + monkeypatch.setattr(host, "_assert_executable_handle", lambda *_args: None) + signals = [] + events = [] + alive = True + + def killpg(pid, value): + nonlocal alive + if value == host.KILL_SIGNAL: + events.append("kill") + signals.append((pid, value)) + alive = False + elif not alive: + events.append("probe") + raise ProcessLookupError + + monkeypatch.setattr(os, "killpg", killpg, raising=False) + + class Process: + pid = 43211 + + def __init__(self, _argv, **_kwargs): + self.calls = 0 + + def wait(self, timeout): + self.calls += 1 + events.append(f"wait:{timeout}") + if self.calls == 1: + raise subprocess.TimeoutExpired("node", timeout) + return -9 + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="timed out"): + host._run_packaged_preflight( + plan, + runtime, + node, + paths, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + popen_factory=Process, + ) + assert signals == [(43211, host.KILL_SIGNAL)] + assert events == ["wait:180", "kill", "wait:30", "probe"] + assert not (paths.work_base / host._runtime_key(plan)).exists() + + +@pytest.mark.parametrize( + ("failure", "expected"), + ( + (OSError("wait failed"), host.Q38LinuxHostRuntimeError), + (KeyboardInterrupt(), KeyboardInterrupt), + ), +) +def test_preflight_wait_failure_kills_reaps_and_proves_group_empty( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: BaseException, + expected: type[BaseException], +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, node = _load_inventory(paths, plan) + runtime = tmp_path / "install" + host._extract_verified_archive( + paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, + plan.runtime_package, + artifacts, + node, + runtime, + ) + (runtime / route.RUNTIME_PACKAGE_NODE_EXECUTABLE).chmod(0o755) + monkeypatch.setattr(host, "_verify_runtime_tree", lambda *_args, **_kwargs: (2, 1)) + monkeypatch.setattr(host, "_assert_executable_handle", lambda *_args: None) + events: list[str] = [] + alive = True + + def killpg(_pid, value): + nonlocal alive + if value == host.KILL_SIGNAL: + events.append("kill") + alive = False + elif not alive: + events.append("probe") + raise ProcessLookupError + + monkeypatch.setattr(os, "killpg", killpg, raising=False) + + class Process: + pid = 43212 + + def __init__(self, _argv, **_kwargs): + self.calls = 0 + + def wait(self, timeout): + self.calls += 1 + events.append(f"wait:{timeout}") + if self.calls == 1: + raise failure + return -9 + + with pytest.raises( + expected, + match="could not start" if expected is host.Q38LinuxHostRuntimeError else None, + ): + host._run_packaged_preflight( + plan, + runtime, + node, + paths, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + popen_factory=Process, + ) + assert events == ["wait:180", "kill", "wait:30", "probe"] + assert not (paths.work_base / host._runtime_key(plan)).exists() + + +def test_preflight_work_setup_is_exact_and_transactional( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + identity = host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001) + chmods: list[tuple[Path, int]] = [] + monkeypatch.setattr( + os, + "chmod", + lambda path, mode: chmods.append((Path(path), mode)), + ) + exact = tmp_path / "exact" + children = host._create_preflight_work(exact, identity) + assert children == (exact / "home", exact / "cache", exact / "tmp") + assert chmods == [ + (exact, 0o711), + (exact / "home", 0o700), + (exact / "cache", 0o700), + (exact / "tmp", 0o700), + ] + host._remove_preflight_work(exact) + + raced = tmp_path / "raced" + raced.mkdir() + sentinel = raced / "active" + sentinel.write_bytes(b"active") + with pytest.raises(FileExistsError): + host._create_preflight_work(raced, identity) + assert sentinel.read_bytes() == b"active" + + failed = tmp_path / "failed" + calls = 0 + + def fail_second_chown(*_args, **_kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("chown failed") + + monkeypatch.setattr(os, "chown", fail_second_chown, raising=False) + with pytest.raises(OSError, match="chown failed"): + host._create_preflight_work(failed, identity) + assert not failed.exists() + + +def test_nonroot_preflight_identity_can_traverse_isolated_parents( + monkeypatch: pytest.MonkeyPatch, +) -> None: + if not sys.platform.startswith("linux") or not hasattr(os, "geteuid") or os.geteuid() != 0: + pytest.skip("native root Linux access semantics are unavailable") + if _NATIVE_CHOWN is None: + pytest.skip("native chown is unavailable") + import pwd + + try: + account = pwd.getpwnam("nobody") + except KeyError: + pytest.skip("a nonroot test identity is unavailable") + identity = host.QualificationIdentity("nobody", account.pw_uid, account.pw_gid) + root = Path(tempfile.mkdtemp(prefix="communityai-q38-access-", dir="/tmp")) + monkeypatch.setattr(os, "chown", _NATIVE_CHOWN) + try: + os.chown(root, 0, 0) + os.chmod(root, 0o711) + home, _cache, _temporary = host._create_preflight_work(root / "plan", identity) + runtime_base = root / "runtime" + internal = runtime_base / "key" / "CommunityAI" / "node" / "_internal" + internal.mkdir(parents=True) + for directory in ( + runtime_base, + runtime_base / "key", + runtime_base / "key" / "CommunityAI", + runtime_base / "key" / "CommunityAI" / "node", + internal, + ): + os.chown(directory, 0, 0) + os.chmod(directory, 0o755) + executable = internal.parent / "CommunityAI-Node" + executable.write_bytes(EXECUTABLE_BYTES) + sidecar = internal / "runtime.bin" + sidecar.write_bytes(SIDECAR_BYTES) + for file, mode in ((executable, 0o755), (sidecar, 0o644)): + os.chown(file, 0, 0) + os.chmod(file, mode) + completed = subprocess.run( + ( + "/bin/sh", + "-c", + 'test -d "$HOME" && test -x "$NODE" && test -r "$SIDECAR" ' + '&& head -c 1 "$SIDECAR" >/dev/null && : > "$HOME/probe"', + ), + env={ + "HOME": str(home), + "NODE": str(executable), + "SIDECAR": str(sidecar), + "PATH": "/usr/bin:/bin", + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=False, + close_fds=True, + preexec_fn=host._preflight_child(identity), + timeout=30, + check=False, + ) + assert completed.returncode == 0 + probe = home / "probe" + assert probe.is_file() + assert probe.stat().st_uid == identity.uid + finally: + shutil.rmtree(root) + + +def test_native_prepared_state_lock_is_root_owned_and_private( + monkeypatch: pytest.MonkeyPatch, +) -> None: + if not sys.platform.startswith("linux") or not hasattr(os, "geteuid") or os.geteuid() != 0: + pytest.skip("native root Linux lock semantics are unavailable") + if _NATIVE_CHOWN is None: + pytest.skip("native chown is unavailable") + state = Path(tempfile.mkdtemp(prefix="communityai-q38-state-", dir="/tmp")) + monkeypatch.setattr(os, "chown", _NATIVE_CHOWN) + monkeypatch.setattr(host, "_prepared_state_lock", _NATIVE_STATE_LOCK) + try: + os.chown(state, 0, 0) + os.chmod(state, 0o700) + with host._prepared_state_lock(state): + lock = state / ".prepared.lock" + metadata = lock.lstat() + assert stat.S_ISREG(metadata.st_mode) + assert metadata.st_uid == metadata.st_gid == 0 + assert stat.S_IMODE(metadata.st_mode) == 0o600 + finally: + shutil.rmtree(state) + + +def test_prepared_record_rejects_bool_integer(tmp_path: Path) -> None: + _paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + value = _prepared_record( + plan, + action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + value["qualification_uid"] = True + value["prepared_record_digest"] = host._prepared_digest(value) + with pytest.raises(host.Q38LinuxHostRuntimeError): + host.validate_prepared_record(value, plan) + + +def test_cleanup_is_exact_and_idempotent( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + destination = host._runtime_destination(plan, paths) + destination.mkdir(parents=True) + (destination / "owned").write_text("x", encoding="utf-8") + work = paths.work_base / host._runtime_key(plan) + work.mkdir(parents=True) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + prepared = _prepared_record( + plan, + action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + paths.prepared_record.parent.mkdir(parents=True) + _write_json(paths.prepared_record, prepared) + stale = paths.prepared_record.parent / ".prepared.interrupted.tmp" + stale.write_bytes(host._canonical_bytes(prepared)) + stale.chmod(0o600) + host.cleanup(paths, **_transport_kwargs(plan), now_unix=plan.deadline_unix + 1) + host.cleanup(paths, **_transport_kwargs(plan), now_unix=plan.deadline_unix + 2) + assert not destination.exists() + assert not work.exists() + assert not paths.prepared_record.exists() + assert not stale.exists() + + +def test_cleanup_rejects_linked_runtime( + tmp_path: Path, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + paths.runtime_base.mkdir() + foreign = tmp_path / "foreign" + foreign.mkdir() + destination = host._runtime_destination(plan, paths) + try: + destination.symlink_to(foreign, target_is_directory=True) + except OSError: + pytest.skip("directory symlink creation is unavailable") + with pytest.raises(host.Q38LinuxHostRuntimeError, match="unsafe"): + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + assert foreign.exists() + + +def test_require_linux_root_rejects_this_windows_host() -> None: + if os.name == "posix" and hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("native root behavior is covered on Linux") + with pytest.raises(host.Q38LinuxHostRuntimeError): + host._require_linux_root() + + +def test_old_start_action_and_runtime_key_are_stale_after_plan_change( + tmp_path: Path, +) -> None: + paths, old_plan, value = _release_and_plan(tmp_path) + value["workers"][0]["machine_id"] = "q38machine-rebound" + _write_json(paths.plan, value) + new_plan = route.load_plan(paths.plan, paths.source_root) + assert host._runtime_key(new_plan) != host._runtime_key(old_plan) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="exact controller action"): + host._load_plan_and_action( + paths.plan, + paths.start_action, + paths.source_root, + expected_action="start_route", + now_unix=1_900_000_000, + ) + + +def test_atomic_prepared_refuses_a_different_existing_result( + tmp_path: Path, +) -> None: + _paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + identity = host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001) + first = _prepared_record( + plan, + action, + identity, + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + path = tmp_path / "state" / "prepared.json" + host._atomic_prepared(path, first, plan) + changed = _prepared_record( + plan, + action, + identity, + host.PreflightResult(0, b"edge-acquire changed help\n", b""), + ) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="another result"): + host._atomic_prepared(path, changed, plan) + assert host._strict_json(path.read_bytes()) == first + + +def test_atomic_prepared_never_replaces_concurrent_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + identity = host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001) + intended = _prepared_record( + plan, + action, + identity, + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + concurrent = _prepared_record( + plan, + action, + identity, + host.PreflightResult(0, b"edge-acquire concurrent help\n", b""), + ) + path = tmp_path / "state" / "prepared.json" + native_link = os.link + + def publish_concurrent(source, destination): + _write_json(Path(destination), concurrent) + return native_link(source, destination) + + monkeypatch.setattr(os, "link", publish_concurrent) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="another result"): + host._atomic_prepared(path, intended, plan) + assert host._strict_json(path.read_bytes()) == concurrent + + +def test_atomic_prepared_recovers_interrupted_temporary( + tmp_path: Path, +) -> None: + _paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + identity = host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001) + intended = _prepared_record( + plan, + action, + identity, + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + path = tmp_path / "state" / "prepared.json" + path.parent.mkdir(parents=True) + stale = path.parent / ".prepared.interrupted.tmp" + stale.write_bytes(host._canonical_bytes(intended)) + stale.chmod(0o600) + + host._atomic_prepared(path, intended, plan) + + assert host._strict_json(path.read_bytes()) == intended + assert not stale.exists() + assert not list(path.parent.glob(".prepared.*.tmp")) + + +def test_stale_cleanup_cannot_delete_newer_prepared_state( + tmp_path: Path, +) -> None: + paths, old_plan, value = _release_and_plan(tmp_path) + old_action = route.action_record( + {"revision": 3, "next_action": "start_route"}, + old_plan, + ) + prepared = _prepared_record( + old_plan, + old_action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + paths.prepared_record.parent.mkdir(parents=True) + _write_json(paths.prepared_record, prepared) + old_destination = host._runtime_destination(old_plan, paths) + old_destination.mkdir(parents=True) + + value["workers"][0]["machine_id"] = "q38machine-rebound" + _write_json(paths.plan, value) + new_plan = route.load_plan(paths.plan, paths.source_root) + _write_json( + paths.cleanup_action, + route.action_record( + {"revision": 8, "next_action": "cleanup_route"}, + new_plan, + ), + ) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="plan binding|prepared record identity"): + host.cleanup(paths, **_transport_kwargs(new_plan), now_unix=new_plan.deadline_unix + 1) + assert old_destination.exists() + assert paths.prepared_record.exists() + + +def test_verified_file_detects_path_replacement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "archive.bin" + target.write_bytes(b"bound-bytes") + backup = tmp_path / "opened.bin" + native_open = os.open + replaced = False + + def replacing_open(path, flags): + nonlocal replaced + descriptor = native_open(path, flags) + if Path(path) == target and not replaced: + try: + target.replace(backup) + target.write_bytes(b"bound-bytes") + except OSError: + os.close(descriptor) + pytest.skip("open-file pathname replacement is unavailable") + replaced = True + return descriptor + + monkeypatch.setattr(os, "open", replacing_open) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="identity changed"): + with host._verified_file( + target, + expected_size=11, + expected_digest=_digest(b"bound-bytes"), + ): + pass + + +def test_archive_entry_bound_applies_during_iteration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, _node = _load_inventory(paths, plan) + monkeypatch.setattr(host, "MAX_ARCHIVE_ENTRIES", 2) + with tarfile.open(paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, "r:gz") as archive: + with pytest.raises(host.Q38LinuxHostRuntimeError, match="entry count"): + host._audit_members(archive, artifacts) + + +def test_archive_expanded_byte_bound_is_absolute( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, _node = _load_inventory(paths, plan) + monkeypatch.setattr(host, "MAX_EXPANDED_BYTES", 1) + with tarfile.open(paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, "r:gz") as archive: + with pytest.raises(host.Q38LinuxHostRuntimeError, match="expanded size"): + host._audit_members(archive, artifacts) + + +def test_archive_rejects_non_file_with_payload_size(tmp_path: Path) -> None: + path = tmp_path / "unsafe.tar" + with tarfile.open(path, "w") as archive: + info = tarfile.TarInfo("CommunityAI/node/link") + info.type = tarfile.SYMTYPE + info.linkname = "CommunityAI/node/CommunityAI-Node" + info.size = 1 + archive.addfile(info) + with tarfile.open(path, "r") as archive: + with pytest.raises(host.Q38LinuxHostRuntimeError, match="mode or size"): + host._audit_members(archive, []) + + +def test_runtime_verification_rejects_extra_ancestor_entry(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + artifacts, node = _load_inventory(paths, plan) + destination = tmp_path / "install" + host._extract_verified_archive( + paths.release_root / route.RUNTIME_PACKAGE_ARCHIVE, + plan.runtime_package, + artifacts, + node, + destination, + ) + (destination / "CommunityAI" / "unexpected").write_bytes(b"x") + with pytest.raises(host.Q38LinuxHostRuntimeError, match="ancestor inventory"): + host._verify_runtime_tree(destination, node, protected=False) + + +def test_preflight_work_cleanup_must_be_proved( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + work = tmp_path / "work" + work.mkdir() + (work / "leftover").write_bytes(b"x") + monkeypatch.setattr(shutil, "rmtree", lambda *_args, **_kwargs: None) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="cleanup is incomplete"): + host._remove_preflight_work(work) + + +def test_transport_source_is_required() -> None: + assert route.LINUX_HOST_TRANSPORT_SOURCE_PATH == "scripts/gateq38_linux_host_transport.py" + assert route.LINUX_HOST_TRANSPORT_SOURCE_PATH in route.REQUIRED_SOURCE_PATHS + + +@pytest.mark.parametrize("kind", ["wrong-key", "wrong-generation", "noncanonical-context", "invalid-boot"]) +def test_prepare_rejects_unbound_transport_inputs_before_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + kind: str, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + arguments = _transport_kwargs(plan) + if kind == "wrong-key": + paths.transport_key.write_bytes(b"x" * transport.KEY_BYTES) + match = "authentication" + elif kind == "wrong-generation": + arguments["expected_generation_digest"] = "sha256:" + "0" * 64 + match = "generation" + elif kind == "noncanonical-context": + context = _transport_context(plan) + paths.instance_context.write_text(json.dumps(context, indent=2) + "\n", encoding="ascii") + match = "transport" + else: + paths.boot_id.write_text("not-a-boot-id\n", encoding="ascii") + match = "boot" + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + with pytest.raises(host.Q38LinuxHostRuntimeError, match=match): + host.prepare( + paths, + **arguments, + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + ) + assert not paths.runtime_base.exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + + +def test_prepared_identity_changes_with_generation_and_boot(tmp_path: Path) -> None: + _paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + identity = host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001) + result = host.PreflightResult(0, b"edge-acquire help\n", b"") + first = _prepared_record(plan, action, identity, result) + resource = _worker_resource(plan) + rebound_context = transport.build_instance_context( + plan, + resource.name, + "987654321", + "2026-09-03T01:21:00+00:00", + issued_at_unix=NOW - 10, + expires_at_unix=NOW + 600, + key=KEY, + ) + rebound = host._prepared_record(plan, action, identity, result, rebound_context, BOOT_ID) + rebooted = host._prepared_record( + plan, + action, + identity, + result, + _transport_context(plan), + "fedcba98-7654-4321-8abc-fedcba987654", + ) + assert ( + len( + { + first["prepared_record_digest"], + rebound["prepared_record_digest"], + rebooted["prepared_record_digest"], + } + ) + == 3 + ) + + +def test_status_builder_rejects_caller_substituted_prepared_digest(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + prepared = _prepared_record( + plan, + action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + prepared["prepared_record_digest"] = "sha256:" + "0" * 64 + inputs = host._load_transport_inputs( + plan, + paths, + **_transport_kwargs(plan), + now_unix=NOW, + ) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="digest changed"): + host.build_prepared_status_envelope( + prepared, + inputs, + plan, + **_transport_kwargs(plan), + revision=1, + published_at_unix=NOW, + ) + + +def test_prepare_status_failure_rolls_back_state_and_new_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + monkeypatch.setattr( + host, + "_atomic_status_locked", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("status publish")), + ) + with pytest.raises(RuntimeError, match="status publish"): + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + assert not host._runtime_destination(plan, paths).exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + assert not list(paths.prepared_record.parent.glob(".*.tmp")) + + +def test_atomic_prepared_removes_new_link_after_validation_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + intended = _prepared_record( + plan, + action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + path = tmp_path / "state" / "prepared.json" + monkeypatch.setattr( + host, + "_accept_existing_prepared", + lambda *_args: (_ for _ in ()).throw(host.Q38LinuxHostRuntimeError("post-link")), + ) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="post-link"): + host._atomic_prepared(path, intended, plan) + assert not path.exists() + assert not list(path.parent.glob(".prepared.*.tmp")) + + +def test_wrong_generation_cleanup_preserves_runtime_and_state(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + destination = host._runtime_destination(plan, paths) + destination.mkdir(parents=True) + (destination / "owned").write_bytes(b"x") + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + prepared = _prepared_record( + plan, + action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + paths.prepared_record.parent.mkdir(parents=True) + _write_json(paths.prepared_record, prepared) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="generation"): + host.cleanup( + paths, + expected_resource_name=str(prepared["resource_name"]), + expected_generation_digest="sha256:" + "0" * 64, + now_unix=NOW, + ) + assert destination.exists() + assert paths.prepared_record.exists() + + +def test_atomic_status_removes_new_link_after_validation_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + action = route.action_record({"revision": 3, "next_action": "start_route"}, plan) + prepared = _prepared_record( + plan, + action, + host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + inputs = host._load_transport_inputs( + plan, + paths, + **_transport_kwargs(plan), + now_unix=NOW, + ) + envelope = host.build_prepared_status_envelope( + prepared, + inputs, + plan, + **_transport_kwargs(plan), + revision=1, + published_at_unix=NOW, + ) + monkeypatch.setattr( + host, + "_accept_existing_status", + lambda *_args, **_kwargs: (_ for _ in ()).throw(host.Q38LinuxHostRuntimeError("post-link")), + ) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="post-link"): + host._atomic_status( + paths.status_envelope, + envelope, + plan, + inputs, + **_transport_kwargs(plan), + now_unix=NOW, + ) + assert not paths.status_envelope.exists() + assert not list(paths.status_envelope.parent.glob(".status.*.tmp")) + + +def test_prepare_then_cleanup_removes_bound_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + host.cleanup(paths, **_transport_kwargs(plan), now_unix=plan.deadline_unix + 1) + assert not host._runtime_destination(plan, paths).exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + + +def test_cleanup_rejects_wrong_generation_without_prepared_state(tmp_path: Path) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + destination = host._runtime_destination(plan, paths) + destination.mkdir(parents=True) + (destination / "owned").write_bytes(b"x") + with pytest.raises(host.Q38LinuxHostRuntimeError, match="generation"): + host.cleanup( + paths, + expected_resource_name=str(_transport_context(plan)["resource_name"]), + expected_generation_digest="sha256:" + "0" * 64, + now_unix=NOW, + ) + assert destination.exists() + assert not paths.prepared_record.exists() + + +@pytest.mark.parametrize("publication_offset", [301, 599]) +def test_prepare_resamples_publication_time( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + publication_offset: int, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + samples = iter((NOW, NOW + publication_offset)) + monkeypatch.setattr(host.time, "time", lambda: next(samples)) + host.prepare( + paths, + **_transport_kwargs(plan), + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + envelope = transport.decode_status_envelope(paths.status_envelope.read_bytes()) + assert envelope["published_at_unix"] == NOW + publication_offset + transport.validate_status_envelope( + envelope, + plan, + key=KEY, + now_unix=NOW + publication_offset, + **_transport_kwargs(plan), + expected_boot_id=BOOT_ID, + ) + + +def test_prepare_rejects_context_that_expires_during_preflight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + samples = iter((NOW, NOW + 600)) + monkeypatch.setattr(host.time, "time", lambda: next(samples)) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="stale"): + host.prepare( + paths, + **_transport_kwargs(plan), + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + assert not host._runtime_destination(plan, paths).exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + + +def test_prepare_and_cleanup_are_serialized_when_prepare_wins( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + operation_lock = threading.Lock() + preflight_entered = threading.Event() + release_preflight = threading.Event() + cleanup_done = threading.Event() + errors: list[BaseException] = [] + + @contextmanager + def exclusive(_parent: Path): + with operation_lock: + yield + + def preflight(*_args): + preflight_entered.set() + assert release_preflight.wait(5) + return host.PreflightResult(0, b"edge-acquire help\n", b"") + + def run_prepare() -> None: + try: + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=preflight, + ) + except BaseException as exc: + errors.append(exc) + + def run_cleanup() -> None: + try: + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + except BaseException as exc: + errors.append(exc) + finally: + cleanup_done.set() + + monkeypatch.setattr(host, "_prepared_state_lock", exclusive) + prepare_thread = threading.Thread(target=run_prepare) + cleanup_thread = threading.Thread(target=run_cleanup) + prepare_thread.start() + assert preflight_entered.wait(5) + cleanup_thread.start() + assert not cleanup_done.wait(0.2) + release_preflight.set() + prepare_thread.join(5) + cleanup_thread.join(5) + assert not prepare_thread.is_alive() + assert not cleanup_thread.is_alive() + assert errors == [] + assert not host._runtime_destination(plan, paths).exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + marker = host._cleanup_marker_path(paths, _transport_context(plan)) + assert marker.exists() + + +def test_cleanup_marker_blocks_late_prepare_when_cleanup_wins( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + operation_lock = threading.Lock() + cleanup_entered = threading.Event() + release_cleanup = threading.Event() + prepare_done = threading.Event() + cleanup_errors: list[BaseException] = [] + prepare_errors: list[BaseException] = [] + native_remove = host._remove_exact_tree + first_remove = True + + @contextmanager + def exclusive(_parent: Path): + with operation_lock: + yield + + def blocking_remove(path: Path, parent: Path) -> None: + nonlocal first_remove + if first_remove: + first_remove = False + cleanup_entered.set() + assert release_cleanup.wait(5) + native_remove(path, parent) + + def run_cleanup() -> None: + try: + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + except BaseException as exc: + cleanup_errors.append(exc) + + def run_prepare() -> None: + try: + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + except BaseException as exc: + prepare_errors.append(exc) + finally: + prepare_done.set() + + monkeypatch.setattr(host, "_prepared_state_lock", exclusive) + monkeypatch.setattr(host, "_remove_exact_tree", blocking_remove) + cleanup_thread = threading.Thread(target=run_cleanup) + prepare_thread = threading.Thread(target=run_prepare) + cleanup_thread.start() + assert cleanup_entered.wait(5) + prepare_thread.start() + assert not prepare_done.wait(0.2) + release_cleanup.set() + cleanup_thread.join(5) + prepare_thread.join(5) + assert not cleanup_thread.is_alive() + assert not prepare_thread.is_alive() + assert cleanup_errors == [] + assert len(prepare_errors) == 1 + assert isinstance(prepare_errors[0], host.Q38LinuxHostRuntimeError) + assert "cleanup is terminal" in str(prepare_errors[0]) + assert not host._runtime_destination(plan, paths).exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + + +def test_cleanup_tombstone_survives_interrupted_deletion_and_blocks_prepare( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + native_remove = host._remove_exact_tree + interrupted = False + + def interrupt_after_delete(path: Path, parent: Path) -> None: + nonlocal interrupted + native_remove(path, parent) + if not interrupted: + interrupted = True + raise KeyboardInterrupt("cleanup interrupted") + + monkeypatch.setattr(host, "_remove_exact_tree", interrupt_after_delete) + with pytest.raises(KeyboardInterrupt, match="cleanup interrupted"): + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + marker = host._cleanup_marker_path(paths, _transport_context(plan)) + assert marker.exists() + assert paths.prepared_record.exists() + assert paths.status_envelope.exists() + with pytest.raises(host.Q38LinuxHostRuntimeError, match="cleanup is terminal"): + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + + monkeypatch.setattr(host, "_remove_exact_tree", native_remove) + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + assert marker.exists() + assert not host._runtime_destination(plan, paths).exists() + assert not paths.prepared_record.exists() + assert not paths.status_envelope.exists() + + +def _prepare_status_fixture( + paths: host.HostPaths, + plan: route.RoutePlan, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(host, "_verify_runtime_tree", _unprotected_verifier) + host.prepare( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + identity=host.QualificationIdentity(host.QUALIFICATION_USER, 1001, 1001), + protector=lambda *_args: None, + preflight=lambda *_args: host.PreflightResult(0, b"edge-acquire help\n", b""), + ) + + +def test_publish_status_sends_exact_protected_envelope( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + sent: list[bytes] = [] + + receipt = host.publish_status( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + sender=sent.append, + ) + + assert sent == [paths.status_envelope.read_bytes()] + envelope = transport.decode_status_envelope(sent[0]) + assert receipt == { + "schema_version": host.SCHEMA_VERSION, + "scope": host.PUBLICATION_SCOPE, + "run_id": plan.run_id, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "resource_name": _transport_kwargs(plan)["expected_resource_name"], + "instance_generation_digest": _transport_kwargs(plan)["expected_generation_digest"], + "context_digest": _transport_context(plan)["context_digest"], + "boot_id": BOOT_ID, + "revision": envelope["revision"], + "prepared_record_digest": envelope["prepared_record_digest"], + "envelope_sha256": _digest(sent[0]), + "envelope_bytes": len(sent[0]), + } + assert KEY not in json.dumps(receipt, sort_keys=True).encode() + + +@pytest.mark.parametrize("target", ["key", "context", "boot", "prepared", "status"]) +def test_publish_status_revalidates_every_protected_input_before_network( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + if target == "key": + paths.transport_key.write_bytes(b"x" * transport.KEY_BYTES) + elif target == "context": + value = _transport_context(plan) + value["instance_id"] = "999" + paths.instance_context.write_bytes(transport.encode_instance_context(value)) + elif target == "boot": + paths.boot_id.write_text("11234567-89ab-4cde-8fab-0123456789ab\n", encoding="ascii") + elif target == "prepared": + value = json.loads(paths.prepared_record.read_text(encoding="utf-8")) + value["preflight_stdout_bytes"] += 1 + _write_json(paths.prepared_record, value) + else: + value = transport.decode_status_envelope(paths.status_envelope.read_bytes()) + value["revision"] += 1 + paths.status_envelope.write_bytes(transport.encode_status_envelope(value)) + sent: list[bytes] = [] + + with pytest.raises(host.Q38LinuxHostRuntimeError): + host.publish_status( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + sender=sent.append, + ) + + assert sent == [] + + +def test_publish_status_requires_prepared_state_before_network( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + paths.prepared_record.unlink() + sent: list[bytes] = [] + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="prepared record"): + host.publish_status( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + sender=sent.append, + ) + + assert sent == [] + + +def test_publish_status_holds_lifecycle_lock_against_cleanup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + operation_lock = threading.Lock() + sender_entered = threading.Event() + release_sender = threading.Event() + cleanup_done = threading.Event() + errors: list[BaseException] = [] + + @contextmanager + def exclusive(_parent: Path): + with operation_lock: + yield + + def sender(_payload: bytes) -> None: + sender_entered.set() + assert release_sender.wait(5) + + def run_publish() -> None: + try: + host.publish_status( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + sender=sender, + ) + except BaseException as exc: + errors.append(exc) + + def run_cleanup() -> None: + try: + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + except BaseException as exc: + errors.append(exc) + finally: + cleanup_done.set() + + monkeypatch.setattr(host, "_prepared_state_lock", exclusive) + publish_thread = threading.Thread(target=run_publish) + cleanup_thread = threading.Thread(target=run_cleanup) + publish_thread.start() + assert sender_entered.wait(5) + cleanup_thread.start() + assert not cleanup_done.wait(0.2) + release_sender.set() + publish_thread.join(5) + cleanup_thread.join(5) + + assert errors == [] + assert not publish_thread.is_alive() + assert not cleanup_thread.is_alive() + assert not paths.status_envelope.exists() + assert not paths.prepared_record.exists() + + +def test_publish_status_rejects_terminal_cleanup_marker( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + native_remove = host._remove_exact_tree + interrupted = False + + def interrupt_after_first_delete(path: Path, parent: Path) -> None: + nonlocal interrupted + native_remove(path, parent) + if not interrupted: + interrupted = True + raise KeyboardInterrupt("simulated cleanup interruption") + + monkeypatch.setattr(host, "_remove_exact_tree", interrupt_after_first_delete) + with pytest.raises(KeyboardInterrupt, match="cleanup interruption"): + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + sent: list[bytes] = [] + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="cleanup is terminal"): + host.publish_status( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + sender=sent.append, + ) + + assert sent == [] + + +class _MetadataResponse: + def __init__( + self, + *, + status: int = 200, + flavor: str | None = "Google", + body: bytes = b"OK", + content_length: str | None = None, + ) -> None: + self.status = status + self._flavor = flavor + self._body = body + self._content_length = content_length + + def getheader(self, name: str) -> str | None: + if name == "Metadata-Flavor": + return self._flavor + if name == "Content-Length": + return self._content_length + return None + + def read(self, maximum: int) -> bytes: + return self._body[:maximum] + + +class _MetadataConnection: + def __init__(self, response: _MetadataResponse) -> None: + self.response = response + self.request_value: tuple[str, str, bytes, dict[str, str]] | None = None + self.closed = False + + def request( + self, + method: str, + path: str, + *, + body: bytes, + headers: dict[str, str], + ) -> None: + self.request_value = (method, path, body, headers) + + def getresponse(self) -> _MetadataResponse: + return self.response + + def close(self) -> None: + self.closed = True + + +def test_guest_attribute_publication_uses_fixed_bounded_metadata_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = b'{"status":"starting"}\n' + connection = _MetadataConnection(_MetadataResponse()) + calls: list[tuple[object, ...]] = [] + monkeypatch.setenv("HTTP_PROXY", "http://attacker.invalid:8080") + monkeypatch.setenv("HTTPS_PROXY", "http://attacker.invalid:8080") + + def factory(*args, **kwargs): + calls.append((*args, kwargs)) + return connection + + host._publish_guest_attribute(payload, connection_factory=factory) + + assert calls == [ + ( + host.METADATA_HOST, + host.METADATA_PORT, + {"timeout": host.METADATA_TIMEOUT_SECONDS}, + ) + ] + assert connection.request_value == ( + "PUT", + host.GUEST_ATTRIBUTE_PATH, + payload, + { + "Metadata-Flavor": "Google", + "Content-Type": "application/octet-stream", + "Content-Length": str(len(payload)), + "Connection": "close", + }, + ) + assert connection.closed is True + + +@pytest.mark.parametrize( + ("response", "error"), + [ + (_MetadataResponse(status=301), "not acknowledged"), + (_MetadataResponse(flavor=None), "not acknowledged"), + ( + _MetadataResponse(body=b"x" * (host.MAX_METADATA_RESPONSE_BYTES + 1)), + "size bound", + ), + ( + _MetadataResponse(content_length=str(host.MAX_METADATA_RESPONSE_BYTES + 1)), + "size bound", + ), + (_MetadataResponse(content_length="invalid"), "length is invalid"), + ], +) +def test_guest_attribute_publication_rejects_unsafe_responses( + response: _MetadataResponse, + error: str, +) -> None: + connection = _MetadataConnection(response) + + with pytest.raises(host.Q38LinuxHostRuntimeError, match=error): + host._publish_guest_attribute( + b"status\n", + connection_factory=lambda *_args, **_kwargs: connection, + ) + + assert connection.closed is True + + +def test_guest_attribute_publication_closes_failed_connection() -> None: + class FailingConnection(_MetadataConnection): + def request(self, *_args, **_kwargs) -> None: + raise OSError("network failure") + + connection = FailingConnection(_MetadataResponse()) + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="publication failed"): + host._publish_guest_attribute( + b"status\n", + connection_factory=lambda *_args, **_kwargs: connection, + ) + + assert connection.closed is True + + +def test_host_runtime_parser_includes_fixed_publish_operation() -> None: + args = host.build_parser().parse_args( + [ + "publish-status", + "--resource-name", + "q38-worker-a", + "--instance-generation-digest", + "sha256:" + "a" * 64, + ] + ) + assert args.operation == "publish-status" + + +def test_publish_status_resamples_time_after_acquiring_lifecycle_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + order: list[str] = [] + sent: list[bytes] = [] + + @contextmanager + def ordered_lock(_parent: Path): + order.append("lock") + yield + + def current_time() -> float: + order.append("time") + return float(NOW + transport.MAX_STATUS_AGE_SECONDS + 1) + + monkeypatch.setattr(host, "_prepared_state_lock", ordered_lock) + monkeypatch.setattr(host.time, "time", current_time) + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="status publication is stale"): + host.publish_status( + paths, + **_transport_kwargs(plan), + sender=sent.append, + ) + + assert order == ["lock", "time"] + assert sent == [] + + +def test_publish_status_failure_preserves_protected_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths, plan, _ = _release_and_plan(tmp_path) + _prepare_status_fixture(paths, plan, monkeypatch) + prepared = paths.prepared_record.read_bytes() + status = paths.status_envelope.read_bytes() + + with pytest.raises(RuntimeError, match="carrier failure"): + host.publish_status( + paths, + **_transport_kwargs(plan), + now_unix=NOW, + sender=lambda _payload: (_ for _ in ()).throw(RuntimeError("carrier failure")), + ) + + assert paths.prepared_record.read_bytes() == prepared + assert paths.status_envelope.read_bytes() == status + assert host._runtime_destination(plan, paths).exists() + + +@pytest.mark.parametrize("declared_length", ["", "+1", "-0", " 1 ", "1_0", "١", 1, b"1"]) +def test_guest_attribute_publication_rejects_noncanonical_content_length( + declared_length: object, +) -> None: + connection = _MetadataConnection(_MetadataResponse(content_length=declared_length)) # type: ignore[arg-type] + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="length is invalid"): + host._publish_guest_attribute( + b"status\n", + connection_factory=lambda *_args, **_kwargs: connection, + ) + + assert connection.closed is True + + +@pytest.mark.parametrize( + "response", + [ + _MetadataResponse(body=b"X", content_length="2"), + _MetadataResponse(body=b"XX", content_length="1"), + ], +) +def test_guest_attribute_publication_rejects_declared_body_length_mismatch( + response: _MetadataResponse, +) -> None: + connection = _MetadataConnection(response) + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="response length changed"): + host._publish_guest_attribute( + b"status\n", + connection_factory=lambda *_args, **_kwargs: connection, + ) + + assert connection.closed is True + + +def test_guest_attribute_publication_rejects_noninteger_success_status() -> None: + connection = _MetadataConnection(_MetadataResponse(status=200.0)) # type: ignore[arg-type] + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="not acknowledged"): + host._publish_guest_attribute( + b"status\n", + connection_factory=lambda *_args, **_kwargs: connection, + ) + + assert connection.closed is True + + +def _delivery_paths(paths: host.HostPaths) -> host.HostPaths: + return replace( + paths, + transport_bundle=paths.plan.parent / "instance-delivery.bin", + ) + + +def _instance_delivery( + plan: route.RoutePlan, + *, + key: bytes = KEY, + epoch: int = 1, + previous_record_digest: str | None = None, +) -> transport.InstanceDelivery: + resource = _worker_resource(plan) + record = route._instance_key_record( + plan, + resource, + INSTANCE_ID, + CREATED, + key=key, + key_epoch=epoch, + issued_at_unix=NOW - 10, + previous_record_digest=previous_record_digest, + ) + return transport.build_instance_delivery( + plan, + route.InstanceGenerationKey(record, key), + now_unix=NOW, + ) + + +def test_instance_delivery_installs_one_atomic_bundle_and_returns_receipt( + tmp_path: Path, +) -> None: + raw_paths, plan, _ = _release_and_plan(tmp_path) + paths = _delivery_paths(raw_paths) + delivery = _instance_delivery(plan) + + receipt = host.install_instance_delivery( + paths, + delivery.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + + assert paths.transport_bundle is not None + assert paths.transport_bundle.read_bytes() == delivery.payload + assert ( + transport.validate_instance_delivery_receipt( + receipt, + delivery, + plan, + now_unix=NOW, + ) + == receipt + ) + paths.instance_context.unlink() + paths.transport_key.unlink() + context, key = host._load_authenticated_context( + plan, + paths, + **_transport_kwargs(plan), + now_unix=NOW, + allow_expired_for_cleanup=False, + ) + assert context["context_digest"] == delivery.record["context_digest"] + assert key == KEY + assert KEY not in json.dumps(receipt, sort_keys=True).encode() + + +def test_instance_delivery_retry_is_idempotent(tmp_path: Path) -> None: + raw_paths, plan, _ = _release_and_plan(tmp_path) + paths = _delivery_paths(raw_paths) + delivery = _instance_delivery(plan) + + first = host.install_instance_delivery( + paths, + delivery.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + second = host.install_instance_delivery( + paths, + delivery.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + + assert second == first + assert paths.transport_bundle is not None + assert paths.transport_bundle.read_bytes() == delivery.payload + + +def test_instance_delivery_failed_rotation_preserves_installed_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw_paths, plan, _ = _release_and_plan(tmp_path) + paths = _delivery_paths(raw_paths) + first_material = _instance_delivery(plan) + host.install_instance_delivery( + paths, + first_material.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + second_material = _instance_delivery( + plan, + key=b"x" * transport.KEY_BYTES, + epoch=2, + previous_record_digest=first_material.record["key_record_digest"], + ) + native_replace = os.replace + + def fail_replace(_source: Path, _target: Path) -> None: + raise OSError("simulated interruption") + + monkeypatch.setattr(os, "replace", fail_replace) + with pytest.raises(host.Q38LinuxHostRuntimeError, match="could not be installed"): + host.install_instance_delivery( + paths, + second_material.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + monkeypatch.setattr(os, "replace", native_replace) + + assert paths.transport_bundle is not None + assert paths.transport_bundle.read_bytes() == first_material.payload + assert not list(paths.transport_bundle.parent.glob(".delivery.*.tmp")) + + +def test_instance_delivery_rotation_is_contiguous_and_replay_safe( + tmp_path: Path, +) -> None: + raw_paths, plan, _ = _release_and_plan(tmp_path) + paths = _delivery_paths(raw_paths) + first = _instance_delivery(plan) + second = _instance_delivery( + plan, + key=b"x" * transport.KEY_BYTES, + epoch=2, + previous_record_digest=first.record["key_record_digest"], + ) + + host.install_instance_delivery( + paths, + first.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + host.install_instance_delivery( + paths, + second.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + + assert paths.transport_bundle is not None + assert paths.transport_bundle.read_bytes() == second.payload + with pytest.raises(host.Q38LinuxHostRuntimeError, match="stale or discontinuous"): + host.install_instance_delivery( + paths, + first.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + + +def test_instance_delivery_rejects_partial_or_mutated_bundle_without_replacement( + tmp_path: Path, +) -> None: + raw_paths, plan, _ = _release_and_plan(tmp_path) + paths = _delivery_paths(raw_paths) + delivery = _instance_delivery(plan) + + for payload in (delivery.payload[:-1], delivery.payload + b"x"): + with pytest.raises(host.Q38LinuxHostRuntimeError): + host.install_instance_delivery( + paths, + payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + assert paths.transport_bundle is not None + assert not paths.transport_bundle.exists() + + +def test_cleanup_tombstone_blocks_late_instance_delivery(tmp_path: Path) -> None: + raw_paths, plan, _ = _release_and_plan(tmp_path) + paths = _delivery_paths(raw_paths) + delivery = _instance_delivery(plan) + host.install_instance_delivery( + paths, + delivery.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + assert paths.transport_bundle is not None + assert not paths.transport_bundle.exists() + host.cleanup(paths, **_transport_kwargs(plan), now_unix=NOW) + + with pytest.raises(host.Q38LinuxHostRuntimeError, match="cleanup is terminal"): + host.install_instance_delivery( + paths, + delivery.payload, + **_transport_kwargs(plan), + now_unix=NOW, + ) + + +def test_install_delivery_is_a_bounded_cli_operation() -> None: + args = host.build_parser().parse_args( + [ + "install-delivery", + "--resource-name", + "worker-instance", + "--instance-generation-digest", + "sha256:" + "1" * 64, + ] + ) + + assert args.operation == "install-delivery" diff --git a/tests/test_gateq38_linux_host_transport.py b/tests/test_gateq38_linux_host_transport.py new file mode 100644 index 000000000..0c0ed55e9 --- /dev/null +++ b/tests/test_gateq38_linux_host_transport.py @@ -0,0 +1,697 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from scripts import gateq38_linux_host_transport as transport, gateq38_route_controller as route + +from tests import test_gateq38_route_controller as route_test + +NOW = 1_900_000_000 +KEY = bytes(range(transport.KEY_BYTES)) +OTHER_KEY = b"x" * transport.KEY_BYTES +BOOT_ID = "01234567-89ab-4cde-8fab-0123456789ab" +PREPARED_DIGEST = "sha256:" + "9" * 64 +INSTANCE_ID = "123456789" +CREATED = "2026-09-03T01:20:00+00:00" + + +@pytest.fixture(autouse=True) +def _trusted_plan_inputs(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(route, "_assert_protected_path", lambda *args, **kwargs: None) + + +def _plan(tmp_path: Path) -> route.RoutePlan: + return route_test._load_plan(tmp_path) + + +def _worker_resource(plan: route.RoutePlan, index: int = 0) -> route.ResourcePlan: + return [item for item in plan.resources if item.kind == "worker_instance"][index] + + +def _bootstrap_resource(plan: route.RoutePlan) -> route.ResourcePlan: + return next(item for item in plan.resources if item.kind == "bootstrap_instance") + + +def _context( + plan: route.RoutePlan, + resource: route.ResourcePlan, + *, + key: bytes = KEY, + instance_id: str = INSTANCE_ID, + created: str = CREATED, +) -> dict[str, object]: + return transport.build_instance_context( + plan, + resource.name, + instance_id, + created, + issued_at_unix=NOW - 10, + expires_at_unix=NOW + 600, + key=key, + ) + + +def _worker_payload(plan: route.RoutePlan, resource: route.ResourcePlan, state: str = "ready") -> dict[str, object]: + assert resource.worker_id is not None + worker = plan.worker_by_id[resource.worker_id] + return { + "state": state, + "machine_id": worker.machine_id, + "peer_id": "Qm" + "a" * 44 if state == "ready" else None, + "source_commit": plan.source_commit, + "plan_digest": plan.plan_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": route._action_id(plan, "start_route"), + "span": worker.span, + "manifest_digest": plan.manifest_digest, + "artifact_bytes": worker.artifact_bytes, + "artifact_set_digest": worker.artifact_set_digest, + "cache_root": worker.cache_root, + } + + +def _bootstrap_payload(plan: route.RoutePlan, state: str = "running") -> dict[str, object]: + return { + "state": state, + "job_id": plan.route_job_id, + "collect_action_id": route._action_id(plan, "collect_route"), + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "worker_plan_digest": plan.worker_plan_digest, + "evidence_digest": None, + "route_record": None, + } + + +def _envelope( + plan: route.RoutePlan, + resource: route.ResourcePlan, + *, + key: bytes = KEY, + state: str = "ready", +) -> dict[str, object]: + context = _context(plan, resource, key=key) + payload = ( + _worker_payload(plan, resource, state) + if resource.kind == "worker_instance" + else _bootstrap_payload(plan, "running") + ) + return transport.build_status_envelope( + context, + payload, + plan, + key=key, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + + +def _validate( + envelope: dict[str, object], + plan: route.RoutePlan, + resource: route.ResourcePlan, + *, + key: bytes = KEY, + now: int = NOW, + minimum_revision: int = 0, + boot_id: str | None = None, +) -> dict[str, object]: + return transport.validate_status_envelope( + envelope, + plan, + key=key, + now_unix=now, + expected_resource_name=resource.name, + expected_generation_digest=envelope["context"]["instance_generation_digest"], + minimum_revision=minimum_revision, + expected_boot_id=boot_id, + ) + + +def test_worker_status_round_trip_binds_exact_instance_and_plan(tmp_path: Path) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + envelope = _envelope(plan, resource) + + decoded = transport.decode_status_envelope(transport.encode_status_envelope(envelope)) + validated = _validate(decoded, plan, resource, boot_id=BOOT_ID) + + assert validated == envelope + assert validated["context"]["resource_name"] == resource.name + assert validated["context"]["instance_generation_digest"] == route.instance_generation_digest( + resource.name, + INSTANCE_ID, + CREATED, + ) + assert validated["payload"]["span"] == plan.worker_by_id[resource.worker_id].span + + +def test_bootstrap_status_round_trip_has_no_worker_identity(tmp_path: Path) -> None: + plan = _plan(tmp_path) + resource = _bootstrap_resource(plan) + envelope = _envelope(plan, resource) + + validated = _validate(envelope, plan, resource) + + assert validated["context"]["role"] == "bootstrap" + assert validated["context"]["worker_id"] is None + assert validated["payload"]["state"] == "running" + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("run_id", "other-run"), + ("source_commit", "b" * 40), + ("plan_digest", "sha256:" + "1" * 64), + ("execution_inventory_digest", "sha256:" + "2" * 64), + ("worker_plan_digest", "sha256:" + "3" * 64), + ("start_action_id", "sha256:" + "4" * 64), + ("collect_action_id", "sha256:" + "5" * 64), + ("project", "other-project"), + ("zone", "us-central1-c"), + ("resource_kind", "bootstrap_instance"), + ("role", "bootstrap"), + ("worker_id", None), + ("instance_id", "987654321"), + ("creation_timestamp", "2026-09-03T01:21:00+00:00"), + ("instance_generation_digest", "sha256:" + "6" * 64), + ], +) +def test_context_substitution_fails_closed(tmp_path: Path, field: str, value: object) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + context = _context(plan, resource) + context[field] = value + + with pytest.raises(transport.Q38LinuxHostTransportError): + transport.validate_instance_context(context, plan, key=KEY, now_unix=NOW) + + +def test_context_rejects_wrong_key_and_cross_instance_replay(tmp_path: Path) -> None: + plan = _plan(tmp_path) + first = _worker_resource(plan, 0) + second = _worker_resource(plan, 1) + context = _context(plan, first) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="authentication"): + transport.validate_instance_context(context, plan, key=OTHER_KEY, now_unix=NOW) + with pytest.raises(transport.Q38LinuxHostTransportError, match="resource"): + transport.validate_instance_context( + context, + plan, + key=KEY, + now_unix=NOW, + expected_resource_name=second.name, + ) + + +@pytest.mark.parametrize( + ("issued", "expires", "now"), + [ + (NOW + 31, NOW + 100, NOW), + (NOW - transport.MAX_CONTEXT_SECONDS - 1, NOW + 1, NOW), + (NOW - 10, NOW, NOW), + (NOW - 10, NOW + transport.MAX_CONTEXT_SECONDS + 1, NOW), + ], +) +def test_context_time_window_fails_closed( + tmp_path: Path, + issued: int, + expires: int, + now: int, +) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + with pytest.raises(transport.Q38LinuxHostTransportError, match="time window|stale"): + context = transport.build_instance_context( + plan, + resource.name, + INSTANCE_ID, + CREATED, + issued_at_unix=issued, + expires_at_unix=expires, + key=KEY, + ) + transport.validate_instance_context(context, plan, key=KEY, now_unix=now) + + +@pytest.mark.parametrize( + ("path", "value"), + [ + (("boot_id",), "not-a-boot-id"), + (("revision",), 0), + (("prepared_record_digest",), "wrong"), + (("payload_digest",), "sha256:" + "0" * 64), + (("envelope_hmac",), "hmac-sha256:" + "0" * 64), + (("payload", "artifact_bytes"), 1), + (("payload", "state"), "absent"), + (("context", "instance_generation_digest"), "sha256:" + "0" * 64), + ], +) +def test_envelope_mutation_fails_closed( + tmp_path: Path, + path: tuple[str, ...], + value: object, +) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + envelope = copy.deepcopy(_envelope(plan, resource)) + target = envelope + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + + with pytest.raises(transport.Q38LinuxHostTransportError): + _validate(envelope, plan, resource) + + +def test_status_rejects_stale_revision_time_and_boot(tmp_path: Path) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + envelope = _envelope(plan, resource) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="revision"): + _validate(envelope, plan, resource, minimum_revision=1) + with pytest.raises(transport.Q38LinuxHostTransportError, match="publication"): + _validate(envelope, plan, resource, now=NOW + transport.MAX_STATUS_AGE_SECONDS + 1) + with pytest.raises(transport.Q38LinuxHostTransportError, match="boot"): + _validate( + envelope, + plan, + resource, + boot_id="11234567-89ab-4cde-8fab-0123456789ab", + ) + + +def test_worker_and_bootstrap_payloads_cannot_cross_roles(tmp_path: Path) -> None: + plan = _plan(tmp_path) + worker = _worker_resource(plan) + bootstrap = _bootstrap_resource(plan) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="worker status"): + transport.build_status_envelope( + _context(plan, worker), + _bootstrap_payload(plan), + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + with pytest.raises(transport.Q38LinuxHostTransportError, match="route-job status"): + transport.build_status_envelope( + _context(plan, bootstrap), + _worker_payload(plan, worker), + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + + +def test_ready_worker_requires_peer_and_non_passed_job_rejects_evidence(tmp_path: Path) -> None: + plan = _plan(tmp_path) + worker = _worker_resource(plan) + worker_payload = _worker_payload(plan, worker) + worker_payload["peer_id"] = None + + with pytest.raises(transport.Q38LinuxHostTransportError, match="peer"): + transport.build_status_envelope( + _context(plan, worker), + worker_payload, + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + + bootstrap = _bootstrap_resource(plan) + job = _bootstrap_payload(plan) + job["evidence_digest"] = "sha256:" + "e" * 64 + with pytest.raises(transport.Q38LinuxHostTransportError, match="exposed evidence"): + transport.build_status_envelope( + _context(plan, bootstrap), + job, + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + + +def test_transport_framing_is_canonical_bounded_and_duplicate_safe(tmp_path: Path) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + envelope = _envelope(plan, resource) + encoded = transport.encode_status_envelope(envelope) + + assert encoded.endswith(b"\n") + assert len(encoded) <= transport.MAX_ENVELOPE_BYTES + assert transport.encode_status_envelope(transport.decode_status_envelope(encoded)) == encoded + + with pytest.raises(transport.Q38LinuxHostTransportError, match="duplicate"): + transport.decode_status_envelope(b'{"schema_version":1,"schema_version":1}\n') + with pytest.raises(transport.Q38LinuxHostTransportError, match="framing"): + transport.decode_status_envelope(encoded + b"\n") + with pytest.raises(transport.Q38LinuxHostTransportError, match="bytes"): + transport.decode_status_envelope(b"x" * (transport.MAX_ENVELOPE_BYTES + 1)) + with pytest.raises(transport.Q38LinuxHostTransportError, match="not canonical"): + transport.decode_status_envelope(encoded.replace(b"{", b"{ ", 1)) + + +@pytest.mark.parametrize( + "payload", + [ + b"[" * 10_000 + b"0" + b"]" * 10_000 + b"\n", + b'{"integer":' + b"9" * 5_000 + b"}\n", + ], +) +def test_bounded_parser_complexity_errors_are_wrapped(payload: bytes) -> None: + assert len(payload) <= transport.MAX_ENVELOPE_BYTES + + with pytest.raises(transport.Q38LinuxHostTransportError, match="transport JSON|canonical"): + transport.decode_status_envelope(payload) + + +def test_status_cannot_predate_its_controller_context(tmp_path: Path) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + context = transport.build_instance_context( + plan, + resource.name, + INSTANCE_ID, + CREATED, + issued_at_unix=NOW, + expires_at_unix=NOW + 600, + key=KEY, + ) + envelope = transport.build_status_envelope( + context, + _worker_payload(plan, resource), + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + envelope["published_at_unix"] = NOW - 1 + envelope["envelope_hmac"] = transport._mac( + b"gateq38-host-status-v1", + transport._envelope_unsigned(envelope), + KEY, + ) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="publication"): + _validate(envelope, plan, resource) + + +@pytest.mark.parametrize("key", [b"", b"x" * 31, b"x" * 33, "x" * 32]) +def test_transport_key_is_exactly_32_bytes(tmp_path: Path, key: object) -> None: + plan = _plan(tmp_path) + resource = _worker_resource(plan) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="key"): + _context(plan, resource, key=key) + + +def test_instance_context_transport_is_canonical_and_bounded(tmp_path: Path) -> None: + plan = _plan(tmp_path) + context = _context(plan, _worker_resource(plan)) + + encoded = transport.encode_instance_context(context) + assert transport.decode_instance_context(encoded) == context + + with pytest.raises(transport.Q38LinuxHostTransportError, match="canonical"): + transport.decode_instance_context(b" " + encoded) + with pytest.raises(transport.Q38LinuxHostTransportError, match="framing"): + transport.decode_instance_context(encoded + b"\n") + + +def test_initial_status_payload_matches_controller_absence_rules(tmp_path: Path) -> None: + plan = _plan(tmp_path) + worker = _worker_resource(plan) + bootstrap = _bootstrap_resource(plan) + + worker_payload = transport.initial_status_payload(_context(plan, worker), plan) + assert worker_payload["state"] == "starting" + assert worker_payload["peer_id"] is None + + bootstrap_payload = transport.initial_status_payload(_context(plan, bootstrap), plan) + assert bootstrap_payload == {field: ("absent" if field == "state" else None) for field in route._ROUTE_JOB_FIELDS} + envelope = transport.build_status_envelope( + _context(plan, bootstrap), + bootstrap_payload, + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + assert _validate(envelope, plan, bootstrap)["payload"] == bootstrap_payload + + +@pytest.mark.parametrize("state", ["starting", "failed"]) +def test_unfinished_worker_cannot_expose_peer(tmp_path: Path, state: str) -> None: + plan = _plan(tmp_path) + worker = _worker_resource(plan) + payload = _worker_payload(plan, worker, state) + payload["peer_id"] = "Qm" + "a" * 44 + + with pytest.raises(transport.Q38LinuxHostTransportError, match="unfinished worker"): + transport.build_status_envelope( + _context(plan, worker), + payload, + plan, + key=KEY, + boot_id=BOOT_ID, + revision=1, + published_at_unix=NOW, + prepared_record_digest=PREPARED_DIGEST, + ) + + +def _delivery_material( + plan: route.RoutePlan, + *, + key: bytes = KEY, + epoch: int = 1, + previous_record_digest: str | None = None, +) -> route.InstanceGenerationKey: + resource = _worker_resource(plan) + record = route._instance_key_record( + plan, + resource, + INSTANCE_ID, + CREATED, + key=key, + key_epoch=epoch, + issued_at_unix=NOW - 10, + previous_record_digest=previous_record_digest, + ) + return route.InstanceGenerationKey(record, key) + + +def test_instance_delivery_round_trip_is_secret_free_outside_payload(tmp_path: Path) -> None: + plan = _plan(tmp_path) + material = _delivery_material(plan) + + delivery = transport.build_instance_delivery(plan, material, now_unix=NOW) + record, context, key = transport.validate_instance_delivery( + delivery, + plan, + now_unix=NOW, + ) + + assert record == dict(delivery.record) + assert context["resource_name"] == _worker_resource(plan).name + assert key == KEY + public = json.dumps(dict(delivery.record), sort_keys=True).encode() + assert KEY not in public + assert KEY.hex().encode() not in public + assert "payload=" not in repr(delivery) + + +def test_instance_delivery_rejects_mutation_and_wrong_generation(tmp_path: Path) -> None: + plan = _plan(tmp_path) + delivery = transport.build_instance_delivery( + plan, + _delivery_material(plan), + now_unix=NOW, + ) + mutated = bytearray(delivery.payload) + mutated[-1] ^= 1 + forged = transport.InstanceDelivery(delivery.record, bytes(mutated)) + + with pytest.raises(transport.Q38LinuxHostTransportError): + transport.validate_instance_delivery(forged, plan, now_unix=NOW) + with pytest.raises(transport.Q38LinuxHostTransportError, match="generation"): + transport.validate_instance_delivery( + delivery, + plan, + now_unix=NOW, + expected_generation_digest="sha256:" + "0" * 64, + ) + + +def test_instance_delivery_rotation_binds_predecessor(tmp_path: Path) -> None: + plan = _plan(tmp_path) + first = _delivery_material(plan) + rotated = _delivery_material( + plan, + key=OTHER_KEY, + epoch=2, + previous_record_digest=first.record["record_digest"], + ) + + delivery = transport.build_instance_delivery(plan, rotated, now_unix=NOW) + record, _context_value, key = transport.validate_instance_delivery( + delivery, + plan, + now_unix=NOW, + ) + + assert record["key_epoch"] == 2 + assert record["previous_key_record_digest"] == first.record["record_digest"] + assert key == OTHER_KEY + + +def test_instance_delivery_rejects_noncanonical_or_trailing_framing(tmp_path: Path) -> None: + plan = _plan(tmp_path) + delivery = transport.build_instance_delivery( + plan, + _delivery_material(plan), + now_unix=NOW, + ) + magic_size = len(transport.DELIVERY_MAGIC) + header_end = delivery.payload.find(b"\n", magic_size) + header = json.loads(delivery.payload[magic_size:header_end]) + noncanonical = ( + transport.DELIVERY_MAGIC + + json.dumps(header, indent=2).encode("ascii") + + b"\n" + + delivery.payload[header_end + 1 :] + ) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="header"): + transport.decode_instance_delivery(noncanonical, plan, now_unix=NOW) + with pytest.raises(transport.Q38LinuxHostTransportError, match="framing"): + transport.decode_instance_delivery(delivery.payload + b"x", plan, now_unix=NOW) + + +def test_instance_delivery_receipt_is_authenticated_and_secret_free(tmp_path: Path) -> None: + plan = _plan(tmp_path) + delivery = transport.build_instance_delivery( + plan, + _delivery_material(plan), + now_unix=NOW, + ) + receipt = transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=NOW, + ) + + assert ( + transport.validate_instance_delivery_receipt( + receipt, + delivery, + plan, + now_unix=NOW, + ) + == receipt + ) + public = json.dumps(receipt, sort_keys=True).encode() + assert KEY not in public + assert KEY.hex().encode() not in public + + changed = copy.deepcopy(receipt) + changed["key_epoch"] += 1 + with pytest.raises(transport.Q38LinuxHostTransportError): + transport.validate_instance_delivery_receipt( + changed, + delivery, + plan, + now_unix=NOW, + ) + + +def test_instance_delivery_receipt_rejects_authenticated_stale_replay(tmp_path: Path) -> None: + plan = _plan(tmp_path) + delivery = transport.build_instance_delivery( + plan, + _delivery_material(plan), + now_unix=NOW, + ) + receipt = transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=NOW, + ) + + with pytest.raises(transport.Q38LinuxHostTransportError, match="receipt is stale"): + transport.validate_instance_delivery_receipt( + receipt, + delivery, + plan, + now_unix=NOW + 900, + ) + + +@pytest.mark.parametrize( + "installed_at_unix", + [ + NOW - transport.MAX_DELIVERY_RECEIPT_AGE_SECONDS - 1, + NOW + transport.MAX_FUTURE_SKEW_SECONDS + 1, + ], +) +def test_instance_delivery_receipt_authenticates_before_time_semantics( + tmp_path: Path, + installed_at_unix: int, +) -> None: + plan = _plan(tmp_path) + delivery = transport.build_instance_delivery( + plan, + _delivery_material(plan), + now_unix=NOW, + ) + receipt = transport.build_instance_delivery_receipt( + delivery, + plan, + installed_at_unix=NOW, + ) + receipt["installed_at_unix"] = installed_at_unix + + with pytest.raises(transport.Q38LinuxHostTransportError, match="receipt digest changed"): + transport.validate_instance_delivery_receipt( + receipt, + delivery, + plan, + now_unix=NOW, + ) + + receipt["receipt_digest"] = transport._receipt_digest_value(receipt) + with pytest.raises(transport.Q38LinuxHostTransportError, match="receipt authentication failed"): + transport.validate_instance_delivery_receipt( + receipt, + delivery, + plan, + now_unix=NOW, + ) diff --git a/tests/test_gateq38_route_controller.py b/tests/test_gateq38_route_controller.py new file mode 100644 index 000000000..26d62c19c --- /dev/null +++ b/tests/test_gateq38_route_controller.py @@ -0,0 +1,2749 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import os +from dataclasses import replace +from pathlib import Path + +import pytest + +from scripts import gateq38_route_controller as route + +REAL_ASSERT_PROTECTED_PATH = route._assert_protected_path +RUN_ID = "q38route-001" +SOURCE_BYTES = b"# production artifact verifier fixture\n" +LEDGER_BYTES = SOURCE_BYTES + ( + b"Q38_ROUTE_RESERVATION run_id=q38route-001 " + b"reservation_id=q38route-reservation-001 maximum_usd=44.00 " + b"deadline_unix=2000000000\n" +) + + +def _source_bytes(relative_path: str) -> bytes: + return LEDGER_BYTES if relative_path == route.READINESS_LEDGER_PATH else SOURCE_BYTES + + +@pytest.fixture(autouse=True) +def _trusted_controller_fixture(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(route, "_assert_protected_path", lambda *args, **kwargs: None) + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, sort_keys=True) + "\n", encoding="utf-8") + + +def _source_root(tmp_path: Path) -> Path: + root = tmp_path / "source" + for relative_path in route.REQUIRED_SOURCE_PATHS: + source = root / relative_path + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(_source_bytes(relative_path)) + return root + + +def _binding(relative_path: str = route.VERIFIER_SOURCE_PATH) -> dict[str, object]: + payload = _source_bytes(relative_path) + return { + "relative_path": relative_path, + "sha256": "sha256:" + hashlib.sha256(payload).hexdigest(), + "byte_size": len(payload), + } + + +def _runtime_package(source_bindings: list[dict[str, object]]) -> dict[str, object]: + record: dict[str, object] = { + "schema_version": route.RUNTIME_PACKAGE_SCHEMA_VERSION, + "scope": route.RUNTIME_PACKAGE_SCOPE, + "platform": route.RUNTIME_PACKAGE_PLATFORM, + "source_commit": "a" * 40, + "source_tree": "d" * 40, + "source_bindings_digest": route._source_bindings_digest(source_bindings), + "release_archive_name": route.RUNTIME_PACKAGE_ARCHIVE, + "release_archive_sha256": "sha256:" + "1" * 64, + "release_archive_bytes": 3_360_000_000, + "checksums_sha256": "sha256:" + "2" * 64, + "checksums_bytes": 100_000, + "provenance_sha256": "sha256:" + "3" * 64, + "provenance_bytes": 100_000, + "desktop_metrics_sha256": "sha256:" + "4" * 64, + "desktop_metrics_bytes": 10_000, + "manifest_digest": route.EXPECTED_MANIFEST_DIGEST, + "manifest_sha256": "sha256:" + "5" * 64, + "manifest_bytes": 50_000, + "node_root": route.RUNTIME_PACKAGE_NODE_ROOT, + "node_executable": route.RUNTIME_PACKAGE_NODE_EXECUTABLE, + "node_executable_sha256": "sha256:" + "6" * 64, + "node_executable_bytes": 10_000_000, + "node_runtime_entry_count": 2_000, + "node_runtime_bytes": 2_500_000_000, + "node_runtime_inventory_digest": "sha256:" + "7" * 64, + "runtime_package_digest": "", + } + record["runtime_package_digest"] = route._runtime_package_digest(record) + return record + + +def _workers() -> list[dict[str, object]]: + result = [] + for index, (span, (artifact_bytes, artifact_digest)) in enumerate(route.EXPECTED_SPANS.items()): + result.append( + { + "worker_id": f"worker-{index}", + "machine_id": f"machine-{index}", + "instance": f"{RUN_ID}-worker-{index}", + "disk": f"{RUN_ID}-worker-{index}-disk", + "span": span, + "artifact_bytes": artifact_bytes, + "artifact_set_digest": artifact_digest, + "cache_root": f"/var/lib/communityai/q38/worker-{index}", + } + ) + return result + + +def _resources(workers: list[dict[str, object]]) -> list[dict[str, object]]: + result = [ + { + "name": f"{RUN_ID}-bootstrap-disk", + "kind": "bootstrap_disk", + "provider": "gcp", + "region": "us-central1", + "worker_id": None, + }, + { + "name": f"{RUN_ID}-bootstrap-instance", + "kind": "bootstrap_instance", + "provider": "gcp", + "region": "us-central1", + "worker_id": None, + }, + { + "name": f"{RUN_ID}-firewall", + "kind": "firewall", + "provider": "gcp", + "region": "us-central1", + "worker_id": None, + }, + { + "name": f"{RUN_ID}-iap-firewall", + "kind": "iap_firewall", + "provider": "gcp", + "region": "us-central1", + "worker_id": None, + }, + ] + for worker in workers: + result.extend( + [ + { + "name": worker["disk"], + "kind": "worker_disk", + "provider": "gcp", + "region": "us-central1", + "worker_id": worker["worker_id"], + }, + { + "name": worker["instance"], + "kind": "worker_instance", + "provider": "gcp", + "region": "us-central1", + "worker_id": worker["worker_id"], + }, + ] + ) + return sorted(result, key=lambda item: str(item["name"])) + + +def _plan_value(*, authorized: bool = True) -> dict[str, object]: + workers = _workers() + source_bindings = [_binding(relative_path) for relative_path in sorted(route.REQUIRED_SOURCE_PATHS)] + return { + "schema_version": route.SCHEMA_VERSION, + "gate": route.GATE, + "run_id": RUN_ID, + "route_job_id": "q38route-job-001", + "source_commit": "a" * 40, + "manifest_digest": route.EXPECTED_MANIFEST_DIGEST, + "model_revision": route.EXPECTED_MODEL_REVISION, + "deadline_unix": 2_000_000_000, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "44.00", + "reservation_recorded": authorized, + "native_auth_revalidated": authorized, + "inventory_revalidated": authorized, + "pricing_revalidated": authorized, + "provisioning_authorized": authorized, + "reservation_id": "q38route-reservation-001", + "reservation_record_path": "reservation.json", + "reservation_record_sha256": "sha256:" + "b" * 64, + "reservation_record_byte_size": 100, + "preflight_record_path": "preflight.json", + "preflight_record_sha256": "sha256:" + "c" * 64, + "preflight_record_byte_size": 100, + "readiness_ledger_sha256": _binding(route.READINESS_LEDGER_PATH)["sha256"], + }, + "source_bindings": source_bindings, + "runtime_package": _runtime_package(source_bindings), + "resources": _resources(workers), + "workers": workers, + } + + +def _load_plan(tmp_path: Path, value: dict[str, object] | None = None) -> route.RoutePlan: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value() if value is None else value) + return route.load_plan(plan_path, source_root) + + +def _route_record( + plan: route.RoutePlan, + workers: dict[str, object], +) -> dict[str, object]: + results = [] + for worker_plan in plan.workers: + worker = workers[worker_plan.worker_id] + results.append( + { + "worker_id": worker_plan.worker_id, + "machine_id": worker_plan.machine_id, + "peer_id": worker["peer_id"], + "span": worker_plan.span, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "artifact_bytes": worker_plan.artifact_bytes, + "artifact_set_digest": worker_plan.artifact_set_digest, + "cache_root": worker_plan.cache_root, + "worker_evidence_digest": "sha256:" + hashlib.sha256(worker_plan.worker_id.encode()).hexdigest(), + } + ) + return { + "schema_version": route.SCHEMA_VERSION, + "result": "passed", + "run_id": plan.run_id, + "job_id": plan.route_job_id, + "collect_action_id": route._action_id(plan, "collect_route"), + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "worker_plan_digest": plan.worker_plan_digest, + "route_span": "0:64", + "session_id": "q38route-session-001", + "route_rpc_evidence_digest": "sha256:" + "d" * 64, + "cleanup_ready": True, + "worker_results": results, + } + + +def _job( + plan: route.RoutePlan, + state: str = "absent", + workers: dict[str, object] | None = None, +) -> dict[str, object]: + if state == "absent": + return { + "state": state, + "job_id": None, + "collect_action_id": None, + "run_id": None, + "plan_digest": None, + "source_commit": None, + "manifest_digest": None, + "worker_plan_digest": None, + "evidence_digest": None, + "route_record": None, + } + record = _route_record(plan, workers) if state == "passed" and workers is not None else None + return { + "state": state, + "job_id": plan.route_job_id, + "collect_action_id": route._action_id(plan, "collect_route"), + "run_id": plan.run_id, + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "worker_plan_digest": plan.worker_plan_digest, + "evidence_digest": route._canonical_digest(record) if record is not None else None, + "route_record": record, + } + + +def _observation( + plan: route.RoutePlan, + *, + resource_state: str = "absent", + worker_state: str = "absent", + job_state: str = "absent", + observed_at: int = 1_900_000_000, +) -> dict[str, object]: + resources: dict[str, object] = {} + for resource in plan.resources: + present = resource_state == "present" + is_instance = present and resource.kind.endswith("instance") + instance_id = str(10_000_000 + len(resources)) if is_instance else None + creation_timestamp = "2026-09-03T01:20:00+00:00" if is_instance else None + generation_digest = ( + route.instance_generation_digest( + resource.name, + instance_id, + creation_timestamp, + ) + if is_instance + else None + ) + resources[resource.name] = { + "present": present, + "kind": resource.kind, + "provider": resource.provider, + "region": resource.region, + "run_id": plan.run_id if present else None, + "source_commit": plan.source_commit if present else None, + "deadline_unix": plan.deadline_unix if present else None, + "plan_digest": plan.plan_digest if present else None, + "start_action_id": route._action_id(plan, "start_route") if present else None, + "worker_id": resource.worker_id if present else None, + "instance_id": instance_id, + "creation_timestamp": creation_timestamp, + "instance_generation_digest": generation_digest, + } + workers: dict[str, object] = {} + for index, worker in enumerate(plan.workers): + present = worker_state != "absent" + workers[worker.worker_id] = { + "state": worker_state, + "machine_id": worker.machine_id if present else None, + "peer_id": f"12D3KooWQwenRoutePeer{index:02d}" if worker_state == "ready" else None, + "source_commit": plan.source_commit if present else None, + "plan_digest": plan.plan_digest if present else None, + "worker_plan_digest": plan.worker_plan_digest if present else None, + "start_action_id": route._action_id(plan, "start_route") if present else None, + "span": worker.span if present else None, + "manifest_digest": plan.manifest_digest if present else None, + "artifact_bytes": worker.artifact_bytes if present else None, + "artifact_set_digest": worker.artifact_set_digest if present else None, + "cache_root": worker.cache_root if present else None, + } + verifier_digest = next( + item["sha256"] for item in plan.source_bindings if item["relative_path"] == route.VERIFIER_SOURCE_PATH + ) + return { + "schema_version": route.SCHEMA_VERSION, + "run_id": plan.run_id, + "observed_at_unix": observed_at, + "protected_bootstrap_running": True, + "artifact_plan_revalidation": { + "verified_at_unix": observed_at - 1, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "model_revision": plan.model_revision, + "index_digest": route.EXPECTED_INDEX_DIGEST, + "block_prefix": route.EXPECTED_BLOCK_PREFIX, + "worker_plan_digest": plan.worker_plan_digest, + "verifier_source_sha256": verifier_digest, + }, + "instance_generations_digest": route.observation_instance_generations_digest( + resources, + plan, + ), + "resources": resources, + "workers": workers, + "route_job": _job(plan, job_state, workers), + } + + +def _advance( + operation: str, + state: dict[str, object], + observation: dict[str, object], + plan: route.RoutePlan, +) -> dict[str, object]: + return route.reconcile( + operation, + state, + observation, + plan, + route_evidence_validated=observation["route_job"]["state"] == "passed", + start_was_issued=not route._all_absent(observation), + ) + + +def test_load_plan_binds_exact_route_and_current_ledger(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + + assert tuple(worker.span for worker in plan.workers) == tuple(route.EXPECTED_SPANS) + assert sum(worker.artifact_bytes for worker in plan.workers) == 24_383_317_332 + assert plan.authorization["combined_cloud_ceiling_usd"] == "100.00" + assert plan.authorization["ledger_committed_before_run_usd"] == "56.00" + assert plan.worker_plan_digest.startswith("sha256:") + assert plan.runtime_package["runtime_package_digest"].startswith("sha256:") + assert route.EXPECTED_BLOCK_PREFIX == "model.language_model.layers" + + +def test_runtime_package_is_immutable_and_bound_to_every_action_identity(tmp_path: Path) -> None: + original_value = _plan_value() + changed_value = copy.deepcopy(original_value) + changed_value["runtime_package"]["node_executable_sha256"] = "sha256:" + "8" * 64 + changed_value["runtime_package"]["runtime_package_digest"] = route._runtime_package_digest( + changed_value["runtime_package"] + ) + original = _load_plan(tmp_path / "original", original_value) + changed = _load_plan(tmp_path / "changed", changed_value) + + assert original.plan_digest != changed.plan_digest + assert original.execution_inventory_digest != changed.execution_inventory_digest + assert route._action_id(original, "start_route") != route._action_id(changed, "start_route") + assert ( + route.action_record(route.initial_state(changed), changed)["runtime_package"] + == changed_value["runtime_package"] + ) + with pytest.raises(TypeError): + original.runtime_package["node_runtime_bytes"] = 1 + with pytest.raises(TypeError): + original.source_bindings[0]["sha256"] = "sha256:" + "0" * 64 + with pytest.raises(TypeError): + original.authorization["maximum_estimate_usd"] = "0.00" + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda value: value.pop("runtime_package"), "plan schema"), + (lambda value: value["runtime_package"].update(extra=True), "runtime_package schema"), + ( + lambda value: value["runtime_package"].update(source_commit="b" * 40), + "source commit changed", + ), + ( + lambda value: value["runtime_package"].update(manifest_digest="sha256:" + "0" * 64), + "manifest binding changed", + ), + ( + lambda value: value["runtime_package"].update(source_bindings_digest="sha256:" + "0" * 64), + "source bindings changed", + ), + ( + lambda value: value["runtime_package"].update(runtime_package_digest="sha256:" + "0" * 64), + "record digest changed", + ), + ( + lambda value: value["runtime_package"].update(node_runtime_bytes=True), + "node_runtime_bytes", + ), + ], +) +def test_load_plan_rejects_runtime_package_substitution( + tmp_path: Path, + mutation, + message: str, +) -> None: + value = _plan_value() + mutation(value) + + with pytest.raises(route.RouteControllerError, match=message): + _load_plan(tmp_path, value) + + +def test_runtime_package_digest_domain_is_distinct() -> None: + value = _plan_value()["runtime_package"] + without_self = {key: item for key, item in value.items() if key != "runtime_package_digest"} + + assert value["runtime_package_digest"] == route._runtime_package_digest(value) + assert value["runtime_package_digest"] != route._canonical_digest(without_self) + serialized = json.dumps(value, allow_nan=False, sort_keys=True, separators=(",", ":")) + "\n" + assert "sha256:" + hashlib.sha256(serialized.encode()).hexdigest() != value["runtime_package_digest"] + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda value: value.update(manifest_digest="sha256:" + "0" * 64), "model binding"), + (lambda value: value.update(model_revision="0" * 40), "model binding"), + ( + lambda value: value["authorization"].update(ledger_committed_before_run_usd="55.00"), + "current combined cloud ledger", + ), + ( + lambda value: value["authorization"].update(maximum_estimate_usd="44.01"), + "current combined cloud ledger", + ), + (lambda value: value["workers"][0].update(artifact_bytes=1), "artifact plan changed"), + ( + lambda value: value["workers"][0].update(artifact_set_digest="sha256:" + "0" * 64), + "artifact plan changed", + ), + (lambda value: value["workers"].reverse(), "canonical exact route"), + (lambda value: value["resources"].reverse(), "sorted by name"), + ( + lambda value: value["workers"][1].update(cache_root=value["workers"][0]["cache_root"]), + "unique cache_root", + ), + ( + lambda value: value["workers"][1].update(machine_id=value["workers"][0]["machine_id"]), + "unique machine_id", + ), + ( + lambda value: value["workers"][0].update(instance=route.PROTECTED_INSTANCE), + "protected bootstrap", + ), + ( + lambda value: value["resources"][0].update(provider="fly"), + "one exact provider", + ), + ( + lambda value: value["resources"][0].update(name="foreign-disk"), + "not run-scoped", + ), + ( + lambda value: [resource.update(provider="fly") for resource in value["resources"]], + "one exact provider", + ), + ( + lambda value: next( + resource for resource in value["resources"] if resource["kind"] == "iap_firewall" + ).update(kind="firewall"), + "resource kind inventory", + ), + ], +) +def test_load_plan_rejects_substitution( + tmp_path: Path, + mutation, + message: str, +) -> None: + value = _plan_value() + mutation(value) + + with pytest.raises(route.RouteControllerError, match=message): + _load_plan(tmp_path, value) + + +@pytest.mark.parametrize( + "name", + [ + "1leading-digit", + f"{RUN_ID}-has.dot", + f"{RUN_ID}-has_under", + "a" * 64, + f"{RUN_ID}-trailing-", + ], +) +def test_load_plan_rejects_non_rfc1035_gcp_resource_names( + tmp_path: Path, + name: str, +) -> None: + value = _plan_value() + value["resources"][0]["name"] = name + + with pytest.raises(route.RouteControllerError, match="resource name is invalid"): + _load_plan(tmp_path, value) + + +def test_load_plan_rejects_missing_execution_source_binding(tmp_path: Path) -> None: + value = _plan_value() + value["source_bindings"] = value["source_bindings"][:-1] + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, value) + + with pytest.raises(route.RouteControllerError, match="exact route execution sources"): + route.load_plan(plan_path, source_root) + + +def test_load_plan_rejects_changed_source_binding(tmp_path: Path) -> None: + source_root = _source_root(tmp_path) + (source_root / route.VERIFIER_SOURCE_PATH).write_bytes(b"changed") + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + + with pytest.raises(route.RouteControllerError, match="source binding"): + route.load_plan(plan_path, source_root) + + +def test_load_plan_rejects_changed_gcp_adapter_binding(tmp_path: Path) -> None: + source_root = _source_root(tmp_path) + (source_root / route.GCP_ADAPTER_SOURCE_PATH).write_bytes(b"changed") + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + + with pytest.raises(route.RouteControllerError, match="source binding"): + route.load_plan(plan_path, source_root) + + +def test_load_plan_rejects_duplicate_json_field(tmp_path: Path) -> None: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + plan_path.write_text('{"schema_version":1,"schema_version":1}\n', encoding="utf-8") + + with pytest.raises(route.RouteControllerError, match="duplicate JSON field"): + route.load_plan(plan_path, source_root) + + +def test_unauthorized_plan_loads_but_start_fails_closed(tmp_path: Path) -> None: + plan = _load_plan(tmp_path, _plan_value(authorized=False)) + observation = _observation(plan) + + with pytest.raises(route.RouteControllerError, match="not authorized"): + _advance("start", route.initial_state(plan), observation, plan) + + +def test_initial_start_emits_one_durable_exact_action(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan) + + started = _advance("start", route.initial_state(plan), observation, plan) + repeated = _advance("start", started, observation, plan) + first_action = route.action_record(started, plan) + repeated_action = route.action_record(repeated, plan) + + assert started["phase"] == "STARTING" + assert started["next_action"] == "start_route" + assert repeated == started + assert first_action == repeated_action + assert first_action["action_id"].startswith("sha256:") + assert first_action["worker_plan_digest"] == plan.worker_plan_digest + assert len(first_action["resources"]) == 12 + assert first_action["resource_specs"] == [route._expected_resource_spec(resource) for resource in plan.resources] + worker_spec = next(spec for spec in first_action["resource_specs"] if spec["kind"] == "worker_instance") + bootstrap_spec = next(spec for spec in first_action["resource_specs"] if spec["kind"] == "bootstrap_instance") + iap_spec = next(spec for spec in first_action["resource_specs"] if spec["kind"] == "iap_firewall") + assert iap_spec["resource_name"] == f"{RUN_ID}-iap-firewall" + assert iap_spec["network"] == route.EXPECTED_NETWORK + assert worker_spec["machine_type"] == "g2-standard-8" + assert worker_spec["accelerator_type"] == "nvidia-l4" + assert worker_spec["accelerator_count"] == 1 + assert bootstrap_spec["machine_type"] == "e2-standard-2" + assert bootstrap_spec["accelerator_type"] == "none" + assert bootstrap_spec["accelerator_count"] == 0 + + +def test_start_reattaches_complete_ready_route_without_recreating(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="ready") + + state = _advance("start", route.initial_state(plan), observation, plan) + + assert state["phase"] == "READY" + assert state["next_action"] == "none" + + +def test_start_reattaches_starting_route_without_recreating(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="starting") + + state = _advance("start", route.initial_state(plan), observation, plan) + + assert state["phase"] == "STARTING" + assert state["next_action"] == "none" + + +def test_partial_reattach_cleans_instead_of_starting(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="starting") + first_resource = next(iter(observation["resources"].values())) + first_resource.update( + present=False, + run_id=None, + source_commit=None, + deadline_unix=None, + plan_digest=None, + start_action_id=None, + worker_id=None, + ) + + state = _advance("start", route.initial_state(plan), observation, plan) + + assert state["phase"] == "CLEANING" + assert state["failure_code"] == "partial-reattach" + assert state["next_action"] == "cleanup_route" + + +def test_starting_acknowledgement_clears_pending_action(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + started = _advance("start", route.initial_state(plan), _observation(plan), plan) + observation = _observation(plan, resource_state="present", worker_state="starting") + + acknowledged = _advance("status", started, observation, plan) + + assert acknowledged["phase"] == "STARTING" + assert acknowledged["next_action"] == "none" + assert acknowledged["revision"] == started["revision"] + 1 + + +def test_starting_becomes_ready_only_with_all_exact_workers(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + started = _advance("start", route.initial_state(plan), _observation(plan), plan) + observation = _observation(plan, resource_state="present", worker_state="ready") + + ready = _advance("status", started, observation, plan) + + assert ready["phase"] == "READY" + assert ready["next_action"] == "none" + + +@pytest.mark.parametrize( + ("field", "replacement"), + [ + ("manifest_digest", "sha256:" + "0" * 64), + ("worker_plan_digest", "sha256:" + "0" * 64), + ("index_digest", "sha256:" + "0" * 64), + ("source_commit", "0" * 40), + ("block_prefix", "layers"), + ("verifier_source_sha256", "sha256:" + "0" * 64), + ], +) +def test_observation_rejects_stale_production_plan_revalidation( + tmp_path: Path, + field: str, + replacement: object, +) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan) + observation["artifact_plan_revalidation"][field] = replacement + + with pytest.raises(route.RouteControllerError, match="plan revalidation"): + route.validate_observation(observation, plan) + + +def test_observation_rejects_future_plan_revalidation(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan) + observation["artifact_plan_revalidation"]["verified_at_unix"] = observation["observed_at_unix"] + 1 + + with pytest.raises(route.RouteControllerError, match="plan revalidation"): + route.validate_observation(observation, plan) + + +@pytest.mark.parametrize("field", ["kind", "provider", "region"]) +def test_observation_rejects_foreign_resource_identity(tmp_path: Path, field: str) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan) + first = next(iter(observation["resources"].values())) + first[field] = "foreign" + + with pytest.raises(route.RouteControllerError, match="resource identity"): + route.validate_observation(observation, plan) + + +def test_observation_rejects_worker_span_substitution_before_collection(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="ready") + worker = next(iter(observation["workers"].values())) + worker["span"] = "16:32" + + with pytest.raises(route.RouteControllerError, match="worker binding"): + route.validate_observation(observation, plan) + + +def test_observation_rejects_duplicate_ready_peer(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="ready") + values = list(observation["workers"].values()) + values[1]["peer_id"] = values[0]["peer_id"] + + with pytest.raises(route.RouteControllerError, match="unique peer"): + route.validate_observation(observation, plan) + + +def test_collect_action_is_durable_until_job_acknowledges(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + ready = _advance( + "start", + route.initial_state(plan), + _observation(plan, resource_state="present", worker_state="ready"), + plan, + ) + observation = _observation(plan, resource_state="present", worker_state="ready") + + collecting = _advance("collect", ready, observation, plan) + repeated = _advance("collect", collecting, observation, plan) + + assert collecting["phase"] == "COLLECTING" + assert collecting["next_action"] == "collect_route" + assert repeated == collecting + assert route.action_record(repeated, plan)["action_id"] == route.action_record(collecting, plan)["action_id"] + + +def test_running_job_acknowledges_collect_action(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + ready = _advance( + "start", + route.initial_state(plan), + _observation(plan, resource_state="present", worker_state="ready"), + plan, + ) + collecting = _advance( + "collect", + ready, + _observation(plan, resource_state="present", worker_state="ready"), + plan, + ) + + acknowledged = _advance( + "status", + collecting, + _observation(plan, resource_state="present", worker_state="ready", job_state="running"), + plan, + ) + + assert acknowledged["phase"] == "COLLECTING" + assert acknowledged["next_action"] == "none" + + +def test_passed_job_binds_evidence_then_requires_cleanup(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + ready = _advance( + "start", + route.initial_state(plan), + _observation(plan, resource_state="present", worker_state="ready"), + plan, + ) + collecting = _advance( + "collect", + ready, + _observation(plan, resource_state="present", worker_state="ready"), + plan, + ) + + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + cleaning = _advance("status", collecting, observation, plan) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["evidence_digest"] == observation["route_job"]["evidence_digest"] + assert cleaning["next_action"] == "cleanup_route" + + +@pytest.mark.parametrize("field", ["run_id", "plan_digest", "source_commit", "manifest_digest", "worker_plan_digest"]) +def test_route_job_rejects_wrong_binding(tmp_path: Path, field: str) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="ready", job_state="passed") + observation["route_job"][field] = "sha256:" + "0" * 64 if "digest" in field else "0" * 40 + + with pytest.raises(route.RouteControllerError, match="route job binding"): + route.validate_observation(observation, plan) + + +def test_failed_job_forces_cleanup(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + observation = _observation(plan, resource_state="present", worker_state="ready", job_state="failed") + state.update( + phase="COLLECTING", + revision=2, + instance_generations_digest=observation["instance_generations_digest"], + ) + + cleaning = _advance("status", state, observation, plan) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["failure_code"] == "qualification-failed" + + +def test_cleanup_retries_same_action_until_all_resources_absent(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update(phase="CLEANING", failure_code="route-failed", next_action="cleanup_route", revision=3) + observation = _observation(plan, resource_state="present", worker_state="failed") + + repeated = _advance("cleanup", state, observation, plan) + + assert repeated == state + assert route.action_record(repeated, plan)["action_id"] == route.action_record(state, plan)["action_id"] + + +def test_cleanup_becomes_passing_terminal_only_after_absence_proof(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update( + phase="CLEANING", + evidence_digest="sha256:" + "e" * 64, + next_action="cleanup_route", + revision=4, + ) + + terminal = _advance("cleanup", state, _observation(plan), plan) + + assert terminal["phase"] == "CLEANED_PASS" + assert terminal["cleanup_verified"] is True + assert terminal["next_action"] == "none" + + +def test_cleanup_without_evidence_is_terminal_failure(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update(phase="CLEANING", failure_code="route-failed", next_action="cleanup_route", revision=2) + + terminal = _advance("cleanup", state, _observation(plan), plan) + + assert terminal["phase"] == "CLEANED_FAILURE" + assert terminal["failure_code"] == "route-failed" + assert terminal["cleanup_verified"] is True + + +def test_deadline_forces_cleanup_then_terminal_failure(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + expired_present = _observation( + plan, + resource_state="present", + worker_state="starting", + observed_at=plan.deadline_unix, + ) + + cleaning = _advance("status", route.initial_state(plan), expired_present, plan) + terminal = _advance( + "cleanup", + cleaning, + _observation(plan, observed_at=plan.deadline_unix + 1), + plan, + ) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["failure_code"] == "run-expired" + assert terminal["phase"] == "CLEANED_FAILURE" + assert terminal["cleanup_verified"] is True + + +def test_terminal_state_retains_latched_generation_after_cleanup(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + present = _observation(plan, resource_state="present", worker_state="starting") + state = route.initial_state(plan) + state.update( + phase="CLEANED_FAILURE", + failure_code="route-failed", + cleanup_verified=True, + instance_generations_digest=present["instance_generations_digest"], + revision=5, + ) + + assert _advance("status", state, _observation(plan), plan) == state + + +def test_terminal_state_rejects_resource_reappearance(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update( + phase="CLEANED_FAILURE", + failure_code="route-failed", + cleanup_verified=True, + revision=5, + ) + + with pytest.raises(route.RouteControllerError, match="returned after terminal"): + _advance( + "status", + state, + _observation(plan, resource_state="present", worker_state="starting"), + plan, + ) + + +def test_cli_recovers_identical_pending_action_after_decision_write_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + plan = route.load_plan(plan_path, source_root) + observation_path = tmp_path / "observation.json" + observation = _observation(plan) + _write_json(observation_path, observation) + state_path = tmp_path / "state.json" + decision_path = tmp_path / "decision.json" + original = route._atomic_json + calls = 0 + + def fail_final_decision(path: Path, value: dict[str, object]) -> None: + nonlocal calls + calls += 1 + if calls == 4: + raise OSError("injected final decision write failure") + original(path, value) + + monkeypatch.setattr(route.time, "time", lambda: 1_900_000_000) + monkeypatch.setattr(route, "revalidate_authorization_evidence", lambda *args, **kwargs: None) + monkeypatch.setattr( + route, + "revalidate_production_artifact_plan", + lambda *args, **kwargs: observation["artifact_plan_revalidation"], + ) + monkeypatch.setattr(route, "_atomic_json", fail_final_decision) + argv = [ + "start", + "--plan", + os.fspath(plan_path), + "--source-root", + os.fspath(source_root), + "--observation", + os.fspath(observation_path), + "--manifest", + os.fspath(tmp_path / "manifest.json"), + "--artifact-root", + os.fspath(tmp_path / "artifacts"), + "--authorization-root", + os.fspath(tmp_path / "authorization"), + "--state", + os.fspath(state_path), + "--decision", + os.fspath(decision_path), + ] + assert route.main(argv) == 2 + persisted = json.loads(state_path.read_text(encoding="utf-8")) + tombstone = json.loads(decision_path.read_text(encoding="utf-8")) + assert persisted["next_action"] == "start_route" + assert tombstone["action"] == "none" + assert tombstone["action_id"] is None + + monkeypatch.setattr(route, "_atomic_json", original) + assert route.main(argv) == 0 + decision = json.loads(decision_path.read_text(encoding="utf-8")) + assert decision["action"] == "start_route" + assert decision["revision"] == persisted["revision"] + + +def test_cli_rejects_overlapping_input_output_paths(tmp_path: Path) -> None: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + plan = route.load_plan(plan_path, source_root) + observation_path = tmp_path / "observation.json" + _write_json(observation_path, _observation(plan)) + + result = route.main( + [ + "start", + "--plan", + os.fspath(plan_path), + "--source-root", + os.fspath(source_root), + "--observation", + os.fspath(observation_path), + "--state", + os.fspath(plan_path), + "--decision", + os.fspath(tmp_path / "decision.json"), + ] + ) + + assert result == 2 + + +def test_atomic_output_rejects_symlink_target(tmp_path: Path) -> None: + target = tmp_path / "target.json" + target.write_text("{}\n", encoding="utf-8") + link = tmp_path / "link.json" + try: + link.symlink_to(target) + except OSError: + pytest.skip("symlink creation is unavailable for this identity") + + with pytest.raises(route.RouteControllerError, match="output target is unsafe"): + route._atomic_json(link, {"ok": True}) + + +def test_authorized_plan_rejects_zero_estimate(tmp_path: Path) -> None: + value = _plan_value() + value["authorization"]["maximum_estimate_usd"] = "0.00" + + with pytest.raises(route.RouteControllerError, match="positive bounded estimate"): + _load_plan(tmp_path, value) + + +@pytest.mark.parametrize("field", ["reservation_record_sha256", "preflight_record_sha256"]) +def test_plan_rejects_unbound_authorization_evidence(tmp_path: Path, field: str) -> None: + value = _plan_value() + value["authorization"][field] = "sha256:" + "x" * 64 + + with pytest.raises(route.RouteControllerError, match=field): + _load_plan(tmp_path, value) + + +def test_start_rejects_stale_production_plan_revalidation(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan) + observation["artifact_plan_revalidation"]["verified_at_unix"] = ( + observation["observed_at_unix"] - route.MAX_PLAN_REVALIDATION_AGE_SECONDS - 1 + ) + + with pytest.raises(route.RouteControllerError, match="artifact plan is stale"): + _advance("start", route.initial_state(plan), observation, plan) + + +def test_cleanup_allows_stale_plan_revalidation_so_teardown_cannot_be_blocked(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update(phase="CLEANING", failure_code="route-failed", next_action="cleanup_route", revision=2) + observation = _observation(plan, resource_state="present", worker_state="failed") + observation["artifact_plan_revalidation"]["verified_at_unix"] = 1 + + repeated = _advance("cleanup", state, observation, plan) + + assert repeated == state + + +@pytest.mark.parametrize("field", ["source_commit", "plan_digest", "worker_plan_digest"]) +def test_observation_rejects_worker_execution_identity_substitution( + tmp_path: Path, + field: str, +) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="ready") + first = next(iter(observation["workers"].values())) + first[field] = "sha256:" + "0" * 64 if "digest" in field else "0" * 40 + + with pytest.raises(route.RouteControllerError, match="worker binding"): + route.validate_observation(observation, plan) + + +def test_observation_rejects_resource_plan_substitution(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, resource_state="present", worker_state="starting") + first = next(iter(observation["resources"].values())) + first["plan_digest"] = "sha256:" + "0" * 64 + + with pytest.raises(route.RouteControllerError, match="resource binding"): + route.validate_observation(observation, plan) + + +def test_state_rejects_evidence_before_verified_job_result(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state["evidence_digest"] = "sha256:" + "e" * 64 + + with pytest.raises(route.RouteControllerError, match="evidence state"): + route.validate_state(state, plan) + + +def _write_authorization_records( + tmp_path: Path, + plan_value: dict[str, object], + *, + checked_at: int = 1_899_999_999, +) -> Path: + root = tmp_path / "authorization" + authorization = plan_value["authorization"] + preview_source_root = _source_root(tmp_path) + preview_path = tmp_path / "authorization-plan-preview.json" + _write_json(preview_path, plan_value) + preview_plan = route.load_plan(preview_path, preview_source_root) + resource_specs = [route._expected_resource_spec(resource) for resource in preview_plan.resources] + resource_costs = [] + for resource, spec in zip(preview_plan.resources, resource_specs): + if resource.kind == "worker_instance": + unit_rate, maximum = "0.80", "8.80" + elif resource.kind == "bootstrap_instance": + unit_rate, maximum = "0.40", "4.40" + elif resource.kind in {"bootstrap_disk", "worker_disk"}: + unit_rate, maximum = "0.08", "0.88" + else: + unit_rate, maximum = "0.00", "0.00" + resource_costs.append( + { + "resource_name": resource.name, + "resource_spec_digest": route._canonical_digest(spec), + "unit_rate_usd": unit_rate, + "quantity": "1.00", + "duration_hours": "11.00", + "maximum_usd": maximum, + } + ) + reservation = { + "schema_version": route.SCHEMA_VERSION, + "reservation_id": authorization["reservation_id"], + "run_id": plan_value["run_id"], + "combined_cloud_ceiling_usd": authorization["combined_cloud_ceiling_usd"], + "ledger_committed_before_run_usd": authorization["ledger_committed_before_run_usd"], + "maximum_estimate_usd": authorization["maximum_estimate_usd"], + "deadline_unix": plan_value["deadline_unix"], + "plan_digest": preview_plan.plan_digest, + "execution_inventory_digest": preview_plan.execution_inventory_digest, + "worker_plan_digest": preview_plan.worker_plan_digest, + "resource_costs": resource_costs, + "readiness_ledger_sha256": authorization["readiness_ledger_sha256"], + "recorded_at_unix": checked_at - 10, + "expires_at_unix": plan_value["deadline_unix"], + "reservation_recorded": True, + } + reservation_path = root / authorization["reservation_record_path"] + _write_json(reservation_path, reservation) + reservation_payload = reservation_path.read_bytes() + authorization["reservation_record_byte_size"] = len(reservation_payload) + authorization["reservation_record_sha256"] = "sha256:" + hashlib.sha256(reservation_payload).hexdigest() + preflight = { + "schema_version": route.SCHEMA_VERSION, + "run_id": plan_value["run_id"], + "source_commit": plan_value["source_commit"], + "plan_digest": preview_plan.plan_digest, + "execution_inventory_digest": preview_plan.execution_inventory_digest, + "worker_plan_digest": preview_plan.worker_plan_digest, + "provider": "gcp", + "resource_names": [resource.name for resource in preview_plan.resources], + "resource_specs": resource_specs, + "pricing_source": "gcp-catalog", + "pricing_currency": "USD", + "pricing_checked_at_unix": checked_at, + "gpu_quota_limit": 4, + "gpu_quota_usage": 0, + "required_gpu_count": 4, + "checked_at_unix": checked_at, + "native_auth_revalidated": True, + "inventory_revalidated": True, + "pricing_revalidated": True, + "provisioning_authorized": True, + "protected_bootstrap_running": True, + "reservation_record_sha256": authorization["reservation_record_sha256"], + } + preflight_path = root / authorization["preflight_record_path"] + _write_json(preflight_path, preflight) + preflight_payload = preflight_path.read_bytes() + authorization["preflight_record_byte_size"] = len(preflight_payload) + authorization["preflight_record_sha256"] = "sha256:" + hashlib.sha256(preflight_payload).hexdigest() + return root + + +def test_authorization_revalidation_opens_exact_fresh_records(tmp_path: Path) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + plan = _load_plan(tmp_path, value) + + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_authorization_revalidation_rejects_record_mutation(tmp_path: Path) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + plan = _load_plan(tmp_path, value) + reservation_path = authorization_root / value["authorization"]["reservation_record_path"] + reservation_path.write_text("{}\n", encoding="utf-8") + + with pytest.raises(route.RouteControllerError, match="record size changed"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_authorization_revalidation_rejects_stale_preflight(tmp_path: Path) -> None: + value = _plan_value() + authorization_root = _write_authorization_records( + tmp_path, + value, + checked_at=1_899_999_000, + ) + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match="preflight record"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +@pytest.mark.parametrize( + ("path", "replacement"), + [ + (("route_span",), "0:63"), + (("cleanup_ready",), False), + (("worker_results", 0, "span"), "16:32"), + (("worker_results", 0, "peer_id"), "12D3KooWSubstitutedPeer000"), + ], +) +def test_passed_route_record_rejects_substitution( + tmp_path: Path, + path: tuple[object, ...], + replacement: object, +) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + target = observation["route_job"]["route_record"] + for part in path[:-1]: + target = target[part] + target[path[-1]] = replacement + observation["route_job"]["evidence_digest"] = route._canonical_digest(observation["route_job"]["route_record"]) + + with pytest.raises(route.RouteControllerError, match="route record"): + route.validate_observation(observation, plan) + + +def test_passed_route_record_digest_rejects_mutation(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + observation["route_job"]["route_record"]["session_id"] = "q38route-session-002" + + with pytest.raises(route.RouteControllerError, match="evidence digest"): + route.validate_observation(observation, plan) + + +def test_cleanup_cannot_be_blocked_by_stale_passed_job_or_bootstrap_loss(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update( + phase="CLEANING", + evidence_digest="sha256:" + "e" * 64, + next_action="cleanup_route", + revision=3, + ) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + observation["artifact_plan_revalidation"]["verified_at_unix"] = 1 + observation["protected_bootstrap_running"] = False + + cleaning = _advance("cleanup", state, observation, plan) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["failure_code"] == "protected-bootstrap-lost" + assert cleaning["next_action"] == "cleanup_route" + + +def test_cli_rejects_output_inside_artifact_root(tmp_path: Path) -> None: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + plan = route.load_plan(plan_path, source_root) + observation_path = tmp_path / "observation.json" + _write_json(observation_path, _observation(plan)) + artifact_root = tmp_path / "artifacts" + artifact_root.mkdir() + + result = route.main( + [ + "start", + "--plan", + os.fspath(plan_path), + "--source-root", + os.fspath(source_root), + "--observation", + os.fspath(observation_path), + "--manifest", + os.fspath(tmp_path / "manifest.json"), + "--artifact-root", + os.fspath(artifact_root), + "--state", + os.fspath(artifact_root / "state.json"), + "--decision", + os.fspath(tmp_path / "decision.json"), + ] + ) + + assert result == 2 + + +def test_production_revalidation_uses_source_bound_module_and_exact_metadata( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from drift.model_manifest import ModelManifest, select_manifest_block_artifacts + + revision = "1" * 40 + artifact_root = tmp_path / "metadata" + artifact_root.mkdir() + config_payload = b"{}" + weight_map = {} + shard_names = [f"span-{index}.safetensors" for index in range(4)] + for block in range(64): + weight_map[f"model.language_model.layers.{block}.weight"] = shard_names[block // 16] + index_payload = json.dumps({"weight_map": weight_map}, sort_keys=True).encode() + (artifact_root / "config.json").write_bytes(config_payload) + (artifact_root / "model.safetensors.index.json").write_bytes(index_payload) + artifacts = [ + { + "role": "config", + "path": "config.json", + "sha256": hashlib.sha256(config_payload).hexdigest(), + "size": len(config_payload), + }, + { + "role": "weight_index", + "path": "model.safetensors.index.json", + "sha256": hashlib.sha256(index_payload).hexdigest(), + "size": len(index_payload), + }, + { + "role": "tokenizer", + "path": "tokenizer.json", + "sha256": hashlib.sha256(b"tokenizer").hexdigest(), + "size": len(b"tokenizer"), + }, + ] + for name in shard_names: + artifacts.append( + { + "role": "weight", + "path": name, + "sha256": hashlib.sha256(name.encode()).hexdigest(), + "size": 1, + } + ) + manifest_value = { + "schema_version": 1, + "name": "Synthetic Qwen route", + "aliases": [], + "source": {"repository": "example/synthetic", "revision": revision}, + "model": { + "architecture": "SyntheticForCausalLM", + "num_blocks": 64, + "context_length": 1024, + "license": "apache-2.0", + "gated": False, + }, + "runtime": { + "implementation": "drift", + "minimum_version": "2.3.0.dev0", + "maximum_version_exclusive": "2.4.0", + "protocol_version": 1, + "tensor_schema": "hidden-states-v1", + "attention_implementation": "eager", + "dtype": "bfloat16", + "quantization": "none", + "adapter_profile": "none", + }, + "artifacts": artifacts, + } + manifest = ModelManifest.from_dict(manifest_value) + manifest_path = tmp_path / "manifest.json" + _write_json(manifest_path, manifest_value) + expected_spans = {} + for index, span in enumerate(("0:16", "16:32", "32:48", "48:64")): + start, end = map(int, span.split(":")) + derived = select_manifest_block_artifacts( + manifest, + block_prefix="model.language_model.layers", + start_block=start, + end_block=end, + weight_map=weight_map, + ) + expected_spans[span] = ( + derived.artifact_bytes, + "sha256:" + derived.artifact_set_digest, + ) + assert derived.artifact_paths[-1] == shard_names[index] + + monkeypatch.setattr(route, "EXPECTED_MANIFEST_DIGEST", manifest.digest_id) + monkeypatch.setattr(route, "EXPECTED_MODEL_REVISION", revision) + monkeypatch.setattr( + route, + "EXPECTED_INDEX_DIGEST", + "sha256:" + hashlib.sha256(index_payload).hexdigest(), + ) + monkeypatch.setattr(route, "EXPECTED_SPANS", expected_spans) + monkeypatch.setattr(route, "EXPECTED_ARTIFACTS_PER_SPAN", 3) + source_root = Path(route.__file__).resolve().parents[1] + value = _plan_value() + source_bindings = [] + for relative_path in sorted(route.REQUIRED_SOURCE_PATHS): + payload = (source_root / relative_path).read_bytes() + source_bindings.append( + { + "relative_path": relative_path, + "sha256": "sha256:" + hashlib.sha256(payload).hexdigest(), + "byte_size": len(payload), + } + ) + value["source_bindings"] = source_bindings + value["runtime_package"]["source_bindings_digest"] = route._source_bindings_digest(source_bindings) + value["runtime_package"]["runtime_package_digest"] = route._runtime_package_digest(value["runtime_package"]) + value["authorization"]["readiness_ledger_sha256"] = next( + binding["sha256"] for binding in source_bindings if binding["relative_path"] == route.READINESS_LEDGER_PATH + ) + plan_path = tmp_path / "synthetic-plan.json" + _write_json(plan_path, value) + plan = route.load_plan(plan_path, source_root) + + record = route.revalidate_production_artifact_plan( + plan, + manifest_path, + artifact_root, + source_root, + verified_at_unix=1_900_000_000, + ) + + assert record["worker_plan_digest"] == plan.worker_plan_digest + assert record["verifier_source_sha256"] == next( + binding["sha256"] for binding in source_bindings if binding["relative_path"] == route.VERIFIER_SOURCE_PATH + ) + + (artifact_root / "model.safetensors.index.json").write_bytes(b"tampered") + with pytest.raises(route.RouteControllerError, match="could not be revalidated"): + route.revalidate_production_artifact_plan( + plan, + manifest_path, + artifact_root, + source_root, + verified_at_unix=1_900_000_000, + ) + + +def _rebind_authorization_record( + plan_value: dict[str, object], + authorization_root: Path, + record_name: str, +) -> None: + authorization = plan_value["authorization"] + record_path = authorization_root / authorization[f"{record_name}_record_path"] + payload = record_path.read_bytes() + authorization[f"{record_name}_record_byte_size"] = len(payload) + authorization[f"{record_name}_record_sha256"] = "sha256:" + hashlib.sha256(payload).hexdigest() + + +def test_trusted_time_rejects_stale_observation(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation(plan, observed_at=1_900_000_000) + + with pytest.raises(route.RouteControllerError, match="stale"): + route.reconcile( + "status", + route.initial_state(plan), + observation, + plan, + now_unix=1_900_000_000 + route.MAX_PLAN_REVALIDATION_AGE_SECONDS + 1, + ) + + +def test_trusted_time_forces_cleanup_even_when_observation_predates_deadline(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="starting", + observed_at=1_900_000_000, + ) + + cleaning = route.reconcile( + "status", + route.initial_state(plan), + observation, + plan, + now_unix=plan.deadline_unix, + ) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["failure_code"] == "run-expired" + assert cleaning["next_action"] == "cleanup_route" + + +def test_state_loss_recovers_exact_passed_evidence_before_cleanup(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + + cleaning = route.reconcile( + "status", + route.initial_state(plan), + observation, + plan, + now_unix=observation["observed_at_unix"], + route_evidence_validated=True, + start_was_issued=True, + ) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["failure_code"] is None + assert cleaning["evidence_digest"] == observation["route_job"]["evidence_digest"] + assert cleaning["next_action"] == "cleanup_route" + + +def test_plan_rejects_readiness_ledger_binding_mismatch(tmp_path: Path) -> None: + value = _plan_value() + value["authorization"]["readiness_ledger_sha256"] = "sha256:" + "0" * 64 + + with pytest.raises(route.RouteControllerError, match="bound to the readiness ledger"): + _load_plan(tmp_path, value) + + +def test_stable_plan_digest_excludes_only_record_self_bindings() -> None: + value = _plan_value() + baseline = route._stable_plan_digest(value) + rebound = copy.deepcopy(value) + rebound["authorization"]["reservation_record_sha256"] = "sha256:" + "0" * 64 + rebound["authorization"]["reservation_record_byte_size"] = 999 + rebound["authorization"]["preflight_record_sha256"] = "sha256:" + "1" * 64 + rebound["authorization"]["preflight_record_byte_size"] = 1_000 + assert route._stable_plan_digest(rebound) == baseline + + rebound["source_bindings"][0]["sha256"] = "sha256:" + "2" * 64 + assert route._stable_plan_digest(rebound) != baseline + + +@pytest.mark.parametrize("field", ["source_bindings", "plan_digest"]) +def test_authorization_revalidation_binds_exact_source_and_plan_identity( + tmp_path: Path, + field: str, +) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + plan = _load_plan(tmp_path, value) + if field == "source_bindings": + bindings = [dict(binding) for binding in plan.source_bindings] + target = next(binding for binding in bindings if binding["relative_path"] == "src/drift/server/server.py") + target["sha256"] = "sha256:" + "0" * 64 + substituted = replace(plan, source_bindings=tuple(bindings)) + else: + substituted = replace(plan, plan_digest="sha256:" + "0" * 64) + + assert substituted.execution_inventory_digest != plan.execution_inventory_digest + with pytest.raises(route.RouteControllerError, match="reservation record"): + route.revalidate_authorization_evidence( + substituted, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_authorization_revalidation_rejects_insufficient_gpu_quota(tmp_path: Path) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + preflight_path = authorization_root / value["authorization"]["preflight_record_path"] + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + preflight["gpu_quota_limit"] = 1 + _write_json(preflight_path, preflight) + _rebind_authorization_record(value, authorization_root, "preflight") + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match="provider preflight record"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ( + lambda record: record["resource_costs"][0].update(maximum_usd="43.99"), + "resource cost was not recomputed", + ), + ( + lambda record: record["resource_costs"][0].update( + resource_name=record["resource_costs"][1]["resource_name"] + ), + "reservation cost inventory is not exact", + ), + ], +) +def test_authorization_revalidation_rejects_cost_substitution( + tmp_path: Path, + mutation, + message: str, +) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + reservation_path = authorization_root / value["authorization"]["reservation_record_path"] + reservation = json.loads(reservation_path.read_text(encoding="utf-8")) + mutation(reservation) + _write_json(reservation_path, reservation) + _rebind_authorization_record(value, authorization_root, "reservation") + + preflight_path = authorization_root / value["authorization"]["preflight_record_path"] + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + preflight["reservation_record_sha256"] = value["authorization"]["reservation_record_sha256"] + _write_json(preflight_path, preflight) + _rebind_authorization_record(value, authorization_root, "preflight") + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match=message): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_authorization_rejects_pricing_shorter_or_longer_than_resource_lifetime( + tmp_path: Path, +) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + reservation_path = authorization_root / value["authorization"]["reservation_record_path"] + reservation = json.loads(reservation_path.read_text(encoding="utf-8")) + cost = next(item for item in reservation["resource_costs"] if item["resource_name"] == f"{RUN_ID}-worker-0") + cost["duration_hours"] = "24.00" + cost["maximum_usd"] = "19.20" + _write_json(reservation_path, reservation) + _rebind_authorization_record(value, authorization_root, "reservation") + + preflight_path = authorization_root / value["authorization"]["preflight_record_path"] + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + preflight["reservation_record_sha256"] = value["authorization"]["reservation_record_sha256"] + _write_json(preflight_path, preflight) + _rebind_authorization_record(value, authorization_root, "preflight") + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match="pricing horizon"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_controller_state_lock_is_exclusive_and_reusable(tmp_path: Path) -> None: + lock_path = tmp_path / ".route-state.json.lock" + + with route._controller_lock(lock_path): + with pytest.raises(route.RouteControllerError, match="another controller invocation"): + with route._controller_lock(lock_path): + pytest.fail("contended invocation acquired the state lock") + + with route._controller_lock(lock_path): + assert lock_path.is_file() + + +def test_terminal_pass_alerts_if_protected_bootstrap_is_lost(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + state = route.initial_state(plan) + state.update( + phase="CLEANED_PASS", + evidence_digest="sha256:" + "e" * 64, + cleanup_verified=True, + revision=5, + ) + observation = _observation(plan) + observation["protected_bootstrap_running"] = False + + with pytest.raises(route.RouteControllerError, match="protected bootstrap was lost"): + route.reconcile( + "status", + state, + observation, + plan, + now_unix=observation["observed_at_unix"], + ) + + +def _write_protected_route_evidence( + tmp_path: Path, + plan: route.RoutePlan, + observation: dict[str, object], +) -> Path: + root = tmp_path / "route-evidence" + root.mkdir() + route_record = observation["route_job"]["route_record"] + rpc = { + "schema_version": route.SCHEMA_VERSION, + "result": "passed", + "run_id": plan.run_id, + "job_id": plan.route_job_id, + "collect_action_id": route._action_id(plan, "collect_route"), + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "worker_plan_digest": plan.worker_plan_digest, + "route_span": "0:64", + "session_id": route_record["session_id"], + } + rpc_path = root / "route-rpc.json" + _write_json(rpc_path, rpc) + route_record["route_rpc_evidence_digest"] = "sha256:" + hashlib.sha256(rpc_path.read_bytes()).hexdigest() + for worker_plan, result in zip(plan.workers, route_record["worker_results"]): + evidence = { + "schema_version": route.SCHEMA_VERSION, + "result": "passed", + "run_id": plan.run_id, + "job_id": plan.route_job_id, + "collect_action_id": route._action_id(plan, "collect_route"), + "plan_digest": plan.plan_digest, + "source_commit": plan.source_commit, + "manifest_digest": plan.manifest_digest, + "worker_plan_digest": plan.worker_plan_digest, + "start_action_id": route._action_id(plan, "start_route"), + "worker_id": worker_plan.worker_id, + "machine_id": worker_plan.machine_id, + "peer_id": result["peer_id"], + "span": worker_plan.span, + "artifact_bytes": worker_plan.artifact_bytes, + "artifact_set_digest": worker_plan.artifact_set_digest, + "cache_root": worker_plan.cache_root, + } + evidence_path = root / f"{worker_plan.worker_id}-evidence.json" + _write_json(evidence_path, evidence) + result["worker_evidence_digest"] = "sha256:" + hashlib.sha256(evidence_path.read_bytes()).hexdigest() + observation["route_job"]["evidence_digest"] = route._canonical_digest(route_record) + _write_json(root / "route-terminal.json", route_record) + return root + + +def test_protected_route_evidence_revalidates_exact_terminal_and_children( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + evidence_root = _write_protected_route_evidence(tmp_path, plan, observation) + validated = route.validate_observation(observation, plan) + + digest = route.revalidate_route_evidence( + plan, + validated, + evidence_root, + tmp_path / "source", + ) + + assert digest == observation["route_job"]["evidence_digest"] + + +def test_protected_route_evidence_rejects_child_mutation(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + evidence_root = _write_protected_route_evidence(tmp_path, plan, observation) + validated = route.validate_observation(observation, plan) + (evidence_root / "worker-0-evidence.json").write_text("{}\n", encoding="utf-8") + + with pytest.raises(route.RouteControllerError, match="worker evidence digest changed"): + route.revalidate_route_evidence( + plan, + validated, + evidence_root, + tmp_path / "source", + ) + + +def test_protected_route_evidence_rejects_extra_file(tmp_path: Path) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + evidence_root = _write_protected_route_evidence(tmp_path, plan, observation) + validated = route.validate_observation(observation, plan) + (evidence_root / "extra.json").write_text("{}\n", encoding="utf-8") + + with pytest.raises(route.RouteControllerError, match="inventory is not exact"): + route.revalidate_route_evidence( + plan, + validated, + evidence_root, + tmp_path / "source", + ) + + +def test_fabricated_embedded_pass_cannot_advance_without_protected_revalidation( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="ready", + job_state="passed", + ) + + with pytest.raises(route.RouteControllerError, match="protected records"): + route.reconcile( + "status", + route.initial_state(plan), + observation, + plan, + now_unix=observation["observed_at_unix"], + start_was_issued=True, + ) + + +def test_authorization_rejects_resource_spec_substitution(tmp_path: Path) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + preflight_path = authorization_root / value["authorization"]["preflight_record_path"] + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + spec = next(item for item in preflight["resource_specs"] if item["kind"] == "worker_instance") + spec["machine_type"] = "substituted-machine" + _write_json(preflight_path, preflight) + _rebind_authorization_record(value, authorization_root, "preflight") + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match="exact launch profile"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +@pytest.mark.parametrize( + ("kind", "field", "replacement"), + [ + ("worker_instance", "accelerator_count", 0), + ("bootstrap_instance", "accelerator_type", "nvidia-l4"), + ("worker_disk", "source_image", "communityai-q38-v1"), + ("firewall", "machine_type", "g2-standard-8"), + ("worker_instance", "zone", "europe-west1-b"), + ("worker_instance", "max_lifetime_seconds", 100_000_002), + ], +) +def test_authorization_rejects_invalid_resource_spec_semantics( + tmp_path: Path, + kind: str, + field: str, + replacement: object, +) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + preflight_path = authorization_root / value["authorization"]["preflight_record_path"] + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + spec = next(item for item in preflight["resource_specs"] if item["kind"] == kind) + spec[field] = replacement + + reservation_path = authorization_root / value["authorization"]["reservation_record_path"] + reservation = json.loads(reservation_path.read_text(encoding="utf-8")) + cost = next(item for item in reservation["resource_costs"] if item["resource_name"] == spec["resource_name"]) + cost["resource_spec_digest"] = route._canonical_digest(spec) + _write_json(reservation_path, reservation) + _rebind_authorization_record(value, authorization_root, "reservation") + + preflight["reservation_record_sha256"] = value["authorization"]["reservation_record_sha256"] + _write_json(preflight_path, preflight) + _rebind_authorization_record(value, authorization_root, "preflight") + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match="exact launch profile"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_authorization_rejects_unprotected_records( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + value = _plan_value() + authorization_root = _write_authorization_records(tmp_path, value) + plan = _load_plan(tmp_path, value) + monkeypatch.setattr(route, "_assert_protected_path", REAL_ASSERT_PROTECTED_PATH) + + with pytest.raises(route.RouteControllerError, match="protection verifier|not protected"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + tmp_path / "source", + now_unix=1_900_000_000, + ) + + +def test_issuance_journal_prevents_paid_start_reissue_after_state_loss( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + plan = route.load_plan(plan_path, source_root) + observation_path = tmp_path / "observation.json" + observation = _observation(plan) + _write_json(observation_path, observation) + state_path = tmp_path / "state.json" + decision_path = tmp_path / "decision.json" + monkeypatch.setattr(route.time, "time", lambda: 1_900_000_000) + authorization_revalidations = 0 + + def revalidate_once(*args, **kwargs) -> None: + nonlocal authorization_revalidations + authorization_revalidations += 1 + if authorization_revalidations > 1: + raise AssertionError("recovery revalidated expired paid-start authorization") + + monkeypatch.setattr(route, "revalidate_authorization_evidence", revalidate_once) + monkeypatch.setattr( + route, + "revalidate_production_artifact_plan", + lambda *args, **kwargs: observation["artifact_plan_revalidation"], + ) + argv = [ + "start", + "--plan", + os.fspath(plan_path), + "--source-root", + os.fspath(source_root), + "--observation", + os.fspath(observation_path), + "--manifest", + os.fspath(tmp_path / "manifest.json"), + "--artifact-root", + os.fspath(tmp_path / "artifacts"), + "--authorization-root", + os.fspath(tmp_path / "authorization"), + "--state", + os.fspath(state_path), + "--decision", + os.fspath(decision_path), + ] + + assert route.main(argv) == 0 + first = json.loads(decision_path.read_text(encoding="utf-8")) + journal_path = state_path.with_name(f".{state_path.name}.issuance.json") + issued = json.loads(journal_path.read_text(encoding="utf-8")) + assert first["action"] == "start_route" + assert issued["status"] == "issued" + + state_path.unlink() + decision_path.unlink() + assert route.main(argv) == 0 + + recovered = json.loads(decision_path.read_text(encoding="utf-8")) + completed = json.loads(journal_path.read_text(encoding="utf-8")) + assert recovered["action"] == "none" + assert completed["status"] == "completed" + assert completed["terminal_phase"] == "CLEANED_FAILURE" + assert completed["failure_code"] == "state-lost-after-start" + assert authorization_revalidations == 1 + + +def test_one_cent_paid_authorization_cannot_bypass_readiness_ledger( + tmp_path: Path, +) -> None: + value = _plan_value() + value["authorization"]["maximum_estimate_usd"] = "0.01" + authorization_root = _write_authorization_records(tmp_path, value) + plan = _load_plan(tmp_path, value) + + with pytest.raises(route.RouteControllerError, match="exact reservation"): + route.revalidate_authorization_evidence( + plan, + authorization_root, + now_unix=1_900_000_000, + ) + + +def test_completed_journal_recovers_pass_after_state_loss( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = _source_root(tmp_path) + plan_path = tmp_path / "plan.json" + _write_json(plan_path, _plan_value()) + plan = route.load_plan(plan_path, source_root) + observation_path = tmp_path / "observation.json" + _write_json(observation_path, _observation(plan)) + state_path = tmp_path / "state.json" + decision_path = tmp_path / "decision.json" + journal_path = state_path.with_name(f".{state_path.name}.issuance.json") + issued = route._issued_journal(plan, 1_899_999_990) + terminal = route.initial_state(plan) + terminal.update( + revision=7, + phase="CLEANED_PASS", + evidence_digest="sha256:" + "e" * 64, + cleanup_verified=True, + ) + _write_json( + journal_path, + route._completed_journal(issued, terminal, 1_899_999_999), + ) + monkeypatch.setattr(route.time, "time", lambda: 1_900_000_000) + argv = [ + "status", + "--plan", + os.fspath(plan_path), + "--source-root", + os.fspath(source_root), + "--observation", + os.fspath(observation_path), + "--state", + os.fspath(state_path), + "--decision", + os.fspath(decision_path), + ] + + assert route.main(argv) == 0 + + recovered = json.loads(state_path.read_text(encoding="utf-8")) + decision = json.loads(decision_path.read_text(encoding="utf-8")) + assert recovered["phase"] == "CLEANED_PASS" + assert recovered["evidence_digest"] == "sha256:" + "e" * 64 + assert recovered["cleanup_verified"] is True + assert decision["action"] == "none" + + +def test_complete_instance_generations_are_latched_and_recreation_forces_cleanup( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path) + starting = _advance( + "start", + route.initial_state(plan), + _observation(plan), + plan, + ) + observation = _observation( + plan, + resource_state="present", + worker_state="starting", + ) + + latched = _advance("status", starting, observation, plan) + + assert latched["phase"] == "STARTING" + assert latched["instance_generations_digest"] == observation["instance_generations_digest"] + assert ( + route.action_record(latched, plan)["instance_generations_digest"] == observation["instance_generations_digest"] + ) + + recreated = copy.deepcopy(observation) + instance = next(item for item in plan.resources if item.kind.endswith("instance")) + resource = recreated["resources"][instance.name] + resource["instance_id"] = str(int(resource["instance_id"]) + 1) + resource["instance_generation_digest"] = route.instance_generation_digest( + instance.name, + resource["instance_id"], + resource["creation_timestamp"], + ) + recreated["instance_generations_digest"] = route.observation_instance_generations_digest( + recreated["resources"], + plan, + ) + + cleaning = _advance("status", latched, recreated, plan) + + assert cleaning["phase"] == "CLEANING" + assert cleaning["failure_code"] == "instance-generation-changed" + assert cleaning["instance_generations_digest"] == latched["instance_generations_digest"] + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("instance_id", None), + ("instance_id", True), + ("instance_id", "0"), + ("instance_id", "not-numeric"), + ("instance_id", "18446744073709551616"), + ("creation_timestamp", None), + ("creation_timestamp", True), + ("creation_timestamp", "2026-09-03"), + ("creation_timestamp", "2026-09-03T01:20:00Z"), + ("creation_timestamp", "2026-13-03T01:20:00+00:00"), + ("creation_timestamp", "2026-09-03T01:20:00+24:00"), + ], +) +def test_instance_generation_fields_fail_closed( + tmp_path: Path, + field: str, + value: object, +) -> None: + plan = _load_plan(tmp_path) + observation = _observation( + plan, + resource_state="present", + worker_state="starting", + ) + instance = next(item for item in plan.resources if item.kind.endswith("instance")) + observation["resources"][instance.name][field] = value + + with pytest.raises(route.RouteControllerError, match="instance generation|provider instance"): + route.validate_observation(observation, plan) + + +def _instance_key_identity(plan: route.RoutePlan) -> tuple[route.ResourcePlan, str, str]: + resource = next(item for item in plan.resources if item.kind == "bootstrap_instance") + return resource, "1234567890123456789", "2026-09-03T17:00:00+00:00" + + +def test_instance_generation_key_is_private_digest_bound_and_idempotent( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + calls: list[int] = [] + + material = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: calls.append(size) or b"a" * size, + ) + reattached = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_001, + key_factory=lambda _size: pytest.fail("idempotent reattachment generated a key"), + ) + + assert calls == [route.INSTANCE_KEY_BYTES] + assert material.key == b"a" * route.INSTANCE_KEY_BYTES + assert reattached.key == material.key + assert dict(reattached.record) == dict(material.record) + assert material.record["key_epoch"] == 1 + assert material.record["key_sha256"] == "sha256:" + hashlib.sha256(material.key).hexdigest() + assert material.record["instance_generation_digest"] == route.instance_generation_digest( + resource.name, + instance_id, + created, + ) + assert set(material.record) == route._INSTANCE_KEY_RECORD_FIELDS + serialized = json.dumps(dict(material.record), sort_keys=True) + assert material.key.hex() not in serialized + assert repr(material.key) not in repr(material) + assert len(list(vault.rglob("*.key"))) == 1 + + +def test_instance_generation_key_creation_recovers_before_activation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + + with monkeypatch.context() as patch: + patch.setattr( + route, + "_replace_instance_key_json", + lambda _path, _value: (_ for _ in ()).throw( + route.RouteControllerError("simulated activation interruption") + ), + ) + with pytest.raises(route.RouteControllerError, match="activation interruption"): + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + + assert len(list(vault.rglob("*.key"))) == 1 + assert len(list(vault.rglob("record-*.json"))) == 1 + assert not list(vault.rglob("active.json")) + recovered = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_001, + key_factory=lambda size: b"b" * size, + ) + + assert recovered.record["key_epoch"] == 1 + assert recovered.key == b"b" * route.INSTANCE_KEY_BYTES + assert [path.read_bytes() for path in vault.rglob("*.key")] == [recovered.key] + assert len(list(vault.rglob("record-*.json"))) == 1 + assert len(list(vault.rglob("active.json"))) == 1 + + +def test_instance_generation_key_rejects_recreated_resource_without_reuse( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + first = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + + with pytest.raises(route.RouteControllerError, match="generation changed"): + route.ensure_instance_generation_key( + plan, + resource.name, + str(int(instance_id) + 1), + created, + vault, + now_unix=1_900_000_001, + key_factory=lambda _size: pytest.fail("recreated resource reused a key"), + ) + + loaded = route.load_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_001, + ) + assert loaded.key == first.key + + +def test_instance_generation_key_rotation_switches_epoch_and_removes_old_bytes( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + first = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + rotated = route.rotate_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_010, + expected_record_digest=first.record["record_digest"], + key_factory=lambda size: b"b" * size, + ) + + assert first.record["key_epoch"] == 1 + assert rotated.record["key_epoch"] == 2 + assert rotated.record["previous_record_digest"] == first.record["record_digest"] + assert rotated.key == b"b" * route.INSTANCE_KEY_BYTES + key_files = list(vault.rglob("*.key")) + assert len(key_files) == 1 + assert key_files[0].read_bytes() == rotated.key + loaded = route.load_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_011, + ) + assert loaded.key == rotated.key + assert loaded.record["record_digest"] == rotated.record["record_digest"] + + retried = route.rotate_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_012, + expected_record_digest=first.record["record_digest"], + key_factory=lambda _size: pytest.fail("rotation retry advanced the epoch"), + ) + assert retried.key == rotated.key + assert dict(retried.record) == dict(rotated.record) + + +def test_instance_generation_key_rotation_recovers_before_activation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + first = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + real_replace = route._replace_instance_key_json + + with monkeypatch.context() as patch: + patch.setattr( + route, + "_replace_instance_key_json", + lambda _path, _value: (_ for _ in ()).throw( + route.RouteControllerError("simulated activation interruption") + ), + ) + with pytest.raises(route.RouteControllerError, match="activation interruption"): + route.rotate_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_010, + expected_record_digest=first.record["record_digest"], + key_factory=lambda size: b"b" * size, + ) + + assert len(list(vault.rglob("*.key"))) == 2 + monkeypatch.setattr(route, "_replace_instance_key_json", real_replace) + recovered = route.rotate_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_011, + expected_record_digest=first.record["record_digest"], + key_factory=lambda size: b"c" * size, + ) + + assert recovered.record["key_epoch"] == 2 + assert recovered.key == b"c" * route.INSTANCE_KEY_BYTES + assert [path.read_bytes() for path in vault.rglob("*.key")] == [recovered.key] + + +def test_instance_generation_key_rotation_recovers_after_activation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + first = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + old_key_path = next(vault.rglob("*.key")) + real_unlink = route._unlink_instance_key_file + + def interrupt_old_key_cleanup(path: Path) -> None: + if path == old_key_path: + raise route.RouteControllerError("simulated old-key cleanup interruption") + real_unlink(path) + + with monkeypatch.context() as patch: + patch.setattr(route, "_unlink_instance_key_file", interrupt_old_key_cleanup) + with pytest.raises(route.RouteControllerError, match="cleanup interruption"): + route.rotate_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_010, + expected_record_digest=first.record["record_digest"], + key_factory=lambda size: b"b" * size, + ) + + assert len(list(vault.rglob("*.key"))) == 2 + recovered = route.rotate_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_011, + expected_record_digest=first.record["record_digest"], + key_factory=lambda _size: pytest.fail("rotation retry generated another key"), + ) + + assert recovered.record["key_epoch"] == 2 + assert recovered.key == b"b" * route.INSTANCE_KEY_BYTES + assert [path.read_bytes() for path in vault.rglob("*.key")] == [recovered.key] + + +def test_instance_generation_key_revocation_is_idempotent_and_missing_key_safe( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + material = route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + key_file = next(vault.rglob("*.key")) + key_file.unlink() + + tombstone = route.cleanup_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_020, + ) + repeated = route.cleanup_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_021, + ) + + assert tombstone == repeated + assert tombstone["last_key_epoch"] == material.record["key_epoch"] + assert tombstone["last_record_digest"] == material.record["record_digest"] + assert not list(vault.rglob("*.key")) + assert not list(vault.rglob("active.json")) + assert len(list(vault.rglob("revoked-*.json"))) == 1 + with pytest.raises(route.RouteControllerError, match="revoked"): + route.load_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_022, + ) + with pytest.raises(route.RouteControllerError, match="revoked"): + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_022, + key_factory=lambda _size: pytest.fail("revoked generation produced a key"), + ) + + +def test_revoked_instance_generation_does_not_block_a_fresh_provider_generation( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + route.revoke_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_010, + ) + + replacement = route.ensure_instance_generation_key( + plan, + resource.name, + str(int(instance_id) + 1), + "2026-09-03T17:01:00+00:00", + vault, + now_unix=1_900_000_020, + key_factory=lambda size: b"b" * size, + ) + + assert replacement.key == b"b" * route.INSTANCE_KEY_BYTES + assert replacement.record["instance_generation_digest"] != route.instance_generation_digest( + resource.name, + instance_id, + created, + ) + + +def test_instance_generation_key_rejects_tampered_record_and_key( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + record_path = next(vault.rglob("record-*.json")) + original = json.loads(record_path.read_text(encoding="utf-8")) + original["key_epoch"] = 2 + record_path.write_text(json.dumps(original, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(route.RouteControllerError): + route.load_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_001, + ) + + +@pytest.mark.parametrize("bad_key", [b"", b"x" * 31, b"x" * 33, "x" * 32]) +def test_instance_generation_key_rejects_invalid_generator_material( + tmp_path: Path, + bad_key: object, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + + with pytest.raises(route.RouteControllerError, match="invalid material"): + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda _size: bad_key, + ) + assert not list(vault.rglob("active.json")) + assert not list(vault.rglob("*.key")) + + +def test_instance_generation_key_rejects_non_instance_and_expired_plan( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + disk = next(item for item in plan.resources if item.kind == "bootstrap_disk") + resource, instance_id, created = _instance_key_identity(plan) + + with pytest.raises(route.RouteControllerError, match="not planned"): + route.ensure_instance_generation_key( + plan, + disk.name, + instance_id, + created, + tmp_path / "vault-a", + now_unix=1_900_000_000, + ) + with pytest.raises(route.RouteControllerError, match="outside the route deadline"): + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + tmp_path / "vault-b", + now_unix=plan.deadline_unix, + ) + + +@pytest.mark.skipif(os.name != "nt", reason="Windows DACL contract") +def test_instance_generation_key_vault_rejects_inheritable_windows_acl( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + resource_root = vault / plan.run_id / resource.name + executable = route.shutil.which("powershell.exe") or route.shutil.which("powershell") + assert executable is not None + completed = route.subprocess.run( + [ + executable, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-Command", + ( + "& { param([string]$path); " + "$acl=[System.IO.Directory]::GetAccessControl($path); " + "$acl.SetAccessRuleProtection($false,$true); " + "[System.IO.Directory]::SetAccessControl($path,$acl) }" + ), + os.fspath(resource_root), + ], + check=False, + shell=False, + stdin=route.subprocess.DEVNULL, + stdout=route.subprocess.DEVNULL, + stderr=route.subprocess.DEVNULL, + timeout=30, + ) + assert completed.returncode == 0 + + with pytest.raises( + route.RouteControllerError, + match="DACL is not protected|ACL is not private", + ): + route.load_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_001, + ) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX permission contract") +def test_instance_generation_key_vault_is_owner_private_on_posix( + tmp_path: Path, +) -> None: + plan = _load_plan(tmp_path / "plan") + resource, instance_id, created = _instance_key_identity(plan) + vault = tmp_path / "vault" + route.ensure_instance_generation_key( + plan, + resource.name, + instance_id, + created, + vault, + now_unix=1_900_000_000, + key_factory=lambda size: b"a" * size, + ) + + for directory in (vault, vault / plan.run_id, vault / plan.run_id / resource.name): + assert directory.stat().st_mode & 0o777 == 0o700 + for file in (vault / plan.run_id / resource.name).iterdir(): + assert file.stat().st_mode & 0o777 == 0o600 diff --git a/tests/test_gateq38_stage_package.py b/tests/test_gateq38_stage_package.py new file mode 100644 index 000000000..1a21263e4 --- /dev/null +++ b/tests/test_gateq38_stage_package.py @@ -0,0 +1,741 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from desktop import build_desktop +from scripts import gateq38_route_controller as controller, gateq38_stage_package as stage + +SOURCE_COMMIT = "a" * 40 +SOURCE_TREE = "b" * 40 +SOURCE_ROOT = Path(__file__).resolve().parents[1] +MANIFEST = SOURCE_ROOT / "manifests" / "candidates" / "qwen3.8-27b-fp8-dequant-eager.json" + + +def _runtime_metrics( + root: Path, + bundle: Path, + summary: dict[str, object], +) -> dict[str, object]: + provenance = json.loads((root / build_desktop.PROVENANCE_NAME).read_text(encoding="utf-8")) + install_platform = summary["install_archive"]["platform"] + executable_suffix = ".exe" if install_platform == "Windows" else "" + bundle_bytes, file_count = build_desktop._directory_metrics(bundle) + node_root = bundle / build_desktop.NODE_DIRECTORY + node_bytes, node_files = build_desktop._directory_metrics(node_root) + return { + "schema_version": 1, + "application": build_desktop.APP_NAME, + "package": "communityai-desktop", + "platform": provenance["build_platform"], + "python": provenance["build_python"], + "bundle_bytes": bundle_bytes, + "file_count": file_count, + "runtime": { + "shell": "pyside", + "framework": "PySide6", + "version": "6.9.0", + }, + "acceptance": { + "api_version": 1, + "model_count": 3, + "worker_actions": 3, + "key_lifecycle": "passed", + "contribution_policy": "passed", + "policy_update": "passed", + "auto_selection": "passed", + }, + "ui_smoke_passed": True, + "onboarding_ui_smoke_passed": True, + "node_sidecar": { + "relative_executable": f"node/CommunityAI-Node{executable_suffix}", + "bundle_bytes": node_bytes, + "file_count": node_files, + "runtime": { + "schema_version": 1, + "application": "CommunityAI-Node", + "drift": "0.1.0", + "torch": "2.6.0+cu124", + "transformers": "4.55.4", + "hivemind": "1.1.12", + "fastapi": "0.116.1", + "uvicorn": "0.35.0", + "keyring": "25.6.0", + "p2pd": f"p2pd{executable_suffix}", + "catalog_bootstrap_schema": 1, + "frozen": True, + }, + "worker_runtime": { + "schema_version": 1, + "application": "CommunityAI-Worker", + "entrypoint": "server", + "server_class": "Server", + "model_loading_performed": False, + "network_join_performed": False, + "throughput_mode": "dry_run", + "training_rpcs_enabled": False, + "process_lifetime_guard_armed": True, + "frozen": True, + }, + "self_test_passed": True, + "worker_self_test_passed": True, + "node_entrypoint_smoke_passed": True, + "worker_entrypoint_smoke_passed": True, + }, + "console_window": install_platform != "Windows", + "signed": False, + "catalog_bootstrap_bundled": provenance["catalog_publication_bundle"] is not None, + "catalog_publication_bundle": provenance["catalog_publication_bundle"], + "release_artifacts": summary, + } + + +def _release_root( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + extra_node: dict[str, bytes] | None = None, + install_platform: str = "Linux", + outside_node_symlink: bool = False, + node_mode: int = 0o755, + publication_evidence: dict[str, object] | None = None, +) -> Path: + build_platform = "Windows-test" if install_platform == "Windows" else "Linux-test" + monkeypatch.setattr(build_desktop.platform, "platform", lambda: build_platform) + root = tmp_path / "release" + bundle = root / build_desktop.APP_NAME + executable_suffix = ".exe" if install_platform == "Windows" else "" + files = { + f"CommunityAI{executable_suffix}": b"desktop\n", + f"node/CommunityAI-Node{executable_suffix}": b"node executable\n", + "node/_internal/python-runtime.bin": b"sidecar\x00", + "node/_internal/drift/runtime.pyc": b"bytecode\x00", + } + files.update(extra_node or {}) + for relative, payload in files.items(): + target = bundle / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(payload) + if outside_node_symlink: + (bundle / "shared.so").write_bytes(b"shared") + try: + (bundle / "node" / "_internal" / "shared-link.so").symlink_to("../../shared.so") + except OSError: + pytest.skip("file symlink creation is unavailable") + (bundle / f"CommunityAI{executable_suffix}").chmod(0o755) + node_executable = bundle / "node" / f"CommunityAI-Node{executable_suffix}" + node_executable.chmod(node_mode) + if install_platform == "Linux": + native_bundle_artifacts = build_desktop._bundle_artifacts + + def linux_bundle_artifacts(bundle_root: Path) -> list[dict[str, object]]: + artifacts = native_bundle_artifacts(bundle_root) + for artifact in artifacts: + if artifact["path"] == "CommunityAI/CommunityAI": + artifact["mode"] = 0o755 + elif artifact["path"] == "CommunityAI/node/CommunityAI-Node": + artifact["mode"] = node_mode + return artifacts + + monkeypatch.setattr(build_desktop, "_bundle_artifacts", linux_bundle_artifacts) + summary = build_desktop._write_release_attestations( + root, + bundle, + source_commit=SOURCE_COMMIT, + source_tree=SOURCE_TREE, + build_workflow="desktop.yaml@refs/heads/test", + build_pyinstaller="6.11.1", + publication_evidence=publication_evidence, + install_platform=install_platform, + ) + build_desktop._write_desktop_metrics( + root, + _runtime_metrics(root, bundle, summary), + ) + build_desktop._verify_release_attestations( + root, + expected_source_commit=SOURCE_COMMIT, + require_metrics=True, + ) + return root + + +def _source_bindings() -> list[dict[str, object]]: + bindings = [] + for relative in sorted(controller.REQUIRED_SOURCE_PATHS): + payload = (SOURCE_ROOT / relative).read_bytes() + bindings.append( + { + "relative_path": relative, + "sha256": stage._sha256(payload), + "byte_size": len(payload), + } + ) + return bindings + + +def _source_context() -> dict[str, object]: + return { + "schema_version": stage.SCHEMA_VERSION, + "scope": stage.SOURCE_CONTEXT_SCOPE, + "source_commit": SOURCE_COMMIT, + "source_tree": SOURCE_TREE, + "source_bindings": _source_bindings(), + } + + +def _validate( + root: Path, + manifest: Path = MANIFEST, + *, + source_commit: str = SOURCE_COMMIT, + source_tree: str = SOURCE_TREE, + protection_verifier: stage.ProtectionVerifier | None = None, +) -> dict[str, object]: + bindings = _source_bindings() + return stage.validate_release_root( + root, + manifest, + expected_source_commit=source_commit, + expected_source_tree=source_tree, + source_root=SOURCE_ROOT, + source_bindings=bindings, + protection_verifier=protection_verifier or (lambda _path, _directory: None), + ) + + +def test_binds_complete_linux_node_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + + record = _validate(root) + + assert record["platform"] == "linux" + assert record["source_commit"] == SOURCE_COMMIT + assert record["source_tree"] == SOURCE_TREE + assert record["source_bindings_digest"] == controller._source_bindings_digest(_source_bindings()) + assert record["manifest_digest"] == controller.EXPECTED_MANIFEST_DIGEST + assert record["manifest_sha256"].startswith("sha256:") + assert record["node_executable"] == "CommunityAI/node/CommunityAI-Node" + assert record["node_runtime_entry_count"] == 3 + assert record["node_runtime_bytes"] == sum( + path.stat().st_size for path in (root / "CommunityAI" / "node").rglob("*") if path.is_file() + ) + assert ( + stage.validate_record( + record, + expected_source_commit=SOURCE_COMMIT, + expected_source_tree=SOURCE_TREE, + expected_manifest_digest=controller.EXPECTED_MANIFEST_DIGEST, + ) + == record + ) + + +def test_accepts_production_sized_release_provenance( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + publication_evidence = { + "schema_version": 1, + "catalog_sequence": 1, + "member_count": 7_000, + "member_digests": {f"catalog/member-{index:05d}-{'x' * 100}.json": "a" * 64 for index in range(7_000)}, + "complete_release_qualification": False, + } + root = _release_root( + tmp_path, + monkeypatch, + publication_evidence=publication_evidence, + ) + provenance = root / build_desktop.PROVENANCE_NAME + assert provenance.stat().st_size > 1_241_883 + assert provenance.stat().st_size < stage.MAX_PROVENANCE_BYTES + + record = _validate(root) + + assert record["provenance_bytes"] == provenance.stat().st_size + assert record["node_runtime_entry_count"] == 3 + + +def test_archive_hashing_uses_bounded_streaming( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + archive_name = "communityai-desktop-linux.tar.gz" + native_read_bytes = Path.read_bytes + native_os_read = os.read + read_sizes: list[int] = [] + + def reject_archive_read_bytes(path: Path) -> bytes: + if path.name == archive_name: + raise AssertionError("release archive must not use Path.read_bytes") + return native_read_bytes(path) + + def bounded_read(descriptor: int, byte_count: int) -> bytes: + read_sizes.append(byte_count) + assert byte_count <= stage.HASH_CHUNK_BYTES + return native_os_read(descriptor, byte_count) + + monkeypatch.setattr(Path, "read_bytes", reject_archive_read_bytes) + monkeypatch.setattr(os, "read", bounded_read) + + record = _validate(root) + + assert record["release_archive_bytes"] == (root / archive_name).stat().st_size + assert read_sizes + assert set(read_sizes) == {stage.HASH_CHUNK_BYTES} + + +def test_archive_replacement_during_streaming_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + archive = root / "communityai-desktop-linux.tar.gz" + replacement = root / "replacement.tar.gz" + replacement.write_bytes(archive.read_bytes()) + native_os_read = os.read + replaced = False + + def replace_then_read(descriptor: int, byte_count: int) -> bytes: + nonlocal replaced + if not replaced: + replaced = True + os.replace(replacement, archive) + return native_os_read(descriptor, byte_count) + + monkeypatch.setattr(os, "read", replace_then_read) + + with pytest.raises(stage.Q38StagePackageError, match="changed|safely"): + _validate(root) + assert replaced + + +@pytest.mark.parametrize( + "node_mode", + (0o100, 0o644, 0o700, 0o757, 0o775, 0o2755, 0o4755), +) +def test_rejects_unsafe_or_inaccessible_node_mode( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + node_mode: int, +) -> None: + root = _release_root(tmp_path, monkeypatch, node_mode=node_mode) + + with pytest.raises(stage.Q38StagePackageError, match="mode is not 0755"): + _validate(root) + + +def test_requires_protection_for_the_complete_release_tree( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + calls: list[tuple[Path, bool]] = [] + + def verify(path: Path, directory: bool) -> None: + calls.append((path.resolve(), directory)) + + _validate(root, protection_verifier=verify) + + assert (MANIFEST.resolve(), False) in calls + assert (root.resolve(), True) in calls + assert ( + (root / "CommunityAI" / "node" / "_internal" / "python-runtime.bin").resolve(), + False, + ) in calls + + def reject_sidecar(path: Path, _directory: bool) -> None: + if path.name == "python-runtime.bin": + raise stage.Q38StagePackageError("sidecar protection failed") + + with pytest.raises(stage.Q38StagePackageError, match="sidecar protection failed"): + _validate(root, protection_verifier=reject_sidecar) + + +@pytest.mark.parametrize("mutation", ["missing", "extra", "changed"]) +def test_release_inventory_mutation_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutation: str, +) -> None: + root = _release_root(tmp_path, monkeypatch) + sidecar = root / "CommunityAI" / "node" / "_internal" / "python-runtime.bin" + if mutation == "missing": + sidecar.unlink() + elif mutation == "extra": + (sidecar.parent / "extra.bin").write_bytes(b"extra") + else: + sidecar.write_bytes(b"changed") + + with pytest.raises(stage.Q38StagePackageError, match="attestations"): + _validate(root) + + +def test_rejects_wrong_source_or_platform( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + + with pytest.raises(stage.Q38StagePackageError, match="attestations"): + _validate(root, source_commit="c" * 40) + + other = _release_root( + tmp_path / "windows", + monkeypatch, + install_platform="Windows", + ) + with pytest.raises(stage.Q38StagePackageError, match="Linux production"): + _validate(other) + + +@pytest.mark.parametrize( + "weight_name", + ( + "layers-0.safetensors", + "model-00001-of-00002.safetensors", + "renamed-weight.gguf", + "pytorch_model-00001-of-00002.bin", + ), +) +def test_rejects_model_weights_inside_node_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + weight_name: str, +) -> None: + root = _release_root( + tmp_path, + monkeypatch, + extra_node={f"node/_internal/{weight_name}": b"weight"}, + ) + + with pytest.raises(stage.Q38StagePackageError, match="model weights"): + _validate(root) + + +def test_rejects_wrong_manifest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + manifest = tmp_path / "manifest.json" + manifest.write_text("{}\n", encoding="utf-8") + + with pytest.raises(stage.Q38StagePackageError, match="manifest"): + _validate(root, manifest) + + +def test_rejects_node_symlink_outside_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + if os.name == "nt": + pytest.skip("POSIX release symlink semantics require native Linux") + root = _release_root( + tmp_path, + monkeypatch, + outside_node_symlink=True, + ) + + with pytest.raises(stage.Q38StagePackageError, match="symlink escapes"): + _validate(root) + + +@pytest.mark.parametrize("target", ("provenance", "archive", "runtime")) +def test_rejects_mutation_after_release_verification( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, +) -> None: + root = _release_root(tmp_path, monkeypatch) + verify = build_desktop._verify_release_attestations + + def verified_then_mutated(*args: object, **kwargs: object) -> dict[str, object]: + result = verify(*args, **kwargs) + if target == "provenance": + path = root / build_desktop.PROVENANCE_NAME + value = json.loads(path.read_text(encoding="utf-8")) + value["artifacts"].append( + { + "kind": "file", + "mode": 0o644, + "path": "CommunityAI/node/_internal/phantom.bin", + "sha256": "0" * 64, + "size_bytes": 1, + } + ) + path.write_bytes(build_desktop._canonical_json(value).encode("utf-8")) + elif target == "archive": + archive = result["install_archive"]["path"] + with (root / archive).open("ab") as stream: + stream.write(b"x") + else: + sidecar = root / "CommunityAI" / "node" / "_internal" / "python-runtime.bin" + sidecar.write_bytes(b"mutated") + return result + + monkeypatch.setattr( + build_desktop, + "_verify_release_attestations", + verified_then_mutated, + ) + + with pytest.raises( + stage.Q38StagePackageError, + match="changed|binding|identity", + ): + _validate(root) + + +def test_rejects_unbound_verifier_source() -> None: + bindings = [ + binding + for binding in _source_bindings() + if binding["relative_path"] + in { + controller.STAGE_PACKAGE_SOURCE_PATH, + controller.DESKTOP_RELEASE_VERIFIER_SOURCE_PATH, + } + ] + + with pytest.raises(stage.Q38StagePackageError, match="not plan-bound"): + stage._assert_verifier_sources(SOURCE_ROOT, bindings[:1]) + + bindings[1]["sha256"] = "sha256:" + "0" * 64 + with pytest.raises(stage.Q38StagePackageError, match="binding changed"): + stage._assert_verifier_sources(SOURCE_ROOT, bindings) + + +def test_record_digest_rejects_substitution( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + record = _validate(root) + record["node_runtime_bytes"] += 1 + + with pytest.raises(stage.Q38StagePackageError, match="record digest"): + stage.validate_record(record) + + +def test_atomic_record_round_trip_and_unsafe_target( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + record = _validate(root) + output = tmp_path / "records" / "runtime.json" + + stage._atomic_record(output, record) + + assert json.loads(output.read_text(encoding="utf-8")) == record + output.unlink() + output.mkdir() + with pytest.raises(stage.Q38StagePackageError, match="target is unsafe"): + stage._atomic_record(output, record) + + +def test_source_context_drives_cli_without_loading_a_final_plan( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + context_path = tmp_path / "source-context.json" + context_path.write_text(json.dumps(_source_context(), sort_keys=True) + "\n", encoding="utf-8") + output = tmp_path / "runtime-package.json" + protected_calls: list[tuple[Path, bool]] = [] + monkeypatch.setattr( + controller, + "_assert_protected_path_from_bindings", + lambda path, _bindings, _root, *, directory: protected_calls.append((path.resolve(), directory)), + ) + monkeypatch.setattr( + controller, + "load_plan", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("final plan must not be loaded")), + ) + + assert ( + stage.main( + [ + "--release-root", + str(root), + "--manifest", + str(MANIFEST), + "--source-context", + str(context_path), + "--source-root", + str(SOURCE_ROOT), + "--output", + str(output), + ] + ) + == 0 + ) + record = json.loads(output.read_text(encoding="utf-8")) + assert record["source_commit"] == SOURCE_COMMIT + assert record["source_tree"] == SOURCE_TREE + assert record["source_bindings_digest"] == controller._source_bindings_digest(_source_bindings()) + assert (context_path.parent.resolve(), True) in protected_calls + assert (context_path.resolve(), False) in protected_calls + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("schema_version", True), + ("scope", "wrong"), + ("source_commit", "0" * 39), + ("source_tree", "0" * 41), + ], +) +def test_source_context_rejects_invalid_identity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, +) -> None: + context = _source_context() + context[field] = value + path = tmp_path / "source-context.json" + path.write_text(json.dumps(context, sort_keys=True) + "\n", encoding="utf-8") + monkeypatch.setattr( + controller, + "_assert_protected_path_from_bindings", + lambda *_args, **_kwargs: None, + ) + + with pytest.raises(stage.Q38StagePackageError, match="identity"): + stage.load_source_context(path, SOURCE_ROOT) + + +def test_source_context_rejects_duplicate_or_incomplete_bindings( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + controller, + "_assert_protected_path_from_bindings", + lambda *_args, **_kwargs: None, + ) + duplicate = tmp_path / "duplicate.json" + duplicate.write_text( + '{"schema_version":1,"schema_version":1}\n', + encoding="utf-8", + ) + with pytest.raises(stage.Q38StagePackageError, match="duplicate JSON field"): + stage.load_source_context(duplicate, SOURCE_ROOT) + + context = _source_context() + context["source_bindings"] = context["source_bindings"][:-1] + incomplete = tmp_path / "incomplete.json" + incomplete.write_text(json.dumps(context, sort_keys=True) + "\n", encoding="utf-8") + with pytest.raises(stage.Q38StagePackageError, match="incomplete"): + stage.load_source_context(incomplete, SOURCE_ROOT) + + +def test_source_context_rejects_mutation_during_protection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "source-context.json" + path.write_text(json.dumps(_source_context(), sort_keys=True) + "\n", encoding="utf-8") + + def mutate_context(protected: Path, *_args: object, **_kwargs: object) -> None: + if protected == path: + protected.write_text(protected.read_text(encoding="utf-8") + " ", encoding="utf-8") + + monkeypatch.setattr(controller, "_assert_protected_path_from_bindings", mutate_context) + + with pytest.raises(stage.Q38StagePackageError, match="changed while validated"): + stage.load_source_context(path, SOURCE_ROOT) + + +def test_record_rejects_source_binding_substitution( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _release_root(tmp_path, monkeypatch) + bindings = _source_bindings() + record = _validate(root) + changed = list(bindings) + changed[0] = dict(changed[0]) + changed[0]["sha256"] = "sha256:" + "0" * 64 + + with pytest.raises(stage.Q38StagePackageError, match="source bindings changed"): + stage.validate_record(record, expected_source_bindings=changed) + + +@pytest.mark.parametrize("target", ("context", "manifest", "release")) +def test_cli_rejects_output_alias_without_overwriting_inputs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target: str, +) -> None: + root = _release_root(tmp_path, monkeypatch) + context_path = tmp_path / "source-context.json" + context_path.write_text(json.dumps(_source_context(), sort_keys=True) + "\n", encoding="utf-8") + manifest_path = tmp_path / "manifest.json" + manifest_path.write_bytes(MANIFEST.read_bytes()) + output = { + "context": context_path, + "manifest": manifest_path, + "release": root / build_desktop.PROVENANCE_NAME, + }[target] + protected = { + context_path: context_path.read_bytes(), + manifest_path: manifest_path.read_bytes(), + root / build_desktop.PROVENANCE_NAME: (root / build_desktop.PROVENANCE_NAME).read_bytes(), + } + + with pytest.raises(SystemExit, match="output"): + stage.main( + [ + "--release-root", + str(root), + "--manifest", + str(manifest_path), + "--source-context", + str(context_path), + "--source-root", + str(SOURCE_ROOT), + "--output", + str(output), + ] + ) + + assert {path: path.read_bytes() for path in protected} == protected + + +def test_output_isolation_rejects_source_root_and_resolved_descendants(tmp_path: Path) -> None: + release_root = tmp_path / "release" + release_root.mkdir() + manifest = tmp_path / "manifest.json" + manifest.write_text("{}\n", encoding="utf-8") + context = tmp_path / "context.json" + context.write_text("{}\n", encoding="utf-8") + + with pytest.raises(stage.Q38StagePackageError, match="protected input root"): + stage._assert_output_isolation( + SOURCE_ROOT / "forbidden-runtime-record.json", + release_root=release_root, + manifest_path=manifest, + source_context_path=context, + source_root=SOURCE_ROOT, + ) + with pytest.raises(stage.Q38StagePackageError, match="protected input root"): + stage._assert_output_isolation( + release_root / "nested" / "runtime-record.json", + release_root=release_root, + manifest_path=manifest, + source_context_path=context, + source_root=SOURCE_ROOT, + ) diff --git a/tests/test_hub_ranges.py b/tests/test_hub_ranges.py new file mode 100644 index 000000000..1fda73f1f --- /dev/null +++ b/tests/test_hub_ranges.py @@ -0,0 +1,133 @@ +import hashlib +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from drift.model_manifest import ManifestArtifactVerifier, ManifestError, ModelManifest + + +@pytest.mark.parametrize( + "mode", + [ + "ranges", + "interrupt", + "transient", + "429", + "401", + "ignore", + "ignored_then_ranges", + "wrong_range", + "corrupt", + "oversize", + ], +) +def test_large_artifact_ranges_preserve_contiguous_resume_and_verify_hash(tmp_path, monkeypatch, mode): + chunk_size = 64 * 1024 + payload = bytes(index % 251 for index in range(600_000)) + source = ModelManifest.load(Path("tests/data/model_manifest_v1_vector.json")).to_dict() + artifact = next(a for a in source["artifacts"] if a["role"] == "weight") + artifact.update(size=len(payload), sha256=hashlib.sha256(payload).hexdigest()) + manifest = ModelManifest.from_dict(source) + verifier = ManifestArtifactVerifier( + manifest, manifest.source.repository, manifest.source.revision, cache_dir=tmp_path + ) + declared = manifest.get_artifact(artifact["path"]) + requests, failures = [], [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + start, end = map(int, self.headers["Range"].removeprefix("bytes=").split("-")) + requests.append((start, end)) + if mode == "ignored_then_ranges" and not failures: + failures.append(start) + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload[:100]) + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + if mode in ("429", "401") and not failures: + failures.append(start) + self.send_response(int(mode)) + self.send_header("Content-Length", "0") + self.end_headers() + return + if mode == "ignore": + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + return + body = payload[start : end + 1] + if mode == "corrupt": + body = bytes([body[0] ^ 255]) + body[1:] + if mode == "oversize": + body += b"!" + self.send_response(206) + self.send_header("Content-Length", str(len(body))) + self.send_header("Content-Range", f"bytes {start + (mode == 'wrong_range')}-{end}/{len(payload)}") + self.end_headers() + if (mode == "interrupt" and start == 2 * chunk_size and len(failures) < 3) or ( + mode == "transient" and start in (0, 2 * chunk_size) and start not in failures + ): + failures.append(start) + self.wfile.write(body[:100]) + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + monkeypatch.setattr("drift.utils.hub_ranges.RANGE_BYTES", chunk_size) + monkeypatch.setattr("huggingface_hub.hf_hub_url", lambda *a, **kw: f"http://127.0.0.1:{server.server_port}/weights") + partial, final, _ = verifier._resumable_paths(declared) + try: + if mode == "ignored_then_ranges": + partial.parent.mkdir(parents=True, exist_ok=True) + partial.write_bytes(payload[: 2 * chunk_size]) + if mode in ("wrong_range", "corrupt", "oversize"): + with pytest.raises(ManifestError): + verifier._resumable_hub_download(declared) + assert not final.exists() + assert not partial.exists() or partial.stat().st_size < len(payload) + return + if mode == "401": + with pytest.raises(ManifestError, match="Interrupted download"): + verifier._resumable_hub_download(declared) + assert len(requests) == 1 + assert not final.exists() + return + if mode == "interrupt": + with pytest.raises(ManifestError, match="Interrupted download"): + verifier._resumable_hub_download(declared) + assert not final.exists() + assert partial.read_bytes() == payload[: 2 * chunk_size] + assert len(failures) == 3 + count = len(requests) + verifier._resumable_hub_download(declared) + assert requests[count][0] == 2 * chunk_size + else: + verifier._resumable_hub_download(declared) + assert final.read_bytes() == payload + assert not partial.exists() + assert all(end - start + 1 <= chunk_size for start, end in requests) + if mode == "transient": + assert failures == [0, 2 * chunk_size] + if mode == "ignored_then_ranges": + assert requests[0][0] == 2 * chunk_size + assert requests[1][0] == 0 + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/test_inference_mode.py b/tests/test_inference_mode.py new file mode 100644 index 000000000..8e501bc1d --- /dev/null +++ b/tests/test_inference_mode.py @@ -0,0 +1,53 @@ +import json + +from fastapi.testclient import TestClient +from test_policy_store import _store + +from drift.node.model_manager import ModelDescriptor, ModelManager, ModelRuntime +from drift.node.server import create_node_app + + +def test_control_mode_change_is_authenticated_persistent_and_preserves_active_answer(tmp_path): + path, supervisor, store = _store(tmp_path) + manager = ModelManager() + remote = {"status": "complete", "covered_blocks": 64, "total_blocks": 64, "peer_count": 4, "source": "discovery"} + local = {"status": "complete", "covered_blocks": 24, "total_blocks": 24, "peer_count": 0, "source": "local"} + manager.register(ModelDescriptor("community"), lambda: ModelRuntime(object(), None), route_health=lambda: remote) + manager.register( + ModelDescriptor("local", execution="local"), lambda: ModelRuntime(object(), None), route_health=lambda: local + ) + manager.configure_auto_selection(["community", "local"]) + app = create_node_app( + manager, + api_keys=["client-secret"], + control_keys=["control-secret"], + worker_supervisor=supervisor, + contribution_policy_store=store, + ) + with TestClient(app) as client: + active = manager.load("auto") + prior = path.read_bytes() + body = {"inference_mode": "local_only", "expected_config_revision": store.snapshot()["config_revision"]} + assert ( + client.put( + "/control/v1/inference-mode", json=body, headers={"Authorization": "Bearer client-secret"} + ).status_code + == 401 + ) + assert path.read_bytes() == prior + headers = {"Authorization": "Bearer control-secret"} + assert client.put("/control/v1/inference-mode", json=body, headers=headers).status_code == 200 + assert json.loads(path.read_text())["inference_mode"] == "local_only" + assert manager.resolve("auto").model_id == "local" + assert active.descriptor.model_id == "community" + assert active.runtime.model is not None + assert client.put("/control/v1/inference-mode", json=body, headers=headers).status_code == 412 + active.release() + assert ( + client.post( + "/v1/completions", + json={"model": "community", "prompt": "hello"}, + headers={"Authorization": "Bearer client-secret"}, + ).status_code + == 503 + ) diff --git a/tests/test_linux_online_installer.py b/tests/test_linux_online_installer.py new file mode 100644 index 000000000..1571ca0e7 --- /dev/null +++ b/tests/test_linux_online_installer.py @@ -0,0 +1,373 @@ +"""Offline, inert checks for the generated Linux installer; never run sudo or APT.""" + +import contextlib +import hashlib +import io +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +INSTALLERS = Path(__file__).resolve().parents[1] / "desktop" / "installers" +sys.path.insert(0, str(INSTALLERS)) + +import linux_online_root as protected +import linux_online_template as online + +PAYLOAD = b"verified package fixture; no executable or Debian archive" + + +def artifact(**updates): + value = { + "platform": "linux-amd64", + "kind": "offline-installer", + "format": "deb", + "version": "0.1.0~alpha.1", + "filename": "communityai_0.1.0~alpha.1_amd64.deb", + "url": "https://downloads.example.com/releases/alpha.1/communityai_0.1.0~alpha.1_amd64.deb", + "sha256": hashlib.sha256(PAYLOAD).hexdigest(), + "size_bytes": len(PAYLOAD), + } + value.update(updates) + return value + + +class Response(io.BytesIO): + def __init__(self, payload=PAYLOAD, *, status=200, url=None, headers=None, failure=None): + super().__init__(payload) + self.status = status + self.url = url or artifact()["url"] + self.headers = {"Content-Length": str(len(payload))} if headers is None else headers + self.failure = failure + + def geturl(self): + return self.url + + def read(self, count=-1): + if self.failure: + raise self.failure + assert count <= online.CHUNK_BYTES + return super().read(count) + + +class Opener: + def __init__(self, response): + self.response = response + self.calls = [] + + def open(self, request, timeout): + self.calls.append((request.full_url, timeout)) + assert request.get_header("Accept-encoding") == "identity" + return self.response + + +def host(monkeypatch, tmp_path, response): + monkeypatch.setattr(online.sys, "platform", "linux") + monkeypatch.setattr(online.platform, "machine", lambda: "x86_64") + monkeypatch.setattr(online.os, "geteuid", lambda: 1000, raising=False) + monkeypatch.setattr(online.shutil, "disk_usage", lambda path: SimpleNamespace(free=10**12)) + monkeypatch.setattr(online, "download_deadline", contextlib.nullcontext) + monkeypatch.setattr(online, "ROOT_HELPER", "inert protected-copy helper fixture") + original_is_file = Path.is_file + monkeypatch.setattr( + Path, + "is_file", + lambda path: path.as_posix() in ("/usr/bin/sudo", "/usr/bin/apt", "/usr/bin/python3") or original_is_file(path), + ) + opener = Opener(response) + monkeypatch.setattr(online.urllib.request, "build_opener", lambda *args: opener) + sentinel = tmp_path / "unrelated.txt" + sentinel.write_text("keep") + return opener, sentinel + + +def test_verified_download_precedes_apt_and_only_owned_staging_is_removed(tmp_path, monkeypatch): + opener, sentinel = host(monkeypatch, tmp_path, Response()) + called = [] + + def install(path, pinned): + called.append(path) + assert path.read_bytes() == PAYLOAD + assert hashlib.sha256(path.read_bytes()).hexdigest() == artifact()["sha256"] + command = online.install_command(path, pinned) + assert command[:4] == ["/usr/bin/sudo", "/usr/bin/python3", "-I", "-c"] + assert command[4] == online.ROOT_HELPER + assert command[5:] == [str(path.resolve()), pinned["filename"], str(len(PAYLOAD)), pinned["sha256"]] + return 0 + + monkeypatch.setattr(online, "install_verified", install) + assert online.run(artifact(), directory=tmp_path) == 0 + assert len(called) == 1 and not called[0].parent.exists() + assert opener.calls == [(artifact()["url"], 30)] + assert list(tmp_path.iterdir()) == [sentinel] + + +@pytest.mark.parametrize( + "response,updates,error", + [ + (lambda: Response(headers={"Content-Length": "999"}), {}, "package size"), + (lambda: Response(headers={"Content-Length": "garbage"}), {}, "package size"), + (lambda: Response(PAYLOAD[:-1], headers={}), {}, "incomplete"), + (lambda: Response(PAYLOAD + b"extra", headers={}), {}, "exceeds"), + (lambda: Response(), {"sha256": "0" * 64}, "SHA-256"), + (lambda: Response(status=206), {}, "exact pinned"), + (lambda: Response(url="https://elsewhere.example.org/payload.deb"), {}, "exact pinned"), + (lambda: Response(headers={"Content-Encoding": "gzip"}), {}, "encoded"), + (lambda: Response(failure=OSError("network failed")), {}, "network failed"), + ], +) +def test_invalid_download_never_invokes_apt_and_cleans_only_owned_files( + tmp_path, monkeypatch, response, updates, error +): + _, sentinel = host(monkeypatch, tmp_path, response()) + + def forbidden(path, pinned): + raise AssertionError("APT must not see unverified bytes") + + monkeypatch.setattr(online, "install_verified", forbidden) + with pytest.raises((online.InstallError, OSError), match=error): + online.run(artifact(**updates), directory=tmp_path) + assert list(tmp_path.iterdir()) == [sentinel] + + +@pytest.mark.parametrize("failure", [KeyboardInterrupt(), online.Cancelled("cancelled")]) +def test_cancelled_download_cleans_partial_without_installing(tmp_path, monkeypatch, failure): + _, sentinel = host(monkeypatch, tmp_path, Response(failure=failure)) + monkeypatch.setattr(online, "install_verified", lambda *args: pytest.fail("must not invoke APT")) + with pytest.raises(type(failure)): + online.run(artifact(), directory=tmp_path) + assert list(tmp_path.iterdir()) == [sentinel] + + +def test_monotonic_download_deadline(tmp_path): + times = iter((0, online.DOWNLOAD_TIMEOUT_SECONDS)) + with pytest.raises(online.InstallError, match="deadline"): + online.download(artifact(), tmp_path / "payload.deb", opener=Opener(Response()), clock=lambda: next(times)) + + +@pytest.mark.parametrize("trigger,error", [(14, online.InstallError), (15, online.Cancelled)]) +def test_download_deadline_signals_cancel_and_restore_handlers(monkeypatch, trigger, error): + previous = {14: object(), 15: object()} + handlers = dict(previous) + timers = [] + + def set_handler(number, handler): + old = handlers[number] + handlers[number] = handler + return old + + monkeypatch.setattr(online.signal, "SIGALRM", 14, raising=False) + monkeypatch.setattr(online.signal, "SIGTERM", 15) + monkeypatch.setattr(online.signal, "ITIMER_REAL", 0, raising=False) + monkeypatch.setattr(online.signal, "signal", set_handler) + monkeypatch.setattr(online.signal, "setitimer", lambda *args: timers.append(args), raising=False) + with pytest.raises(error): + with online.download_deadline(): + handlers[trigger](trigger, None) + assert handlers == previous + assert timers == [(0, online.DOWNLOAD_TIMEOUT_SECONDS), (0, 0)] + + +@pytest.mark.parametrize("code", [301, 302, 303, 307, 308]) +def test_redirect_handler_never_follows_another_url(code): + with pytest.raises(online.InstallError, match="redirected"): + online.NoRedirects().redirect_request(None, None, code, "redirect", {}, "https://other.example.org/file") + + +def test_download_only_retains_exact_verified_package(tmp_path, monkeypatch): + host(monkeypatch, tmp_path, Response()) + monkeypatch.setattr(online, "install_verified", lambda *args: pytest.fail("download-only cannot invoke APT")) + assert online.run(artifact(), directory=tmp_path, download_only=True) == 0 + files = list(tmp_path.glob("communityai-online-*/*.deb")) + assert len(files) == 1 and files[0].read_bytes() == PAYLOAD + + +@pytest.mark.parametrize("condition", ["root", "wrong_arch", "disk_space"]) +def test_preflight_failure_never_opens_network(tmp_path, monkeypatch, condition): + opener, sentinel = host(monkeypatch, tmp_path, Response()) + if condition == "root": + monkeypatch.setattr(online.os, "geteuid", lambda: 0) + elif condition == "wrong_arch": + monkeypatch.setattr(online.platform, "machine", lambda: "aarch64") + else: + monkeypatch.setattr(online.shutil, "disk_usage", lambda path: SimpleNamespace(free=0)) + with pytest.raises(online.InstallError): + online.run(artifact(), directory=tmp_path) + assert not opener.calls and list(tmp_path.iterdir()) == [sentinel] + + +def test_unconfirmed_apt_interruption_preserves_its_input(tmp_path, monkeypatch): + host(monkeypatch, tmp_path, Response()) + + def interrupted(path, pinned): + raise KeyboardInterrupt() + + monkeypatch.setattr(online, "install_verified", interrupted) + with pytest.raises(KeyboardInterrupt): + online.run(artifact(), directory=tmp_path) + assert len(list(tmp_path.glob("communityai-online-*/*.deb"))) == 1 + + +def test_failed_apt_exit_is_reported_and_completed_staging_removed(tmp_path, monkeypatch): + _, sentinel = host(monkeypatch, tmp_path, Response()) + monkeypatch.setattr(online, "install_verified", lambda *args: 100) + with pytest.raises(online.InstallError, match="status 100"): + online.run(artifact(), directory=tmp_path) + assert list(tmp_path.iterdir()) == [sentinel] + + +def test_install_waits_for_apt_without_killing_on_interrupt(tmp_path): + waits = iter((KeyboardInterrupt(), 130)) + + class Child: + def wait(self): + value = next(waits) + if isinstance(value, BaseException): + raise value + return value + + calls = [] + assert ( + online.install_verified( + tmp_path / "package.deb", artifact(), popen=lambda command: calls.append(command) or Child() + ) + == 130 + ) + assert calls == [online.install_command(tmp_path / "package.deb", artifact())] + + +def test_builder_embeds_pinned_manifest_and_generated_script_is_standalone(tmp_path): + from build_linux_online import build + + manifest = tmp_path / "release.json" + manifest.write_text(json.dumps({"schema_version": 1, "artifacts": {"linux-amd64": artifact()}})) + output = tmp_path / "communityai-online.py" + build(manifest, output) + namespace = {"__name__": "installer_fixture"} + exec(compile(output.read_text(), str(output), "exec"), namespace) + assert namespace["ARTIFACT"] == artifact() + assert namespace["ROOT_HELPER"] == (INSTALLERS / "linux_online_root.py").read_text() + assert "release_downloads" not in output.read_text() + metadata = json.loads(output.with_suffix(".py.json").read_text()) + assert metadata["sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() + assert metadata["size_bytes"] == output.stat().st_size + assert metadata["offline_installer"] == artifact() + assert metadata["live_download_verified"] is False + completed = subprocess.run([sys.executable, str(output), "--help"], capture_output=True, text=True, timeout=10) + assert completed.returncode == 0 and "--download-only" in completed.stdout + with pytest.raises(FileExistsError): + build(manifest, output) + + +def protected_host(monkeypatch): + monkeypatch.setattr(protected.os, "geteuid", lambda: 0, raising=False) + # Windows fixtures exercise bytes and ordering. Linux O_NOFOLLOW/O_NONBLOCK + # and actual root ownership remain an explicit platform qualification limit. + if sys.platform == "win32": + monkeypatch.setattr(protected.os, "O_NOFOLLOW", 0, raising=False) + monkeypatch.setattr(protected.os, "O_NONBLOCK", 0, raising=False) + monkeypatch.setattr(protected, "copy_deadline", contextlib.nullcontext) + monkeypatch.setattr(protected.shutil, "disk_usage", lambda path: SimpleNamespace(free=10**12)) + + +def test_protected_copy_is_independent_of_user_file_after_verification(tmp_path, monkeypatch): + protected_host(monkeypatch) + source = tmp_path / "source.deb" + source.write_bytes(PAYLOAD) + calls = [] + + def apt(command): + package = Path(command[-1]) + assert command[:2] == ["/usr/bin/apt", "install"] + assert package != source and package.read_bytes() == PAYLOAD + source.write_bytes(b"changed by an ordinary-user writer") + assert package.read_bytes() == PAYLOAD + calls.append(package) + return SimpleNamespace(wait=lambda: 0) + + assert ( + protected.run(source, artifact()["filename"], len(PAYLOAD), artifact()["sha256"], directory=tmp_path, popen=apt) + == 0 + ) + assert len(calls) == 1 and not calls[0].parent.exists() + assert source.exists() + + +@pytest.mark.parametrize("payload", [PAYLOAD[:-1], b"x" * len(PAYLOAD), PAYLOAD + b"extra"]) +def test_changed_package_during_sudo_wait_never_reaches_apt(tmp_path, monkeypatch, payload): + protected_host(monkeypatch) + source = tmp_path / "source.deb" + source.write_bytes(payload) + with pytest.raises(ValueError, match="regular file|verification"): + protected.run( + source, + artifact()["filename"], + len(PAYLOAD), + artifact()["sha256"], + directory=tmp_path, + popen=lambda command: pytest.fail("APT must not receive changed bytes"), + ) + assert list(tmp_path.iterdir()) == [source] + + +def test_protected_helper_preserves_copy_if_apt_exit_is_unconfirmed(tmp_path, monkeypatch): + protected_host(monkeypatch) + source = tmp_path / "source.deb" + source.write_bytes(PAYLOAD) + + def interrupted(): + raise KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + protected.run( + source, + artifact()["filename"], + len(PAYLOAD), + artifact()["sha256"], + directory=tmp_path, + popen=lambda command: SimpleNamespace(wait=interrupted), + ) + assert len(list(tmp_path.glob("communityai-install-*/*.deb"))) == 1 + + +def test_protected_helper_retains_input_if_spawn_is_interrupted(tmp_path, monkeypatch): + protected_host(monkeypatch) + source = tmp_path / "source.deb" + source.write_bytes(PAYLOAD) + + def uncertain_spawn(command): + raise KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + protected.run( + source, + artifact()["filename"], + len(PAYLOAD), + artifact()["sha256"], + directory=tmp_path, + popen=uncertain_spawn, + ) + assert len(list(tmp_path.glob("communityai-install-*/*.deb"))) == 1 + + +@pytest.mark.skipif(sys.platform != "linux", reason="Actual Linux O_NOFOLLOW behavior") +def test_protected_helper_rejects_symlink_source_on_linux(tmp_path, monkeypatch): + protected_host(monkeypatch) + source = tmp_path / "source.deb" + source.write_bytes(PAYLOAD) + linked = tmp_path / "linked.deb" + linked.symlink_to(source) + with pytest.raises(OSError): + protected.run( + linked, + artifact()["filename"], + len(PAYLOAD), + artifact()["sha256"], + directory=tmp_path, + popen=lambda command: pytest.fail("APT must not see a symlink source"), + ) + assert source.read_bytes() == PAYLOAD and linked.is_symlink() + assert not list(tmp_path.glob("communityai-install-*")) diff --git a/tests/test_lm_head.py b/tests/test_lm_head.py new file mode 100644 index 000000000..6ea62e5ff --- /dev/null +++ b/tests/test_lm_head.py @@ -0,0 +1,23 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from drift.client.lm_head import LMHead + + +@pytest.mark.parametrize("supports_bf16", [False, True]) +def test_auto_head_uses_torch_probe_without_importing_cpufeature(supports_bf16): + config = SimpleNamespace(vocab_size=19, hidden_size=8, use_chunked_forward="auto", chunked_forward_step=7) + with patch.dict("sys.modules", {"cpufeature": None}), patch.object( + torch.cpu, "_is_avx512_bf16_supported", return_value=supports_bf16 + ): + head = LMHead(config).to(dtype=torch.bfloat16) + assert head.use_chunked_forward is not supports_bf16 + torch.manual_seed(3) + head.weight.copy_(torch.randn_like(head.weight)) + inputs = torch.randn(1, 3, 8, dtype=torch.bfloat16) + output = head(inputs) + reference = torch.nn.functional.linear(inputs.float(), head.weight.float()) + assert torch.allclose(output.float(), reference, atol=0.03, rtol=0.01) diff --git a/tests/test_loading_diagnostics.py b/tests/test_loading_diagnostics.py index b3c6520d8..05d3fc116 100644 --- a/tests/test_loading_diagnostics.py +++ b/tests/test_loading_diagnostics.py @@ -1,10 +1,15 @@ from types import SimpleNamespace +import pytest import torch from torch import nn from drift.client.lm_head import LMHead -from drift.server.from_pretrained import _find_unconsumed_checkpoint_keys, _load_state_dict_from_local_file +from drift.server.from_pretrained import ( + _find_unconsumed_checkpoint_keys, + _load_state_dict_from_local_file, + dequantize_finegrained_fp8_state_dict, +) from drift.utils.asyncio import patch_hivemind_task_cleanup, safe_cancel_task_if_running @@ -26,6 +31,33 @@ def test_legacy_rotary_frequency_is_derived_but_other_state_stays_strict(): ] +def test_finegrained_fp8_checkpoint_weights_dequantize_from_their_scale_grid(): + fp8_dtype = getattr(torch, "float8_e4m3fn", None) + if fp8_dtype is None: + pytest.skip("torch build has no float8_e4m3fn dtype") + state_dict = { + "proj.weight": torch.ones(4, 4, dtype=fp8_dtype), + "proj.weight_scale_inv": torch.tensor([[2.0, 4.0], [0.5, 1.0]]), + "norm.weight": torch.arange(4, dtype=torch.bfloat16), + } + + loaded = dequantize_finegrained_fp8_state_dict(state_dict, output_dtype=torch.bfloat16) + + expected = torch.tensor( + [[2.0, 2.0, 4.0, 4.0], [2.0, 2.0, 4.0, 4.0], [0.5, 0.5, 1.0, 1.0], [0.5, 0.5, 1.0, 1.0]], + dtype=torch.bfloat16, + ) + assert torch.equal(loaded["proj.weight"], expected) + assert loaded["proj.weight"].dtype == torch.bfloat16 + assert torch.equal(loaded["norm.weight"], state_dict["norm.weight"]) + assert "proj.weight_scale_inv" not in loaded + + +def test_finegrained_fp8_checkpoint_rejects_an_orphan_scale(): + with pytest.raises(ValueError, match="has no matching"): + dequantize_finegrained_fp8_state_dict({"proj.weight_scale_inv": torch.ones(1, 1)}, output_dtype=torch.bfloat16) + + def test_safetensors_block_load_copies_selected_tensors_out_of_the_file_mapping(monkeypatch): selected = torch.arange(4, dtype=torch.float32) unrelated = torch.ones(2) diff --git a/tests/test_local_inference.py b/tests/test_local_inference.py new file mode 100644 index 000000000..1ccacde99 --- /dev/null +++ b/tests/test_local_inference.py @@ -0,0 +1,148 @@ +from dataclasses import replace +from types import SimpleNamespace + +import pytest +import torch + +from drift.node.config import NodeConfig, NodeConfigError, NodeModelConfig +from drift.node.local_inference import ( + LocalInferenceModel, + local_device, + local_route_observer, + make_local_manifest_loader, +) +from drift.node.model_manager import ModelDescriptor, ModelManager, ModelRuntime + + +def manifest(weight_bytes=1024): + return SimpleNamespace( + runtime=SimpleNamespace(quantization="none", adapter_profile="none"), + model=SimpleNamespace(num_blocks=24, context_length=1024), + artifacts=[SimpleNamespace(size=weight_bytes)], + artifacts_for_roles=lambda roles: [SimpleNamespace(size=weight_bytes)], + ) + + +def test_first_install_local_config_needs_no_public_peers(tmp_path): + config = NodeConfig.from_dict( + { + "schema_version": 1, + "models": [{"manifest": "qwen.json", "initial_peers": [], "execution": "local"}], + "inference_mode": "local_only", + }, + base_dir=tmp_path, + ) + assert config.models[0].execution == "local" + assert config.inference_mode == "local_only" + with pytest.raises(NodeConfigError, match="at least one"): + NodeModelConfig.from_dict({"manifest": "qwen.json", "initial_peers": []}, base_dir=tmp_path, index=0) + + +def test_budget_rejects_model_before_device_or_download_work(tmp_path, monkeypatch): + config = NodeModelConfig(tmp_path / "qwen.json", (), execution="local", local_max_memory_bytes=1024) + monkeypatch.setattr(torch.cuda, "is_available", lambda: pytest.fail("must reject before device selection")) + with pytest.raises(MemoryError, match="budget"): + local_device(config, manifest()) + assert local_route_observer(manifest(), config)()["status"] == "unavailable" + + +def test_auto_device_uses_ram_when_other_apps_occupy_gpu(tmp_path, monkeypatch): + config = NodeModelConfig(tmp_path / "qwen.json", (), execution="local") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device: (0, 8 * 1024**3)) + import psutil + + monkeypatch.setattr(psutil, "virtual_memory", lambda: SimpleNamespace(available=16 * 1024**3)) + assert local_device(config, manifest()) == "cpu" + with pytest.raises(MemoryError, match="free GPU"): + local_device(replace(config, local_device="cuda:0"), manifest()) + + +def test_local_runtime_rejects_unbounded_request_and_closes(tmp_path): + class Model: + def generate(self, ids, **kwargs): + assert kwargs["max_time"] == 120 + return torch.cat([ids, torch.tensor([[7]])], dim=1) + + config = NodeModelConfig(tmp_path / "qwen.json", (), execution="local", local_max_context=8, local_max_new_tokens=3) + runtime = LocalInferenceModel(Model(), config, manifest(), "cpu") + assert runtime.generate(torch.tensor([[1, 2]]), max_new_tokens=1).tolist() == [[1, 2, 7]] + with pytest.raises(ValueError, match="token budget"): + runtime.generate(torch.tensor([[1]]), max_new_tokens=4) + with pytest.raises(ValueError, match="context budget"): + runtime.generate(torch.ones((1, 8), dtype=torch.long), max_new_tokens=1) + runtime.close() + with pytest.raises(RuntimeError, match="closed"): + runtime.generate(torch.tensor([[1]]), max_new_tokens=1) + + +def test_fallback_rechecks_gpu_after_download_and_uses_cpu_if_sharing_fills_it(tmp_path, monkeypatch): + import psutil + from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer + + from drift.model_manifest import ModelManifest + + model_manifest = ModelManifest.load("manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json") + config = NodeModelConfig(tmp_path / "qwen.json", (), execution="local") + free = [8 * 1024**3] + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device: (free[0], 8 * 1024**3)) + monkeypatch.setattr(psutil, "virtual_memory", lambda: SimpleNamespace(available=16 * 1024**3)) + + class Verifier: + snapshot_root = tmp_path + + def __init__(self, *args, **kwargs): + pass + + def ensure_startup_metadata(self, **kwargs): + pass + + def ensure_path(self, path): + # Another process claimed the GPU while the local files downloaded. + free[0] = 0 + + devices = [] + model = SimpleNamespace(get_memory_footprint=lambda: 1024) + model.eval = lambda: model + + def load_model(*args, **kwargs): + devices.append(kwargs["device_map"]) + return model + + monkeypatch.setattr("drift.node.local_inference.ManifestArtifactVerifier", Verifier) + monkeypatch.setattr(AutoConfig, "from_pretrained", lambda *args, **kwargs: SimpleNamespace()) + monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", load_model) + monkeypatch.setattr(AutoTokenizer, "from_pretrained", lambda *args, **kwargs: object()) + runtime = make_local_manifest_loader(model_manifest, config)() + try: + assert devices == [{"": "cpu"}] + assert runtime.route_health()["device"] == "cpu" + finally: + runtime.close() + + +def test_auto_moves_up_and_down_without_replacing_active_runtime(): + remote = {"status": "incomplete", "covered_blocks": 0, "total_blocks": 64, "peer_count": 0, "source": "discovery"} + local = {"status": "complete", "covered_blocks": 24, "total_blocks": 24, "peer_count": 0, "source": "local"} + manager = ModelManager(max_loaded_models=2) + manager.register( + ModelDescriptor("local-qwen", execution="local"), + lambda: ModelRuntime("local", None), + route_health=lambda: local, + ) + manager.register(ModelDescriptor("qwen38"), lambda: ModelRuntime("remote", None), route_health=lambda: remote) + manager.configure_auto_selection(["local-qwen", "qwen38"]) + old = manager.load("auto") + assert old.runtime.model == "local" + remote.update(status="complete", covered_blocks=64, peer_count=4) + with manager.load("auto") as new: + assert new.runtime.model == "remote" + assert old.runtime.model == "local" + remote.update(status="incomplete", covered_blocks=48) + assert manager.resolve("auto").model_id == "local-qwen" + manager.configure_auto_selection(["qwen38", "local-qwen"], local_only=True) + remote.update(status="complete", covered_blocks=64) + assert manager.resolve("auto").model_id == "local-qwen" + old.release() + manager.shutdown() diff --git a/tests/test_login_startup_linux.py b/tests/test_login_startup_linux.py new file mode 100644 index 000000000..81e49674f --- /dev/null +++ b/tests/test_login_startup_linux.py @@ -0,0 +1,84 @@ +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "qualify_login_startup_linux.py" +spec = importlib.util.spec_from_file_location("qualify_login_startup_linux", SCRIPT) +replay = importlib.util.module_from_spec(spec) +spec.loader.exec_module(replay) + + +class Node: + def __init__(self, name="", role="application", children=(), pid=0): + self.name, self.role, self.nodes, self.pid = name, role, list(children), pid + + @property + def childCount(self): + return len(self.nodes) + + def getChildAtIndex(self, index): + return self.nodes[index] + + def getRoleName(self): + return self.role + + def get_process_id(self): + return self.pid + + +def test_pid_selection_does_not_trust_application_title(): + wrong = Node("CommunityAI", pid=42) + owned = Node("CommunityAI", pid=99) + assert replay.select_application(Node(children=[wrong, owned]), 99) is owned + with pytest.raises(RuntimeError, match="exactly one"): + replay.select_application(Node(children=[wrong]), 99) + + +def test_checkbox_lookup_requires_unique_name_and_role(): + title = Node(replay.CHECKBOX_NAME, "label") + checkbox = Node(replay.CHECKBOX_NAME, "check box") + app = Node(children=[title, Node(children=[checkbox])]) + assert replay.find_named(app, replay.CHECKBOX_NAME, {"check box"}) is checkbox + app.nodes.append(Node(replay.CHECKBOX_NAME, "check box")) + with pytest.raises(RuntimeError, match="found 2"): + replay.find_named(app, replay.CHECKBOX_NAME, {"check box"}) + + +def test_malformed_or_cyclic_accessibility_tree_is_bounded(monkeypatch): + monkeypatch.setattr(replay, "MAX_ACCESSIBLE_NODES", 4) + cyclic = Node() + cyclic.nodes.append(cyclic) + with pytest.raises(RuntimeError, match="tree exceeds"): + replay.find_named(cyclic, "missing", {"check box"}) + + +@pytest.mark.parametrize("display", [None, "", ":0", ":0.0", "localhost:10.0", "host:99", ":99; command"]) +def test_inherited_or_nonlocal_display_is_refused(display): + with pytest.raises(RuntimeError, match="private numbered"): + replay.private_display(display) + + +def test_private_xvfb_display_format(): + assert replay.private_display(":99") == ":99" + + +def test_qt_controls_can_expose_both_press_and_toggle_without_ambiguity(): + class Action: + nActions = 3 + selected = [] + + def getName(self, index): + return ["Toggle", "Press", "SetFocus"][index] + + def doAction(self, index): + self.selected.append(index) + return True + + action = Action() + control = type("Control", (), {"queryAction": lambda self: action})() + assert replay.invoke_action(control, "Press") == "Press" + assert replay.invoke_action(control, "Toggle") == "Toggle" + assert action.selected == [1, 0] + with pytest.raises(RuntimeError, match="Expected one"): + replay.invoke_action(control, "UnreportedAction") diff --git a/tests/test_login_startup_windows_cycle.py b/tests/test_login_startup_windows_cycle.py new file mode 100644 index 000000000..8452e01e1 --- /dev/null +++ b/tests/test_login_startup_windows_cycle.py @@ -0,0 +1,149 @@ +"""Cycle finalization without launching a GUI or changing native user state.""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "desktop" / "src")) +from qualify_login_startup_windows_cycle import CredentialMissingError, finalize_cycle +from qualify_login_startup_windows_state import RunStateConflict, RunValueGuard + + +class Guard: + def __init__(self, calls, error=None): + self.calls, self.error = calls, error + + def restore(self): + self.calls.append("restore") + if self.error: + raise self.error + return False + + +class Store: + def __init__(self, calls, *, delete_error=None, read_error=None, remains=False): + self.calls, self.delete_error, self.read_error, self.remains = calls, delete_error, read_error, remains + + def delete(self): + self.calls.append("delete") + if self.delete_error: + raise self.delete_error + + def get(self): + self.calls.append("get") + if self.read_error: + raise self.read_error + if self.remains: + return "qualification-sentinel" + raise CredentialMissingError("qualification account absent") + + +def result_with(stopped=True): + return {"result": "passed", "phases": [{"owned_processes_stopped": stopped}]} + + +def finish(tmp_path, result, guard, store, created=True): + finalize_cycle(result, tmp_path, guard, store, created) + assert json.loads((tmp_path / "result.json").read_text()) == result + + +def test_original_is_restored_before_private_credential_is_removed(tmp_path): + calls = [] + result = result_with() + finish(tmp_path, result, Guard(calls), Store(calls)) + assert result["result"] == "passed" + assert result["original_run_state_restored"] is True + assert result["private_credential_removed"] is True + assert calls == ["restore", "delete", "get"] + + +@pytest.mark.parametrize("stopped", [False, None, 1]) +def test_failed_or_unknown_shutdown_prevents_restoration_and_credential_deletion(tmp_path, stopped): + calls = [] + result = result_with(stopped) + finish(tmp_path, result, Guard(calls), Store(calls)) + assert result["result"] == "failed" + assert result["original_run_state_restored"] is False + assert result["private_credential_retained_for_live_processes"] is True + assert calls == [] + + +def test_one_unverified_phase_prevents_restore_even_when_other_phase_stopped(tmp_path): + calls = [] + result = result_with() + result["phases"].append({}) + finish(tmp_path, result, Guard(calls), Store(calls)) + assert result["result"] == "failed" + assert result["owned_processes_stopped"] is False + assert calls == [] + + +def test_prelaunch_failure_still_restores_and_cleans_without_claiming_acceptance(tmp_path): + calls = [] + result = {"result": "failed", "phases": []} + finish(tmp_path, result, Guard(calls), Store(calls)) + assert result["result"] == "failed" + assert result["original_run_state_restored"] is True + assert result["private_credential_removed"] is True + assert calls == ["restore", "delete", "get"] + + +@pytest.mark.parametrize("error", [RunStateConflict("qualification conflict"), OSError("qualification restore error")]) +def test_restoration_failure_retains_evidence_and_still_cleans_stopped_runtime_credential(tmp_path, error): + calls = [] + result = result_with() + finish(tmp_path, result, Guard(calls, error), Store(calls)) + assert result["result"] == "failed" + assert result["original_run_state_restored"] is False + assert result["restoration_error_type"] == type(error).__name__ + assert result["private_credential_removed"] is True + assert calls == ["restore", "delete", "get"] + + +@pytest.mark.parametrize("problem", ["delete", "read", "remains"]) +def test_credential_cleanup_error_downgrades_pass_and_preserves_result(tmp_path, problem): + calls = [] + result = result_with() + store = Store( + calls, + delete_error=OSError("qualification delete error") if problem == "delete" else None, + read_error=OSError("qualification read error") if problem == "read" else None, + remains=problem == "remains", + ) + finish(tmp_path, result, Guard(calls), store) + assert result["result"] == "failed" + assert result["original_run_state_restored"] is True + assert result["private_credential_removed"] is False + + +def test_uncreated_credential_is_never_touched(tmp_path): + calls = [] + result = result_with() + finish(tmp_path, result, Guard(calls), Store(calls), created=False) + assert calls == ["restore"] + + +def test_actual_state_guard_conflict_leaves_unrelated_run_value_untouched(tmp_path): + calls = [] + values = [None] + + def write(value): + calls.append("registry write") + values[0] = value + + def delete(): + calls.append("registry delete") + values[0] = None + + guard = RunValueGuard(lambda: values[0], write, delete) + guard.expect(("qualification executable", 1)) + values[0] = ("concurrent executable", 1) + result = result_with() + finish(tmp_path, result, guard, Store(calls)) + assert result["result"] == "failed" + assert result["restoration_error_type"] == "RunStateConflict" + assert values[0] == ("concurrent executable", 1) + assert calls == ["delete", "get"] diff --git a/tests/test_login_startup_windows_safety.py b/tests/test_login_startup_windows_safety.py new file mode 100644 index 000000000..51723c993 --- /dev/null +++ b/tests/test_login_startup_windows_safety.py @@ -0,0 +1,189 @@ +"""Qualification cleanup checks without launching Qt or touching user state.""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "desktop" / "src")) +import qualify_login_startup_windows as replay + + +class Store: + def __init__(self, *, delete_error=None, read_error=None, remains=False): + self.delete_error, self.read_error, self.remains = delete_error, read_error, remains + self.calls = [] + + def delete(self): + self.calls.append("delete") + if self.delete_error: + raise self.delete_error + return True + + def get(self): + self.calls.append("get") + if self.read_error: + raise self.read_error + if self.remains: + return "test-only-sentinel" + raise replay.CredentialMissingError("test account absent") + + +def finish(tmp_path, store, *, stopped=True, created=True, original=None, read_run=lambda: None): + result = {"result": "passed", "owned_processes_stopped": stopped} + replay.finalize_evidence(result, tmp_path, store, created, original, [(12, 120)], read_run=read_run) + assert json.loads((tmp_path / "result.json").read_text()) == json.loads(json.dumps(result)) + return result + + +def test_success_requires_credential_absence_after_owned_process_cleanup(tmp_path): + store = Store() + result = finish(tmp_path, store) + assert result["result"] == "passed" + assert result["run_entry_unchanged"] is True + assert result["private_credential_removed"] is True + assert store.calls == ["delete", "get"] + + +@pytest.mark.parametrize("stopped", [False, None]) +def test_live_or_unverified_runtime_preserves_private_credential(tmp_path, stopped): + store = Store() + result = finish(tmp_path, store, stopped=stopped) + assert store.calls == [] + assert not result.get("private_credential_removed", False) + assert result["result"] == "failed" + + +def test_uncreated_credential_is_never_deleted(tmp_path): + store = Store() + finish(tmp_path, store, created=False) + assert store.calls == [] + + +@pytest.mark.parametrize("proof", [None, "False", "unknown"]) +def test_missing_job_cleanup_proof_retains_credential_despite_recorded_pid_absence(tmp_path, proof): + if proof is not None: + (tmp_path / "helper-result.txt").write_text(f"job_cleanup_verified={proof}\n") + result = {"result": "passed", "owned_processes_stopped": True, "helper_containment_required": True} + store = Store() + replay.finalize_evidence(result, tmp_path, store, True, None, [], read_run=lambda: None) + saved = json.loads((tmp_path / "result.json").read_text()) + assert saved["result"] == "failed" + assert saved["owned_processes_stopped"] is False + assert saved["private_credential_retained_for_live_processes"] is True + assert store.calls == [] + + +def test_verified_emergency_job_cleanup_allows_credential_removal_but_cannot_pass(tmp_path): + (tmp_path / "helper-result.txt").write_text("result=failed\njob_cleanup_verified=True\n") + result = {"result": "failed", "owned_processes_stopped": True, "helper_containment_required": True} + store = Store() + replay.finalize_evidence(result, tmp_path, store, True, None, [], read_run=lambda: None) + assert result["result"] == "failed" + assert result["private_credential_removed"] is True + assert store.calls == ["delete", "get"] + + +@pytest.mark.parametrize( + "store", + [ + Store(delete_error=RuntimeError("delete failed")), + Store(read_error=RuntimeError("read failed")), + Store(remains=True), + ], +) +def test_credential_cleanup_failure_cannot_pass_or_skip_evidence(tmp_path, store): + result = finish(tmp_path, store) + assert result["result"] == "failed" + assert not result.get("private_credential_removed", False) + + +def test_registry_read_failure_keeps_evidence_and_still_cleans_owned_credential(tmp_path): + def denied(): + raise PermissionError("registry unreadable") + + store = Store() + result = finish(tmp_path, store, read_run=denied) + assert result["result"] == "failed" + assert result["private_credential_removed"] is True + assert store.calls == ["delete", "get"] + + +@pytest.mark.parametrize("current", [None, ("original", 2), ("changed", 1)]) +def test_registry_check_preserves_exact_original_value_and_type(tmp_path, current): + result = finish(tmp_path, Store(), original=("original", 1), read_run=lambda: current) + assert result["result"] == "failed" + assert result["run_entry_unchanged"] is False + + +class Process: + def __init__(self, pid, created, children=()): + self.pid, self.created, self.descendants = pid, created, children + self.children_read = False + + def create_time(self): + return self.created + + def is_running(self): + return True + + def status(self): + return replay.psutil.STATUS_RUNNING + + def children(self, recursive=False): + assert recursive + self.children_read = True + return self.descendants + + +def processes(monkeypatch, values): + lookup = {value.pid: value for value in values} + + def find(pid): + try: + return lookup[pid] + except KeyError: + raise replay.psutil.NoSuchProcess(pid) from None + + monkeypatch.setattr(replay.psutil, "Process", find) + + +def test_missing_or_reused_gui_pid_is_not_treated_as_owned(monkeypatch): + processes(monkeypatch, [Process(12, 999)]) + assert not replay.owned_gui_is_live(None) + assert not replay.owned_gui_is_live((10, 100)) + assert not replay.owned_gui_is_live((12, 120)) + assert replay.owned_gui_is_live((12, 999)) + + +def test_descendant_discovery_continues_when_initial_helper_has_exited(monkeypatch): + grandchild = Process(13, 130) + child = Process(12, 120, [grandchild]) + processes(monkeypatch, [child, grandchild]) + identities = [(10, 100), (12, 120)] + replay.refresh_owned_identities(identities) + assert (13, 130) in identities + assert child.children_read + + +def test_pid_reuse_does_not_adopt_unrelated_descendants(monkeypatch): + unrelated = Process(13, 130) + reused = Process(12, 999, [unrelated]) + processes(monkeypatch, [reused, unrelated]) + identities = [(12, 120)] + replay.refresh_owned_identities(identities) + assert set(identities) == {(12, 120)} + assert not reused.children_read + + +def test_unreadable_process_ownership_is_not_reported_as_absent(monkeypatch): + def denied(_): + raise replay.psutil.AccessDenied() + + monkeypatch.setattr(replay.psutil, "Process", denied) + with pytest.raises(replay.psutil.AccessDenied): + replay.owned_gui_is_live((12, 120)) + with pytest.raises(replay.psutil.AccessDenied): + replay.refresh_owned_identities([(12, 120)]) diff --git a/tests/test_login_startup_windows_state.py b/tests/test_login_startup_windows_state.py new file mode 100644 index 000000000..87a752584 --- /dev/null +++ b/tests/test_login_startup_windows_state.py @@ -0,0 +1,163 @@ +"""Exercise Run backup/restore without touching a registry or launching an app.""" + +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from qualify_login_startup_windows_state import RunStateConflict, RunStateVerificationError, RunValueGuard + +ORIGINAL = ("original executable --started-at-login", 2) +QUALIFICATION = ("qualified executable --started-at-login", 1) + + +class Registry: + def __init__(self, value): + self.value = deepcopy(value) + self.calls = [] + + def read(self): + self.calls.append("read") + return deepcopy(self.value) + + def write(self, value): + self.calls.append(("write", deepcopy(value))) + self.value = deepcopy(value) + + def delete(self): + self.calls.append("delete") + self.value = None + + +def guard_for(registry): + return RunValueGuard(registry.read, registry.write, registry.delete) + + +def test_original_absence_is_restored_by_deleting_only_the_expected_value(): + registry = Registry(None) + guard = guard_for(registry) + guard.expect(QUALIFICATION) + registry.value = QUALIFICATION + assert guard.restore() is True + assert registry.value is None + assert registry.calls == ["read", "read", "delete", "read"] + + +@pytest.mark.parametrize("current", [QUALIFICATION, None]) +def test_exact_original_data_and_type_are_restored_from_enabled_or_disabled_state(current): + registry = Registry(ORIGINAL) + guard = guard_for(registry) + guard.expect(QUALIFICATION) + guard.expect(None) + registry.value = current + assert guard.restore() is True + assert registry.value == ORIGINAL + assert registry.calls[-2:] == [("write", ORIGINAL), "read"] + + +@pytest.mark.parametrize("original", [None, ORIGINAL]) +def test_unchanged_original_is_verified_without_any_write(original): + registry = Registry(original) + guard = guard_for(registry) + assert guard.restore() is False + assert registry.calls == ["read", "read", "read"] + + +@pytest.mark.parametrize("unrelated", [("another executable", 1), (QUALIFICATION[0], 2), None]) +def test_unknown_concurrent_data_type_or_unregistered_absence_is_not_overwritten(unrelated): + registry = Registry(ORIGINAL) + guard = guard_for(registry) + guard.expect(QUALIFICATION) + registry.value = unrelated + with pytest.raises(RunStateConflict): + guard.restore() + assert registry.value == unrelated + assert registry.calls == ["read", "read"] + + +def test_snapshot_and_expectations_are_not_changed_through_mutable_value_aliases(): + registry = Registry((["original"], 7)) + guard = guard_for(registry) + expected = (["qualification"], 7) + guard.expect(expected) + expected[0].append("unrelated") + guard.original[0].append("unrelated") + registry.value = (["qualification"], 7) + guard.restore() + assert registry.value == (["original"], 7) + + +def test_verification_is_read_only_and_detects_registry_type_mismatch(): + registry = Registry(QUALIFICATION) + guard = guard_for(registry) + guard.verify(QUALIFICATION) + with pytest.raises(RunStateVerificationError): + guard.verify((QUALIFICATION[0], 2)) + assert registry.calls == ["read", "read", "read"] + + +def test_restore_does_not_claim_success_when_delete_silently_fails(): + registry = Registry(None) + guard = RunValueGuard(registry.read, registry.write, lambda: registry.calls.append("delete")) + guard.expect(QUALIFICATION) + registry.value = QUALIFICATION + with pytest.raises(RunStateVerificationError): + guard.restore() + assert registry.calls[-2:] == ["delete", "read"] + + +def test_restore_verifies_final_state_even_when_write_reports_an_error(): + registry = Registry(ORIGINAL) + + def write_then_fail(value): + registry.write(value) + raise OSError("qualification write error") + + guard = RunValueGuard(registry.read, write_then_fail, registry.delete) + guard.expect(QUALIFICATION) + registry.value = QUALIFICATION + with pytest.raises(OSError, match="qualification write error"): + guard.restore() + assert registry.value == ORIGINAL + assert registry.calls[-2:] == [("write", ORIGINAL), "read"] + + +def test_failed_read_cannot_trigger_a_restore_write(): + registry = Registry(ORIGINAL) + available = True + + def read(): + if not available: + raise PermissionError("qualification read error") + return registry.read() + + guard = RunValueGuard(read, registry.write, registry.delete) + guard.expect(QUALIFICATION) + registry.value = QUALIFICATION + available = False + with pytest.raises(PermissionError): + guard.restore() + assert registry.value == QUALIFICATION + assert registry.calls == ["read"] + + +def test_concurrent_change_during_final_verification_is_reported_without_second_write(): + registry = Registry(None) + reads = 0 + + def read(): + nonlocal reads + reads += 1 + if reads == 3: + registry.value = ("concurrent executable", 1) + return registry.read() + + guard = RunValueGuard(read, registry.write, registry.delete) + guard.expect(QUALIFICATION) + registry.value = QUALIFICATION + with pytest.raises(RunStateVerificationError): + guard.restore() + assert registry.value == ("concurrent executable", 1) + assert registry.calls.count("delete") == 1 diff --git a/tests/test_manifest_cache_budget.py b/tests/test_manifest_cache_budget.py new file mode 100644 index 000000000..9c0ae27f4 --- /dev/null +++ b/tests/test_manifest_cache_budget.py @@ -0,0 +1,45 @@ +from types import SimpleNamespace + +import pytest + +from drift.utils import disk_cache + + +def _cache_info(monkeypatch, *, size=0, files=()): + revisions = [SimpleNamespace(files=files)] if files else [] + repos = [SimpleNamespace(revisions=revisions)] if revisions else [] + monkeypatch.setattr( + disk_cache.huggingface_hub, "scan_cache_dir", lambda _: SimpleNamespace(size_on_disk=size, repos=repos) + ) + + +def test_manifest_snapshots_and_partials_count_toward_download_budget(tmp_path, monkeypatch): + _cache_info(monkeypatch) + snapshot = tmp_path / "manifest-artifacts" / "model" / "snapshot" + partials = snapshot.parent / "partial" + snapshot.mkdir(parents=True) + partials.mkdir() + (snapshot / "weights").write_bytes(b"x" * 24) + (partials / "next.part").write_bytes(b"x" * 8) + with pytest.raises(RuntimeError, match="Insufficient disk space"): + disk_cache.free_disk_space_for(16, cache_dir=tmp_path, max_disk_space=40, os_quota=0) + assert (snapshot / "weights").stat().st_size == 24 + assert (partials / "next.part").stat().st_size == 8 + + +def test_shared_hub_and_manifest_blob_is_counted_once_and_cannot_be_evicted(tmp_path, monkeypatch): + import os + + blob = tmp_path / "blob" + blob.write_bytes(b"x" * 32) + snapshot = tmp_path / "manifest-artifacts" / "model" / "snapshot" + snapshot.mkdir(parents=True) + os.link(blob, snapshot / "weights") + entry = SimpleNamespace(blob_path=blob, file_path=tmp_path / "pointer", size_on_disk=32, blob_last_accessed=0) + entry.file_path.write_bytes(b"pointer") + _cache_info(monkeypatch, size=32, files=[entry]) + disk_cache.free_disk_space_for(8, cache_dir=tmp_path, max_disk_space=40, os_quota=0) + with pytest.raises(RuntimeError, match="Insufficient disk space"): + disk_cache.free_disk_space_for(16, cache_dir=tmp_path, max_disk_space=40, os_quota=0) + assert blob.exists() and (snapshot / "weights").exists() + assert entry.file_path.exists() diff --git a/tests/test_manifest_resume_validation.py b/tests/test_manifest_resume_validation.py new file mode 100644 index 000000000..e36b851f4 --- /dev/null +++ b/tests/test_manifest_resume_validation.py @@ -0,0 +1,33 @@ +import pytest + +from drift.utils.hub_ranges import RANGE_BYTES +from scripts.validate_manifest_resume import validate_resume_observations + + +def test_accepts_exact_parallel_bounded_resume_ranges_in_response_order(): + size = RANGE_BYTES * 3 + observations = [ + {"requested_range": f"bytes={start}-{end}", "status": 206, "content_range": f"bytes {start}-{end}/{size}"} + for start, end in [ + (RANGE_BYTES + 100, RANGE_BYTES * 2 + 99), + (100, RANGE_BYTES + 99), + (RANGE_BYTES * 2 + 100, size - 1), + ] + ] + validate_resume_observations(observations, prefix_size=100, artifact_size=size) + for altered in ( + observations[:-1], + observations + observations[:1], + [{**item, "status": 200} for item in observations], + ): + with pytest.raises(RuntimeError): + validate_resume_observations(altered, prefix_size=100, artifact_size=size) + + +def test_small_artifact_requires_exact_open_ended_resume(): + valid = [{"requested_range": "bytes=100-", "status": 206, "content_range": "bytes 100-199/200"}] + validate_resume_observations(valid, prefix_size=100, artifact_size=200) + with pytest.raises(RuntimeError): + validate_resume_observations( + [{**valid[0], "content_range": "bytes 99-199/200"}], prefix_size=100, artifact_size=200 + ) diff --git a/tests/test_measured_model_selection.py b/tests/test_measured_model_selection.py new file mode 100644 index 000000000..4b27b5100 --- /dev/null +++ b/tests/test_measured_model_selection.py @@ -0,0 +1,139 @@ +import threading +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from test_catalog_bootstrap import _release_documents + +from drift.model_catalog import CatalogRung +from drift.node.model_manager import ModelDescriptor, ModelManager, ModelManagerClosedError, ModelRuntime +from drift.node.model_selection import MeasuredModelSelector, RouteProbeService +from drift.node.route_health import _coverage_health + + +def test_peer_disjoint_routes_and_largest_peer_loss(): + complete = _coverage_health([{1, 3}, {1, 3}, {2, 4}, {2, 4}], updated_age=0) + assert complete["minimum_replicas"] == complete["independent_routes"] == 2 + assert complete["replicas_after_largest_peer_loss"] == 1 + # Two replicas per block do not imply two routes with independent peers. + overlapping = _coverage_health([{1, 2}, {2, 3}, {1, 3}], updated_age=0) + assert overlapping["minimum_replicas"] == 2 + assert overlapping["independent_routes"] == 1 + assert _coverage_health([{1}, {2}], updated_age=0)["replicas_after_largest_peer_loss"] == 0 + assert complete["coverage_fingerprint"] != overlapping["coverage_fingerprint"] + + +def test_probe_soak_staleness_latency_and_peer_change_gate_promotion(): + _, envelope, _ = _release_documents() + catalog = envelope.signed + digest = catalog.models[0].manifest_digest + health = _coverage_health([{1, 2}, {1, 2}], updated_age=0.5) + now = [2_000_000_000.0] + selector = MeasuredModelSelector(catalog, lambda _: health, clock=lambda: now[0]) + assert selector.selection()[0] is None + target = selector.probe_target() + assert target == (digest, health["coverage_fingerprint"]) + assert selector.record_probe(digest, target[1], first_token_seconds=0.2, completion_tokens=3, duration_seconds=1) + assert selector.selection()[0] is None # Coverage has not soaked yet. + for _ in range(6): + now[0] += 10 + selector.selection() + assert selector.selection()[0].manifest_digest == digest + health["last_updated_age"] = 31 + assert selector.selection()[0] is None + health["last_updated_age"] = 0 + target = selector.probe_target() + selector.record_probe(digest, target[1], first_token_seconds=3, completion_tokens=3, duration_seconds=4) + for _ in range(6): + now[0] += 10 + selector.selection() + assert selector.selection()[0] is None # Real latency misses signed budget. + health.update(_coverage_health([{3, 4}, {3, 4}], updated_age=0)) + assert not selector.record_probe( + digest, target[1], first_token_seconds=0.1, completion_tokens=3, duration_seconds=1 + ) + assert selector.selection()[0] is None + + +def test_best_effort_is_an_explicit_catalog_policy_and_local_needs_no_peer_probe(): + _, envelope, _ = _release_documents() + rung = replace( + envelope.signed.rungs[0], + minimum_replicas=1, + minimum_independent_routes=1, + minimum_surviving_replicas=0, + minimum_soak_seconds=0, + ) + assert CatalogRung.from_dict(rung.to_dict(), index=0).minimum_surviving_replicas == 0 + local, remote = envelope.signed.models + catalog = replace(envelope.signed, rungs=(rung,), models=(replace(local, execution="local"), remote)) + health = _coverage_health([{1}, {2}], updated_age=0) + + def read(digest): + assert digest == remote.manifest_digest + return health + + selector = MeasuredModelSelector(catalog, read, clock=lambda: 2_000_000_000) + digest, fingerprint = selector.probe_target() + selector.record_probe(digest, fingerprint, first_token_seconds=0.1, completion_tokens=3, duration_seconds=1) + assert selector.selection()[0] == remote + + +def test_catalog_restart_waits_for_lease_and_atomically_rejects_new_loads(): + manager = ModelManager() + closed = [] + manager.register( + ModelDescriptor("local"), lambda: ModelRuntime(object(), object(), close=lambda: closed.append(True)) + ) + lease = manager.load("local") + assert not manager.begin_idle_restart() + assert lease.runtime.model is not None + lease.release() + assert manager.begin_idle_restart() + with pytest.raises(ModelManagerClosedError): + manager.load("local") + manager.shutdown() + assert closed == [True] + + +def test_busy_generation_keeps_observing_fresh_coverage_and_still_rejects_real_gaps(): + _, envelope, _ = _release_documents() + now = [2_000_000_000.0] + health = _coverage_health([{1, 2}, {1, 2}], updated_age=0) + selector = MeasuredModelSelector(envelope.signed, lambda _: health, clock=lambda: now[0]) + digest, fingerprint = selector.probe_target() + selector.record_probe(digest, fingerprint, first_token_seconds=0.1, completion_tokens=3, duration_seconds=1) + observed, gap_seen = threading.Event(), threading.Event() + reads = [] + + def read(_): + now[0] += 5 # Fresh observations throughout a generation longer than 30 seconds. + reads.append(now[0]) + if len(reads) >= 20: + observed.set() + if health["status"] == "unknown": + gap_seen.set() + return dict(health) + + selector._health_reader = read + + class BusyManager: + inference_mode = "auto" + + def snapshots(self): + return [SimpleNamespace(active_requests=1)] + + def load(self, _): + raise AssertionError("A busy generation must not start another probe") + + service = RouteProbeService(BusyManager(), selector, period=0.005) + service.start() + try: + assert observed.wait(2), "Route observations stopped while generation was busy" + assert selector.selection()[0] is not None + assert selector._measurements[digest].samples == 1 + health["status"] = "unknown" + assert gap_seen.wait(2) + assert selector.selection()[0] is None + finally: + service.close() diff --git a/tests/test_model_catalog.py b/tests/test_model_catalog.py index f4a7520a8..d608d09a6 100644 --- a/tests/test_model_catalog.py +++ b/tests/test_model_catalog.py @@ -114,11 +114,14 @@ def observation( ) -def test_catalog_requires_two_options_and_one_primary_per_rung(): +def test_catalog_allows_one_primary_without_a_standby_and_rejects_two_primaries(): source = catalog_dict() - source["models"] = source["models"][:1] + source["models"][2:] - with pytest.raises(ModelCatalogError, match="at least two model options"): - ModelCatalog.from_dict(source) + source["models"] = [source["models"][0], source["models"][2]] + parsed = ModelCatalog.from_dict(source) + assert [model.role for model in parsed.models] == ["primary", "primary"] + selected, _ = select_highest_eligible_model(parsed, [observation("a"), observation("c")], now=NOW) + assert selected is not None + assert selected.manifest_digest == digest("c") source = catalog_dict() source["models"][1]["role"] = "primary" diff --git a/tests/test_model_manifest.py b/tests/test_model_manifest.py index ae42ec788..de1e2e298 100644 --- a/tests/test_model_manifest.py +++ b/tests/test_model_manifest.py @@ -1,5 +1,6 @@ import hashlib import json +import os from pathlib import Path from types import SimpleNamespace @@ -11,8 +12,11 @@ ModelManifest, create_manifest_from_snapshot, resolve_manifest_loading, + select_manifest_block_artifacts, ) +from drift.server import from_pretrained as from_pretrained_module from drift.server.handler import TransformerConnectionHandler +from drift.server.server import _scoped_manifest_artifact_verifier def test_qwen3_first_rung_candidate_is_exactly_pinned(): @@ -64,6 +68,77 @@ def test_qwen3_5_edge_primary_candidate_is_exactly_pinned(): } +def test_qwen3_8_27b_bf16_reference_is_exactly_pinned(): + reference = Path(__file__).resolve().parents[1] / "manifests" / "reference" / "qwen3.8-27b-bfloat16-eager.json" + manifest = ModelManifest.load(reference) + + assert manifest.name == "Qwen3.8 27B BF16 Reference" + assert manifest.aliases == ("qwen3.8-27b-bf16-reference",) + assert manifest.source.repository == "Qwen/Qwen3.8-27B" + assert manifest.source.revision == "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + assert manifest.model.architecture == "Qwen3_5ForConditionalGeneration" + assert manifest.model.num_blocks == 64 + assert manifest.model.context_length == 262144 + assert manifest.model.license == "apache-2.0" + assert manifest.model.gated is False + assert manifest.runtime.dtype == "bfloat16" + assert manifest.runtime.attention_implementation == "eager" + assert manifest.runtime.quantization == "none" + assert manifest.digest_id == "sha256:3d70e5be1eb079143b82a139e12823529d1294810f1df0265ba6aa10e7a48c0e" + assert len(manifest.artifacts) == 25 + assert sum(artifact.size for artifact in manifest.artifacts) == 55_586_035_522 + assert sum(artifact.size for artifact in manifest.artifacts if artifact.role == "weight") == 55_563_006_776 + + +def test_qwen3_8_27b_fp8_dequant_candidate_is_exactly_pinned(): + candidate = Path(__file__).resolve().parents[1] / "manifests" / "candidates" / "qwen3.8-27b-fp8-dequant-eager.json" + manifest = ModelManifest.load(candidate) + + assert manifest.name == "Qwen3.8 27B FP8 Dequant" + assert manifest.aliases == ("qwen3.8-27b", "qwen3.8-27b-fp8") + assert manifest.source.repository == "Qwen/Qwen3.8-27B-FP8" + assert manifest.source.revision == "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a" + assert manifest.model.architecture == "Qwen3_5ForConditionalGeneration" + assert manifest.model.num_blocks == 64 + assert manifest.model.context_length == 262144 + assert manifest.model.license == "apache-2.0" + assert manifest.model.gated is False + assert manifest.runtime.dtype == "bfloat16" + assert manifest.runtime.quantization == "fp8_dequant" + assert manifest.digest_id == "sha256:c4dfe76969bd769bf4b6bd28d08961a97eb2d73d588187c8dd4b9aa40b1055a4" + assert len(manifest.artifacts) == 73 + assert sum(artifact.size for artifact in manifest.artifacts) == 30_889_967_831 + assert sum(artifact.size for artifact in manifest.artifacts if artifact.role == "weight") == 30_866_866_928 + + +def test_qwen3_8_16_block_worker_plan_excludes_outside_mtp_and_other_layers(): + candidate = Path(__file__).resolve().parents[1] / "manifests" / "candidates" / "qwen3.8-27b-fp8-dequant-eager.json" + manifest = ModelManifest.load(candidate) + weight_map = {f"model.language_model.layers.{index}.weight": f"layers-{index}.safetensors" for index in range(64)} + weight_map.update( + { + "model.visual.weight": "outside.safetensors", + "model.mtp.weight": "mtp.safetensors", + } + ) + + plan = select_manifest_block_artifacts( + manifest, + block_prefix="model.language_model.layers", + start_block=16, + end_block=32, + weight_map=weight_map, + ) + + assert plan.artifact_bytes == 6_095_829_389 + assert plan.artifact_paths == ( + "config.json", + *(f"layers-{index}.safetensors" for index in range(16, 32)), + "model.safetensors.index.json", + ) + assert {"outside.safetensors", "mtp.safetensors", "tokenizer.json"}.isdisjoint(plan.artifact_paths) + + def test_gemma4_edge_standby_candidate_is_exactly_pinned(): candidate = Path(__file__).resolve().parents[1] / "manifests" / "candidates" / "gemma-4-e2b-it-bfloat16-eager.json" manifest = ModelManifest.load(candidate) @@ -169,6 +244,385 @@ def create_test_snapshot_manifest(root: Path) -> ModelManifest: ) +def create_block_plan_snapshot( + root: Path, *, index_payload: bytes | None = None +) -> tuple[ModelManifest, dict[str, bytes]]: + if index_payload is None: + index_payload = json.dumps( + { + "metadata": {"format": "pt"}, + "weight_map": { + "model.layers.0.attn.weight": "shared.safetensors", + "model.layers.1.attn.weight": "shared.safetensors", + "model.layers.2.attn.weight": "layer-2.safetensors", + "model.embed_tokens.weight": "outside.safetensors", + "model.mtp.weight": "mtp.safetensors", + }, + }, + sort_keys=True, + ).encode() + payloads = { + "config.json": b"{}", + "model.safetensors.index.json": index_payload, + "shared.safetensors": b"shared", + "layer-2.safetensors": b"layer two", + "outside.safetensors": b"outside", + "mtp.safetensors": b"mtp", + "tokenizer.json": b"tokenizer", + } + roles = { + "config.json": "config", + "model.safetensors.index.json": "weight_index", + "tokenizer.json": "tokenizer", + } + for path, payload in payloads.items(): + (root / path).write_bytes(payload) + source = manifest_dict() + source["model"]["num_blocks"] = 3 + source["artifacts"] = [ + { + "role": roles.get(path, "weight"), + "path": path, + "sha256": hashlib.sha256(payload).hexdigest(), + "size": len(payload), + } + for path, payload in payloads.items() + ] + return ModelManifest.from_dict(source), payloads + + +def test_block_artifact_plan_deduplicates_shared_shards_and_excludes_unassigned_files(tmp_path): + manifest, payloads = create_block_plan_snapshot(tmp_path) + verifier = ManifestArtifactVerifier( + manifest, + manifest.source.repository, + manifest.source.revision, + artifact_root=tmp_path, + ) + + first = verifier.plan_block_artifacts(block_prefix="model.layers", start_block=0, end_block=2) + second = verifier.plan_block_artifacts(block_prefix="model.layers", start_block=1, end_block=3) + + assert first.artifact_paths == ( + "config.json", + "model.safetensors.index.json", + "shared.safetensors", + ) + assert first.artifact_bytes == sum(len(payloads[path]) for path in first.artifact_paths) + assert len(first.artifact_set_digest) == 64 + assert second.artifact_paths == ( + "config.json", + "layer-2.safetensors", + "model.safetensors.index.json", + "shared.safetensors", + ) + assert first.artifact_set_digest != second.artifact_set_digest + assert {"outside.safetensors", "mtp.safetensors", "tokenizer.json"}.isdisjoint(first.artifact_paths) + + +def test_unsharded_block_plan_counts_the_single_checkpoint_once(): + source = manifest_dict() + source["artifacts"][0]["path"] = "model.safetensors" + manifest = ModelManifest.from_dict(source) + + plan = select_manifest_block_artifacts( + manifest, + block_prefix="model.layers", + start_block=1, + end_block=7, + ) + + assert plan.artifact_paths == ("config.json", "model.safetensors") + assert plan.artifact_bytes == 4 + assert "tokenizer.json" not in plan.artifact_paths + + +def test_manifested_server_builds_an_exact_worker_scope(monkeypatch, tmp_path): + manifest, _ = create_block_plan_snapshot(tmp_path) + accesses = [] + original_ensure_path = ManifestArtifactVerifier.ensure_path + + def audited_ensure_path(self, path, **kwargs): + accesses.append((path, self.allowed_paths)) + assert self.allowed_paths is not None + assert path in self.allowed_paths + return original_ensure_path(self, path, **kwargs) + + monkeypatch.setattr(ManifestArtifactVerifier, "ensure_path", audited_ensure_path) + verifier = _scoped_manifest_artifact_verifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=False, + cache_dir=str(tmp_path), + max_disk_space=10_000, + block_prefix="model.layers", + block_indices=(0, 1), + artifact_root=tmp_path, + ) + + assert accesses + assert accesses[0][1] == frozenset({"config.json", "model.safetensors.index.json"}) + assert verifier.allowed_paths == frozenset({"config.json", "model.safetensors.index.json", "shared.safetensors"}) + monkeypatch.setattr(ManifestArtifactVerifier, "ensure_path", original_ensure_path) + with pytest.raises(ManifestError, match="outside this worker artifact plan"): + verifier.ensure_path("outside.safetensors") + with pytest.raises(ManifestError, match="contiguous block span"): + _scoped_manifest_artifact_verifier( + manifest, + repository=manifest.source.repository, + revision=manifest.source.revision, + token=False, + cache_dir=str(tmp_path), + max_disk_space=10_000, + block_prefix="model.layers", + block_indices=(0, 2), + artifact_root=tmp_path, + ) + + +def test_manifested_server_binds_acknowledged_worker_artifact_plan_before_weights(monkeypatch, tmp_path): + manifest, _ = create_block_plan_snapshot(tmp_path) + plan = select_manifest_block_artifacts( + manifest, + block_prefix="model.layers", + start_block=0, + end_block=2, + weight_map=json.loads((tmp_path / "model.safetensors.index.json").read_text(encoding="utf-8"))["weight_map"], + ) + weight_accesses = [] + original_ensure_path = ManifestArtifactVerifier.ensure_path + + def audited_ensure_path(self, path, **kwargs): + if manifest.get_artifact(path).role == "weight": + weight_accesses.append(path) + return original_ensure_path(self, path, **kwargs) + + monkeypatch.setattr(ManifestArtifactVerifier, "ensure_path", audited_ensure_path) + common = { + "repository": manifest.source.repository, + "revision": manifest.source.revision, + "token": False, + "cache_dir": str(tmp_path), + "max_disk_space": 10_000, + "block_prefix": "model.layers", + "block_indices": (0, 1), + "artifact_root": tmp_path, + "expected_manifest_digest": manifest.digest_id, + "expected_block_indices": "0:2", + "expected_artifact_bytes": plan.artifact_bytes, + "expected_artifact_set_digest": plan.artifact_set_digest, + "expected_cache_root": str(tmp_path.resolve()), + } + + verifier = _scoped_manifest_artifact_verifier(manifest, **common) + + assert verifier.allowed_paths == frozenset(plan.artifact_paths) + assert weight_accesses == [] + + for field, value, message in ( + ("expected_manifest_digest", "sha256:" + "0" * 64, "manifest digest"), + ("expected_block_indices", "1:2", "block span"), + ("expected_artifact_bytes", plan.artifact_bytes + 1, "byte count"), + ("expected_artifact_set_digest", "0" * 64, "plan digest"), + ("expected_cache_root", str(tmp_path.resolve() / "other"), "cache root"), + ): + changed = dict(common) + changed[field] = value + with pytest.raises(ManifestError, match=message): + _scoped_manifest_artifact_verifier(manifest, **changed) + assert weight_accesses == [] + + incomplete = dict(common) + incomplete["expected_artifact_set_digest"] = None + with pytest.raises(ManifestError, match="supplied together"): + _scoped_manifest_artifact_verifier(manifest, **incomplete) + assert weight_accesses == [] + + single_block_plan = select_manifest_block_artifacts( + manifest, + block_prefix="model.layers", + start_block=0, + end_block=1, + weight_map=json.loads((tmp_path / "model.safetensors.index.json").read_text(encoding="utf-8"))["weight_map"], + ) + same_artifacts_different_span = dict( + common, + block_indices=(1,), + expected_block_indices="0:1", + expected_artifact_bytes=single_block_plan.artifact_bytes, + expected_artifact_set_digest=single_block_plan.artifact_set_digest, + ) + with pytest.raises(ManifestError, match="block span"): + _scoped_manifest_artifact_verifier(manifest, **same_artifacts_different_span) + assert weight_accesses == [] + + +def test_module_container_checks_worker_plan_before_constructing_join_announcer(monkeypatch): + from drift.server import server as server_module + + events = [] + + def reject_plan(*args, **kwargs): + events.append("plan") + raise ManifestError("plan rejected") + + def construct_announcer(*args, **kwargs): + events.append("announcer") + raise AssertionError("announcer must not be constructed") + + monkeypatch.setattr(server_module, "_scoped_manifest_artifact_verifier", reject_plan) + monkeypatch.setattr(server_module, "ModuleAnnouncerThread", construct_announcer) + + with pytest.raises(ManifestError, match="plan rejected"): + server_module.ModuleContainer.create( + dht=None, + dht_prefix="test", + converted_model_name_or_path="org/model", + block_config=SimpleNamespace(block_prefix="model.layers"), + attn_cache_bytes=1, + server_info=SimpleNamespace(), + model_info=SimpleNamespace(), + block_indices=[0], + min_batch_size=1, + max_batch_size=1, + max_chunk_size_bytes=1, + max_alloc_timeout=1, + torch_dtype=None, + cache_dir=None, + max_disk_space=None, + device="cpu", + compression=None, + update_period=1, + expiration=None, + revision="a" * 40, + token=False, + quant_type=None, + tensor_parallel_devices=(None,), + ) + + assert events == ["plan"] + + +def test_worker_artifact_scope_fails_before_unassigned_access(tmp_path): + manifest, _ = create_block_plan_snapshot(tmp_path) + verifier = ManifestArtifactVerifier( + manifest, + manifest.source.repository, + manifest.source.revision, + artifact_root=tmp_path, + ) + plan = verifier.plan_block_artifacts(block_prefix="model.layers", start_block=0, end_block=2) + verifier.restrict_to_paths(plan.artifact_paths) + + assert verifier.ensure_path("shared.safetensors", allowed_roles={"weight"}) == tmp_path / "shared.safetensors" + for path in ("layer-2.safetensors", "outside.safetensors", "mtp.safetensors", "tokenizer.json"): + with pytest.raises(ManifestError, match="outside this worker artifact plan"): + verifier.ensure_path(path) + with pytest.raises(ManifestError, match="outside this worker artifact plan"): + verifier.partial_size(path) + with pytest.raises(ManifestError, match="outside this worker artifact plan"): + verifier.verify_resolved_file(tmp_path / path) + + +@pytest.mark.parametrize( + "weight_map,match", + [ + ({"model.layers.0.weight": "../shared.safetensors"}, "non-normalized"), + ({"model.layers.0.weight": "missing.safetensors"}, "not declared"), + ({"model.layers.0.weight": "tokenizer.json"}, "non-checkpoint role"), + ({"model.layers.0.weight": "shared.safetensors"}, "block prefix"), + ], +) +def test_block_artifact_plan_rejects_unsafe_or_incomplete_index_maps(tmp_path, weight_map, match): + manifest, _ = create_block_plan_snapshot(tmp_path) + + with pytest.raises(ManifestError, match=match): + select_manifest_block_artifacts( + manifest, + block_prefix="model.layers", + start_block=0, + end_block=2, + weight_map=weight_map, + ) + + +def test_weight_index_parser_rejects_duplicate_json_keys(tmp_path): + duplicate = ( + b'{"weight_map":{"model.layers.0.weight":"shared.safetensors",' + b'"model.layers.0.weight":"shared.safetensors"}}' + ) + manifest, _ = create_block_plan_snapshot(tmp_path, index_payload=duplicate) + verifier = ManifestArtifactVerifier( + manifest, + manifest.source.repository, + manifest.source.revision, + artifact_root=tmp_path, + ) + + with pytest.raises(ManifestError, match="duplicate object key"): + verifier.load_weight_map() + + +def test_manifested_block_loader_consumes_the_strict_in_memory_weight_map(monkeypatch, tmp_path): + manifest, _ = create_block_plan_snapshot(tmp_path) + verifier = ManifestArtifactVerifier( + manifest, + manifest.source.repository, + manifest.source.revision, + artifact_root=tmp_path, + ) + index_path = tmp_path / "model.safetensors.index.json" + index_reads = [] + original_read_bytes = Path.read_bytes + + def tracked_read_bytes(path): + if path == index_path: + index_reads.append(path) + return original_read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", tracked_read_bytes) + plan = verifier.plan_block_artifacts(block_prefix="model.layers", start_block=0, end_block=2) + verifier.restrict_to_paths(plan.artifact_paths) + loaded = [] + + def fake_load(_model, filename, *, block_prefix, **_kwargs): + loaded.append(filename) + return {f"{block_prefix}weight": object()} + + def forbid_permissive_open(*_args, **_kwargs): + raise AssertionError("manifested loader reopened the index pathname") + + monkeypatch.setattr(from_pretrained_module, "_load_state_dict_from_repo_file", fake_load) + monkeypatch.setattr(from_pretrained_module, "open", forbid_permissive_open, raising=False) + + state_dict = from_pretrained_module._load_state_dict_from_repo( + manifest.source.repository, + "model.layers.0.", + revision=manifest.source.revision, + token=False, + cache_dir=str(tmp_path), + artifact_verifier=verifier, + ) + + assert set(state_dict) == {"weight"} + assert state_dict["weight"] is not None + assert loaded == ["shared.safetensors"] + assert index_reads == [index_path] + weight_map = verifier.load_weight_map() + assert weight_map is verifier.load_weight_map() + with pytest.raises(TypeError): + weight_map["model.layers.0.weight"] = "outside.safetensors" + + +def test_manifest_rejects_case_colliding_artifact_paths(): + source = manifest_dict() + source["artifacts"].append({"role": "weight", "path": "Weights.bin", "sha256": "4" * 64, "size": 4}) + + with pytest.raises(ManifestError, match="collide case-insensitively"): + ModelManifest.from_dict(source) + + def test_digest_and_namespace_are_canonical_and_order_independent(): source = manifest_dict() first = ModelManifest.from_dict(source) @@ -258,6 +712,50 @@ def test_wrapper_manifest_validation_uses_preserved_source_architecture(): ) +def test_manifest_rejects_unimplemented_prequantized_source_profile(): + manifest = ModelManifest.from_dict(manifest_dict()) + + with pytest.raises(ManifestError, match="pre-quantized 'fp8'.*explicit compatible profile"): + manifest.validate_model_config( + SimpleNamespace( + architectures=["LlamaForCausalLM"], + num_hidden_layers=8, + max_position_embeddings=2048, + _source_quantization_method="fp8", + ) + ) + + +def test_manifest_accepts_finegrained_fp8_dequant_profile(): + source = manifest_dict() + source["runtime"]["quantization"] = "fp8_dequant" + manifest = ModelManifest.from_dict(source) + + manifest.validate_model_config( + SimpleNamespace( + architectures=["LlamaForCausalLM"], + num_hidden_layers=8, + max_position_embeddings=2048, + _source_quantization_method="fp8", + ) + ) + + +def test_manifest_rejects_fp8_dequant_profile_without_fp8_source_metadata(): + source = manifest_dict() + source["runtime"]["quantization"] = "fp8_dequant" + manifest = ModelManifest.from_dict(source) + + with pytest.raises(ManifestError, match="fp8_dequant.*requires source config quant_method='fp8'"): + manifest.validate_model_config( + SimpleNamespace( + architectures=["LlamaForCausalLM"], + num_hidden_layers=8, + max_position_embeddings=2048, + ) + ) + + def test_artifact_verification(tmp_path): files = { "config.json": b"c", @@ -414,7 +912,8 @@ def cache_miss(*args, **kwargs): monkeypatch.setattr( "huggingface_hub.hf_hub_url", lambda *args, **kwargs: f"http://127.0.0.1:{server.server_port}/weights.bin" ) - monkeypatch.setattr("drift.utils.disk_cache.free_disk_space_for", lambda *args, **kwargs: None) + reservations = [] + monkeypatch.setattr("drift.utils.disk_cache.free_disk_space_for", lambda size, **kwargs: reservations.append(size)) try: with pytest.raises(ManifestError, match="Interrupted download"): verifier.ensure_path("weights.bin", allowed_roles={"weight"}) @@ -425,6 +924,7 @@ def cache_miss(*args, **kwargs): assert verifier.ensure_path("weights.bin", allowed_roles={"weight"}) == final.absolute() assert final.read_bytes() == resumed_payload assert InterruptOnceHandler.requests == [None, f"bytes={1024 * 1024}-"] + assert reservations == [len(resumed_payload), len(resumed_payload) - 1024 * 1024] finally: server.shutdown() server.server_close() @@ -461,6 +961,34 @@ def replace_after_release(source, destination): assert not partial.exists() +@pytest.mark.skipif(os.name != "nt", reason="Win32 extended-length paths are Windows-specific") +def test_resumable_manifest_paths_work_beyond_legacy_windows_max_path(tmp_path): + from drift.utils.file_lock import file_lock + + manifest = ModelManifest.from_dict(manifest_dict()) + cache = tmp_path / ("cache-" + "x" * 96) + verifier = ManifestArtifactVerifier( + manifest, + manifest.source.repository, + manifest.source.revision, + cache_dir=cache, + ) + + partial, final, lock = verifier._resumable_paths(manifest.get_artifact("weights.bin")) + assert len(str(cache.absolute() / "manifest-artifacts" / manifest.digest / "partial")) > 248 + assert str(partial).startswith("\\\\?\\") + assert str(lock).startswith("\\\\?\\") + + partial.parent.mkdir(parents=True, exist_ok=True) + partial.write_bytes(b"partial") + with file_lock(lock, exclusive=True): + final.parent.mkdir(parents=True, exist_ok=True) + final.write_bytes(b"final") + + assert partial.read_bytes() == b"partial" + assert final.read_bytes() == b"final" + + def test_mixed_cached_and_downloaded_artifacts_share_one_snapshot_root(tmp_path, monkeypatch): from huggingface_hub.utils import LocalEntryNotFoundError @@ -672,6 +1200,147 @@ def test_server_cli_applies_manifest_profile(tmp_path, monkeypatch): assert resolved["model_manifest"].digest == resolved["dht_prefix"].removeprefix("drift-m1-") +def test_server_cli_requires_complete_internal_worker_artifact_claims(tmp_path, monkeypatch): + from drift.cli import run_server + + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest_dict()), encoding="utf-8") + manifest = ModelManifest.load(path) + parser = run_server.build_parser(bound_worker=True) + assert run_server._uses_bound_worker_parser(["--expected_manifest_digest", manifest.digest_id]) + base = [ + "org/tiny-test", + "--new_swarm", + "--model_manifest", + str(path), + "--identity_path", + str(tmp_path / "worker.key"), + "--block_indices", + "0:1", + "--cache_dir", + str(tmp_path.resolve()), + "--increase_file_limit", + "0", + ] + claim_args = [ + "--expected_manifest_digest", + manifest.digest_id, + "--expected_block_indices", + "0:1", + "--expected_artifact_bytes", + "4", + "--expected_artifact_set_digest", + "a" * 64, + "--expected_cache_root", + str(tmp_path.resolve()), + ] + monkeypatch.setattr(run_server, "tie_child_processes_to_this_process", lambda: None) + monkeypatch.setattr(run_server, "log_version", lambda: None) + monkeypatch.setattr(run_server, "Server", lambda **kwargs: kwargs) + (tmp_path / "config.yml").write_text( + "custom_module_path: injected.py\nallow_training_rpcs: true\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + + unbound = vars(run_server.build_parser().parse_args(base + claim_args)) + assert unbound["custom_module_path"] == "injected.py" + with pytest.raises(ManifestError, match="source-bound internal parser"): + run_server.server_from_args(unbound) + + args = vars(parser.parse_args(base + claim_args)) + assert args["custom_module_path"] is None + assert args["allow_training_rpcs"] is False + args.pop("config", None) + resolved = run_server.server_from_args(args) + assert resolved["expected_manifest_digest"] == manifest.digest_id + assert resolved["expected_block_indices"] == "0:1" + assert resolved["expected_artifact_bytes"] == 4 + assert resolved["expected_artifact_set_digest"] == "a" * 64 + assert resolved["expected_cache_root"] == str(tmp_path.resolve()) + + incomplete = vars(parser.parse_args(base + claim_args[:-2])) + incomplete.pop("config", None) + with pytest.raises(ManifestError, match="supplied together"): + run_server.server_from_args(incomplete) + + mismatched = list(claim_args) + mismatched[1] = "sha256:" + "0" * 64 + invalid = vars(parser.parse_args(base + mismatched)) + invalid.pop("config", None) + with pytest.raises(ManifestError, match="manifest digest"): + run_server.server_from_args(invalid) + + no_span = [value for value in base if value not in ("--block_indices", "0:1")] + invalid = vars(parser.parse_args(no_span + claim_args)) + invalid.pop("config", None) + with pytest.raises(ManifestError, match="explicit --block_indices"): + run_server.server_from_args(invalid) + + wrong_span = list(base) + wrong_span[wrong_span.index("0:1")] = "1:2" + invalid = vars(parser.parse_args(wrong_span + claim_args)) + invalid.pop("config", None) + with pytest.raises(ManifestError, match="block span"): + run_server.server_from_args(invalid) + + relative_cache = list(base) + relative_cache[relative_cache.index(str(tmp_path.resolve()))] = "." + invalid = vars(parser.parse_args(relative_cache + claim_args)) + invalid.pop("config", None) + with pytest.raises(ManifestError, match="canonical absolute --cache_dir"): + run_server.server_from_args(invalid) + + no_cache = list(base) + cache_option = no_cache.index("--cache_dir") + del no_cache[cache_option : cache_option + 2] + invalid = vars(parser.parse_args(no_cache + claim_args)) + invalid.pop("config", None) + with pytest.raises(ManifestError, match="explicit canonical --cache_dir"): + run_server.server_from_args(invalid) + + with pytest.raises(SystemExit): + parser.parse_args(base + claim_args + ["--config", str(tmp_path / "config.yml")]) + + unsafe = vars(parser.parse_args(base + claim_args + ["--allow_training_rpcs"])) + with pytest.raises(ManifestError, match="forbid custom modules"): + run_server.server_from_args(unsafe) + + +def test_server_cli_selects_fp8_dequant_loader_from_manifest(tmp_path, monkeypatch): + from drift.cli import run_server + from drift.utils.convert_block import QuantType + + source = manifest_dict() + source["runtime"]["dtype"] = "bfloat16" + source["runtime"]["quantization"] = "fp8_dequant" + path = tmp_path / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + args = vars( + run_server.build_parser().parse_args( + [ + "org/tiny-test", + "--new_swarm", + "--model_manifest", + str(path), + "--identity_path", + str(tmp_path / "worker.key"), + "--increase_file_limit", + "0", + ] + ) + ) + args.pop("config", None) + + monkeypatch.setattr(run_server, "tie_child_processes_to_this_process", lambda: None) + monkeypatch.setattr(run_server, "log_version", lambda: None) + monkeypatch.setattr(run_server, "Server", lambda **kwargs: kwargs) + resolved = run_server.server_from_args(args) + + assert resolved["torch_dtype"] == "bfloat16" + assert resolved["quant_type"] is QuantType.FP8_DEQUANT + + def test_server_cli_derives_repository_from_manifest(tmp_path, monkeypatch): from drift.cli import run_server diff --git a/tests/test_node.py b/tests/test_node.py index 4243493ed..c11753df1 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -121,6 +121,7 @@ def test_node_status_requires_auth_and_reports_lazy_model(): "denied_models": [], "max_disk_space": None, "max_vram": None, + "max_processing_percent": 100.0, "max_bandwidth_mbps": None, "max_power_watts": None, "pause_timeout": 10.0, @@ -264,6 +265,8 @@ def snapshots(self): "resource_admitted": True, "resource_reason": None, "resource_suspended": False, + "intent_published": False, + "remote_acknowledged": False, "max_disk_bytes": 100 * 1024**3, "max_vram_bytes": 4 * 1024**3, "vram_pool_bytes": 8 * 1024**3, @@ -313,7 +316,10 @@ def shutdown(self): with TestClient(app) as client: assert client.get("/control/v1/workers").status_code == 401 assert client.get("/control/v1/workers", headers={"Authorization": "Bearer client-secret"}).status_code == 401 - assert client.get("/control/v1/workers", headers=headers).json()["workers"][0]["state"] == "paused" + private_worker = client.get("/control/v1/workers", headers=headers).json()["workers"][0] + assert private_worker["state"] == "paused" + assert private_worker["intent_published"] is False + assert private_worker["remote_acknowledged"] is False status = client.get("/control/v1/status", headers=headers).json() assert status["workers"] == [{"id": "worker", "model": "model", "state": "paused", "desired_running": False}] status_worker = status["contribution"]["workers"][0] diff --git a/tests/test_node_config.py b/tests/test_node_config.py index aebd13b4b..eb11bec5f 100644 --- a/tests/test_node_config.py +++ b/tests/test_node_config.py @@ -1,6 +1,7 @@ import json import sys import time +from dataclasses import replace from datetime import datetime, timezone from pathlib import Path @@ -11,12 +12,14 @@ _build_automatic_placement_service, _build_model_manager, _build_worker_supervisor, + _can_retain_acknowledged_plan, _load_persisted_and_runtime_config, _merge_cached_initial_peers, + _placement_decision_key, _prepare_route_identity, _reuse_runtime_initial_peers, ) -from drift.model_manifest import ModelManifest +from drift.model_manifest import ManifestError, ModelManifest from drift.node.config import ( NODE_CONFIG_SCHEMA_VERSION, ContributionPolicyConfig, @@ -29,6 +32,8 @@ from drift.node.contribution_planner import ( MAX_AUTOMATIC_PLACEMENT_BLOCKS, MAX_AUTOMATIC_PLACEMENT_CANDIDATES, + PlacementArtifactPlan, + PlacementCandidate, PlacementDecision, PlacementPlan, PlacementRegistry, @@ -40,6 +45,38 @@ from drift.protocol_identity import NodeIdentity, ProtocolSecurityError +@pytest.mark.parametrize("mutation", ["none", "policy", "budget", "age", "never-seen", "digest", "blocks"]) +def test_discovery_gap_retention_requires_current_exact_admission(mutation): + decision = PlacementDecision("model", "sha256:" + "1" * 64, "1:2", 100, (0, 0), 1, "test", "2" * 64) + candidate = PlacementCandidate( + "model", + decision.manifest_digest, + 0, + False, + 100, + 2, + {"status": "unknown", "last_known_status": "incomplete", "last_updated_age": 1.0}, + artifact_plans=(PlacementArtifactPlan(1, 2, 100, "2" * 64),), + max_artifact_bytes=100, + ) + if mutation == "policy": + candidate = replace(candidate, policy_reason="denied") + elif mutation == "budget": + candidate = replace(candidate, max_artifact_bytes=99) + elif mutation == "age": + candidate = replace(candidate, health={**candidate.health, "last_updated_age": 91}) + elif mutation == "never-seen": + candidate = replace(candidate, health={"status": "unknown", "last_updated_age": 0}) + elif mutation == "digest": + candidate = replace(candidate, artifact_plans=(PlacementArtifactPlan(1, 2, 100, "3" * 64),)) + assert run_node_module._recent_gap_preserves_artifact_claim( + candidate, + decision, + 2 if mutation == "blocks" else 1, + maximum_age=90, + ) is (mutation == "none") + + def _config_dict(**overrides): source = { "schema_version": 1, @@ -242,7 +279,7 @@ def fake_make_loader(manifest, **kwargs): loader_calls.append((manifest.digest_id, kwargs)) return lambda: ModelRuntime(object(), object()) - monkeypatch.setattr("drift.cli.run_node.make_manifest_loader", fake_make_loader) + monkeypatch.setattr("drift.cli.run_node.make_text_peer_loader", fake_make_loader) config = NodeConfig( schema_version=1, max_loaded_models=1, @@ -268,10 +305,9 @@ def fake_make_loader(manifest, **kwargs): assert manager.auto_selection_snapshot()["status"] == "unavailable" assert loader_calls[0][1]["initial_peers"] == ("peer-one",) assert loader_calls[1][1]["initial_peers"] == ("peer-two",) - assert loader_calls[1][1]["cache_dir"] == "cache" + assert all(descriptor.selected_whole_shard_bytes == 0 for descriptor in descriptors) assert loader_calls[1][1]["request_timeout"] == 9 - assert loader_calls[1][1]["max_retries"] == 4 - assert all(call[1]["token"] == "provider-token" for call in loader_calls) + assert all("cache_dir" not in call[1] and "token" not in call[1] for call in loader_calls) assert discovery.snapshot(first.digest_id)["status"] == "unknown" assert discovery._states[first.digest_id].target.cache_scope == ("shipped-one",) assert discovery._states[second.digest_id].target.cache_scope == ("shipped-two",) @@ -284,12 +320,13 @@ def fake_make_loader(manifest, **kwargs): manager.shutdown() -def test_worker_supervisor_command_is_pinned_to_configured_manifest(monkeypatch, tmp_path): +@pytest.mark.parametrize("public_port", [None, 43210]) +def test_worker_supervisor_command_is_pinned_to_configured_manifest(monkeypatch, tmp_path, public_port): manifest = ModelManifest.load("tests/data/model_manifest_v1_vector.json") manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig( @@ -303,6 +340,9 @@ def test_worker_supervisor_command_is_pinned_to_configured_manifest(monkeypatch, identity_path=tmp_path / "worker.key", num_blocks=2, throughput=1.25, + port=31330, + public_ip="203.0.113.4", + public_port=public_port, ), ), ) @@ -317,6 +357,13 @@ def test_worker_supervisor_command_is_pinned_to_configured_manifest(monkeypatch, assert str(manifest_path) in launch.command assert launch.command[launch.command.index("--num_blocks") + 1] == "2" assert launch.command[launch.command.index("--throughput") + 1] == "1.25" + assert launch.command[launch.command.index("--port") + 1] == "31330" + if public_port is None: + assert launch.command[launch.command.index("--public_ip") + 1] == "203.0.113.4" + assert "--announce_maddrs" not in launch.command + else: + assert launch.command[launch.command.index("--announce_maddrs") + 1] == "/ip4/203.0.113.4/tcp/43210" + assert "--public_ip" not in launch.command assert "provider-token" not in launch.command assert launch.environment == (("HF_TOKEN", "provider-token"),) supervisor.shutdown() @@ -334,7 +381,7 @@ def test_automatic_worker_waits_then_binds_exact_model_and_block_range(monkeypat manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig.from_dict( @@ -376,25 +423,168 @@ def test_automatic_worker_waits_then_binds_exact_model_and_block_range(monkeypat replica_counts=(0,), score=100, reason="selected 1:2 from fresh verified coverage", + artifact_set_digest="a" * 64, ) - placed = _build_worker_supervisor( + unacknowledged = _build_worker_supervisor( config, manager, automatic_placements={ "automatic": PlacementPlan(decision, decision.reason, 1), }, ) + unacknowledged_launch = unacknowledged.launches[0] + assert unacknowledged_launch.policy_admitted is False + assert unacknowledged_launch.intent_published is False + assert unacknowledged_launch.remote_acknowledged is False + assert "not remotely acknowledged" in unacknowledged_launch.policy_reason + unacknowledged.shutdown() + + unbound_decision = PlacementDecision( + model_id=manifest.name, + manifest_digest=manifest.digest_id, + block_indices="1:2", + artifact_bytes=decision.artifact_bytes, + replica_counts=(0,), + score=100, + reason=decision.reason, + ) + unbound = _build_worker_supervisor( + config, + manager, + automatic_placements={ + "automatic": PlacementPlan( + unbound_decision, + unbound_decision.reason, + 1, + intent_published=True, + remote_acknowledged=True, + ), + }, + ) + unbound_launch = unbound.launches[0] + assert unbound_launch.policy_admitted is False + assert "no exact artifact-set binding" in unbound_launch.policy_reason + assert "--expected_artifact_set_digest" not in unbound_launch.command + assert unbound_launch.placement_manifest_digest is None + unbound.shutdown() + + placed = _build_worker_supervisor( + config, + manager, + automatic_placements={ + "automatic": PlacementPlan( + decision, + decision.reason, + 1, + intent_published=True, + remote_acknowledged=True, + ), + }, + ) launch = placed.launches[0] assert launch.model_id == manifest.name assert launch.policy_admitted is True assert launch.automatic is True assert launch.block_indices == "1:2" + assert launch.intent_published is True + assert launch.remote_acknowledged is True assert launch.command[launch.command.index("--block_indices") + 1] == "1:2" + assert launch.command[launch.command.index("--expected_manifest_digest") + 1] == manifest.digest_id + assert launch.command[launch.command.index("--expected_block_indices") + 1] == "1:2" + assert launch.command[launch.command.index("--expected_artifact_bytes") + 1] == str(decision.artifact_bytes) + assert launch.command[launch.command.index("--expected_artifact_set_digest") + 1] == "a" * 64 + expected_cache_root = str(Path(run_node_module.DEFAULT_CACHE_DIR).expanduser().resolve()) + assert launch.command[launch.command.index("--cache_dir") + 1] == expected_cache_root + assert launch.command[launch.command.index("--expected_cache_root") + 1] == expected_cache_root + assert launch.placement_manifest_digest == manifest.digest_id + assert launch.placement_artifact_bytes == decision.artifact_bytes + assert launch.placement_artifact_set_digest == "a" * 64 + assert launch.placement_cache_root == expected_cache_root assert "--num_blocks" not in launch.command placed.shutdown() manager.shutdown() +def test_retained_placement_requires_an_unexpired_lease_for_the_current_identity(tmp_path): + worker = WorkerConfig( + worker_id="automatic", + model="auto", + identity_path=tmp_path / "worker.key", + num_blocks=1, + ) + decision = PlacementDecision( + model_id="qwen", + manifest_digest="sha256:" + "1" * 64, + block_indices="1:2", + artifact_bytes=123, + replica_counts=(0,), + score=100, + reason="selected", + artifact_set_digest="2" * 64, + ) + previous = PlacementPlan( + decision, + decision.reason, + 1, + intent_published=True, + remote_acknowledged=True, + ) + identity_key_id = "sha256:" + "3" * 64 + lease = { + "decision_key": _placement_decision_key(worker, decision, identity_key_id), + "expires_at": 100.0, + } + + assert _can_retain_acknowledged_plan(previous, decision, worker, identity_key_id, lease, now=99.0) + assert not _can_retain_acknowledged_plan(previous, decision, worker, identity_key_id, lease, now=100.0) + assert not _can_retain_acknowledged_plan( + previous, + decision, + worker, + identity_key_id, + {**lease, "expires_at": float("inf")}, + now=99.0, + ) + assert not _can_retain_acknowledged_plan( + previous, + decision, + worker, + identity_key_id, + {**lease, "expires_at": float("nan")}, + now=99.0, + ) + assert not _can_retain_acknowledged_plan(previous, decision, worker, "sha256:" + "4" * 64, lease, now=99.0) + + changed_decision = PlacementDecision( + model_id="qwen", + manifest_digest=decision.manifest_digest, + block_indices="0:1", + artifact_bytes=122, + replica_counts=(1,), + score=99, + reason="smaller eligible span", + artifact_set_digest="5" * 64, + ) + assert not _can_retain_acknowledged_plan(previous, changed_decision, worker, identity_key_id, lease, now=99.0) + + rotated = WorkerConfig( + worker_id="automatic", + model="auto", + identity_path=tmp_path / "rotated.key", + num_blocks=1, + ) + assert not _can_retain_acknowledged_plan(previous, decision, rotated, identity_key_id, lease, now=99.0) + + changed_throughput = WorkerConfig( + worker_id="automatic", + model="auto", + identity_path=worker.identity_path, + num_blocks=1, + throughput=2.5, + ) + assert not _can_retain_acknowledged_plan(previous, decision, changed_throughput, identity_key_id, lease, now=99.0) + + @pytest.mark.parametrize( "pause_while_waiting, publish_succeeds, expected_desired", [(False, True, True), (True, True, False), (False, False, None)], @@ -408,12 +598,20 @@ def test_automatic_placement_service_reconciles_fresh_coverage_into_supervision( route_identity = NodeIdentity.create(tmp_path / "route-demand.key") second_authority = NodeIdentity.create(tmp_path / "second-route-demand.key") authority_roots = tuple(sorted((route_identity.key_id, second_authority.key_id))) + dht_time = [2_000.0] + monkeypatch.setattr(run_node_module, "get_dht_time", lambda: dht_time[0]) monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config_source = _config_dict( - models=[{"manifest": str(manifest_path), "initial_peers": ["peer-one"]}], + models=[ + { + "manifest": str(manifest_path), + "initial_peers": ["peer-one"], + "cache_dir": "model-cache", + } + ], auto_model_priority=[manifest.digest_id], route_demand_authority_roots=list(authority_roots), workers=[ @@ -435,6 +633,18 @@ def test_automatic_placement_service_reconciles_fresh_coverage_into_supervision( config_path.write_text(json.dumps(config_source), encoding="utf-8") config = NodeConfig.load(config_path) manager, _, discovery = _build_model_manager(config, token=None) + selected_artifact_bytes = 4_242 + artifact_plans = tuple( + PlacementArtifactPlan(index, index + 1, selected_artifact_bytes + index, f"{index + 1:064x}") + for index in range(manifest.model.num_blocks) + ) + planning_cache_dirs = [] + + def fake_artifact_plans(*args, **kwargs): + planning_cache_dirs.append(kwargs["cache_dir"]) + return artifact_plans + + monkeypatch.setattr(run_node_module, "_manifest_artifact_plans", fake_artifact_plans) counts = [1] * manifest.model.num_blocks counts[1] = 0 state = discovery._states[manifest.digest_id] @@ -510,6 +720,7 @@ def publish_intent(digest_id, source): service.reconcile_once() + assert planning_cache_dirs == [tmp_path / "model-cache"] launch = supervisor.launches[0] snapshot = supervisor.snapshot("automatic") assert len(publish_calls) == 1 @@ -519,7 +730,16 @@ def publish_intent(digest_id, source): assert published["payload"]["manifest_digest"] == manifest.digest assert published["payload"]["start_block"] == 1 assert published["payload"]["end_block"] == 2 - assert "private_path" not in published["payload"]["resource_claims"] + resource_claims = published["payload"]["resource_claims"] + assert set(resource_claims) == { + "schema_version", + "artifact_bytes", + "block_count", + "throughput_milli_rps", + } + assert resource_claims["artifact_bytes"] == selected_artifact_bytes + 1 + assert resource_claims["artifact_bytes"] != sum(artifact.size for artifact in manifest.artifacts) + assert "private_path" not in resource_claims assert len(demand_calls) == 1 demand = demand_calls[0][1] assert demand["kind"] == "route_demand" @@ -538,6 +758,10 @@ def publish_intent(digest_id, source): assert launch.model_id == manifest.name assert launch.block_indices == "1:2" assert launch.policy_admitted is True + assert launch.intent_published is True + assert launch.remote_acknowledged is True + assert snapshot["intent_published"] is True + assert snapshot["remote_acknowledged"] is True assert "local demand bucket 1" in launch.placement_reason assert snapshot["desired_running"] is expected_desired assert snapshot["operator_paused"] is pause_while_waiting @@ -545,10 +769,85 @@ def publish_intent(digest_id, source): else: assert launch.model_id == "auto" assert launch.policy_admitted is False + assert launch.intent_published is False + assert launch.remote_acknowledged is False assert "signed placement intent" in launch.policy_reason assert snapshot["pid"] is None assert registry.snapshot()["automatic"].decision is None + if publish_succeeds: + # A verified immutable plan must survive later metadata-cache contention; + # otherwise a discovery tick can stop an admitted worker mid-download. + def unavailable_metadata(*args, **kwargs): + raise ManifestError("metadata temporarily unavailable during worker acquisition") + + monkeypatch.setattr(run_node_module, "_manifest_artifact_plans", unavailable_metadata) + service.reconcile_once() + assert registry.snapshot()["automatic"].decision.block_indices == "1:2" + assert supervisor.launches[0].policy_admitted is True + assert len(planning_cache_dirs) == 1 + monkeypatch.setattr(run_node_module, "_manifest_artifact_plans", fake_artifact_plans) + + # A short discovery interruption preserves an already acknowledged exact + # claim; it must not kill a loading worker or renew the old lease. + prior = registry.snapshot()["automatic"] + state.last_error = "seed temporarily unavailable" + service.reconcile_once() + assert registry.snapshot()["automatic"] == prior + assert supervisor.launches[0].policy_admitted is True + assert len(publish_calls) == 1 + state.last_error = None + + config_source["workers"][0]["throughput"] = 2.5 + config_path.write_text(json.dumps(config_source), encoding="utf-8") + service.reconcile_once() + assert len(publish_calls) == 2 + throughput_record = publish_calls[-1][1] + assert throughput_record["payload"]["resource_claims"]["throughput_milli_rps"] == 2_500 + throughput_launch = supervisor.launches[0] + assert throughput_launch.command[throughput_launch.command.index("--throughput") + 1] == "2.5" + + rotated_identity = NodeIdentity.create(tmp_path / "rotated-automatic.key") + (tmp_path / "automatic.key").chmod(0o600) + (tmp_path / "rotated-automatic.key").replace(tmp_path / "automatic.key") + service.reconcile_once() + assert len(publish_calls) == 3 + assert publish_calls[-1][1]["key_id"] == rotated_identity.key_id + + config_source["contribution_policy"]["max_disk_space"] = f"{selected_artifact_bytes}B" + config_path.write_text(json.dumps(config_source), encoding="utf-8") + failed_records = [] + monkeypatch.setattr( + discovery, + "publish_intent", + lambda _digest, source: failed_records.append(source) or False, + ) + service.reconcile_once() + assert failed_records[-1]["payload"]["start_block"] == 0 + assert registry.snapshot()["automatic"].decision is None + assert supervisor.launches[0].policy_admitted is False + + config_source["contribution_policy"]["max_disk_space"] = "1GiB" + config_path.write_text(json.dumps(config_source), encoding="utf-8") + monkeypatch.setattr(discovery, "publish_intent", publish_intent) + service.reconcile_once() + assert registry.snapshot()["automatic"].decision.block_indices == "1:2" + + dht_time[0] = 2_481.0 + monkeypatch.setattr(discovery, "publish_intent", lambda *_args, **_kwargs: False) + service.reconcile_once() + assert registry.snapshot()["automatic"].decision is not None + + dht_time[0] = 2_600.0 + service.reconcile_once() + assert registry.snapshot()["automatic"].decision is None + assert supervisor.launches[0].policy_admitted is False + + dht_time[0] = 2_601.0 + monkeypatch.setattr(discovery, "publish_intent", publish_intent) + service.reconcile_once() + assert registry.snapshot()["automatic"].decision is not None + original_candidates = run_node_module._automatic_placement_candidates remote_consumption = [] @@ -558,11 +857,13 @@ def capture_remote_consumption(*args, **kwargs): monkeypatch.setattr(run_node_module, "_automatic_placement_candidates", capture_remote_consumption) config_source["route_demand_authority_roots"] = ["sha256:" + "0" * 64, "sha256:" + "f" * 64] + config_source["workers"][0]["cache_dir"] = "worker-cache" config_path.write_text(json.dumps(config_source), encoding="utf-8") demand_calls.clear() service.reconcile_once() assert demand_calls == [] assert remote_consumption == [False] + assert planning_cache_dirs[-1] == tmp_path / "worker-cache" service.close() supervisor.shutdown() @@ -670,7 +971,7 @@ def test_worker_supervisor_enforces_resolved_model_policy_and_disk_ceiling(monke manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig.from_dict( @@ -723,7 +1024,7 @@ def test_disabled_contribution_policy_blocks_auto_start_and_control_start(monkey manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig( @@ -760,7 +1061,7 @@ def test_denied_model_cannot_be_started_through_an_alias(monkeypatch, tmp_path): manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig( @@ -803,7 +1104,7 @@ def test_contribution_policy_rejects_alias_based_allow_deny_conflicts(monkeypatc manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig( @@ -888,7 +1189,7 @@ def test_accelerator_worker_inherits_tighter_resolved_vram_limit(monkeypatch, tm manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) monkeypatch.setattr("drift.cli.run_node.get_device_total_memory", lambda device: 16 * 1024**3) @@ -909,6 +1210,7 @@ def test_accelerator_worker_inherits_tighter_resolved_vram_limit(monkeypatch, tm "sharing_enabled": True, "max_disk_space": "1GiB", "max_vram": "75%", + "max_processing_percent": 25, }, ), base_dir=tmp_path, @@ -921,18 +1223,67 @@ def test_accelerator_worker_inherits_tighter_resolved_vram_limit(monkeypatch, tm assert launch.vram_pool_bytes == 12 * 1024**3 assert launch.vram_device == "cuda:0" assert launch.command[launch.command.index("--max_device_memory") + 1] == str(8 * 1024**3) + assert launch.command[launch.command.index("--max_processing_percent") + 1] == "25.0" + assert launch.command[launch.command.index("--processing_budget_path") + 1] == str( + tmp_path / ".worker.key.processing-budget" + ) assert supervisor.snapshot("gpu-worker")["max_vram_bytes"] == 8 * 1024**3 supervisor.shutdown() manager.shutdown() +@pytest.mark.parametrize("sharing_percent", [25, 100]) +def test_fallback_never_reduces_worker_sharing_budget(monkeypatch, tmp_path, sharing_percent): + manifest = ModelManifest.load("tests/data/model_manifest_v1_vector.json") + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") + monkeypatch.setattr("drift.cli.run_node.get_device_total_memory", lambda device: 8 * 1024**3) + local_manifest = Path("manifests/candidates/qwen3.5-0.8b-local-bfloat16-eager.json").resolve() + config = NodeConfig.from_dict( + _config_dict( + models=[ + {"manifest": str(manifest_path), "initial_peers": ["peer-one"]}, + { + "manifest": str(local_manifest), + "initial_peers": [], + "execution": "local", + "local_max_memory": "16GiB", + }, + ], + workers=[ + { + "id": "gpu-worker", + "model": manifest.name, + "identity_path": "worker.key", + "num_blocks": 2, + "device": "cuda:0", + } + ], + contribution_policy={"sharing_enabled": True, "max_disk_space": "1GiB", "max_vram": f"{sharing_percent}%"}, + ), + base_dir=tmp_path, + ) + manager, _, _ = _build_model_manager(config, token=None) + supervisor = _build_worker_supervisor(config, manager) + try: + launch = supervisor.launches[0] + expected = 8 * 1024**3 * sharing_percent // 100 + assert launch.policy_admitted + assert launch.max_vram_bytes == expected + assert launch.vram_pool_bytes == expected + assert launch.command[launch.command.index("--max_device_memory") + 1] == str(expected) + finally: + supervisor.shutdown() + manager.shutdown() + + def test_power_monitor_is_scoped_to_each_cuda_workers_device(monkeypatch, tmp_path): manifest = ModelManifest.load("tests/data/model_manifest_v1_vector.json") manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) monkeypatch.setattr("drift.cli.run_node.get_device_total_memory", lambda device: 16 * 1024**3) @@ -987,7 +1338,7 @@ def test_accelerator_worker_requires_node_wide_vram_pool(monkeypatch, tmp_path): manifest_path = tmp_path / "manifest.json" manifest_path.write_text(manifest.canonical_json(), encoding="utf-8") monkeypatch.setattr( - "drift.cli.run_node.make_manifest_loader", + "drift.cli.run_node.make_text_peer_loader", lambda *args, **kwargs: lambda: ModelRuntime(object(), object()), ) config = NodeConfig.from_dict( diff --git a/tests/test_node_hardware_status.py b/tests/test_node_hardware_status.py new file mode 100644 index 000000000..f945b83d7 --- /dev/null +++ b/tests/test_node_hardware_status.py @@ -0,0 +1,64 @@ +from types import SimpleNamespace + +import torch + +from drift.node.hardware_status import HardwareStatus + + +def config(*, local=True, worker_device=None): + return SimpleNamespace( + workers=[SimpleNamespace(device=worker_device)], + models=[SimpleNamespace(execution="local", local_device="auto", local_max_memory_bytes=3 * 1024**3)] + if local + else [], + ) + + +def test_gpu_names_and_budget_exist_before_worker_placement(monkeypatch): + monkeypatch.setattr("drift.node.hardware_status.cpu_name", lambda: "AMD Ryzen 9 5900X") + monkeypatch.setattr("drift.utils.hardware.auto_detect_device", lambda: "cuda") + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: "NVIDIA GeForce RTX 3070") + monkeypatch.setattr("drift.utils.hardware.get_device_total_memory", lambda device: 8 * 1024**3) + status = HardwareStatus(config()) + full = status.snapshot({"sharing_enabled": False, "max_vram": "100%"}) + assert full["cpu_name"] == "AMD Ryzen 9 5900X" + assert full["gpu_name"] == "NVIDIA GeForce RTX 3070" + assert full["device"] == "cuda:0" + assert full["gpu_total_bytes"] == 8 * 1024**3 + assert full["sharing_vram_bytes"] == 8 * 1024**3 + assert full["processing_percent"] == 100 + half = status.snapshot({"sharing_enabled": False, "max_vram": "25%", "max_processing_percent": 30}) + assert half["sharing_vram_bytes"] == 2 * 1024**3 + assert half["processing_percent"] == 30 + # Changing saved limits must not probe or initialize the GPU again. + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: (_ for _ in ()).throw(AssertionError())) + assert status.snapshot({"sharing_enabled": False, "max_vram": "1GiB"})["sharing_vram_bytes"] == 1024**3 + + +def test_cpu_only_and_unavailable_accelerator_are_not_invented(monkeypatch): + monkeypatch.setattr("drift.node.hardware_status.cpu_name", lambda: "Intel Core i7-12700") + monkeypatch.setattr("drift.utils.hardware.auto_detect_device", lambda: "cpu") + cpu = HardwareStatus(config()).snapshot({"sharing_enabled": False}) + assert cpu["cpu_name"] == "Intel Core i7-12700" + assert cpu["gpu_name"] is None + assert cpu["sharing_vram_bytes"] is None + assert cpu["device"] == "cpu" + monkeypatch.setattr("drift.utils.hardware.auto_detect_device", lambda: "cuda") + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: (_ for _ in ()).throw(RuntimeError())) + unavailable = HardwareStatus(config()).snapshot({"sharing_enabled": False}) + assert unavailable["device"] == "unknown" + assert unavailable["gpu_total_bytes"] is None + assert unavailable["sharing_vram_bytes"] is None + + +def test_explicit_cpu_sharing_still_reports_installed_gpu(monkeypatch): + monkeypatch.setattr("drift.node.hardware_status.cpu_name", lambda: "Intel Core i7") + monkeypatch.setattr("drift.utils.hardware.auto_detect_device", lambda: "cuda") + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: "NVIDIA RTX 2070 SUPER") + monkeypatch.setattr("drift.utils.hardware.get_device_total_memory", lambda device: 8 * 1024**3) + status = HardwareStatus(config(worker_device="cpu")).snapshot({"sharing_enabled": False}) + assert status["device"] == "cpu" + assert status["gpu_device"] == "cuda:0" + assert status["gpu_name"] == "NVIDIA RTX 2070 SUPER" + assert status["gpu_total_bytes"] == 8 * 1024**3 + assert status["sharing_vram_bytes"] == 0 diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py index d9efaca08..6ba9c9470 100644 --- a/tests/test_openai_api.py +++ b/tests/test_openai_api.py @@ -111,6 +111,43 @@ def test_chat_completion_non_stream(api): assert api.model.last_gen_kwargs["do_sample"] is False +@pytest.mark.parametrize("thinking", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +def test_chat_reasoning_switch_reaches_template_only(thinking, stream): + class RecordingTokenizer(FakeTokenizer): + def apply_chat_template(self, messages, *, enable_thinking, **kwargs): + self.thinking = enable_thinking + return super().apply_chat_template(messages, **kwargs) + + tokenizer, model = RecordingTokenizer(), FakeModel() + client = TestClient(create_app(model, tokenizer, model_name="fake/model")) + response = client.post( + "/v1/chat/completions", + json={ + "messages": [{"role": "user", "content": "Answer briefly"}], + "enable_thinking": thinking, + "stream": stream, + "max_tokens": 3, + }, + ) + assert response.status_code == 200 + assert tokenizer.thinking is thinking + assert "enable_thinking" not in model.last_gen_kwargs + + +@pytest.mark.parametrize("thinking", ["false", 0, {}, []]) +def test_chat_reasoning_switch_rejects_non_boolean_values(api, thinking): + response = api.client.post( + "/v1/chat/completions", + json={ + "messages": [{"role": "user", "content": "hi"}], + "enable_thinking": thinking, + }, + ) + assert response.status_code == 422 + assert api.model.last_gen_kwargs is None + + def test_requested_model_must_resolve_exactly(api): response = api.client.post( "/v1/chat/completions", diff --git a/tests/test_policy_store.py b/tests/test_policy_store.py index 351ea9e1c..a8d6aedb4 100644 --- a/tests/test_policy_store.py +++ b/tests/test_policy_store.py @@ -106,6 +106,7 @@ def _complete_policy(**overrides): "denied_models": [], "max_disk_space": "12GiB", "max_vram": "50%", + "max_processing_percent": 100.0, "max_bandwidth_mbps": 25.0, "max_power_watts": 150.0, "pause_timeout": 7.0, diff --git a/tests/test_processing_budget.py b/tests/test_processing_budget.py new file mode 100644 index 000000000..b7d1e53e6 --- /dev/null +++ b/tests/test_processing_budget.py @@ -0,0 +1,108 @@ +import threading +import time + +import pytest + +from drift.node.config import ContributionPolicyConfig, NodeConfigError +from drift.server.processing_budget import ProcessingBudget + + +@pytest.mark.parametrize("value", [0, -1, 101, True, None, "50", float("nan"), float("inf")]) +def test_rejects_invalid_processing_limits(value): + with pytest.raises(ValueError): + ProcessingBudget(value) + with pytest.raises(NodeConfigError): + ContributionPolicyConfig.from_dict({"sharing_enabled": False, "max_processing_percent": value}) + + +def test_old_configs_default_to_full_processing_without_enabling_sharing(): + policy = ContributionPolicyConfig.from_dict({"sharing_enabled": False}) + assert policy.max_processing_percent == 100 + assert not policy.sharing_enabled + + +@pytest.mark.parametrize("percent,rest", [(25, 6), (50, 2), (100, 0)]) +def test_accounts_for_completed_device_work_and_rest(percent, rest): + events = [] + now = [0.0] + + class Stop: + def is_set(self): + return False + + def wait(self, seconds): + events.append(("wait", seconds)) + now[0] += seconds + + def compute(): + events.append("compute") + now[0] += 2 + return "result" + + budget = ProcessingBudget(percent, stop=Stop(), clock=lambda: now[0]) + assert budget.run(compute, synchronize=lambda: events.append("sync")) == "result" + assert now[0] == 2 + rest + assert events == (["compute"] if percent == 100 else ["sync", "compute", "sync", ("wait", rest)]) + + +def test_shutdown_interrupts_long_cooldown_and_releases_shared_lock(tmp_path): + stop = threading.Event() + budget = ProcessingBudget(1, path=tmp_path / "node-budget", stop=stop) + entered = threading.Event() + + def compute(): + entered.set() + time.sleep(0.03) + + thread = threading.Thread(target=lambda: budget.run(compute)) + thread.start() + assert entered.wait(2) + stop.set() + thread.join(timeout=1) + assert not thread.is_alive() + assert ProcessingBudget(50, path=budget.path).run(lambda: 42) == 42 + + +def test_failed_compute_still_cools_down(): + now = iter([0.0, 2.0]) + waits = [] + + class Stop: + def is_set(self): + return False + + def wait(self, seconds): + waits.append(seconds) + + def fail(): + raise RuntimeError("failed step") + + with pytest.raises(RuntimeError, match="failed step"): + ProcessingBudget(50, stop=Stop(), clock=lambda: next(now)).run(fail) + assert waits == [2.0] + + +def test_shared_workers_cannot_overlap_compute_or_cooldown(tmp_path): + # Real OS locks, independent limiter instances, competing worker threads. + events = [] + barrier = threading.Barrier(2) + + def worker(): + budget = ProcessingBudget(25, path=tmp_path / "shared") + barrier.wait() + + def compute(): + start = time.monotonic() + time.sleep(0.03) + events.append((start, time.monotonic())) + + budget.run(compute) + + threads = [threading.Thread(target=worker) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + assert not thread.is_alive() + first, second = sorted(events) + assert second[0] >= first[1] + (first[1] - first[0]) * 2.9 diff --git a/tests/test_protocol_identity.py b/tests/test_protocol_identity.py index caa7c5bc1..e2d0ba013 100644 --- a/tests/test_protocol_identity.py +++ b/tests/test_protocol_identity.py @@ -682,3 +682,56 @@ def test_replay_guard_enforces_active_entry_and_byte_limits(tmp_path): oversized_path.write_text(" " * (MAX_REPLAY_HISTORY_BYTES + 1), encoding="utf-8") with pytest.raises(ProtocolSecurityError, match="byte limit"): ReplayGuard(oversized_path) + + +def test_dht_snapshot_uses_newest_signed_span_without_weakening_replay(tmp_path): + identity = make_identity(tmp_path) + now = time.time() - 2 + older, newer = make_server_info(), make_server_info() + sign_server_info(identity, older, now=now, sequence=1) + sign_server_info(identity, newer, now=now + 1, sequence=2) + uids = [f"{DHT_PREFIX}.{i}" for i in range(2)] + guard = ReplayGuard() + + def lookup(values): + class Snapshot: + async def get_many(self, requested, expiration_time, num_workers): + return { + uid: SimpleNamespace(value={identity.peer_id.to_base58(): SimpleNamespace(value=value.to_tuple())}) + if value + else None + for uid, value in zip(uids, values) + } + + return asyncio.run( + _get_remote_module_infos( + SimpleNamespace(num_workers=None), + Snapshot(), + uids, + None, + MANIFEST_DIGEST, + EXECUTION_PROFILE, + RevocationStore(), + guard, + None, + True, + ) + ) + + # Renewal writes are not atomic across the two DHT keys. Both orderings + # describe one usable authenticated span, including a temporarily absent key. + for values in ([newer, older], [older, newer], [newer, None]): + assert all(identity.peer_id in item.servers for item in lookup(values)) + assert all(not item.servers for item in lookup([older, older])) + # A newer worker placement removes block 0; stale copies cannot bring it back. + moved = make_server_info() + moved.start_block = 1 + sign_server_info(identity, moved, now=now + 1.5, sequence=3) + result = lookup([newer, moved]) + assert not result[0].servers + assert identity.peer_id in result[1].servers + # Equivocation at the newest generation excludes the peer for the whole view. + fork = make_server_info() + fork.throughput = 3.0 + sign_server_info(identity, fork, now=now + 1.5, sequence=3) + assert all(not item.servers for item in lookup([fork, moved])) diff --git a/tests/test_public_worker_health.py b/tests/test_public_worker_health.py index 32f7dddfc..efa740165 100644 --- a/tests/test_public_worker_health.py +++ b/tests/test_public_worker_health.py @@ -14,6 +14,30 @@ MANIFEST_DIGEST = "sha256:" + "a" * 64 +def test_live_container_converts_wire_digest_for_public_health(tmp_path): + """The DHT wire hash is bare hex; public health requires the sha256: ID.""" + from types import SimpleNamespace + + from drift.server.server import ModuleContainer + + alive = SimpleNamespace(is_alive=lambda: True) + target = tmp_path / "health.json" + container = SimpleNamespace( + admission_state=SimpleNamespace(snapshot=_admission), + dht_announcer=alive, + conn_handlers=[alive], + runtime=SimpleNamespace(pools=[alive]), + ready=SimpleNamespace(is_set=lambda: True), + server_info=SimpleNamespace(manifest_digest="a" * 64, start_block=16, end_block=32), + health_state_path=target, + ) + + assert ModuleContainer.is_healthy(container) is True + payload = json.loads(target.read_text()) + assert payload["worker_healthy"] is True + assert payload["route"] == {"manifest_digest": MANIFEST_DIGEST, "start_block": 16, "end_block": 32} + + def _admission(**overrides): snapshot = { "active_sessions": 1, diff --git a/tests/test_qualification_harness.py b/tests/test_qualification_harness.py index e3c1f32f8..77a8cec03 100644 --- a/tests/test_qualification_harness.py +++ b/tests/test_qualification_harness.py @@ -7,6 +7,7 @@ import scripts.smoke_tinyllama_local_swarm as local_swarm from drift.model_manifest import ModelManifest +from drift.utils.convert_block import QuantType from scripts.qualify_model_manifest import ( DEFAULT_QUALIFICATION_PROMPT, build_parser as build_qualification_parser, @@ -259,6 +260,32 @@ def test_manifest_smoke_passes_runtime_cache_to_the_distributed_client(): assert "cache_dir=args.cache_dir," in model_kwargs +def test_manifest_smoke_uses_the_exact_fp8_worker_profile(monkeypatch): + candidate = REPOSITORY_ROOT / "manifests" / "candidates" / "qwen3.8-27b-fp8-dequant-eager.json" + manifest = ModelManifest.load(candidate) + created = {} + + monkeypatch.setattr(local_swarm, "ServerInfo", lambda **kwargs: kwargs) + + def capture_create(**kwargs): + created.update(kwargs) + return "worker" + + monkeypatch.setattr(local_swarm.ModuleContainer, "create", capture_create) + + worker = local_swarm.create_qualification_worker( + manifest=manifest, + server_info_kwargs={"throughput": 1.0}, + container_kwargs={"dht_prefix": manifest.dht_prefix}, + ) + + assert worker == "worker" + assert created["server_info"] == {"throughput": 1.0, "quant_type": "fp8_dequant"} + assert created["quant_type"] is QuantType.FP8_DEQUANT + assert created["dht_prefix"] == manifest.dht_prefix + assert local_swarm.manifest_quant_type(None) is QuantType.NONE + + def test_hub_snapshot_cache_is_inferred_narrowly(tmp_path): hub = tmp_path / "hub" snapshot = hub / "models--Qwen--Qwen3-1.7B" / "snapshots" / ("a" * 40) diff --git a/tests/test_qualification_image_contract.py b/tests/test_qualification_image_contract.py index f5c5a2939..2e99849bd 100644 --- a/tests/test_qualification_image_contract.py +++ b/tests/test_qualification_image_contract.py @@ -239,7 +239,9 @@ def test_standalone_image_verifier_needs_no_prepare_only_helper(tmp_path: Path): check=False, capture_output=True, text=True, - timeout=10, + # A cold Torch/Transformers import can exceed ten seconds on macOS CI. + # This checks standalone verification, not an import-latency target. + timeout=30, ) assert result.returncode == 0, result.stderr diff --git a/tests/test_qualify_catalog_desktop.py b/tests/test_qualify_catalog_desktop.py new file mode 100644 index 000000000..d002a5522 --- /dev/null +++ b/tests/test_qualify_catalog_desktop.py @@ -0,0 +1,58 @@ +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "qualify_catalog_desktop.py" +sys.path.insert(0, str(SCRIPT.parents[1] / "desktop" / "src")) +spec = importlib.util.spec_from_file_location("qualify_catalog_desktop", SCRIPT) +replay = importlib.util.module_from_spec(spec) +spec.loader.exec_module(replay) + + +def test_default_replay_cannot_inherit_a_visible_qt_platform(): + args = replay.build_parser().parse_args(["--desktop", "CommunityAI.exe", "--output", "new-state"]) + assert args.visible_ui is False + with patch.dict(replay.os.environ, {"QT_QPA_PLATFORM": "windows", "KEEP_TEST_ENV": "value"}): + environment = replay.desktop_environment(args.visible_ui) + assert environment["QT_QPA_PLATFORM"] == "offscreen" + assert environment["KEEP_TEST_ENV"] == "value" + assert replay.os.environ["QT_QPA_PLATFORM"] == "windows" + + +def test_native_windows_require_explicit_visible_qualification_option(): + args = replay.build_parser().parse_args(["--desktop", "CommunityAI.exe", "--output", "new-state", "--visible-ui"]) + with patch.dict(replay.os.environ, {"QT_QPA_PLATFORM": "offscreen"}): + assert replay.desktop_environment(args.visible_ui, system="Windows")["QT_QPA_PLATFORM"] == "windows" + + +def test_linux_replay_stays_offscreen_and_selects_its_packaged_node(): + assert replay.desktop_environment(False, system="Linux")["QT_QPA_PLATFORM"] == "offscreen" + assert replay.node_executable(Path("bundle/CommunityAI"), "Linux") == Path("bundle/node/CommunityAI-Node") + assert replay.node_executable(Path("bundle/CommunityAI.exe"), "Windows") == Path("bundle/node/CommunityAI-Node.exe") + with pytest.raises(ValueError, match="offscreen"): + replay.desktop_environment(True, system="Linux") + + +def test_linux_replay_requires_ordinary_uid_without_windows_calls(monkeypatch): + monkeypatch.setattr(replay.platform, "system", lambda: "Linux") + monkeypatch.setattr(replay.os, "geteuid", lambda: 1000, raising=False) + monkeypatch.setattr(replay.ctypes, "windll", None, raising=False) + assert replay.qualification_platform() == "Linux" + assert replay.visible_window(1234) is False + monkeypatch.setattr(replay.os, "geteuid", lambda: 0) + with pytest.raises(RuntimeError, match="ordinary Linux user"): + replay.qualification_platform() + + +def test_windows_replay_still_refuses_elevation(monkeypatch): + monkeypatch.setattr(replay.platform, "system", lambda: "Windows") + admin = SimpleNamespace(IsUserAnAdmin=lambda: True) + monkeypatch.setattr(replay.ctypes, "windll", SimpleNamespace(shell32=admin), raising=False) + with pytest.raises(RuntimeError, match="non-elevated Windows"): + replay.qualification_platform() + admin.IsUserAnAdmin = lambda: False + assert replay.qualification_platform() == "Windows" diff --git a/tests/test_qwen3_5_multimodal_loading.py b/tests/test_qwen3_5_multimodal_loading.py index 834f56723..365b223f7 100644 --- a/tests/test_qwen3_5_multimodal_loading.py +++ b/tests/test_qwen3_5_multimodal_loading.py @@ -1,4 +1,4 @@ -"""Offline loading tests for the Qwen3.5 multimodal wrapper's text tower.""" +"""Offline loading tests for the Qwen3.5/Qwen3.8 multimodal wrapper's text tower.""" import os @@ -103,6 +103,121 @@ def test_qwen3_5_wrapper_detection_and_dispatch(wrapper_checkpoint, text_only_ch assert AutoDistributedConfig.from_pretrained(text_only_checkpoint).block_prefix == "model.layers" +def test_qwen3_8_27b_release_shape_dispatches_offline(tmp_path): + """Lock the public 27B release's architecture contract without downloading its weights.""" + from transformers.models.qwen3_5 import Qwen3_5Config, Qwen3_5VisionConfig + + text_config = DistributedQwen3_5Config( + vocab_size=248320, + hidden_size=5120, + intermediate_size=17408, + num_hidden_layers=64, + num_attention_heads=24, + num_key_value_heads=4, + head_dim=256, + linear_num_key_heads=16, + linear_num_value_heads=48, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_conv_kernel_dim=4, + full_attention_interval=4, + max_position_embeddings=262144, + tie_word_embeddings=False, + ) + outer_config = Qwen3_5Config( + text_config=text_config.to_dict(), + vision_config=Qwen3_5VisionConfig( + depth=27, + hidden_size=1152, + intermediate_size=4304, + num_heads=16, + out_hidden_size=5120, + num_position_embeddings=2304, + ).to_dict(), + tie_word_embeddings=False, + ) + outer_config.architectures = ["Qwen3_5ForConditionalGeneration"] + outer_config.save_pretrained(tmp_path) + + config = AutoDistributedConfig.from_pretrained(tmp_path) + + assert isinstance(config, DistributedQwen3_5Config) + assert config._source_architectures == ("Qwen3_5ForConditionalGeneration",) + assert config.block_prefix == "model.language_model.layers" + assert config.num_hidden_layers == 64 + assert config.hidden_size == 5120 + assert config.vocab_size == 248320 + assert config.layer_types.count("linear_attention") == 48 + assert config.layer_types.count("full_attention") == 16 + + +def test_fp8_source_config_dequantizes_a_real_block_and_runs_forward(tmp_path, monkeypatch): + """Cover source config dispatch through the production block loader and forward path.""" + import importlib + + from transformers.models.qwen3_5 import Qwen3_5Config, Qwen3_5ForCausalLM, Qwen3_5VisionConfig + + from drift.server.block_utils import get_model_block + + text_config = _tiny_text_config() + text_config.num_hidden_layers = 1 + text_config.layer_types = text_config.layer_types[:1] + torch.manual_seed(1) + reference = Qwen3_5ForCausalLM(text_config).eval() + outer_config = Qwen3_5Config( + text_config=text_config.to_dict(), + vision_config=Qwen3_5VisionConfig( + depth=1, + hidden_size=32, + intermediate_size=64, + num_heads=4, + out_hidden_size=64, + num_position_embeddings=16, + ).to_dict(), + tie_word_embeddings=True, + quantization_config={ + "quant_method": "fp8", + "activation_scheme": "dynamic", + "weight_block_size": [128, 128], + }, + ) + outer_config.architectures = ["Qwen3_5ForConditionalGeneration"] + outer_config.save_pretrained(tmp_path) + + config = AutoDistributedConfig.from_pretrained(tmp_path) + assert config._source_quantization_method == "fp8" + + checkpoint, expected_state = {}, {} + for name, tensor in reference.model.layers[0].state_dict().items(): + if tensor.is_floating_point() and tensor.ndim >= 2: + quantized = tensor.to(torch.float8_e4m3fn) + checkpoint[name] = quantized + checkpoint[f"{name}_scale_inv"] = torch.ones((*tensor.shape[:-2], 1, 1), dtype=torch.float32) + expected_state[name] = quantized.to(torch.float32) + else: + checkpoint[name] = tensor.detach().clone() + expected_state[name] = tensor.detach().clone() + + loader_module = importlib.import_module("drift.server.from_pretrained") + monkeypatch.setattr(loader_module, "_load_state_dict_from_repo", lambda *args, **kwargs: checkpoint) + + actual_block = load_pretrained_block( + str(tmp_path), + 0, + config=config, + torch_dtype=torch.float32, + quant_type=QuantType.FP8_DEQUANT, + ).eval() + expected_block = get_model_block(config, layer_idx=0).eval() + expected_block.load_state_dict(expected_state) + + hidden_states = torch.randn(1, 4, config.hidden_size) + with torch.inference_mode(): + (actual,) = actual_block(hidden_states) + (expected,) = expected_block(hidden_states) + assert torch.allclose(actual, expected, atol=1e-6), (actual - expected).abs().max() + + def test_qwen3_5_cache_budget_includes_fixed_recurrent_state(): config = _tiny_text_config() strategy = config.kv_cache_strategy diff --git a/tests/test_qwen_formation_runner.py b/tests/test_qwen_formation_runner.py new file mode 100644 index 000000000..ec02cf835 --- /dev/null +++ b/tests/test_qwen_formation_runner.py @@ -0,0 +1,283 @@ +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from qwen_formation_node import LOCAL, REMOTE, coverage, coverage_observed, node_config, selected +from run_qwen_formation import ROOT, FormationRun, validate_config + +from drift.node.config import NodeConfig + + +def config(): + return json.loads((ROOT / "config/qwen_formation.json").read_text()) + + +def test_formation_configuration_only_supplies_capacity(tmp_path): + value = node_config( + ROOT, tmp_path, {"peers": ["/ip4/127.0.0.1/tcp/31330"], "capacity_blocks": 16, "ip": "127.0.0.1"} + ) + parsed = NodeConfig.from_dict(value, base_dir=ROOT) + assert parsed.workers[0].model == "auto" + assert parsed.workers[0].num_blocks == 16 + assert parsed.workers[0].block_indices is None + assert parsed.inference_mode == "auto" + assert value["catalog_path"] == str(ROOT / "public-alpha/catalog-qwen-v2/catalog.signed.json") + + +@pytest.mark.parametrize( + "field,value", + [ + ("spans", ["0:16"]), + ("block_indices", "0:16"), + ("worker_machine_type", "e2-highmem-4"), + ("max_duration_seconds", 86400), + ("zone", "europe-west1-b"), + ], +) +def test_formation_rejects_assignments_or_unbounded_topology(field, value): + proposed = config() + proposed[field] = value + with pytest.raises(ValueError): + validate_config(proposed) + + +def test_host_configuration_cannot_smuggle_an_assignment(tmp_path): + with pytest.raises(ValueError, match="assigned"): + node_config(ROOT, tmp_path, {"span": "0:16"}) + + +def test_desktop_policy_gates_startup_without_disabling_restart(tmp_path): + value = node_config( + ROOT, + tmp_path, + { + "peers": ["/ip4/127.0.0.1/tcp/31330"], + "capacity_blocks": 16, + "ip": "203.0.113.4", + "public_port": 43210, + "desktop_driven_sharing": True, + }, + ) + worker = NodeConfig.from_dict(value, base_dir=ROOT).workers[0] + assert worker.enabled is True + assert value["contribution_policy"]["sharing_enabled"] is False + assert worker.block_indices is None + assert worker.port == 31330 + assert worker.public_port == 43210 + + +@pytest.mark.parametrize("automatic_start", [False, True]) +def test_desktop_replays_literal_start_after_policy_save(tmp_path, automatic_start): + from types import SimpleNamespace + + sys.path.insert(0, str(ROOT / "desktop/src")) + from qwen_formation_desktop import FormationDesktop + + identity = "a" * 24 + (tmp_path / "desktop-command.json").write_text( + json.dumps({"id": identity, "action": "start-sharing", "source": "local"}) + ) + calls = [] + contribution = {"policy": {"sharing_enabled": True}, "intent_enabled": automatic_start} + + def pause(): + calls.append("model-pause") + contribution["intent_enabled"] = False + + def start(): + calls.append("master-start") + contribution["intent_enabled"] = True + + checkbox = SimpleNamespace( + accessibleName=lambda: "Share compute with Qwen", + isChecked=lambda: contribution["intent_enabled"], + isEnabled=lambda: True, + click=pause, + ) + window = SimpleNamespace( + _controller=object(), + _busy=False, + _snapshot={"contribution": contribution, "workers": [{"model": "Qwen", "desired_running": True}]}, + _page_buttons=[None, None, SimpleNamespace(click=lambda: None)], + findChildren=lambda kind: [checkbox], + master_share_button=SimpleNamespace( + isEnabled=lambda: True, + text=lambda: "Pause sharing" if contribution["intent_enabled"] else "Start sharing", + click=start, + ), + ) + automation = FormationDesktop(tmp_path) + automation.window = window + automation.qt = {"QCheckBox": object} + automation.tick() + if automatic_start: + assert calls == ["model-pause"] + assert identity not in automation.clicked + automation.tick() + assert calls == (["model-pause", "master-start"] if automatic_start else ["master-start"]) + assert identity in automation.clicked + + +@pytest.mark.parametrize( + "patch", [{"public_port": 0}, {"public_port": 65536}, {"public_port": True}, {"port": None}, {"public_ip": None}] +) +def test_public_tunnel_configuration_rejects_unusable_endpoints(tmp_path, patch): + from drift.node.config import NodeConfigError + + value = node_config( + ROOT, + tmp_path, + {"peers": ["/ip4/127.0.0.1/tcp/31330"], "capacity_blocks": 16, "ip": "203.0.113.4", "public_port": 43210}, + ) + value["workers"][0].update(patch) + with pytest.raises(NodeConfigError): + NodeConfig.from_dict(value, base_dir=ROOT) + + +@pytest.mark.parametrize("zone", ["us-central1-b", "us-central1-c", "us-central1-f"]) +def test_capacity_retry_can_use_another_approved_zone(zone): + proposed = config() + proposed["zone"] = zone + validate_config(proposed) + + +@pytest.mark.parametrize("machine", ["c3-highmem-4", "n2-highmem-4"]) +def test_capacity_retry_keeps_the_same_cpu_and_memory_profile(machine): + proposed = config() + proposed["worker_machine_type"] = machine + validate_config(proposed) + + +def test_exact_manifest_selection_is_required(): + assert selected({"auto_selection": {"status": "selected", "manifest_digest": LOCAL}}, "local") + assert not selected({"auto_selection": {"status": "selected", "manifest_digest": REMOTE}}, "local") + assert not selected({"auto_selection": {"status": "waiting", "manifest_digest": REMOTE}}, "community") + + +def test_unknown_discovery_cannot_satisfy_a_positive_coverage_checkpoint(): + snapshot = {"models": [{"manifest_digest": REMOTE, "route": {"status": "unknown", "covered_blocks": None}}]} + assert not coverage(snapshot) >= 32 + assert not coverage_observed(snapshot) + + +def test_initial_discovery_requires_a_fresh_observation_even_with_no_workers(): + route = {"status": "incomplete", "covered_blocks": 0, "last_updated_age": 1} + snapshot = {"models": [{"manifest_digest": REMOTE, "route": route}]} + assert coverage_observed(snapshot) + route["last_updated_age"] = 61 + assert not coverage_observed(snapshot) + + +@pytest.mark.parametrize( + "stage", + [ + "preflight", + "bundle", + "create_firewalls", + "create_gcp", + "stage", + "wait_setup", + "start_job", + "start_participant", + "start_desktop", + "exercise", + ], +) +def test_failure_always_records_outcome_and_cleans_after_mutation(tmp_path, monkeypatch, stage): + import run_qwen_formation as module + + run = FormationRun(tmp_path / "q38af-test", config()) + events = [] + + def step(name, result=None): + def execute(*args, **kwargs): + events.append(name) + if name == stage: + raise RuntimeError("injected " + name) + return result + + return execute + + for name in ( + "preflight", + "bundle", + "create_firewalls", + "stage", + "wait_setup", + "start_job", + "start_participant", + "start_desktop", + "write_remote", + "cloud", + "exercise", + ): + monkeypatch.setattr(run, name, step(name)) + run.config["admin_ip"] = "127.0.0.1" + monkeypatch.setattr(module.MixedRun, "create_gcp", step("create_gcp")) + monkeypatch.setattr(run, "wait_file", step("wait_file", {"peers": []})) + monkeypatch.setattr(run, "host_config", lambda *a: {}) + monkeypatch.setattr(run, "stop_desktop", step("stop_desktop")) + monkeypatch.setattr(run, "capture", step("capture")) + monkeypatch.setattr(run, "cleanup", step("cleanup", {"verified": True})) + result = run.run() + assert result["result"] == "failed" + assert result["error"] == "RuntimeError: injected " + stage + assert "stop_desktop" in events + assert ("cleanup" in events) == (stage not in {"preflight", "bundle"}) + assert json.loads((run.path / "result.json").read_text())["result"] == "failed" + assert json.loads((run.path / "qualification/run-state.json").read_text())["result"] == "failed" + + +def test_diagnostics_failure_cannot_prevent_cloud_cleanup(tmp_path, monkeypatch): + run = FormationRun(tmp_path / "q38af-test", config()) + monkeypatch.setattr(run, "preflight", lambda: None) + monkeypatch.setattr(run, "bundle", lambda: None) + monkeypatch.setattr(run, "create_firewalls", lambda: (_ for _ in ()).throw(RuntimeError("mutation"))) + monkeypatch.setattr(run, "capture", lambda: (_ for _ in ()).throw(RuntimeError("diagnostics"))) + calls = [] + monkeypatch.setattr(run, "cleanup", lambda: calls.append("cleanup") or {"verified": True}) + result = run.run() + assert calls == ["cleanup"] + assert result["diagnostic_error"] == "diagnostics" + + +def test_assignment_seed_uses_persistent_identity_not_installation_path(tmp_path, monkeypatch): + from types import SimpleNamespace + + from drift.cli import run_node + + # Same conventional basename on different machines must not be a common seed. + paths = [tmp_path / name / "worker-identity.key" for name in ("a", "b")] + seeds = [] + monkeypatch.setattr(run_node, "AutomaticContributionPlanner", lambda **kwargs: seeds.append(kwargs["jitter_seed"])) + for path in [*paths, paths[0]]: + worker = SimpleNamespace(worker_id="automatic", model="auto", num_blocks=16, identity_path=path) + config = SimpleNamespace(workers=[worker], discovery_update_period=5, route_demand_authority_roots=()) + run_node._build_automatic_placement_service( + config, None, None, None, None, token=None, config_path=None, peer_cache=None + ) + assert seeds[0] != seeds[1] + assert seeds[0] == seeds[2] + + +def test_unavailable_contribution_identity_does_not_abort_local_service(tmp_path, monkeypatch): + from types import SimpleNamespace + + from drift.cli import run_node + + def unavailable(*args): + raise PermissionError("key temporarily inaccessible") + + monkeypatch.setattr(run_node.NodeIdentity, "ensure", unavailable) + worker = SimpleNamespace( + worker_id="automatic", model="auto", num_blocks=16, identity_path=tmp_path / "identity.key" + ) + config = SimpleNamespace(workers=[worker], discovery_update_period=5, route_demand_authority_roots=()) + service = run_node._build_automatic_placement_service( + config, None, None, None, None, token=None, config_path=None, peer_cache=None + ) + assert service is not None + assert run_node._automatic_placement_seed(worker) != run_node._automatic_placement_seed(worker) diff --git a/tests/test_qwen_full_inference_gcp.py b/tests/test_qwen_full_inference_gcp.py new file mode 100644 index 000000000..fd70cc8d4 --- /dev/null +++ b/tests/test_qwen_full_inference_gcp.py @@ -0,0 +1,157 @@ +"""Evidence checks for a full real route and same-session worker replacement.""" +import copy +import hashlib +import json +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from run_qwen_full_inference_gcp import ( + MANIFEST_DIGEST, + REVISION, + CommandError, + LauncherLock, + SwarmRun, + process_exists, + validate_result, + validate_route, +) + + +def evidence(): + route = [dict(start=i * 16, end=(i + 1) * 16, peer_id=f"peer-{i}") for i in range(4)] + after = copy.deepcopy(route) + after[1]["peer_id"] = "replacement" + return { + "result": "passed", + "manifest_digest": MANIFEST_DIGEST, + "model_revision": REVISION, + "baseline": {"route": route, "token_ids": [11, 22, 33]}, + "recovery": { + "before_route": route, + "after_route": after, + "token_ids": [11, 22, 33], + "matches_baseline": True, + "same_session": True, + "position_before": 5, + "position_after": 7, + }, + } + + +class FullInferenceEvidenceTests(unittest.TestCase): + def test_stage_copy_retries_the_same_verified_file(self): + run = SwarmRun.__new__(SwarmRun) + run.config = {"zone": "test-zone"} + run.deadline = float("inf") + run.cloud = Mock(side_effect=[RuntimeError("connection closed"), None]) + run.event = Mock() + with patch("run_qwen_full_inference_gcp.time.sleep"): + run.scp("worker", Path("source.tar.gz"), "/tmp/source.tar.gz") + self.assertEqual(run.cloud.call_count, 2) + self.assertEqual(run.cloud.call_args_list[0], run.cloud.call_args_list[1]) + + def test_resume_rejects_an_unchanged_worker_or_restarted_client_before_mutating(self): + for replace_worker in (False, True): + with self.subTest(replace_worker=replace_worker), tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "q38-resume-test" + run = SwarmRun(path, {"max_duration_seconds": 21600, "zone": "test-zone"}) + (path / "events.jsonl").write_text(json.dumps({"time": time.time()}) + "\n") + (path / "source.tar.gz").write_bytes(b"frozen source") + (path / "source-inventory.json").write_text( + json.dumps({"bundle_sha256": hashlib.sha256(b"frozen source").hexdigest()}) + ) + (path / "baseline.json").write_text(json.dumps(evidence()["baseline"])) + (path / "recovery-ready.json").write_text(json.dumps({"route": evidence()["baseline"]["route"]})) + (path / "client-before-recovery.json").write_text(json.dumps({"client_pid": 444})) + instances = [] + for index, name in enumerate(run.names): + instance = { + "id": str(index), + "creationTimestamp": "original", + "labels": {"q38-run": run.run_id}, + "networkInterfaces": [{"networkIP": "192.0.2.1"}], + } + (path / (name + "-instance-original.json")).write_text(json.dumps(instance)) + instances.append(instance) + if replace_worker: + instances[2]["id"] = "new-generation" + run.cloud_json = Mock(side_effect=instances) + run.ssh = Mock(return_value=subprocess.CompletedProcess([], 0, "555\n", "")) + run.stage = Mock() + run.cleanup = Mock() + with self.assertRaisesRegex(ValueError, "client process|middle worker"): + run.resume_replacement() + run.stage.assert_not_called() + run.cleanup.assert_not_called() + + def test_monitor_timeout_does_not_abort_inference(self): + run = SwarmRun.__new__(SwarmRun) + run.config = {"zone": "test-zone"} + run.cloud = Mock(side_effect=CommandError("transport timeout")) + run.event = Mock() + self.assertIsNone(run.read("worker", "health.json")) + run.event.assert_called_once() + with self.assertRaises(CommandError): + run.ssh("worker", "a mutation") + + def test_lock_probe_does_not_kill_the_active_launcher(self): + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + self.assertTrue(process_exists(child.pid)) + self.assertIsNone(child.poll()) + with tempfile.TemporaryDirectory() as directory: + lock = Path(directory) / "launcher.lock" + lock.write_text(str(child.pid)) + with self.assertRaisesRegex(RuntimeError, "already running"): + with LauncherLock(lock): + self.fail("active launcher lock was ignored") + self.assertIsNone(child.poll()) + finally: + child.terminate() + child.wait(timeout=10) + self.assertFalse(process_exists(child.pid)) + + def test_complete_replacement_passes(self): + validate_result(evidence(), "peer-1") + + def test_missing_final_block_rejected(self): + route = evidence()["baseline"]["route"] + route[-1]["end"] = 63 + with self.assertRaises(ValueError): + validate_route(route) + + def test_gap_and_duplicate_workers_rejected(self): + for field, value in [("start", 17), ("peer_id", "peer-0")]: + route = evidence()["baseline"]["route"] + route[1][field] = value + with self.assertRaises(ValueError): + validate_route(route) + + def test_unchanged_peer_is_not_recovery(self): + value = evidence() + value["recovery"]["after_route"] = value["recovery"]["before_route"] + with self.assertRaises(ValueError): + validate_result(value, "peer-1") + + def test_changed_tokens_or_restarted_session_rejected(self): + for field, val in [("token_ids", [11, 22, 44]), ("same_session", False), ("position_after", 5)]: + value = evidence() + value["recovery"][field] = val + with self.assertRaises(ValueError): + validate_result(value, "peer-1") + + def test_unbound_model_rejected(self): + value = evidence() + value["model_revision"] = "another-revision" + with self.assertRaises(ValueError): + validate_result(value, "peer-1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qwen_mixed_inference.py b/tests/test_qwen_mixed_inference.py new file mode 100644 index 000000000..8524992ca --- /dev/null +++ b/tests/test_qwen_mixed_inference.py @@ -0,0 +1,99 @@ +"""Keep mixed provisioning behind the completed CPU proof.""" +import copy +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from run_qwen_mixed_inference import MixedRun, mixed_quota_requirements, require_cpu_proof +from test_qwen_full_inference_gcp import evidence + + +class MixedInferenceTests(unittest.TestCase): + def test_cpu_family_quotas_and_supported_disks_match_created_instances(self): + config = json.loads((Path(__file__).resolve().parents[1] / "config/qwen_mixed_inference.json").read_text()) + config["disk_gb"] = 80 + for machine, e2, standard, ssd in [("e2-highmem-4", 12, 240, 100), ("c3-highmem-4", 4, 80, 260)]: + config["worker_machine_type"] = machine + quotas = mixed_quota_requirements(config) + self.assertEqual(quotas["E2_CPUS"], e2) + self.assertEqual(quotas["CPUS_ALL_REGIONS"], 20) + self.assertEqual(quotas["DISKS_TOTAL_GB"], standard) + self.assertEqual(quotas["SSD_TOTAL_GB"], ssd) + self.assertEqual(quotas.get("C3_CPUS", 0), 8 if machine.startswith("c3") else 0) + with tempfile.TemporaryDirectory() as directory: + run = MixedRun(Path(directory) / "q38m-test", config) + run.cloud = Mock() + run.cloud_json = Mock( + return_value={ + "networkInterfaces": [{"networkIP": "10.0.0.1", "accessConfigs": [{"natIP": "192.0.2.1"}]}] + } + ) + run.create_gcp(run.names[3], machine) + args = run.cloud.call_args.args[0] + self.assertIn( + "--boot-disk-type=" + ("pd-balanced" if machine.startswith("c3") else "pd-standard"), args + ) + self.assertEqual("--network-interface" in args, machine.startswith("c3")) + if machine.startswith("c3"): + self.assertIn("nic-type=GVNIC,subnet=" + config["subnet"], args) + + def test_quota_estimate_rejects_unaccounted_topology_changes(self): + config = json.loads((Path(__file__).resolve().parents[1] / "config/qwen_mixed_inference.json").read_text()) + for key, value in [ + ("worker_machine_type", "c3-highmem-8"), + ("client_machine_type", "e2-standard-8"), + ("gpu_machine_type", "g2-standard-16"), + ("spans", ["0:64"]), + ]: + with self.assertRaises(ValueError): + mixed_quota_requirements(dict(config, **{key: value})) + + def test_only_complete_cpu_recovery_and_cleanup_unlock_mixed(self): + value = {"result": "passed", "topology": "gcp-cpu", "cleanup": {"verified": True}, "evidence": evidence()} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "replacement.json").write_text(json.dumps({"lost_peer": "peer-1"})) + (path / "result.json").write_text(json.dumps(value)) + self.assertEqual(require_cpu_proof(path)["run_id"], path.name) + for field, replacement in [ + ("result", "failed"), + ("cleanup", {"verified": False}), + ("topology", "gcp-l4-azure-t4-cpu"), + ]: + changed = copy.deepcopy(value) + changed[field] = replacement + (path / "result.json").write_text(json.dumps(changed)) + with self.assertRaises(ValueError): + require_cpu_proof(path) + + def test_inference_alone_does_not_unlock_mixed(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "result.json").write_text(json.dumps({"result": "passed", "evidence": evidence()})) + with self.assertRaises(OSError): + require_cpu_proof(path) + + def test_gpu_and_cpu_runtime_profiles_share_the_four_spans(self): + with tempfile.TemporaryDirectory() as directory: + run = MixedRun( + Path(directory) / "q38m-test", + {"max_duration_seconds": 21600, "ubuntu_driver_version": "580.173.02-0ubuntu0.24.04.1"}, + ) + run.public_ips = {name: f"192.0.2.{i + 1}" for i, name in enumerate(run.names)} + for i, name in enumerate(run.names[1:]): + config = run.host_config(name, f"{i * 16}:{(i + 1) * 16}", ["peer"]) + self.assertEqual(config["device"], "cuda" if i < 2 else "cpu") + self.assertFalse(config["run_recovery"]) + setup = run.setup_source(name, "a" * 64) + self.assertIn("/whl/cu124" if i < 2 else "/whl/cpu", setup) + self.assertEqual(" gpu_probe" in setup, i < 2) + self.assertEqual("nvidia-driver-580-server=" in setup, i < 2) + self.assertIn(run.run_id + "-public", run.firewalls) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qwen_offline_http.py b/tests/test_qwen_offline_http.py new file mode 100644 index 000000000..86e55d685 --- /dev/null +++ b/tests/test_qwen_offline_http.py @@ -0,0 +1,36 @@ +import sys +from pathlib import Path + +import httpx +import pytest +import requests + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from qwen_offline_http import HttpDownloadBlocker + + +def test_download_blocker_rejects_both_http_libraries_and_https_tunnels(): + blocker = HttpDownloadBlocker() + try: + proxies = {"http": blocker.url, "https": blocker.url} + assert requests.get("http://hub-offline.invalid", proxies=proxies, timeout=3).status_code == 403 + with pytest.raises(requests.exceptions.ProxyError): + requests.get("https://hub-offline.invalid", proxies=proxies, timeout=3) + with httpx.Client(proxy=blocker.url, timeout=3, trust_env=False) as client: + assert client.get("http://hub-offline.invalid").status_code == 403 + with pytest.raises(httpx.ProxyError): + client.get("https://hub-offline.invalid") + assert blocker.denied_requests == 4 + env = blocker.environment( + {"http_proxy": "inherited", "HTTPS_PROXY": "inherited", "no_proxy": "*", "KEEP": "ok"} + ) + assert env == dict( + HTTP_PROXY=blocker.url, + HTTPS_PROXY=blocker.url, + ALL_PROXY=blocker.url, + NO_PROXY="127.0.0.1,localhost,::1", + KEEP="ok", + ) + finally: + blocker.close() + assert not blocker.thread.is_alive() diff --git a/tests/test_qwen_packaged_worker_action.py b/tests/test_qwen_packaged_worker_action.py new file mode 100644 index 000000000..08303492f --- /dev/null +++ b/tests/test_qwen_packaged_worker_action.py @@ -0,0 +1,65 @@ +import json +import sys +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import qwen_packaged_worker_action as action_module + + +@pytest.mark.parametrize("failure", ["expired", "finished", "wrong-label", "wrong-tag", "wrong-name", None]) +def test_worker_action_requires_live_window_and_actual_instance_ownership(tmp_path, monkeypatch, failure): + monkeypatch.setattr(action_module, "RUNS", tmp_path) + path = tmp_path / "q38pm-test" + path.mkdir() + (path / "packaged-client-ready.json").write_text( + json.dumps( + { + "run_id": path.name, + "deadline_unix": time.time() + (-10 if failure == "expired" else 600), + } + ) + ) + (path / "provider-config.json").write_text('{"zone":"us-central1-b"}') + if failure == "finished": + (path / "result.json").write_text("{}") + commands = [] + + class Run: + names = ["unused"] * 3 + [path.name + "-w2"] + + def __init__(self, *args): + pass + + def cloud_json(self, args): + return { + "name": "communityai-bootstrap-1" if failure == "wrong-name" else self.names[3], + "labels": {"q38-run": "another-run" if failure == "wrong-label" else path.name}, + "tags": {"items": [] if failure == "wrong-tag" else [path.name]}, + } + + def ssh(self, name, command): + from types import SimpleNamespace + + commands.append((name, command)) + return SimpleNamespace(stdout="MainPID=0\nActiveState=inactive\nKillMode=control-group\n") + + monkeypatch.setattr(action_module, "MixedProductRun", Run) + if failure is not None: + with pytest.raises(ValueError): + action_module.worker_action(path, "stop") + assert commands == [] + else: + result = action_module.worker_action(path, "stop") + assert result["instance"] == path.name + "-w2" + assert commands[1] == (path.name + "-w2", "sudo systemctl stop q38-worker") + + +def test_worker_action_rejects_unowned_paths_and_other_commands(tmp_path, monkeypatch): + monkeypatch.setattr(action_module, "RUNS", tmp_path) + with pytest.raises(ValueError): + action_module.worker_action(tmp_path.parent / "elsewhere", "stop") + with pytest.raises(ValueError): + action_module.worker_action(tmp_path / "q38pm-test", "delete") diff --git a/tests/test_qwen_product_recovery.py b/tests/test_qwen_product_recovery.py new file mode 100644 index 000000000..060c665ad --- /dev/null +++ b/tests/test_qwen_product_recovery.py @@ -0,0 +1,63 @@ +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import qwen_product_recovery as recovery + + +@pytest.mark.parametrize( + "state,pid,kill_mode,stopped", + [ + ("inactive", 0, "control-group", True), + ("failed", 0, "control-group", True), + ("failed", 12, "control-group", False), + ("active", 0, "control-group", False), + ("inactive", 0, "process", False), + ], +) +def test_signal_terminated_unit_counts_as_stopped_only_without_a_live_process(state, pid, kill_mode, stopped): + value = f"MainPID={pid}\nActiveState={state}\nKillMode={kill_mode}\n" + assert recovery.worker_is_stopped(value) is stopped + + +@pytest.mark.parametrize("new_receipt", [False, True]) +def test_stale_control_receipt_cannot_release_a_new_recovery_attempt(tmp_path, monkeypatch, new_receipt): + path = tmp_path / "product-worker-stopped.json" + path.write_text(json.dumps({"recovery_nonce": "previous-attempt"})) + clock = [0] + monkeypatch.setattr(recovery.time, "monotonic", lambda: clock[0]) + + def sleep(seconds): + clock[0] += seconds + if new_receipt: + path.write_text(json.dumps({"recovery_nonce": "current-attempt"})) + + monkeypatch.setattr(recovery.time, "sleep", sleep) + if new_receipt: + assert recovery.wait_recovery_control(path, "current-attempt", 2)["recovery_nonce"] == "current-attempt" + assert clock[0] == 1 + else: + with pytest.raises(TimeoutError): + recovery.wait_recovery_control(path, "current-attempt", 2) + + +@pytest.mark.parametrize("fault", [None, "missing", "stale", "wrong-peer"]) +def test_recovery_evidence_must_acknowledge_this_fault_and_replacement(fault): + evidence = { + "worker_stopped_acknowledgement": {"recovery_nonce": "current", "peer_id": "old-peer"}, + "worker_replaced_acknowledgement": {"recovery_nonce": "current", "peer_id": "new-peer"}, + } + if fault == "missing": + evidence.pop("worker_stopped_acknowledgement") + if fault == "stale": + evidence["worker_stopped_acknowledgement"]["recovery_nonce"] = "previous" + if fault == "wrong-peer": + evidence["worker_replaced_acknowledgement"]["peer_id"] = "old-peer" + if fault: + with pytest.raises(RuntimeError, match="recovery acknowledgements"): + recovery.require_recovery_acknowledgements(evidence, "current", "old-peer", "new-peer") + else: + recovery.require_recovery_acknowledgements(evidence, "current", "old-peer", "new-peer") diff --git a/tests/test_qwen_product_runners.py b/tests/test_qwen_product_runners.py new file mode 100644 index 000000000..17ca5fa43 --- /dev/null +++ b/tests/test_qwen_product_runners.py @@ -0,0 +1,139 @@ +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from run_qwen_product_gcp import ROOT, ProductRun +from run_qwen_product_mixed import MixedProductRun + + +def test_mixed_product_failure_and_failed_diagnostics_still_clean_both_providers(tmp_path, monkeypatch): + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + run = MixedProductRun(tmp_path / "q38pm-test", config) + calls = [] + for name in ("preflight", "bundle", "create_firewalls", "create_gcp"): + monkeypatch.setattr(run, name, lambda *args, **kwargs: None) + monkeypatch.setattr(run, "az_json", lambda *args: {"registrationState": "Registered"}) + + def fail(*args, **kwargs): + raise RuntimeError("simulated provider failure") + + monkeypatch.setattr(run, "create_azure", fail) + monkeypatch.setattr(run, "capture_product", fail) + monkeypatch.setattr(run, "cleanup", lambda: calls.append("both-providers") or {"verified": True}) + result = run.run() + assert result["result"] == "failed" + assert result["cleanup"]["verified"] + assert "diagnostic_error" in result + assert calls == ["both-providers"] + + +def test_mixed_product_stages_api_and_gpu_dependencies(tmp_path): + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + run = MixedProductRun(tmp_path / "q38pm-test", config) + coordinator = run.setup_source(run.names[0], "0" * 64) + worker = run.setup_source(run.names[1], "0" * 64) + assert "source[api]" in coordinator + assert "nvidia-driver-580-server" in worker + assert "cu124" in worker + assert run.bundle.__func__ is ProductRun.bundle + + +@pytest.mark.parametrize("source_finished", [False, True]) +def test_failed_source_can_hold_loaded_workers_for_independent_package_without_passing( + tmp_path, monkeypatch, source_finished +): + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + config["packaged_client_wait_seconds"] = 60 + run = MixedProductRun(tmp_path / "q38pm-test", config) + calls = [] + for name in ( + "preflight", + "bundle", + "create_firewalls", + "create_gcp", + "create_azure", + "finish_network", + "enable_packaged_client", + "stage", + "wait_setup", + "start_job", + "start_product", + "capture_product", + ): + monkeypatch.setattr(run, name, lambda *args, **kwargs: None) + monkeypatch.setattr(run, "az_json", lambda *args: {"registrationState": "Registered"}) + monkeypatch.setattr(run, "wait_file", lambda *args, **kwargs: {"peers": []}) + monkeypatch.setattr(run, "read", lambda *args: {"result": "failed"} if source_finished else None) + monkeypatch.setattr(run, "wait_packaged_client", lambda: calls.append("package") or {"result": "passed"}) + monkeypatch.setattr(run, "cleanup", lambda: calls.append("cleanup") or {"verified": True}) + (run.path / "workers.json").write_text('{"workers": []}') + + def fail(): + raise RuntimeError("source transition assertion") + + monkeypatch.setattr(run, "exercise_workers", fail) + result = run.run() + assert result["result"] == "failed" + assert "source transition assertion" in result["error"] + assert result["cleanup"]["verified"] + assert calls == (["package", "cleanup"] if source_finished else ["cleanup"]) + + +@pytest.mark.parametrize("receipt", ["passed", "failed", "wrong-run", "not-stopped", "timeout"]) +def test_packaged_wait_requires_bound_result_and_always_has_a_deadline(tmp_path, receipt): + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + config["packaged_client_wait_seconds"] = 1 + run = MixedProductRun(tmp_path / "q38pm-test", config) + if receipt == "timeout": + run.started = 0 + else: + (run.path / "packaged-client-result.json").write_text( + json.dumps( + { + "run_id": "another-run" if receipt == "wrong-run" else run.run_id, + "node_stopped": receipt != "not-stopped", + "result": "failed" if receipt == "failed" else "passed", + } + ) + ) + if receipt == "passed": + assert run.wait_packaged_client()["result"] == "passed" + else: + with pytest.raises((RuntimeError, ValueError, TimeoutError)): + run.wait_packaged_client() + + +def test_source_recovery_rejects_a_result_completed_before_the_injected_outage(tmp_path, monkeypatch): + config = json.loads((ROOT / "config/qwen_full_inference_gcp.json").read_text()) + run = ProductRun(tmp_path / "q38p-test", config) + monkeypatch.setattr(run, "start_job", lambda *args: None) + monkeypatch.setattr( + run, + "ssh", + lambda *args, **kwargs: type( + "Reply", (), {"stdout": "MainPID=0\nActiveState=inactive\nKillMode=control-group\n"} + )(), + ) + calls = [] + + def wait_file(name, filename, **kwargs): + calls.append(filename) + if filename == "worker.json": + return {"peer_id": "replacement" if kwargs.get("predicate") else name} + if filename == "product-ready-for-loss.json": + return {"ready": True, "recovery_nonce": "test-outage"} + if filename == "product-local-after-loss.json": + return {"recovery_nonce": "test-outage"} + if filename == "product-result.json": + # The old harness accepted this even if a spontaneous local fallback + # and recovery had completed while cloud control was unavailable. + return {"result": "passed"} + return {} + + monkeypatch.setattr(run, "wait_file", wait_file) + with pytest.raises(RuntimeError, match="recovery acknowledgements"): + run.exercise_workers() diff --git a/tests/test_qwen_product_test.py b/tests/test_qwen_product_test.py new file mode 100644 index 000000000..684e218d0 --- /dev/null +++ b/tests/test_qwen_product_test.py @@ -0,0 +1,393 @@ +import concurrent.futures +import json +import sys +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import qwen_product_provenance as provenance +import run_qwen_product_test as launcher +from report_qwen_product import report, summarize + + +def put(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +@pytest.fixture +def receipts(tmp_path): + run, output = tmp_path / "q38pm-test", tmp_path / "package" + completion = lambda model: { + "response": {"model": model, "usage": {"completion_tokens": 3}, "choices": [{"text": "Paris"}]} + } + remote, local = "Qwen3.8 27B FP8 Dequant", "Qwen3.5-0.8B-Local" + selection = lambda **kwargs: {"auto_selection": kwargs} + phases = [ + dict( + hub_offline=offline, + node_stopped=True, + node_sha256="a" * 64, + community_completion=completion(remote), + community_chat=completion(remote), + local_only_completion=completion(local), + http_downloads_blocked=offline, + ) + for offline in (False, True) + ] + phases[0]["worker_outage"] = { + "before_stop_status": selection(model=remote), + "fallback_status": selection(source="local"), + "recovered_status": selection(model=remote), + "local_completion": completion(local), + "community_after_rejoin": completion(remote), + "stop": { + "instance": run.name + "-w2", + "action": "stop", + "observed_at_unix": 1, + "after": "MainPID=0\nActiveState=inactive\nKillMode=control-group\n", + }, + "restart": {"instance": run.name + "-w2", "action": "start", "observed_at_unix": 2}, + } + packaged = dict( + run_id=run.name, + result="passed", + node_stopped=True, + phases=phases, + node_sha256="a" * 64, + catalog_scope="public policy", + remote_cache_source="seeded cache", + ) + source = dict( + result="passed", + worker_stopped_acknowledgement={"recovery_nonce": "test", "peer_id": "old"}, + worker_replaced_acknowledgement={"recovery_nonce": "test", "peer_id": "new"}, + ) + final = dict( + result="passed", run_id=run.name, packaged_client=packaged, evidence=source, cleanup={"verified": True} + ) + put(run / "result.json", final) + put(run / "packaged-client-result.json", packaged) + put(output / "result.json", packaged) + put( + run / "provider-config.json", + dict(client_machine_type="e2-standard-4", gpu_machine_type="g2-standard-8", worker_machine_type="c3-highmem-4"), + ) + for suffix, machine in zip( + ("c", "w0", "w2", "w3"), ("e2-standard-4", "g2-standard-8", "c3-highmem-4", "c3-highmem-4") + ): + put( + run / f"{run.name}-{suffix}-instance.json", + {"name": run.name + "-" + suffix, "machineType": "zones/test/" + machine}, + ) + # This used to break the manual report's broad *-instance.json glob. + put(run / f"{run.name}-w1-instance.json", {"azure": True}) + (run / "source.tar.gz").write_bytes(b"test archive") + put(run / "source-inventory.json", {"files": {}, "bundle_sha256": provenance.sha256(run / "source.tar.gz")}) + return run, output + + +def test_report_accepts_both_providers_without_parsing_azure_as_gcp(receipts): + value = summarize(*receipts) + assert value["result"] == "passed" + assert len(value["gcp_instances"]) == 4 + + +@pytest.mark.parametrize("fault", [None, "failed", "changed-inputs", "changed-inventory"]) +def test_standalone_reporting_cannot_override_failed_launcher_provenance(receipts, fault): + run, output = receipts + (run / "launcher-source.tar.gz").write_bytes(b"launch archive") + put( + run / "launcher-source.json", + { + "run_id": run.name, + "files": {}, + "archive_sha256": provenance.sha256(run / "launcher-source.tar.gz"), + "inputs": {"node": {"sha256": "a" * 64}}, + }, + ) + put( + run / "launcher-result.json", + { + "run_id": run.name, + "result": "failed" if fault == "failed" else "passed", + "inputs_unchanged": fault != "changed-inputs", + "source_inventory_sha256": "wrong" + if fault == "changed-inventory" + else provenance.sha256(run / "launcher-source.json"), + }, + ) + result = report(run, output, run / "standalone-report.json") + assert result["result"] == ("passed" if fault is None else "failed") + + +@pytest.mark.parametrize( + "fault", + ["run-id", "cleanup", "receipt", "offline", "already-local", "wrong-worker", "nonce", "topology", "archive"], +) +def test_report_rejects_incomplete_or_unbound_success(receipts, fault): + run, output = receipts + final = launcher.read(run / "result.json") + packaged = final["packaged_client"] + if fault == "run-id": + final["run_id"] = "another-run" + elif fault == "cleanup": + final["cleanup"]["verified"] = False + elif fault == "offline": + packaged["phases"][1]["http_downloads_blocked"] = False + elif fault == "already-local": + packaged["phases"][0]["worker_outage"]["before_stop_status"]["auto_selection"]["model"] = "local" + elif fault == "wrong-worker": + packaged["phases"][0]["worker_outage"]["stop"]["instance"] = "communityai-bootstrap-1" + elif fault == "nonce": + final["evidence"]["worker_replaced_acknowledgement"]["recovery_nonce"] = "stale" + elif fault == "topology": + put(run / f"{run.name}-w2-instance.json", {"name": run.name + "-w2", "machineType": "e2-highmem-4"}) + elif fault == "archive": + (run / "source.tar.gz").write_bytes(b"changed") + put(run / "result.json", final) + put(output / "result.json", packaged) + if fault != "receipt": + put(run / "packaged-client-result.json", packaged) + else: + put(run / "packaged-client-result.json", {"run_id": run.name, "result": "failed"}) + value = report(run, output, run / "report.json") + assert value["result"] == "failed" + with pytest.raises(ValueError, match="Preserve"): + report(run, output, run / "report.json") + + +@pytest.mark.parametrize("change", ["source", "new-source", "input"]) +def test_launch_inventory_preserves_bytes_and_rejects_changes(tmp_path, monkeypatch, change): + root, run = tmp_path / "repo", tmp_path / "run" + run.mkdir() + for name in ("pyproject.toml", "README.md", "LICENSE", "desktop/pyproject.toml", "scripts/runner.py"): + p = root / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("before") + (root / ".publisher-secrets").mkdir() + (root / ".publisher-secrets/key.pem").write_text("must never enter the archive") + item = tmp_path / "input.json" + item.write_text("{}") + monkeypatch.setattr(provenance.subprocess, "run", lambda *a, **k: type("Reply", (), {"stdout": "a" * 40})()) + inventory = provenance.snapshot(root, run, {"input": item}) + provenance.verify_snapshot(root, run, inventory) + assert not any("publisher-secrets" in name for name in inventory["files"]) + target = {"source": root / "scripts/runner.py", "new-source": root / "scripts/new.py", "input": item}[change] + target.write_text("after") + with pytest.raises(ValueError, match="changed during"): + provenance.verify_snapshot(root, run, inventory) + + +def test_package_verification_catches_changed_dependency(tmp_path): + node, dll = tmp_path / "CommunityAI/node.exe", tmp_path / "CommunityAI/dependency.dll" + node.parent.mkdir() + node.write_bytes(b"entrypoint") + dll.write_bytes(b"dependency") + digest = provenance.sha256(node) + p = tmp_path / "provenance.json" + put( + p, + { + "artifacts": [ + { + "path": f.relative_to(tmp_path).as_posix(), + "kind": "file", + "size_bytes": f.stat().st_size, + "sha256": provenance.sha256(f), + } + for f in (node, dll) + ] + }, + ) + assert provenance.verify_package(node, p, digest)["verified_files"] == 2 + extra = node.parent / "unexpected.dll" + extra.write_bytes(b"unlisted") + with pytest.raises(ValueError, match="unlisted"): + provenance.verify_package(node, p, digest) + extra.unlink() + dll.write_bytes(b"wrong-code") + with pytest.raises(ValueError, match="provenance mismatch"): + provenance.verify_package(node, p, digest) + + +def test_main_creates_fresh_runs_and_accepts_preflight_observation_without_changing_inputs(tmp_path, monkeypatch): + import run_qwen_mixed_inference + import run_qwen_product_mixed + + root = tmp_path / "repo" + root.mkdir() + for name in ("pyproject.toml", "README.md", "LICENSE", "desktop/pyproject.toml"): + p = root / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("source") + node = root / "CommunityAI/node.exe" + node.parent.mkdir() + node.write_bytes(b"package") + p = root / "provenance.json" + put( + p, + { + "artifacts": [ + {"path": "CommunityAI/node.exe", "kind": "file", "size_bytes": 7, "sha256": provenance.sha256(node)} + ] + }, + ) + proof = root / "proof" + put(proof / "result.json", {"result": "passed"}) + put(proof / "replacement.json", {}) + put(root / "cache.json", {"result": "passed"}) + put(root / "cloud.json", {}) + (root / "cache").mkdir() + config = root / "replay.json" + put( + config, + dict( + node=str(node), + node_sha256=provenance.sha256(node), + package_provenance=str(p), + local_cache=str(root / "cache"), + remote_cache=str(root / "cache"), + cache_provenance=str(root / "cache.json"), + cloud_config=str(root / "cloud.json"), + ), + ) + seen = [] + + class Preflight: + def __init__(self, path, config): + self.path, self.config = path, config + seen.append(path) + + def preflight(self): + self.config["admin_ip"] = "192.0.2.1" + put(self.path / "provider-config.json", self.config) + + monkeypatch.setattr(launcher, "ROOT", root) + monkeypatch.setattr(run_qwen_product_mixed, "RUNS", root / "runs") + monkeypatch.setattr(run_qwen_product_mixed, "MixedProductRun", Preflight) + monkeypatch.setattr(run_qwen_mixed_inference, "require_cpu_proof", lambda path: {}) + monkeypatch.setattr(launcher.subprocess, "run", lambda *a, **k: type("Reply", (), {"stdout": "a" * 40})()) + args = ["--config", str(config), "--cpu-proof", str(proof), "--preflight-only"] + assert launcher.main(args) == launcher.main(args) == 0 + assert len(set(seen)) == 2 + for path in seen: + assert launcher.read(path / "provider-config.json")["admin_ip"] == "192.0.2.1" + assert "admin_ip" not in launcher.read(path / "requested-provider-config.json") + assert not (path / "result.json").exists() # Preflight is never a generation pass. + + +@pytest.mark.parametrize("failure", ["spawn", "timeout", "exit"]) +def test_packaged_failure_unblocks_cloud_cleanup(tmp_path, monkeypatch, failure): + run = tmp_path / "q38pm-test" + put(run / "packaged-client-ready.json", {"run_id": run.name, "deadline_unix": time.time() + 3600}) + future = concurrent.futures.Future() + calls = [] + + class Process: + pid = 123 + returncode = 1 + + def poll(self): + return None if failure == "timeout" else 1 + + def start(*args, **kwargs): + if failure == "spawn": + raise OSError("could not start qualifier") + if failure == "timeout": + future.set_result({"result": "failed"}) + return Process() + + monkeypatch.setattr(launcher.subprocess, "Popen", start) + monkeypatch.setattr(launcher, "stop_process_tree", lambda p: calls.append(p.pid) or True) + with pytest.raises((OSError, RuntimeError, TimeoutError)): + launcher.execute_packaged( + run, + tmp_path / "out", + {"local_cache": "local", "remote_cache": "remote"}, + {"node": tmp_path / "node.exe", "cache_provenance": tmp_path / "cache.json"}, + future, + ) + receipt = launcher.read(run / "packaged-client-result.json") + assert receipt["run_id"] == run.name and receipt["result"] == "failed" + assert receipt["node_stopped"] is (failure != "exit") + assert calls == ([123] if failure == "timeout" else []) + + +def test_timeout_cleanup_stops_a_real_owned_process_and_child(tmp_path): + import subprocess + + import psutil + + pid_file = tmp_path / "child-pid.txt" + program = ( + "import subprocess,sys,time; from pathlib import Path; " + "child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(90)']); " + "Path(sys.argv[1]).write_text(str(child.pid)); time.sleep(90)" + ) + parent = subprocess.Popen( + [sys.executable, "-c", program, str(pid_file)], creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0) + ) + try: + deadline = time.monotonic() + 10 + while not pid_file.exists() and time.monotonic() < deadline: + time.sleep(0.02) + child = psutil.Process(int(pid_file.read_text())) + assert launcher.stop_process_tree(parent) + assert parent.poll() is not None + assert not child.is_running() + finally: + if parent.poll() is None: + launcher.stop_process_tree(parent) + + +@pytest.mark.parametrize("fail", [False, True]) +def test_orchestration_automatically_hands_off_reports_and_waits_for_cleanup(receipts, monkeypatch, fail): + run_path, output = receipts + inventory_path = run_path / "launcher-source.json" + # The report's source binding is separately tested; this test exercises orchestration. + inventory_path.write_text("{}") + calls = [] + + class Run: + path, run_id = run_path, run_path.name + + def run(self): + put(self.path / "packaged-client-ready.json", {"run_id": self.run_id, "deadline_unix": time.time() + 30}) + while not (self.path / "handoff-done").exists(): + time.sleep(0.01) + calls.append("cleanup") + return {"result": "passed", "cleanup": {"verified": True}} + + def execute(*args): + calls.append("package") + (run_path / "handoff-done").touch() + if fail: + raise RuntimeError("package failed") + return 0 + + monkeypatch.setattr(launcher, "verify_snapshot", lambda *a: None) + monkeypatch.setattr(launcher, "verify_package", lambda *a: None) + + def record(run, output, destination, *, launcher): + assert calls == ["package", "cleanup"] + assert (run / "launcher-result.json").exists() + assert destination.name == "product-report.json" + calls.append("report") + return launcher + + monkeypatch.setattr(launcher, "report", record) + result = launcher.orchestrate( + Run(), + output, + {"node_sha256": "a" * 64}, + {"node": Path("node"), "package_provenance": Path("prov")}, + {}, + {}, + execute=execute, + ) + assert result["result"] == ("failed" if fail else "passed") + assert calls == ["package", "cleanup", "report"] diff --git a/tests/test_qwen_qualification.py b/tests/test_qwen_qualification.py new file mode 100644 index 000000000..7fab27b55 --- /dev/null +++ b/tests/test_qwen_qualification.py @@ -0,0 +1,234 @@ +import json +import os +import shutil +import subprocess +import sys +import threading +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import qwen_qualification as qualification +import run_qwen_qualification as launcher +from qwen_product_provenance import sha256 +from run_qwen_product_mixed import MixedProductRun +from test_qwen_product_test import receipts + + +def put(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +@pytest.mark.parametrize("result_name,exit_code", [("passed", 0), ("failed", 1)]) +@pytest.mark.parametrize("old_marker", [None, "not even valid JSON"]) +def test_gate13_style_launcher_starts_only_a_fresh_run(tmp_path, monkeypatch, result_name, exit_code, old_marker): + run_id = "q38pm-new-test" + runs_root = tmp_path / ".gate13-runs/qwen-product-mixed" + old = runs_root / "q38pm-old" + old.mkdir(parents=True) + (old / "result.json").write_text('{"result":"failed"}') + if old_marker is not None: + (runs_root / "active.json").write_text(old_marker) + calls = [] + + class Run: + def __init__(self, **kwargs): + assert kwargs["output_root"] == runs_root / run_id + assert kwargs["cloud_config"] == {"project": "test-project"} + + def run(self): + calls.append(run_id) + return {"result": result_name, "events": [], "duration_seconds": 0} + + monkeypatch.setattr(launcher, "__file__", str(tmp_path / "scripts/run_qwen_qualification.py")) + monkeypatch.setattr(launcher, "_new_run_id", lambda: run_id) + monkeypatch.setattr(launcher, "_inputs", lambda root: ({}, {}, {"project": "test-project"})) + monkeypatch.setattr(launcher, "QwenQualification", Run) + assert launcher.main([]) == exit_code + assert calls == [run_id] + assert (old / "result.json").read_text() == '{"result":"failed"}' + assert not (runs_root / "launcher.lock").exists() + if old_marker is not None: + assert (runs_root / "active.json").read_text() == old_marker + + +def test_no_arguments_contract_rejects_before_loading_config_or_touching_cloud(monkeypatch): + monkeypatch.setattr(launcher, "_inputs", lambda root: pytest.fail("must not read inputs")) + assert launcher.main(["--resume"]) == 2 + + +@pytest.mark.parametrize( + "fail_at", + [None, "preflight", "package", "bundle", "create", "stage", "source", "client", "interrupt", "cleanup", "report"], +) +def test_ordered_route_client_cleanup_and_failure_records(receipts, monkeypatch, fail_at): + path, old_output = receipts + packaged = json.loads((old_output / "result.json").read_text()) + source = json.loads((path / "result.json").read_text())["evidence"] + # Use the synthetic response payloads as a fake provider's output. The new + # run must generate its own raw result and receive its own client receipt. + (path / "result.json").unlink() + (path / "packaged-client-result.json").unlink() + calls = [] + thread_ids = [] + config = json.loads((ROOT / "config/qwen_mixed_inference.json").read_text()) + config.update(worker_machine_type="c3-highmem-4", packaged_client_wait_seconds=3600, max_duration_seconds=10800) + put(path / "provider-config.json", config) + put( + path / "workers.json", + { + "workers": [ + {"hardware": {"gpu_name": "L4"}}, + {"hardware": {"gpu_name": "T4"}}, + {"hardware": {"device": "cpu"}}, + {"hardware": {"device": "cpu"}}, + ] + }, + ) + + def step(name): + calls.append(name) + thread_ids.append(threading.get_ident()) + if fail_at == name: + raise RuntimeError("injected " + name) + + def capture_source(root, run, inputs): + (run / "launcher-source.tar.gz").write_bytes(b"launcher archive") + value = { + "run_id": run.name, + "files": {}, + "archive_sha256": sha256(run / "launcher-source.tar.gz"), + "inputs": {"node": {"sha256": "a" * 64}}, + } + put(run / "launcher-source.json", value) + return value + + def package(*args): + step("package") + return {"verified_files": 1, "node_sha256": "a" * 64} + + def source_test(self): + step("source") + return {"result": "passed", "evidence": source} + + def client(run, output, config, inputs): + assert (run / "packaged-client-ready.json").is_file() + step("client") + if fail_at == "interrupt": + raise KeyboardInterrupt("injected client interrupt") + put(output / "result.json", packaged) + put(run / "packaged-client-result.json", packaged) + return 0 + + def cleanup(self): + step("cleanup") + return {"verified": True} + + def report(*args, **kwargs): + step("report") + from report_qwen_product import report as real_report + + return real_report(*args, **kwargs) + + monkeypatch.setattr(qualification, "snapshot", capture_source) + monkeypatch.setattr(qualification, "verify_snapshot", lambda *args: None) + monkeypatch.setattr(qualification, "verify_package", package) + monkeypatch.setattr(qualification, "execute_packaged", client) + monkeypatch.setattr(qualification, "report", report) + monkeypatch.setattr( + qualification.LoggedRunner, + "run", + lambda self, argv, **kwargs: None if argv[0] == sys.executable else pytest.fail("unexpected provider call"), + ) + monkeypatch.setattr(MixedProductRun, "preflight", lambda self: step("preflight")) + monkeypatch.setattr(MixedProductRun, "bundle", lambda self: step("bundle")) + monkeypatch.setattr(MixedProductRun, "create_firewalls", lambda self: step("create")) + monkeypatch.setattr(MixedProductRun, "stage", lambda self, *args: step("stage")) + monkeypatch.setattr(MixedProductRun, "exercise_workers", source_test) + monkeypatch.setattr(MixedProductRun, "cleanup", cleanup) + monkeypatch.setattr(MixedProductRun, "az_json", lambda self, *args: {"registrationState": "Registered"}) + monkeypatch.setattr(MixedProductRun, "wait_file", lambda self, *args, **kwargs: {"peers": []}) + # A source failure before its final receipt never launches the package test. + monkeypatch.setattr(MixedProductRun, "read", lambda self, *args: None) + for method in ( + "create_gcp", + "create_azure", + "finish_network", + "enable_packaged_client", + "wait_setup", + "start_job", + "start_product", + "capture_product", + ): + monkeypatch.setattr(MixedProductRun, method, lambda *args, **kwargs: None) + provenance = path / "package-provenance.json" + put(provenance, {}) + result = qualification.QwenQualification( + root=ROOT, + output_root=path, + cloud_config=config, + packaged_config={"node_sha256": "a" * 64}, + inputs={"node": path / "node.exe", "package_provenance": provenance}, + ).run() + assert result["result"] == ("passed" if fail_at is None else "failed") + assert len(set(thread_ids)) == 1 # Ordered execution, including the packaged callback and cleanup. + assert json.loads((path / "qualification/result.json").read_text()) == result + assert json.loads((path / "qualification/run-state.json").read_text())["result"] == result["result"] + if fail_at in ("preflight", "package", "bundle"): + assert "create" not in calls and "cleanup" not in calls + else: + assert calls.count("cleanup") == 1 + if fail_at is None: + assert calls == [ + "preflight", + "package", + "bundle", + "create", + *(["stage"] * 5), + "source", + "client", + "cleanup", + "package", + "report", + ] + assert result["cleanup"]["result"] == "passed" + assert set(result["clients"]) == {"windows"} + assert json.loads((path / "result.json").read_text())["scope"] == "production-node-model-transitions" + else: + assert result["failure_reason"] + assert any(e["phase"] == "FAILURE" for e in result["events"]) + if fail_at == "source": + assert ( + next(e for e in result["events"] if e["phase"] == "FAILURE")["details"]["failed_phase"] + == "SOURCE_RUNNING" + ) + + +@pytest.mark.skipif(os.name != "nt", reason="Windows one-click shell") +@pytest.mark.parametrize("exit_code", [0, 1, 2]) +def test_cmd_preserves_exit_code_and_handles_paths_with_spaces(tmp_path, exit_code): + root = tmp_path / "repo with spaces" + (root / "scripts").mkdir(parents=True) + cmd = root / "Run Qwen Qualification.cmd" + shutil.copyfile(ROOT / cmd.name, cmd) + (root / "scripts/run_qwen_qualification.py").write_text( + "import sys\nprint('launched test entrypoint')\nsys.exit(" + str(exit_code) + ")\n" + ) + result = subprocess.run( + # cmd.exe has its own quoting grammar; a list would insert backslashes + # before these quotes using the C-runtime argv rules. + f'cmd.exe /d /s /c ""{cmd}""', + cwd=tmp_path, + env=dict(os.environ, COMMUNITYAI_TEST_PYTHON=sys.executable), + input="\n", + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode == exit_code, result.stdout + result.stderr + assert "launched test entrypoint" in result.stdout + assert f"finished with exit code {exit_code}" in result.stdout diff --git a/tests/test_qwen_reference_gcp.py b/tests/test_qwen_reference_gcp.py new file mode 100644 index 000000000..6ba113867 --- /dev/null +++ b/tests/test_qwen_reference_gcp.py @@ -0,0 +1,35 @@ +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from run_qwen_reference_gcp import ROOT, ReferenceRun + + +def test_reference_failure_and_diagnostic_failure_still_clean_up(tmp_path, monkeypatch): + config = json.loads((ROOT / "config/qwen_full_inference_gcp.json").read_text()) + run = ReferenceRun(tmp_path / "q38r-test", config) + assert len(run.names) == 1 + calls = [] + for name in ("preflight", "bundle", "create_firewalls", "create", "stage", "wait_setup"): + monkeypatch.setattr(run, name, lambda *args: None) + monkeypatch.setattr(run, "ssh", lambda *args, **kwargs: SimpleNamespace(stdout="", stderr="")) + + def fail(*args, **kwargs): + raise RuntimeError("simulated host failure") + + monkeypatch.setattr(run, "wait_file", fail) + monkeypatch.setattr(run, "read", fail) + + def cleanup(): + calls.append("cleanup") + return {"verified": True} + + monkeypatch.setattr(run, "cleanup", cleanup) + result = run.run() + assert result["result"] == "failed" + assert result["cleanup"]["verified"] + assert "diagnostic_error" in result + assert calls == ["cleanup"] + assert json.loads((run.path / "result.json").read_text())["cleanup"]["verified"] diff --git a/tests/test_qwen_sharing_harness.py b/tests/test_qwen_sharing_harness.py new file mode 100644 index 000000000..a4ff01769 --- /dev/null +++ b/tests/test_qwen_sharing_harness.py @@ -0,0 +1,29 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from qualify_qwen_sharing_product import worker_is_ready + + +def test_restart_readiness_requires_current_span_and_a_new_runtime_ready_event(): + old = "Sep 05 22:00:00.000 [INFO] Connection handlers are ready, starting the runtime" + new = "Sep 05 22:01:00.000 [INFO] Connection handlers are ready, starting the runtime" + worker = { + "state": "running", + "remote_acknowledged": True, + "model": "Qwen", + "block_indices": "6:7", + "recent_logs": [old], + } + counts = [0] * 64 + counts[59] = 1 + status = {"models": [{"id": "Qwen", "route": {"status": "incomplete", "replica_counts": counts}}]} + assert not worker_is_ready(status, worker) + counts[6] = 1 + assert worker_is_ready(status, worker) + assert not worker_is_ready(status, worker, {old}) + worker["recent_logs"].append(new) + assert worker_is_ready(status, worker, {old}) + status["models"][0]["route"]["status"] = "unknown" + assert not worker_is_ready(status, worker, {old}) diff --git a/tests/test_route_health.py b/tests/test_route_health.py index 15fda2bc4..3f9681523 100644 --- a/tests/test_route_health.py +++ b/tests/test_route_health.py @@ -85,3 +85,5 @@ def test_module_info_health_ignores_joining_workers(): assert health["status"] == "incomplete" assert health["replica_counts"] == [1, 0] assert health["missing_blocks"] == [1] + assert health["joining_counts"] == [1, 1] + assert next(peer for peer in health["peers"] if peer["peer_id"] == str(joining))["joining_blocks"] == [0, 1] diff --git a/tests/test_run_gate13_gcp.py b/tests/test_run_gate13_gcp.py new file mode 100644 index 000000000..36004ee72 --- /dev/null +++ b/tests/test_run_gate13_gcp.py @@ -0,0 +1,69 @@ +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import run_gate13_gcp as launcher +from gate13_gcp_provider import GcpConfig + +RUN_ID = "g13-20260905-000000-abcd" + + +@pytest.mark.parametrize("result_name,exit_code", [("passed", 0), ("failed", 1)]) +@pytest.mark.parametrize("old_marker", [None, "not even valid JSON"]) +def test_launcher_starts_only_a_new_run_without_historic_recovery( + tmp_path, monkeypatch, result_name, exit_code, old_marker +): + config = GcpConfig.load(ROOT / "config" / "gate13_gcp.json") + runs_root = tmp_path / ".gate13-runs" / "gcp" + old_run = runs_root / "g13-20260904-000000-aaaa" + old_run.mkdir(parents=True) + old_evidence = old_run / "result.json" + old_evidence.write_text('{"result":"failed"}\n', encoding="utf-8") + active_path = runs_root / "active.json" + if old_marker is not None: + active_path.write_text(old_marker, encoding="utf-8") + calls = [] + + def make_provider(**kwargs): + calls.append(("provider", kwargs["run_id"])) + return object() + + class CurrentRun: + def __init__(self, **kwargs): + assert kwargs["run_id"] == RUN_ID + + def run(self): + calls.append(("run", RUN_ID)) + return { + "result": result_name, + "cleanup": {"result": result_name}, + "events": [], + "duration_seconds": 0, + "failure_reason": "test failure" if result_name == "failed" else None, + } + + monkeypatch.setattr(launcher, "__file__", str(tmp_path / "scripts" / "run_gate13_gcp.py")) + monkeypatch.setattr(launcher.GcpConfig, "load", lambda _path: config) + monkeypatch.setattr(launcher, "_new_run_id", lambda: RUN_ID) + monkeypatch.setattr(launcher, "_provider", make_provider) + monkeypatch.setattr(launcher, "Gate13CloudOrchestrator", CurrentRun) + monkeypatch.setattr( + launcher.LoggedRunner, + "run", + lambda *_args, **_kwargs: pytest.fail("must not issue historic recovery commands"), + ) + + assert launcher.main([]) == exit_code + assert calls == [("provider", RUN_ID), ("run", RUN_ID)] + assert json.loads((runs_root / RUN_ID / "provider-config.json").read_text())["project"] == config.project + assert old_evidence.read_text() == '{"result":"failed"}\n' + if old_marker is None: + assert not active_path.exists() + else: + assert active_path.read_text() == old_marker + assert not (runs_root / "launcher.lock").exists() diff --git a/tests/test_runtime_archive_hardlinks.py b/tests/test_runtime_archive_hardlinks.py new file mode 100644 index 000000000..c2e90920a --- /dev/null +++ b/tests/test_runtime_archive_hardlinks.py @@ -0,0 +1,217 @@ +"""Tiny offline fixtures for all consumers of the normalized Linux archive.""" + +from __future__ import annotations + +import copy +import hashlib +import io +import os +import stat +import sys +import tarfile +import unittest +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "desktop/src")) +sys.path.insert(0, str(ROOT / "scripts")) +from desktop import build_desktop as builder +from scripts import gate13_linux_packaged_lifecycle as lifecycle, gateq38_linux_host_runtime as host + +PAYLOAD = b"native runtime library fixture\n" * 7 +A = "CommunityAI/node/_internal/a.so" +B = "CommunityAI/node/_internal/b.so" +C = "CommunityAI/node/_internal/c.so" + + +class RuntimeArchiveHardlinkTests(unittest.TestCase): + def setUp(self): + self.temporary = TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + probe = self.root / "mode" + probe.write_bytes(b"") + self.mode = stat.S_IMODE(probe.stat().st_mode) + self.artifacts = [ + { + "path": name, + "kind": "file", + "sha256": hashlib.sha256(PAYLOAD).hexdigest(), + "size_bytes": len(PAYLOAD), + "mode": self.mode, + } + for name in (A, B, C) + ] + + def archive(self, *, hardlinks=True, mutate=None): + entries = [] + for name in (A, B, C): + info = tarfile.TarInfo(name) + info.mode = self.mode + if hardlinks and name != A: + info.type = tarfile.LNKTYPE + info.linkname = A + else: + info.size = len(PAYLOAD) + entries.append((info, PAYLOAD if info.isfile() else None)) + if mutate: + mutate(entries) + archive = self.root / "runtime.tar.gz" + with tarfile.open(archive, "w:gz") as target: + for info, payload in entries: + target.addfile(info, io.BytesIO(payload) if payload is not None else None) + return archive + + def auditors(self, archive, artifacts=None): + artifacts = self.artifacts if artifacts is None else artifacts + + def build(): + builder._verify_tar_install_archive(archive, artifacts) + + def packaged(): + with tarfile.open(archive, "r:gz") as source: + lifecycle._audit_tar_payload(source, artifacts) + + def hosted(): + with tarfile.open(archive, "r:gz") as source: + host._audit_members(source, [host.Artifact(**item, link_target=None) for item in artifacts]) + + return build, packaged, hosted + + def test_old_regular_payloads_and_new_backward_hardlinks_both_verify(self): + for hardlinks in (False, True): + archive = self.archive(hardlinks=hardlinks) + for auditor in self.auditors(archive): + with self.subTest(hardlinks=hardlinks, auditor=auditor.__name__): + auditor() + with tarfile.open(archive, "r:gz") as source: + members = source.getmembers() + # The old allowlist rejected the new representation despite an + # identical attested logical inventory; all three consumers now accept it. + old_type_check = all(member.isfile() for member in members) + self.assertEqual(old_type_check, not hardlinks) + self.assertEqual(sum(member.size for member in members), len(PAYLOAD) * (1 if hardlinks else 3)) + + def test_writer_preserves_inode_groups_as_direct_backward_links(self): + bundle = self.root / "CommunityAI" + first = bundle / "node/_internal/a.so" + first.parent.mkdir(parents=True) + first.write_bytes(PAYLOAD) + os.link(first, first.with_name("b.so")) + os.link(first, first.with_name("c.so")) + artifacts = builder._bundle_artifacts(bundle) + entries = builder._install_archive_entries(bundle, artifacts) + archive = self.root / "writer.tar.gz" + builder._write_tar_install_archive(archive, entries) + builder._verify_tar_install_archive(archive, entries) + with tarfile.open(archive, "r:gz") as source: + self.assertTrue(source.getmember(A).isfile()) + for name in (B, C): + self.assertTrue(source.getmember(name).islnk()) + self.assertEqual(source.getmember(name).linkname, A) + + def test_unsafe_targets_chains_and_metadata_changes_fail_every_auditor(self): + cases = { + "traversal": lambda entries: setattr(entries[1][0], "linkname", "CommunityAI/../outside"), + "absolute": lambda entries: setattr(entries[1][0], "linkname", "/etc/passwd"), + "backslash": lambda entries: setattr(entries[1][0], "linkname", A.replace("/", "\\")), + "noncanonical": lambda entries: setattr(entries[1][0], "linkname", A + "/"), + "missing": lambda entries: setattr(entries[1][0], "linkname", A + "missing"), + "forward": lambda entries: setattr(entries[1][0], "linkname", C), + "chain": lambda entries: setattr(entries[2][0], "linkname", B), + "mode": lambda entries: setattr(entries[1][0], "mode", 0o755), + "privileged_mode": lambda entries: setattr(entries[1][0], "mode", self.mode | 0o4000), + "size": lambda entries: setattr(entries[1][0], "size", 1), + "symlink_target": lambda entries: setattr(entries[0][0], "type", tarfile.SYMTYPE), + "duplicate": lambda entries: setattr(entries[2][0], "name", B), + "tamper": lambda entries: entries.__setitem__(0, (entries[0][0], b"x" * len(PAYLOAD))), + } + for name, mutate in cases.items(): + archive = self.archive(mutate=mutate) + for auditor in self.auditors(archive): + with self.subTest(case=name, auditor=auditor.__name__): + with self.assertRaises((RuntimeError, host.Q38LinuxHostRuntimeError, lifecycle.LifecycleRunError)): + auditor() + + def test_hardlinks_require_identical_attested_hash_size_and_mode(self): + archive = self.archive() + for key, value in (("sha256", "a" * 64), ("size_bytes", len(PAYLOAD) + 1), ("mode", 0o755)): + artifacts = copy.deepcopy(self.artifacts) + artifacts[1][key] = value + for auditor in self.auditors(archive, artifacts): + with self.subTest(field=key, auditor=auditor.__name__): + with self.assertRaises((RuntimeError, host.Q38LinuxHostRuntimeError, lifecycle.LifecycleRunError)): + auditor() + + def test_both_extractors_keep_hardlinks_with_identical_logical_files(self): + archive = self.archive() + audit = SimpleNamespace(archive=archive, artifacts=self.artifacts) + installed = lifecycle._extract_package(audit, self.root / "installed") + self.assertTrue(os.path.samefile(installed / "node/_internal/a.so", installed / "node/_internal/b.so")) + + @contextmanager + def verified_file(path, **kwargs): + with path.open("rb") as stream: + yield stream, b"" + + artifacts = [host.Artifact(**item, link_target=None) for item in self.artifacts] + destination = self.root / "hosted" + package = { + "release_archive_bytes": archive.stat().st_size, + "release_archive_sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "node_root": "CommunityAI/node", + } + with patch.object(host, "_assert_root_managed"), patch.object(host, "_verified_file", verified_file): + host._extract_verified_archive(archive, package, artifacts, artifacts, destination) + for base in (self.root / "installed", destination): + paths = [base / name for name in (A, B, C)] + self.assertTrue(all(path.read_bytes() == PAYLOAD and not path.is_symlink() for path in paths)) + self.assertEqual(len({path.stat().st_ino for path in paths}), 1) + + def test_lifecycle_rejects_tampered_link_before_writing_payload_and_cleans_stage(self): + archive = self.archive(mutate=lambda entries: setattr(entries[1][0], "linkname", "../outside")) + destination = self.root / "rejected" + with self.assertRaises(lifecycle.LifecycleRunError): + lifecycle._extract_package(SimpleNamespace(archive=archive, artifacts=self.artifacts), destination) + self.assertFalse(destination.exists()) + self.assertFalse((self.root / "outside").exists()) + + def test_node_extraction_rejects_a_hardlink_to_the_desktop_inventory_and_cleans_stage(self): + # A full bundle link can be valid while its target is absent from the + # node-only extraction. Such a link must never be followed or copied. + desktop_path = "CommunityAI/_internal/a.so" + + def move_target(entries): + entries[0][0].name = desktop_path + for info, _ in entries[1:]: + info.linkname = desktop_path + + archive = self.archive(mutate=move_target) + raw_artifacts = copy.deepcopy(self.artifacts) + raw_artifacts[0]["path"] = desktop_path + artifacts = [host.Artifact(**item, link_target=None) for item in raw_artifacts] + destination = self.root / "rejected" + + @contextmanager + def verified_file(path, **kwargs): + with path.open("rb") as stream: + yield stream, b"" + + package = { + "release_archive_bytes": archive.stat().st_size, + "release_archive_sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "node_root": "CommunityAI/node", + } + with patch.object(host, "_assert_root_managed"), patch.object(host, "_verified_file", verified_file): + with self.assertRaisesRegex(host.Q38LinuxHostRuntimeError, "leaves the runtime inventory"): + host._extract_verified_archive(archive, package, artifacts, artifacts[1:], destination) + self.assertFalse(destination.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_server_admission.py b/tests/test_server_admission.py index f63391e4a..2e1b0b94f 100644 --- a/tests/test_server_admission.py +++ b/tests/test_server_admission.py @@ -584,7 +584,7 @@ def test_manifested_container_publishes_machine_readable_aggregate_health(tmp_pa container.admission_state = AdmissionState.local(_policy()) container.health_state_path = tmp_path / "health.json" container.server_info = SimpleNamespace( - manifest_digest="sha256:" + "a" * 64, + manifest_digest="a" * 64, start_block=0, end_block=24, ) diff --git a/tests/test_server_memory_budget.py b/tests/test_server_memory_budget.py index 9ffefa517..df3df79db 100644 --- a/tests/test_server_memory_budget.py +++ b/tests/test_server_memory_budget.py @@ -1,3 +1,4 @@ +import argparse from types import SimpleNamespace import pytest @@ -6,6 +7,27 @@ from drift.server.block_utils import get_block_size from drift.server.server import Server, parse_block_indices from drift.utils.convert_block import QuantType +from drift.utils.resource_limits import DEVICE_MEMORY_BUDGET_EXIT_CODE, DeviceMemoryBudgetError + + +def test_cli_distinguishes_memory_budget_rejection(monkeypatch, capsys): + from drift.cli import run_server + + parser = SimpleNamespace( + parse_args=lambda: SimpleNamespace(), + prog="CommunityAI-Node server", + ) + parser.exit = argparse.ArgumentParser().exit + monkeypatch.setattr(run_server, "build_parser", lambda **kwargs: parser) + + def reject(args): + raise DeviceMemoryBudgetError("Configured blocks exceed the VRAM budget") + + monkeypatch.setattr(run_server, "server_from_args", reject) + with pytest.raises(SystemExit) as stopped: + run_server.main() + assert stopped.value.code == DEVICE_MEMORY_BUDGET_EXIT_CODE + assert "VRAM budget" in capsys.readouterr().err def _budget_server(*, tensor_parallel_devices=(torch.device("cuda:0"),)): @@ -85,3 +107,23 @@ def __init__(self, config, layer_idx=0): assert first == 4 assert third == 12 + + +def test_fp8_dequant_memory_budget_uses_execution_dtype(): + class Block(torch.nn.Module): + def __init__(self, config, layer_idx=0): + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(16)) + + config = SimpleNamespace(block_class=Block, torch_dtype=torch.bfloat16) + + assert ( + get_block_size( + config, + "memory", + dtype=torch.bfloat16, + quant_type=QuantType.FP8_DEQUANT, + eps=0, + ) + == 32 + ) diff --git a/tests/test_text_generation.py b/tests/test_text_generation.py new file mode 100644 index 000000000..1d3ebbe22 --- /dev/null +++ b/tests/test_text_generation.py @@ -0,0 +1,188 @@ +"""Long chat prompts and overlapping answer/title requests on text peers.""" + +import asyncio +import contextlib +import threading +import queue +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from torch import nn + +from drift.models.qwen3_5.model import DistributedQwen3_5ForCausalLM +from drift.server.text_generation import TextGenerationEngine +from drift.api.server import _RequestCancelled +from test_qwen3_5_block import _tiny_config, _wrapped_blocks + + +def test_chunked_generation_preserves_remote_cache_and_full_prompt(): + from transformers.models.qwen3_5 import Qwen3_5ForCausalLM + from drift.models.qwen3_5.cache import Qwen3_5HybridCache + + torch.set_num_threads(1) + cfg = _tiny_config() + # Include a second full-attention block. Its isolated cache must not use + # the first full-attention block's (empty) cache to size its causal mask. + cfg.num_hidden_layers = 8 + cfg.layer_types = ["linear_attention"] * 3 + ["full_attention"] + cfg.layer_types *= 2 + cfg.max_position_embeddings = 1024 + torch.manual_seed(7) + reference = Qwen3_5ForCausalLM(cfg).eval() + + class LocalBlocks(nn.Module): + """Actual worker blocks/caches behind the distributed generation interface.""" + + def __init__(self, config, **kwargs): + super().__init__() + self.config = config + self.blocks = nn.ModuleList(_wrapped_blocks(config, reference.model)) + self.active_session = None + self.sizes = [] + + @property + def position(self): + return self.active_session.position + + @contextlib.contextmanager + def inference_session(self, max_length): + self.active_session = SimpleNamespace(position=0, output_ids=None) + self.strategies = [Qwen3_5HybridCache(cfg, module=b) for b in self.blocks] + self.caches = [ + [ + torch.zeros(d.shape, dtype=d.dtype) + for d in s.get_cache_descriptors( + 1, + max_length, + dtype=torch.float32, + devices=[torch.device("cpu")], + shard_num_heads=[cfg.num_attention_heads], + ) + ] + for s in self.strategies + ] + try: + yield self.active_session + finally: + self.active_session = None + + def forward(self, hidden, **kwargs): + self.sizes.append(hidden.shape[1]) + for block, strategy, cache in zip(self.blocks, self.strategies, self.caches): + past = strategy.select_layer_past(cache, self.position, num_shards=1) + hidden, new_states = block(hidden, layer_past=past, use_cache=True) + strategy.update_cache(cache, new_states, self.position) + self.active_session.position += hidden.shape[1] + return hidden + + with patch("drift.models.qwen3_5.model.RemoteSequential", LocalBlocks): + actual = DistributedQwen3_5ForCausalLM(cfg).eval() + actual.model.embed_tokens.load_state_dict(reference.model.embed_tokens.state_dict()) + actual.model.norm.load_state_dict(reference.model.norm.state_dict()) + actual.lm_head.load_state_dict(reference.lm_head.state_dict()) + for block, reference_block in zip(actual.model.layers.blocks, reference.model.layers): + block.load_state_dict(reference_block.state_dict()) + prompt = torch.randint(3, cfg.vocab_size, (1, 533)) + with torch.inference_mode(): + expected = reference.generate(prompt, max_new_tokens=3, do_sample=False, eos_token_id=None) + result = actual.generate(prompt, max_new_tokens=3, do_sample=False, eos_token_id=None, prefill_chunk_size=64) + assert torch.equal(result, expected) + assert actual.model.layers.sizes == [64] * 8 + [21, 1, 1] + assert torch.equal(result[:, :533], prompt) + + +def test_overlapping_requests_queue_and_cancel_without_parallel_generation(): + async def check(): + engine = TextGenerationEngine(None, request_timeout=10) + started, release = threading.Event(), threading.Event() + seen = [] + + def generate(payload, key, cancel, out): + seen.append(key) + started.set() + assert release.wait(5) + out.put({"type": "done"}) + + engine._generate = generate + payload = lambda n: {"request_id": str(n) * 32, "chat": True} + first = engine.stream(payload(1), "peer") + first_task = asyncio.create_task(anext(first)) + assert await asyncio.to_thread(started.wait, 2) + second = engine.stream(payload(2), "peer") + assert (await anext(second))["type"] == "heartbeat" + third = engine.stream(payload(3), "peer") + assert (await anext(third))["type"] == "heartbeat" + fourth = engine.stream(payload(4), "peer") + assert (await anext(fourth))["code"] == "busy" + duplicate = engine.stream(payload(1), "peer") + assert (await anext(duplicate))["code"] == "busy" + engine.cancel("3" * 32, "peer") + await third.aclose() + release.set() + await first_task + await first.aclose() + frames = [frame async for frame in second] + assert frames[-1]["type"] == "done" + assert seen == [("peer", "1" * 32), ("peer", "2" * 32)] + engine.close() + + asyncio.run(check()) + + +def test_disconnect_stops_prefill_before_the_next_chunk_and_removes_hook(): + cancel = _RequestCancelled() + + class Model(nn.Module): + calls = 0 + + def forward(self, inputs): + self.calls += 1 + cancel.event.set() + return inputs + + def generate(self, inputs, **kwargs): + for part in inputs.split(64, dim=1): + self(part) + raise AssertionError("Cancelled prefill must stop before finishing") + + model = Model() + tokenizer = SimpleNamespace( + eos_token_id=None, apply_chat_template=lambda *a, **kw: {"input_ids": torch.ones(1, 533).long()} + ) + engine = TextGenerationEngine(SimpleNamespace(model=model, tokenizer=tokenizer)) + output = queue.Queue() + engine._generate( + {"chat": True, "body": {"model": "auto", "messages": [{"role": "user", "content": "hi"}]}}, + ("peer", "request"), + cancel, + output, + ) + assert model.calls == 1 + assert not model._forward_pre_hooks + assert output.empty() + + +def test_chat_stops_on_tokenizer_turn_end_as_well_as_model_end_of_text(): + captured = {} + + class Model(nn.Module): + generation_config = SimpleNamespace(eos_token_id=248044) + + def generate(self, inputs, **kwargs): + captured.update(kwargs) + return inputs + + tokenizer = SimpleNamespace( + eos_token_id=248046, apply_chat_template=lambda *a, **kw: {"input_ids": torch.ones(1, 8).long()} + ) + engine = TextGenerationEngine(SimpleNamespace(model=Model(), tokenizer=tokenizer)) + output = queue.Queue() + engine._generate( + {"chat": True, "body": {"model": "auto", "messages": [{"role": "user", "content": "hi"}]}}, + ("peer", "request"), + _RequestCancelled(), + output, + ) + assert captured["eos_token_id"] == [248044, 248046] + assert output.get_nowait()["finish_reason"] == "stop" diff --git a/tests/test_text_mesh.py b/tests/test_text_mesh.py new file mode 100644 index 000000000..f5d8a1209 --- /dev/null +++ b/tests/test_text_mesh.py @@ -0,0 +1,302 @@ +"""Consumer acceptance: no local model, full text responses, real peer transport.""" + +import asyncio +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import httpx + +from drift.api.server import create_app +from drift.model_manifest import ModelManifest +from drift.node.model_manager import ModelDescriptor, ModelManager, ModelRuntime +from drift.protocol_identity import NodeIdentity, ProtocolSecurityError, RevocationStore +from drift.text_mesh import ( + TextPeerClient, + TextPeerUnavailable, + TextPeerProtocol, + announcement_key, + create_text_announcement, + decode, + verify_text_announcement, +) + +MANIFEST = "manifests/candidates/qwen3.8-27b-fp8-dequant-eager.json" + + +class FakeTextClient: + def __init__(self): + self.requests = [] + self.closed_requests = 0 + + async def stream(self, body, *, chat): + self.requests.append((body, chat)) + try: + yield {"type": "heartbeat"} + yield {"type": "delta", "text": "Hello "} + yield {"type": "delta", "text": "from peers."} + yield { + "type": "done", + "finish_reason": "stop", + "usage": {"prompt_tokens": 3, "completion_tokens": 3, "total_tokens": 6}, + } + finally: + self.closed_requests += 1 + + +class ConsumerTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.manager = ModelManager() + self.peer = FakeTextClient() + self.health = { + "status": "complete", + "covered_blocks": 64, + "total_blocks": 64, + "peer_count": 4, + "chat_ready": True, + } + self.manager.register( + ModelDescriptor("Community", selected_whole_shard_bytes=0), + lambda: ModelRuntime(None, None, text_client=self.peer), + route_health=lambda: dict(self.health), + ) + self.manager.register( + ModelDescriptor("Local fallback", execution="local"), + lambda: (_ for _ in ()).throw(AssertionError("Local model must not load")), + route_health=lambda: { + "status": "complete", + "covered_blocks": 1, + "total_blocks": 1, + "peer_count": 0, + "source": "local", + }, + ) + self.manager.configure_auto_selection(["Community", "Local fallback"]) + self.client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(model_manager=self.manager)), base_url="http://test" + ) + + async def asyncTearDown(self): + await self.client.aclose() + self.manager.shutdown() + + async def test_fresh_consumer_uses_community_with_all_local_weight_loading_forbidden(self): + with patch( + "drift.model_manifest.ManifestArtifactVerifier.ensure_path", side_effect=AssertionError("No download") + ): + result = await self.client.post( + "/v1/chat/completions", json={"model": "auto", "messages": [{"role": "user", "content": "Hello"}]} + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()["model"], "Community") + self.assertEqual(result.json()["choices"][0]["message"]["content"], "Hello from peers.") + self.assertEqual(result.json()["usage"]["total_tokens"], 6) + self.assertEqual(self.manager.snapshots()[0].selected_whole_shard_bytes, 0) + self.assertTrue(all(item.active_requests == 0 for item in self.manager.snapshots())) + + async def test_streaming_and_plain_completions_keep_openai_shape(self): + result = await self.client.post( + "/v1/chat/completions", + json={"model": "auto", "messages": [{"role": "user", "content": "Hi"}], "stream": True}, + ) + self.assertIn('"content": "Hello "', result.text) + self.assertIn("data: [DONE]", result.text) + result = await self.client.post("/v1/completions", json={"model": "auto", "prompt": "Hi"}) + self.assertEqual(result.json()["choices"][0]["text"], "Hello from peers.") + self.assertEqual(self.peer.closed_requests, 2) + + async def test_fallback_only_when_community_path_unavailable_then_returns(self): + self.assertEqual(self.manager.resolve("auto").model_id, "Community") + self.health.update(chat_ready=False) + self.assertEqual(self.manager.resolve("auto").model_id, "Local fallback") + self.health.update(chat_ready=True, covered_blocks=63, status="incomplete") + self.assertEqual(self.manager.resolve("auto").model_id, "Local fallback") + self.health.update(covered_blocks=64, status="complete") + self.assertEqual(self.manager.resolve("auto").model_id, "Community") + self.manager.configure_auto_selection(["Community", "Local fallback"], local_only=True) + self.assertEqual(self.manager.resolve("auto").model_id, "Local fallback") + + async def test_disconnect_after_role_chunk_releases_unstarted_peer_request(self): + from drift.api.server import ChatCompletionRequest + from drift.api.text_response import text_peer_response + + loaded = self.manager.load("auto") + response = await text_peer_response( + loaded, + ChatCompletionRequest(model="auto", messages=[{"role": "user", "content": "Hi"}], stream=True), + chat=True, + semaphore=asyncio.Semaphore(1), + ) + await anext(response.body_iterator) + await response.body_iterator.aclose() + self.assertTrue(all(item.active_requests == 0 for item in self.manager.snapshots())) + + async def test_node_status_reaches_desktop_with_no_download_and_live_block_grid(self): + from dataclasses import replace + + from communityai_desktop.client import NodeClient + from communityai_desktop.controller import DesktopController + from drift.node.server import create_node_app + + manifest = ModelManifest.load(MANIFEST) + manager = ModelManager() + descriptor = replace(ModelDescriptor.from_manifest(manifest), selected_whole_shard_bytes=0) + health = {**self.health, "source": "discovery", "replica_counts": [1] * 64, "last_updated_age": 0.0} + manager.register( + descriptor, lambda: ModelRuntime(None, None, text_client=self.peer), route_health=lambda: health + ) + manager.configure_auto_selection([descriptor.model_id]) + try: + app = create_node_app(manager, api_keys=["api-test"], control_keys=["control-test"]) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as api: + response = await api.get("/control/v1/status", headers={"Authorization": "Bearer control-test"}) + self.assertEqual(response.status_code, 200) + control = NodeClient("http://127.0.0.1:8080", "control-test") + with patch.object(control, "_request", return_value=response.json()), patch.object( + control, "list_keys", return_value=[] + ): + view = DesktopController(control).snapshot() + model = view["models"][0] + self.assertTrue(model["route_complete"]) + self.assertTrue(model["auto_selected"]) + self.assertIsNone(model["download_progress"]) + self.assertEqual(model["covered_blocks"], 64) + self.assertEqual(model["health"]["replica_counts"], [1] * 64) + finally: + manager.shutdown() + + +class PeerStartupTests(unittest.TestCase): + def test_readiness_discovery_starts_without_a_consumer_request(self): + from drift.server.text_peer import TextPeerService + + discovered = [] + route = SimpleNamespace(start_discovery=lambda: discovered.append(True)) + runtime = SimpleNamespace( + model=SimpleNamespace(transformer=SimpleNamespace(h=SimpleNamespace(sequence_manager=route))), + close=lambda: None, + ) + manifest = ModelManifest.load(MANIFEST) + service = TextPeerService( + SimpleNamespace(peer_id="peer"), + SimpleNamespace(peer_id="peer"), + manifest, + initial_peers=[], + cache_dir="unused", + ) + + async def serve(_runtime): + self.assertTrue(discovered, "A text peer must discover blocks before advertising readiness") + + with patch("drift.server.text_peer.make_manifest_loader", return_value=lambda: runtime), patch.object( + service, "_serve", side_effect=serve + ): + service._run() + self.assertIsNone(service.error) + + +class IdentityTests(unittest.TestCase): + def test_service_signature_binds_model_identity_expiry_and_revocation(self): + manifest = ModelManifest.load(MANIFEST) + with tempfile.TemporaryDirectory() as directory: + identity = NodeIdentity.ensure(Path(directory) / "peer.key") + source = create_text_announcement( + manifest, identity, max_context_tokens=2048, max_output_tokens=512, now=1000 + ).to_dict() + verify_text_announcement(source, manifest, identity.peer_id, now=1001) + with self.assertRaises(ProtocolSecurityError): + verify_text_announcement(source, manifest, identity.peer_id, now=1041) + with self.assertRaises(ProtocolSecurityError): + verify_text_announcement( + source, + manifest, + identity.peer_id, + now=1001, + revocations=RevocationStore(revoked_key_ids={identity.key_id}), + ) + source["payload"]["max_output_tokens"] = 10000 + with self.assertRaises(ProtocolSecurityError): + verify_text_announcement(source, manifest, identity.peer_id, now=1001) + + def test_malformed_and_oversized_requests_rejected(self): + for data in (b'{"x":1,"x":2}', b'{"x":NaN}', b"x" * 65537): + with self.assertRaises(ValueError): + decode(data) + + +class RealTransportTests(unittest.IsolatedAsyncioTestCase): + async def test_text_only_roundtrip_over_authenticated_peer_transport(self): + from hivemind import DHT + + manifest = ModelManifest.load(MANIFEST) + with tempfile.TemporaryDirectory() as directory: + identity = NodeIdentity.ensure(Path(directory) / "peer.key") + server = await asyncio.to_thread( + DHT, + initial_peers=[], + host_maddrs=["/ip4/127.0.0.1/tcp/0"], + identity_path=str(Path(directory) / "peer.key"), + start=True, + tls=True, + ) + p2p = await server.replicate_p2p() + + class Engine: + def __init__(self): + self.cancelled = asyncio.Event() + + async def stream(self, payload, peer): + if payload["body"]["prompt"] == "busy": + yield {"type": "error", "code": "busy", "message": "This community peer is busy"} + return + yield {"type": "delta", "text": payload["body"]["prompt"] + " via mesh"} + if payload["body"]["prompt"] == "cancel": + await self.cancelled.wait() + return + yield { + "type": "done", + "finish_reason": "stop", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + + def cancel(self, request_id, peer): + self.cancelled.set() + return True + + engine = Engine() + protocol = TextPeerProtocol(manifest, engine) + client = None + try: + await protocol.add_p2p_handlers(p2p) + record = create_text_announcement(manifest, identity, max_context_tokens=2048, max_output_tokens=512) + await asyncio.to_thread( + server.store, + announcement_key(manifest), + record.to_dict(), + subkey=str(identity.peer_id), + expiration_time=record.payload["expires_at_ms"] / 1000, + ) + peers = [str(a) for a in server.get_visible_maddrs()] + client = await asyncio.to_thread(TextPeerClient, manifest, initial_peers=peers) + frames = [frame async for frame in client.stream({"prompt": "Hello"}, chat=False)] + self.assertEqual(frames[0]["text"], "Hello via mesh") + self.assertEqual(frames[-1]["type"], "done") + with self.assertRaisesRegex(TextPeerUnavailable, "This community peer is busy"): + _ = [frame async for frame in client.stream({"prompt": "busy"}, chat=False)] + stream = client.stream({"prompt": "cancel"}, chat=False) + first = await anext(stream) + self.assertEqual(first["text"], "cancel via mesh") + await stream.aclose() + await asyncio.wait_for(engine.cancelled.wait(), 3) + finally: + if client is not None: + await asyncio.to_thread(client.close) + await protocol.remove_p2p_handlers(p2p) + await p2p.shutdown() + await asyncio.to_thread(server.shutdown) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_windows_download_helper.py b/tests/test_windows_download_helper.py new file mode 100644 index 000000000..72c8235db --- /dev/null +++ b/tests/test_windows_download_helper.py @@ -0,0 +1,435 @@ +"""Tiny real HTTP fixtures for the separately compiled Windows downloader test build. + +Production is independently compiled without the loopback/test timing symbols. +No public requests, full installers, models or visible windows are launched. +""" + +import ctypes +import hashlib +import http.server +import os +import re +import socket +import stat +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "desktop/installers/WindowsDownload.cs" +PAYLOAD = bytes(range(256)) * 1024 +HASH = hashlib.sha256(PAYLOAD).hexdigest() + + +def creation_time(pid): + from ctypes import wintypes + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel.OpenProcess.restype = wintypes.HANDLE + kernel.GetProcessTimes.argtypes = [wintypes.HANDLE] + [ctypes.POINTER(wintypes.FILETIME)] * 4 + kernel.GetProcessTimes.restype = wintypes.BOOL + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + handle = kernel.OpenProcess(0x1000, False, pid) + assert handle, ctypes.get_last_error() + try: + values = [wintypes.FILETIME() for _ in range(4)] + assert kernel.GetProcessTimes(handle, *(ctypes.byref(value) for value in values)) + return (values[0].dwHighDateTime << 32) | values[0].dwLowDateTime + finally: + kernel.CloseHandle(handle) + + +class Fixture(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_GET(self): + server = self.server + server.requests.append(dict(self.headers)) + mode = server.mode + offset = int(self.headers.get("Range", "bytes=0-")[6:-1]) + if mode == "retry_exhausted": + self.send_response(503) + self.send_header("Content-Length", "0") + self.end_headers() + return + if mode == "redirect": + self.send_response(302) + self.send_header("Location", "/redirect-target") + self.send_header("Content-Length", "0") + self.end_headers() + return + drop = mode in {"resume", "wrong_start", "wrong_total", "wrong_end", "ignored_range"} and offset == 0 + resumed = offset > 0 and mode != "ignored_range" + self.send_response(206 if resumed else 200) + if resumed: + first = offset + (mode == "wrong_start") + last = len(PAYLOAD) - 1 - (mode == "wrong_end") + total = len(PAYLOAD) + (mode == "wrong_total") + self.send_header("Content-Range", f"bytes {first}-{last}/{total}") + length = len(PAYLOAD) - offset + if mode == "bad_length": + length -= 1 + self.send_header("Content-Length", str(length)) + if mode == "encoding": + self.send_header("Content-Encoding", "gzip") + self.end_headers() + try: + if mode == "stall": + self.wfile.write(PAYLOAD[:1024]) + self.wfile.flush() + server.entered.set() + server.release.wait(10) + return + if drop: + self.wfile.write(PAYLOAD[:90000]) + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + return + if mode == "slow": + for start in range(offset, len(PAYLOAD), 4096): + self.wfile.write(PAYLOAD[start : start + 4096]) + self.wfile.flush() + time.sleep(0.02) + else: + self.wfile.write(PAYLOAD[offset:]) + except (OSError, ConnectionError): + pass + + +@unittest.skipUnless(sys.platform == "win32", "Requires Windows .NET Framework and Win32 process handles") +class WindowsDownloadTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.compiled = tempfile.TemporaryDirectory(prefix="communityai-download-tests-") + cls.build = Path(cls.compiled.name) + compiler = Path(os.environ["WINDIR"]) / "Microsoft.NET/Framework64/v4.0.30319/csc.exe" + if not compiler.is_file(): + raise unittest.SkipTest("Windows .NET Framework compiler unavailable") + cls.executables = {} + for name, symbols in ( + ("production", []), + ("test", ["WINDOWS_DOWNLOAD_TEST"]), + ("deadline", ["WINDOWS_DOWNLOAD_TEST", "WINDOWS_DOWNLOAD_SHORT_DEADLINE"]), + ): + executable = cls.build / (name + ".exe") + command = [str(compiler), "/nologo", "/target:winexe", "/out:" + str(executable)] + if symbols: + command.append("/define:" + ";".join(symbols)) + command.append(str(SOURCE)) + result = subprocess.run(command, capture_output=True, timeout=30) + if result.returncode: + raise AssertionError(result.stdout.decode(errors="replace")) + cls.executables[name] = executable + probe_source = cls.build / "RangeProbe.cs" + probe_source.write_text( + """using System; +using System.Reflection; +internal static class RangeProbe { + private static int Main(string[] args) { + try { + typeof(WindowsDownload).GetMethod("ValidateRangeHeader", BindingFlags.NonPublic | BindingFlags.Static) + .Invoke(null, new object[] { args[0], Int64.Parse(args[1]), Int64.Parse(args[2]) }); + return 0; + } catch { return 1; } + } +} +""" + ) + cls.range_probe = cls.build / "range-probe.exe" + subprocess.run( + [ + str(compiler), + "/nologo", + "/target:winexe", + "/main:RangeProbe", + "/out:" + str(cls.range_probe), + str(SOURCE), + str(probe_source), + ], + capture_output=True, + timeout=30, + check=True, + ) + + @classmethod + def tearDownClass(cls): + cls.compiled.cleanup() + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="communityai-download-case-") + self.directory = Path(self.temporary.name) + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Fixture) + self.server.daemon_threads = True + self.server.mode = "full" + self.server.requests = [] + self.server.entered = threading.Event() + self.server.release = threading.Event() + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.children = [] + + def tearDown(self): + self.server.release.set() + for child in self.children: + if child.poll() is None: + child.kill() + child.wait(timeout=5) + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=3) + self.temporary.cleanup() + + def launch(self, mode="full", binary="test", sha=HASH, parent=None, parent_time=None, size=None): + self.server.mode = mode + self.output = self.directory / "package.exe" + self.progress = self.directory / "progress.txt" + self.cancel = self.directory / "cancel" + parent = os.getpid() if parent is None else parent + argv = [ + str(self.executables[binary]), + f"http://127.0.0.1:{self.server.server_port}/package.exe", + str(self.output), + str(len(PAYLOAD) if size is None else size), + sha, + str(self.progress), + str(self.cancel), + str(parent), + str(creation_time(parent) if parent_time is None else parent_time), + ] + child = subprocess.Popen(argv, creationflags=subprocess.CREATE_NO_WINDOW) + self.children.append(child) + return child + + def finish(self, child, expected=0, progress_verified=True): + self.assertEqual(child.wait(timeout=12), expected) + progress = self.progress.read_text() + self.assertRegex(progress, r"\A[0-9]+\|[0-9]+\|[0-5]\Z") + self.assertFalse(Path(str(self.progress) + ".new").exists()) + if expected == 0: + self.assertEqual(self.output.read_bytes(), PAYLOAD) + if progress_verified: + self.assertEqual(progress, f"{len(PAYLOAD)}|{len(PAYLOAD)}|3") + else: + self.assertFalse(self.output.exists()) + detail = Path(str(self.progress) + ".error").read_text(encoding="utf-8") + self.assertTrue(0 < len(detail) <= 1024) + self.assertNotIn("http://", detail) + self.assertNotIn("\n", detail) + + def test_exact_download_and_identity_encoding(self): + self.finish(self.launch()) + self.assertEqual(len(self.server.requests), 1) + self.assertEqual(self.server.requests[0]["Accept-Encoding"], "identity") + self.assertEqual(self.server.requests[0]["User-Agent"], "CommunityAI-Online-Installer/1") + + def test_interrupted_stream_resumes_exact_offset(self): + self.finish(self.launch("resume")) + self.assertEqual(len(self.server.requests), 2) + self.assertEqual(self.server.requests[1]["Range"], "bytes=90000-") + + def test_wrong_range_start_is_rejected(self): + self.finish(self.launch("wrong_start"), 1) + self.assertEqual(len(self.server.requests), 2) + + def test_wrong_range_end_is_rejected(self): + self.finish(self.launch("wrong_end"), 1) + + def test_wrong_range_total_is_rejected(self): + self.finish(self.launch("wrong_total"), 1) + + def test_ignored_resume_range_is_rejected(self): + self.finish(self.launch("ignored_range"), 1) + + def test_wrong_size_is_rejected_before_acceptance(self): + self.finish(self.launch("bad_length"), 1) + + def test_hash_mismatch_removes_download(self): + self.finish(self.launch(sha="0" * 64), 1) + + def test_redirect_is_never_followed(self): + self.finish(self.launch("redirect"), 1) + self.assertEqual(len(self.server.requests), 1) + + def test_content_encoding_is_rejected(self): + self.finish(self.launch("encoding"), 1) + + def test_retries_have_a_finite_request_budget(self): + self.finish(self.launch("retry_exhausted"), 1) + self.assertEqual(len(self.server.requests), 8) + + def test_production_build_rejects_loopback_http_without_request(self): + child = self.launch(binary="production") + self.assertEqual(child.wait(timeout=5), 1) + self.assertFalse(self.server.requests) + self.assertFalse(self.output.exists()) + + def test_parent_creation_mismatch_prevents_request(self): + self.finish(self.launch(parent_time=creation_time(os.getpid()) + 1), 1) + self.assertFalse(self.server.requests) + + def test_cancellation_aborts_a_blocked_response(self): + child = self.launch("stall") + self.assertTrue(self.server.entered.wait(5)) + started = time.monotonic() + self.cancel.write_text("cancel") + self.finish(child, 3) + self.assertLess(time.monotonic() - started, 2) + + def test_parent_exit_aborts_a_blocked_response(self): + parent = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], creationflags=subprocess.CREATE_NO_WINDOW + ) + self.children.append(parent) + child = self.launch("stall", parent=parent.pid) + self.assertTrue(self.server.entered.wait(5)) + parent.terminate() + parent.wait(timeout=5) + self.finish(child, 3) + + def test_monotonic_deadline_aborts_stalled_download(self): + child = self.launch("stall", binary="deadline") + self.finish(child, 2) + + def test_existing_output_is_preserved(self): + existing = self.directory / "package.exe" + existing.write_bytes(b"original") + child = self.launch() + self.assertEqual(child.wait(timeout=5), 1) + self.assertEqual(existing.read_bytes(), b"original") + self.assertFalse(self.server.requests) + + def test_progress_survives_transient_reader_delete_lock(self): + from ctypes import wintypes + + child = self.launch("slow") + until = time.monotonic() + 5 + while not self.progress.exists() and time.monotonic() < until: + time.sleep(0.01) + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [ + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ] + kernel.CreateFileW.restype = wintypes.HANDLE + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + handle = kernel.CreateFileW(str(self.progress), 0x80000000, 1, None, 3, 0x80, None) + self.assertNotEqual(handle, wintypes.HANDLE(-1).value) + try: + time.sleep(0.45) + finally: + kernel.CloseHandle(handle) + self.finish(child) + + def test_persistent_progress_reader_lock_cannot_abort_verified_download(self): + from ctypes import wintypes + + child = self.launch("slow") + until = time.monotonic() + 5 + while not self.progress.exists() and time.monotonic() < until: + time.sleep(0.005) + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [ + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ] + kernel.CreateFileW.restype = wintypes.HANDLE + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + handle = kernel.CreateFileW(str(self.progress), 0x80000000, 1, None, 3, 0x80, None) + self.assertNotEqual(handle, wintypes.HANDLE(-1).value) + started = time.monotonic() + try: + # Hold through helper exit, including all final progress updates. + self.assertEqual(child.wait(timeout=12), 0) + self.assertGreater(time.monotonic() - started, 1) + finally: + kernel.CloseHandle(handle) + self.finish(child, progress_verified=False) + warning = Path(str(self.progress) + ".warning").read_text() + self.assertRegex(warning, r"Skipped progress frame: [A-Za-z]+ HRESULT [0-9A-F]{8}") + + def test_access_denied_progress_cannot_abort_verified_download(self): + progress = self.directory / "progress.txt" + progress.write_text(f"0|{len(PAYLOAD)}|0") + os.chmod(progress, stat.S_IREAD) + try: + self.finish(self.launch(), progress_verified=False) + self.assertEqual(progress.read_text(), f"0|{len(PAYLOAD)}|0") + warning = Path(str(progress) + ".warning").read_text() + self.assertIn("HRESULT 80070005", warning) + finally: + os.chmod(progress, stat.S_IREAD | stat.S_IWRITE) + + def test_unavailable_progress_never_bypasses_hash_rejection(self): + progress = self.directory / "progress.txt" + progress.write_text(f"0|{len(PAYLOAD)}|0") + os.chmod(progress, stat.S_IREAD) + try: + self.finish(self.launch(sha="0" * 64), 1) + self.assertEqual(progress.read_text(), f"0|{len(PAYLOAD)}|0") + detail = Path(str(progress) + ".error").read_text() + self.assertIn("SHA-256 or size mismatch", detail) + finally: + os.chmod(progress, stat.S_IREAD | stat.S_IWRITE) + + def test_progress_is_atomic_and_limited_to_four_updates_per_second(self): + child = self.launch("slow") + stamps = set() + snapshots = [] + while child.poll() is None: + try: + snapshots.append(self.progress.read_text()) + stamps.add(self.progress.stat().st_mtime_ns) + except (FileNotFoundError, PermissionError): + pass + time.sleep(0.005) + self.finish(child) + self.assertTrue(all(re.fullmatch(r"[0-9]+\|[0-9]+\|[0-5]", value) for value in snapshots)) + ordered = sorted(stamps) + self.assertGreaterEqual(len(ordered), 3) + self.assertTrue(all(after - before >= 240_000_000 for before, after in zip(ordered, ordered[1:]))) + + def test_range_validation_preserves_offsets_above_two_gib(self): + expected = 3 * 1024**3 + 17 + offset = 2 * 1024**3 + 123 + result = subprocess.run( + [str(self.range_probe), f"bytes {offset}-{expected - 1}/{expected}", str(offset), str(expected)], + creationflags=subprocess.CREATE_NO_WINDOW, + timeout=5, + ) + self.assertEqual(result.returncode, 0) + + def test_range_validation_rejects_overflow_and_truncation(self): + expected = 3 * 1024**3 + 17 + offset = 2 * 1024**3 + 123 + for header in (f"bytes 123-{expected - 1}/{expected}", "bytes 0-9223372036854775808/9223372036854775809"): + with self.subTest(header=header): + result = subprocess.run( + [str(self.range_probe), header, str(offset), str(expected)], + creationflags=subprocess.CREATE_NO_WINDOW, + timeout=5, + ) + self.assertEqual(result.returncode, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_windows_online_integration.py b/tests/test_windows_online_integration.py new file mode 100644 index 000000000..4fc8b9654 --- /dev/null +++ b/tests/test_windows_online_integration.py @@ -0,0 +1,393 @@ +"""Real hidden Inno downloads of a tiny harmless child from a loopback fixture. + +Only the separately compiled helper enables WINDOWS_DOWNLOAD_TEST. No product +installer, public transfer, visible window, model or native credential is used. +""" + +import ctypes +import hashlib +import http.server +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "desktop/installers/communityai-online.iss" +HELPER = ROOT / "desktop/installers/WindowsDownload.cs" +CHILD_SOURCE = r""" +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +internal static class HarmlessChild { + private static int Main(string[] args) { + string directory = null; + foreach (string arg in args) + if (arg.StartsWith("/DIR=", StringComparison.OrdinalIgnoreCase)) directory = arg.Substring(5); + if (directory == null || !Directory.Exists(directory)) return 99; + string executable = Process.GetCurrentProcess().MainModule.FileName; + File.WriteAllLines(Path.Combine(directory, "arguments.txt"), args); + File.WriteAllText(Path.Combine(directory, "executable.txt"), executable); + File.WriteAllText(Path.Combine(directory, "started"), "started"); + Thread.Sleep(1200); + File.WriteAllText(Path.Combine(directory, "finished"), File.Exists(executable) ? "alive" : "missing"); + return 7; + } +} +""" + + +class DownloadFixture(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_GET(self): + server = self.server + server.requests.append(dict(self.headers)) + offset = int(self.headers.get("Range", "bytes=0-")[6:-1]) + self.send_response(206 if offset else 200) + if offset: + self.send_header("Content-Range", f"bytes {offset}-{len(server.payload) - 1}/{len(server.payload)}") + self.send_header("Content-Length", str(len(server.payload) - offset)) + self.end_headers() + try: + if server.mode == "held": + self.wfile.write(server.payload[offset : offset + 32]) + self.wfile.flush() + server.entered.set() + server.release.wait(10) + elif server.mode == "resume" and not offset: + self.wfile.write(server.payload[: server.cut]) + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + elif server.mode == "slow": + step = max(1, len(server.payload) // 40) + for start in range(offset, len(server.payload), step): + self.wfile.write(server.payload[start : start + step]) + self.wfile.flush() + time.sleep(0.075) + else: + self.wfile.write(server.payload[offset:]) + except OSError: + pass + + +class OwnedProcess: + """Retain the opened process handle; never terminate by a reused PID.""" + + def __init__(self, process, kernel, wintypes): + self.pid = process.pid + self.kernel = kernel + self.handle = kernel.OpenProcess(0x100000 | 0x1000 | 1, False, self.pid) + if not self.handle: + raise OSError(ctypes.get_last_error(), "OpenProcess") + try: + values = [wintypes.FILETIME() for _ in range(4)] + if not kernel.GetProcessTimes(self.handle, *(ctypes.byref(value) for value in values)): + raise OSError(ctypes.get_last_error(), "GetProcessTimes") + self.created = (values[0].dwHighDateTime << 32) | values[0].dwLowDateTime + if abs(self.created / 10_000_000 - 11_644_473_600 - process.create_time()) > 0.002: + raise RuntimeError("Process identity changed before its handle was retained") + self.arguments = process.cmdline() + except BaseException: + kernel.CloseHandle(self.handle) + raise + + def stopped(self): + status = self.kernel.WaitForSingleObject(self.handle, 0) + if status not in (0, 258): + raise OSError(ctypes.get_last_error(), "WaitForSingleObject") + return status == 0 + + def terminate(self): + if not self.stopped() and not self.kernel.TerminateProcess(self.handle, 91): + raise OSError(ctypes.get_last_error(), "TerminateProcess") + + def close(self): + self.kernel.CloseHandle(self.handle) + + def exit_code(self): + code = ctypes.c_uint32() + if not self.kernel.GetExitCodeProcess(self.handle, ctypes.byref(code)): + raise OSError(ctypes.get_last_error(), "GetExitCodeProcess") + return code.value + + +@unittest.skipUnless(sys.platform == "win32", "Requires Windows Inno and .NET Framework") +class WindowsOnlineIntegrationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + from ctypes import wintypes + + import psutil + + cls.psutil = psutil + cls.wintypes = wintypes + cls.kernel = ctypes.WinDLL("kernel32", use_last_error=True) + cls.kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + cls.kernel.OpenProcess.restype = wintypes.HANDLE + cls.kernel.GetProcessTimes.argtypes = [wintypes.HANDLE] + [ctypes.POINTER(wintypes.FILETIME)] * 4 + cls.kernel.GetProcessTimes.restype = wintypes.BOOL + cls.kernel.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] + cls.kernel.WaitForSingleObject.restype = wintypes.DWORD + cls.kernel.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT] + cls.kernel.TerminateProcess.restype = wintypes.BOOL + cls.kernel.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + cls.kernel.GetExitCodeProcess.restype = wintypes.BOOL + cls.kernel.CloseHandle.argtypes = [wintypes.HANDLE] + cls.kernel.CloseHandle.restype = wintypes.BOOL + cls.kernel.CreateFileW.argtypes = [ + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ] + cls.kernel.CreateFileW.restype = wintypes.HANDLE + explicit = os.environ.get("COMMUNITYAI_INNO_COMPILER") + cls.inno = Path(explicit or shutil.which("ISCC.exe") or ROOT / ".gate13-runs/inno-setup-6/ISCC.exe") + cls.csc = Path(os.environ["WINDIR"]) / "Microsoft.NET/Framework64/v4.0.30319/csc.exe" + if not cls.inno.is_file() or not cls.csc.is_file(): + raise unittest.SkipTest("Set COMMUNITYAI_INNO_COMPILER to Inno6.7.3; Framework csc is also required") + cls.compiled = tempfile.TemporaryDirectory(prefix="communityai-inno-fixture-") + cls.addClassCleanup(cls.compiled.cleanup) + cls.build = Path(cls.compiled.name) + cls.helper = cls.build / "WindowsDownload.exe" + child_source = cls.build / "HarmlessChild.cs" + child_source.write_text(CHILD_SOURCE, encoding="utf-8") + cls.child = cls.build / "harmless-child.exe" + for source, output, defines in ( + (HELPER, cls.helper, ["/define:WINDOWS_DOWNLOAD_TEST"]), + (child_source, cls.child, []), + ): + result = subprocess.run( + [str(cls.csc), "/nologo", "/target:winexe", "/out:" + str(output), *defines, str(source)], + capture_output=True, + timeout=30, + creationflags=subprocess.CREATE_NO_WINDOW, + ) + if result.returncode: + raise AssertionError(result.stdout.decode(errors="replace")) + cls.payload = cls.child.read_bytes() + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="communityai-inno-case-") + self.directory = Path(self.temporary.name) + self.markers = self.directory / "child markers" + self.markers.mkdir() + self.temp_root = self.directory / "inno-temp" + self.temp_root.mkdir() + self.owned = {} + self.outer = None + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), DownloadFixture) + self.server.daemon_threads = True + self.server.payload = self.payload + self.server.cut = len(self.payload) // 2 + self.server.requests = [] + self.server.mode = "resume" + self.server.entered = threading.Event() + self.server.release = threading.Event() + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.addCleanup(self.cleanup) + result = subprocess.run( + [ + str(self.inno), + "/Qp", + "/DAppVersion=0.1.0-test", + f"/DInstallerUrl=http://127.0.0.1:{self.server.server_port}/harmless-child.exe", + "/DInstallerFilename=harmless-child.exe", + "/DInstallerSha256=" + hashlib.sha256(self.payload).hexdigest(), + "/DInstallerSize=" + str(len(self.payload)), + "/DDownloadHelper=" + str(self.helper), + "/DOutputPath=" + str(self.directory), + "/DPublisherName=Local integration fixture", + str(SCRIPT), + ], + capture_output=True, + timeout=30, + creationflags=subprocess.CREATE_NO_WINDOW, + ) + self.assertEqual(result.returncode, 0, (result.stdout + result.stderr).decode(errors="replace")) + self.wrapper = self.directory / "communityai-0.1.0-test-windows-online-setup.exe" + + def remember(self, process): + if process.pid in self.owned: + return + self.owned[process.pid] = OwnedProcess(process, self.kernel, self.wintypes) + + def refresh(self): + for owned in list(self.owned.values()): + if owned.stopped(): + continue + try: + parent = self.psutil.Process(owned.pid) + for process in parent.children(recursive=True): + try: + self.remember(process) + except (self.psutil.NoSuchProcess, OSError): + pass + except self.psutil.NoSuchProcess: + pass + + def wait_until(self, predicate, seconds=12): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + self.refresh() + if predicate(): + return + time.sleep(0.02) + log = self.directory / "wrapper.log" + detail = log.read_text(errors="replace")[-6000:] if log.exists() else "No wrapper log" + self.fail("Timed out waiting for owned fixture state. " + detail) + + def launch(self, mode="resume"): + self.server.mode = mode + self.forwarded = [ + "/VERYSILENT", + "/SUPPRESSMSGBOXES", + "/NORESTART", + "/SP-", + "/CURRENTUSER", + "/DIR=" + str(self.markers), + "/GROUP=CommunityAI local fixture", + ] + environment = os.environ.copy() + environment.update(TEMP=str(self.temp_root), TMP=str(self.temp_root)) + self.outer = subprocess.Popen( + [str(self.wrapper), *self.forwarded, "/LOG=" + str(self.directory / "wrapper.log")], + env=environment, + creationflags=subprocess.CREATE_NO_WINDOW, + ) + self.remember(self.psutil.Process(self.outer.pid)) + + def helper_process(self): + return next( + ( + owned + for owned in self.owned.values() + if owned.arguments and Path(owned.arguments[0]).name == "WindowsDownload.exe" + ), + None, + ) + + def assert_finished(self, expected=None): + self.wait_until(lambda: self.outer.poll() is not None) + if expected is not None: + self.assertEqual(self.outer.returncode, expected) + self.wait_until(lambda: all(owned.stopped() for owned in self.owned.values()), seconds=5) + + def cleanup(self): + self.server.release.set() + self.refresh() + for owned in reversed(list(self.owned.values())): + owned.terminate() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and not all(owned.stopped() for owned in self.owned.values()): + time.sleep(0.02) + stopped = all(owned.stopped() for owned in self.owned.values()) + for owned in self.owned.values(): + owned.close() + if self.outer is not None: + self.outer.wait(timeout=5) + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=3) + if stopped: + self.temporary.cleanup() + self.assertTrue(stopped, "Only owned fixture processes were stopped; temporary evidence retained on failure") + + def test_resume_handoff_arguments_exit_code_and_temporary_lifetime(self): + self.launch() + self.wait_until(lambda: (self.markers / "started").exists()) + self.assertIsNone(self.outer.poll()) + downloaded = Path((self.markers / "executable.txt").read_text()) + self.assertTrue(downloaded.is_relative_to(self.temp_root)) + self.assertTrue(downloaded.is_file()) + self.assertEqual((self.markers / "arguments.txt").read_text().splitlines(), self.forwarded + ["/LOG"]) + self.assertEqual(len(self.server.requests), 2) + self.assertEqual(self.server.requests[1]["Range"], f"bytes={self.server.cut}-") + self.assert_finished(expected=7) + self.assertEqual((self.markers / "finished").read_text(), "alive") + self.assertFalse(downloaded.exists()) + self.assertFalse(downloaded.parent.exists()) + self.assertIsNotNone(self.helper_process()) + + def test_cancel_file_stops_held_download_without_child(self): + self.launch("held") + self.wait_until(lambda: self.server.entered.is_set() and self.helper_process() is not None) + helper = self.helper_process() + cancel = Path(helper.arguments[6]) + output = Path(helper.arguments[2]) + self.assertTrue(cancel.is_relative_to(self.temp_root)) + cancel.write_text("cancel", encoding="ascii") + self.assert_finished() + self.assertNotEqual(self.outer.returncode, 0) + self.assertEqual(helper.exit_code(), 3) + self.assertFalse((self.markers / "started").exists()) + self.assertFalse(output.exists()) + + def test_long_progress_reader_lock_does_not_block_verified_handoff(self): + self.launch("slow") + self.wait_until( + lambda: self.helper_process() is not None and Path(self.helper_process().arguments[5]).is_file() + ) + progress = Path(self.helper_process().arguments[5]) + self.assertTrue(progress.is_relative_to(self.temp_root)) + handle = self.kernel.CreateFileW(str(progress), 0x80000000, 1, None, 3, 0x80, None) + self.assertNotEqual(handle, self.wintypes.HANDLE(-1).value) + started = time.monotonic() + try: + # Inno's real reader permits reads but denies delete sharing. Keep + # that lock beyond the helper's former one-second fatal threshold, + # and observe the warning proving a publication attempt failed. + self.wait_until( + lambda: time.monotonic() - started >= 1.4 and Path(str(progress) + ".warning").exists(), + seconds=5, + ) + self.assertIsNone(self.outer.poll()) + finally: + self.kernel.CloseHandle(handle) + # Release before the child exits so the fixture cannot itself prevent + # Inno from deleting the temporary directory during normal cleanup. + self.wait_until(lambda: (self.markers / "started").exists()) + downloaded = Path((self.markers / "executable.txt").read_text()) + self.assertTrue(downloaded.is_relative_to(self.temp_root)) + self.assertTrue(downloaded.is_file()) + self.assertIsNone(self.outer.poll()) + self.assertEqual((self.markers / "arguments.txt").read_text().splitlines(), self.forwarded + ["/LOG"]) + self.assert_finished(expected=7) + self.assertEqual((self.markers / "finished").read_text(), "alive") + self.assertFalse(downloaded.exists()) + self.assertFalse(downloaded.parent.exists()) + + def test_engine_exit_kills_owned_helper_without_child(self): + self.launch("held") + self.wait_until(lambda: self.server.entered.is_set() and self.helper_process() is not None) + helper = self.helper_process() + engine_pid = int(helper.arguments[7]) + self.assertIn(engine_pid, self.owned) + engine = self.owned[engine_pid] + self.assertEqual(engine.created, int(helper.arguments[8])) + engine.terminate() + self.wait_until(helper.stopped, seconds=3) + self.assert_finished() + self.assertNotEqual(self.outer.returncode, 0) + self.assertFalse((self.markers / "started").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_supervisor.py b/tests/test_worker_supervisor.py index ace3326f2..d58e8d1e0 100644 --- a/tests/test_worker_supervisor.py +++ b/tests/test_worker_supervisor.py @@ -1,10 +1,17 @@ +import os +import signal +import socket +import subprocess import sys import threading import time +from dataclasses import replace from types import SimpleNamespace +from unittest.mock import Mock import pytest +from drift.node import worker_supervisor as worker_module from drift.node.worker_supervisor import ( SystemBandwidthMonitor, WorkerLaunch, @@ -14,6 +21,40 @@ WorkerSupervisor, WorkerSupervisorSettings, ) +from drift.utils.resource_limits import DEVICE_MEMORY_BUDGET_EXIT_CODE + + +def test_memory_budget_rejection_waits_for_changed_configuration(): + launch = WorkerLaunch( + "worker", + "model", + (sys.executable, "-c", f"raise SystemExit({DEVICE_MEMORY_BUDGET_EXIT_CODE})"), + restart_backoff=0.01, + ) + supervisor = WorkerSupervisor([launch], poll_period=0.01, stop_timeout=2) + try: + supervisor.start_service() + supervisor.start_worker("worker") + _wait_for(lambda: supervisor.snapshot("worker")["last_exit_code"] == DEVICE_MEMORY_BUDGET_EXIT_CODE, timeout=10) + rejected = supervisor.snapshot("worker") + assert rejected["state"] == "paused" and rejected["pid"] is None + assert rejected["desired_running"] and rejected["resource_suspended"] + assert not rejected["resource_admitted"] and "VRAM budget" in rejected["resource_reason"] + time.sleep(0.1) # Several monitor ticks must not retry an unchanged budget. + assert supervisor.snapshot("worker")["restart_count"] == rejected["restart_count"] + supervisor.pause_worker_for_reconfiguration("worker") + supervisor.replace_launch(launch, start=True) + assert supervisor.snapshot("worker")["resource_suspended"] + supervisor.pause_worker_for_reconfiguration("worker") + corrected = replace(launch, command=(sys.executable, "-c", "import time; time.sleep(30)")) + supervisor.replace_launch(corrected, start=True) + resumed = supervisor.snapshot("worker") + assert resumed["state"] == "running" and resumed["resource_admitted"] + supervisor.pause_worker("worker") + assert supervisor.snapshot("worker")["operator_paused"] + assert supervisor.snapshot("worker")["pid"] is None + finally: + supervisor.shutdown() def _wait_for(predicate, timeout=2): @@ -25,6 +66,146 @@ def _wait_for(predicate, timeout=2): raise AssertionError("condition was not reached before timeout") +def _automatic_server_command(block_indices, binding): + return ( + sys.executable, + "-m", + "drift.cli", + "server", + "org/model", + "--block_indices", + block_indices, + "--expected_manifest_digest", + binding["placement_manifest_digest"], + "--expected_block_indices", + block_indices, + "--expected_artifact_bytes", + str(binding["placement_artifact_bytes"]), + "--expected_artifact_set_digest", + binding["placement_artifact_set_digest"], + "--expected_cache_root", + binding["placement_cache_root"], + "--cache_dir", + binding["placement_cache_root"], + ) + + +def _sleep_popen(command, **kwargs): + return subprocess.Popen((sys.executable, "-c", "import time; time.sleep(30)"), **kwargs) + + +def test_placement_explanation_changes_do_not_change_worker_assignment(): + from drift.node.contribution_planner import AutomaticContributionPlanner, PlacementCandidate + + planner = AutomaticContributionPlanner(num_blocks=35, jitter_seed="gate13-gemma") + candidate = PlacementCandidate( + "gemma", + "sha256:" + "a" * 64, + 0, + True, + 1234, + 35, + {"status": "incomplete", "last_updated_age": 0, "replica_counts": [0] * 35}, + ) + first = planner.plan([candidate], sharing_enabled=True, now=0) + covered = replace(candidate, health={"status": "complete", "last_updated_age": 0, "replica_counts": [1] * 35}) + refreshed = planner.plan([covered], sharing_enabled=True, now=901) + assert first.decision.block_indices == refreshed.decision.block_indices == "0:35" + assert first.reason != refreshed.reason + binding = { + "placement_manifest_digest": "sha256:" + "a" * 64, + "placement_artifact_bytes": 1234, + "placement_artifact_set_digest": "b" * 64, + "placement_cache_root": os.path.realpath(sys.prefix), + } + initial = WorkerLaunch( + "automatic", + "gemma", + _automatic_server_command("0:35", binding), + automatic=True, + block_indices="0:35", + intent_published=True, + remote_acknowledged=True, + placement_reason=first.reason, + **binding, + ) + # The reconciler compares these launches after the 15-minute residency. + assert initial == replace(initial, placement_reason=refreshed.reason) + assert initial == replace(initial, placement_reason="same range; local demand bucket 1") + assert initial != replace(initial, policy_admitted=False, policy_reason="coverage stale") + assert initial != replace( + initial, + block_indices="0:34", + command=_automatic_server_command("0:34", binding), + ) + + +@pytest.mark.parametrize("platform", ["linux", "win32"]) +@pytest.mark.parametrize("exit_mode", ["graceful", "timeout", "crashed"]) +def test_worker_group_cleanup_is_linux_only(monkeypatch, platform, exit_mode): + monkeypatch.setattr(worker_module, "sys", SimpleNamespace(platform=platform)) + monkeypatch.setattr(worker_module, "signal", SimpleNamespace(SIGKILL=9)) + kill_group = Mock() + monkeypatch.setattr(worker_module.os, "killpg", kill_group, raising=False) + process = Mock(pid=12345, stdout=None) + process.poll.return_value = None + process.wait.return_value = 0 + popen = Mock(return_value=process) + launch = WorkerLaunch("worker", "model", ("node", "server"), auto_restart=False) + supervisor = WorkerSupervisor([launch], popen=popen) + supervisor.start_worker("worker") + assert popen.call_args.kwargs.get("start_new_session", False) is (platform == "linux") + if exit_mode == "crashed": + process.poll.return_value = 1 + assert supervisor.snapshot("worker")["state"] == "crashed" + else: + if exit_mode == "timeout": + process.wait.side_effect = [subprocess.TimeoutExpired("worker", 10), -9] + supervisor.pause_worker("worker") + process.terminate.assert_called_once() + assert process.kill.call_count == int(exit_mode == "timeout") + if platform == "linux": + kill_group.assert_called_once_with(process.pid, 9) + else: + kill_group.assert_not_called() + supervisor.shutdown() + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires Linux process groups") +@pytest.mark.parametrize("crash", [False, True]) +def test_linux_worker_exit_releases_descendant_listening_port(crash): + child = ( + "import signal,socket,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); " + "s=socket.socket(); s.bind(('127.0.0.1',0)); s.listen(); " + "print(s.getsockname()[1],flush=True); time.sleep(30)" + ) + parent = "import subprocess,sys,time; " f"subprocess.Popen([sys.executable,'-c',{child!r}]); time.sleep(30)" + launch = WorkerLaunch("worker", "model", (sys.executable, "-c", parent), auto_restart=False) + supervisor = WorkerSupervisor([launch], stop_timeout=1, poll_period=0.01) + try: + supervisor.start_worker("worker") + _wait_for(lambda: bool(supervisor.snapshot("worker")["recent_logs"])) + snapshot = supervisor.snapshot("worker") + port = int(snapshot["recent_logs"][0]) + if crash: + os.kill(snapshot["pid"], signal.SIGKILL) + _wait_for(lambda: supervisor.snapshot("worker")["state"] == "crashed") + else: + supervisor.pause_worker("worker") + + def port_released(): + with socket.socket() as probe: + try: + probe.bind(("127.0.0.1", port)) + except OSError: + return False + return True + + _wait_for(port_released) + finally: + supervisor.shutdown() + + def test_supervisor_reconfigures_only_after_persistence_and_while_idle(): initial = WorkerLaunch( "worker", @@ -58,7 +239,106 @@ def test_supervisor_reconfigures_only_after_persistence_and_while_idle(): supervisor.shutdown() +def test_automatic_worker_requires_exact_remote_intent_acknowledgement(): + common = { + "worker_id": "automatic", + "model_id": "model", + "command": (sys.executable, "-c", "raise SystemExit(0)"), + "automatic": True, + "block_indices": "0:1", + "placement_reason": "selected", + } + with pytest.raises(ValueError, match="remotely acknowledged"): + WorkerLaunch(**common) + with pytest.raises(ValueError, match="remote acknowledgement"): + WorkerLaunch(**common, policy_admitted=False, policy_reason="blocked", intent_published=True) + with pytest.raises(ValueError, match="manual workers"): + WorkerLaunch( + "manual", + "model", + common["command"], + intent_published=True, + remote_acknowledged=True, + ) + with pytest.raises(ValueError, match="configured together"): + WorkerLaunch( + **common, + policy_admitted=False, + policy_reason="blocked", + placement_manifest_digest="sha256:" + "a" * 64, + ) + + +def test_automatic_worker_binds_artifact_claims_to_exact_server_command(): + binding = { + "placement_manifest_digest": "sha256:" + "a" * 64, + "placement_artifact_bytes": 1234, + "placement_artifact_set_digest": "b" * 64, + "placement_cache_root": os.path.realpath(sys.prefix), + } + admitted = { + "worker_id": "automatic", + "model_id": "model", + "automatic": True, + "block_indices": "0:1", + "placement_reason": "selected", + "intent_published": True, + "remote_acknowledged": True, + **binding, + } + command = _automatic_server_command("0:1", binding) + WorkerLaunch(command=command, **admitted) + frozen_command = (sys.executable, "server", *command[4:]) + WorkerLaunch(command=frozen_command, **admitted) + + fake_executable = os.path.join(os.path.dirname(sys.executable), "not-the-node-executable") + with pytest.raises(ValueError, match="current node executable"): + WorkerLaunch(command=(fake_executable, *command[1:]), **admitted) + with pytest.raises(ValueError, match="current node executable"): + WorkerLaunch(command=(fake_executable, *frozen_command[1:]), **admitted) + + with pytest.raises(ValueError, match="forbidden server option"): + WorkerLaunch(command=(sys.executable, "-c", "raise SystemExit(0)"), **admitted) + + wrong_span = list(command) + wrong_span[wrong_span.index("--block_indices") + 1] = "1:2" + with pytest.raises(ValueError, match="mismatched --block_indices"): + WorkerLaunch(command=tuple(wrong_span), **admitted) + + missing_cache_claim = list(command) + expected_cache = missing_cache_claim.index("--expected_cache_root") + del missing_cache_claim[expected_cache : expected_cache + 2] + with pytest.raises(ValueError, match="exactly one --expected_cache_root"): + WorkerLaunch(command=tuple(missing_cache_claim), **admitted) + + with pytest.raises(ValueError, match="exactly one --expected_manifest_digest"): + WorkerLaunch( + command=command + ("--expected_manifest_digest", binding["placement_manifest_digest"]), + **admitted, + ) + + with pytest.raises(ValueError, match="must not use --num_blocks"): + WorkerLaunch(command=command + ("--num_blocks", "1"), **admitted) + + for forbidden in ( + ("-c", "config.yml"), + ("--config", "config.yml"), + ("--custom_module_path", "custom.py"), + ("--allow_training_rpcs",), + ("--token", "secret"), + ("--use_auth_token",), + ): + with pytest.raises(ValueError, match="forbidden server option"): + WorkerLaunch(command=command + forbidden, **admitted) + + def test_supervisor_replaces_one_paused_automatic_assignment_and_autostarts(): + artifact_binding = { + "placement_manifest_digest": "sha256:" + "a" * 64, + "placement_artifact_bytes": 1234, + "placement_artifact_set_digest": "b" * 64, + "placement_cache_root": os.path.realpath(sys.prefix), + } initial = WorkerLaunch( "automatic", "auto", @@ -72,16 +352,27 @@ def test_supervisor_replaces_one_paused_automatic_assignment_and_autostarts(): updated = WorkerLaunch( "automatic", "model", - (sys.executable, "-c", "import time; time.sleep(30)"), + _automatic_server_command("2:3", artifact_binding), auto_start=True, policy_admitted=True, automatic=True, block_indices="2:3", placement_reason="selected least-covered block", + intent_published=True, + remote_acknowledged=True, + **artifact_binding, ) - supervisor = WorkerSupervisor([initial], stop_timeout=2, poll_period=0.01) + supervisor = WorkerSupervisor([initial], stop_timeout=2, poll_period=0.01, popen=_sleep_popen) supervisor.start_service() + supervisor.pause_worker("automatic") + assert supervisor.snapshot("automatic")["operator_paused"] is True + assert supervisor.start_worker("automatic") is False + pending = supervisor.snapshot("automatic") + assert pending["operator_paused"] is False + assert pending["desired_running"] is False + assert pending["pid"] is None # Pending placement must never launch a worker. + assert supervisor.replace_launch(updated) is True _wait_for(lambda: supervisor.snapshot("automatic")["state"] == "running") snapshot = supervisor.snapshot("automatic") @@ -89,6 +380,11 @@ def test_supervisor_replaces_one_paused_automatic_assignment_and_autostarts(): assert snapshot["automatic"] is True assert snapshot["block_indices"] == "2:3" assert snapshot["placement_reason"] == "selected least-covered block" + assert snapshot["intent_published"] is True + assert snapshot["remote_acknowledged"] is True + assert "placement_manifest_digest" not in snapshot + assert "placement_artifact_set_digest" not in snapshot + assert "placement_cache_root" not in snapshot with pytest.raises(WorkerReconfigurationBusyError, match="pause contribution worker"): supervisor.replace_launch(initial) @@ -97,12 +393,15 @@ def test_supervisor_replaces_one_paused_automatic_assignment_and_autostarts(): migrated = WorkerLaunch( "automatic", "other-model", - updated.command, + _automatic_server_command("3:4", artifact_binding), auto_start=True, policy_admitted=True, automatic=True, block_indices="3:4", placement_reason="coverage changed", + intent_published=True, + remote_acknowledged=True, + **artifact_binding, ) assert supervisor.replace_launch(migrated, start=True) is True _wait_for(lambda: supervisor.snapshot("automatic")["state"] == "running") @@ -115,12 +414,15 @@ def test_supervisor_replaces_one_paused_automatic_assignment_and_autostarts(): paused_update = WorkerLaunch( "automatic", "third-model", - updated.command, + _automatic_server_command("4:5", artifact_binding), auto_start=True, policy_admitted=True, automatic=True, block_indices="4:5", placement_reason="coverage changed again", + intent_published=True, + remote_acknowledged=True, + **artifact_binding, ) assert supervisor.replace_launch(paused_update, start=True) is True snapshot = supervisor.snapshot("automatic")