test(e2e): drive the Lemonade recovery scenario with a planted runtime

.github/workflows/ci.yml

@@ -631,20 +631,16 @@ jobs:
         # Windows install lifecycle (packaging + native-crypto verify + user-PATH
         # + loopback HTTP + isolated smoke + uninstall) as opt-in @lifecycle E2E.
         #
-        # Hand the suite the binaries the Build step already produced. Without
-        # this `cargo xtask e2e` builds `rocm`/`rocmd` for itself, and because it
-        # adds `--features rocm/e2e-test-hooks` the feature resolution differs
-        # from the Build step's — so cargo recompiles the whole release graph
-        # rather than reusing it. That second release build cost 3-4 minutes on
-        # every run, on the job that alone determines when CI goes green.
+        # Hand the suite the binaries the Build step already produced, so
+        # `cargo xtask e2e` skips its own build entirely. Relying on cargo to
+        # notice the artifacts are fresh is not enough: any divergence between
+        # the two invocations (profile, package set, RUSTFLAGS, target dir)
+        # re-resolves the graph and recompiles it. That is what happened here —
+        # a feature-set mismatch cost a second 3-4 minute release build on every
+        # run, on the job that alone determines when CI goes green.
         #
-        # Deliberately NOT built with `rocm/e2e-test-hooks`, unlike the prebuilt
-        # lanes in e2e-selfhosted.yml / nightly.yml. Those run the full suite,
-        # whose scripted failure seams are compiled out without the feature.
-        # `E2E_ONLY_LIFECYCLE` restricts this lane to @lifecycle scenarios, none
-        # of which touch a seam — and this lane packages and installs the binary
-        # through the real installer, so it should ship exactly what a release
-        # ships rather than a build carrying test hooks.
+        # This lane packages and installs the binary through the real installer,
+        # so it ships exactly what a release ships.
         env:
           E2E_INCLUDE_LIFECYCLE: "1"
           E2E_ONLY_LIFECYCLE: "1"

.github/workflows/e2e-selfhosted.yml

@@ -315,14 +315,11 @@ jobs:
           # pre-warm and suite so xtask does not rebuild. Honors
           # CARGO_TARGET_DIR set above.
           #
-          # `--features rocm/e2e-test-hooks` must match what `cargo xtask e2e`
-          # builds when it builds for itself. The suite's deterministic failure
-          # seams (e.g. the scripted Lemonade backend-install failure) are
-          # compiled out without it, so a pre-built binary that omits the feature
-          # leaves those scenarios unable to reach their premise — they then fail
-          # as regressions on whichever lane happens to select them. Every lane
-          # that pre-builds and exports ROCM_CLI_BINARY must pass it.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # Keep this invocation identical to the one `cargo xtask e2e` runs when
+          # it builds for itself. Any divergence (profile, package set, features)
+          # re-resolves the graph and makes xtask recompile everything instead of
+          # reusing what this step just built.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -480,9 +477,9 @@ jobs:
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
           # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite.
-          # See the e2e-gpu lane for why the e2e-test-hooks feature must match
-          # what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane for why this must match what `cargo xtask e2e`
+          # would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -648,9 +645,9 @@ jobs:
           # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite.
           # This job does not set CARGO_TARGET_DIR, so the binaries land in
           # the default target\release (fall back to it when the env var is unset).
-          # See the e2e-gpu lane for why the e2e-test-hooks feature must match
-          # what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane for why this must match what `cargo xtask e2e`
+          # would build for itself.
+          cargo build --release -p rocm -p rocmd
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" }
           $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe"
@@ -981,9 +978,9 @@ jobs:
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
           # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite.
-          # See the e2e-gpu lane for why the e2e-test-hooks feature must match
-          # what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane for why this must match what `cargo xtask e2e`
+          # would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -1157,9 +1154,9 @@ jobs:
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
-          # See the e2e-gpu lane for why the e2e-test-hooks feature must match
-          # what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane for why this must match what `cargo xtask e2e`
+          # would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -1286,9 +1283,9 @@ jobs:
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
-          # See the e2e-gpu lane for why the e2e-test-hooks feature must match
-          # what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane for why this must match what `cargo xtask e2e`
+          # would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 

.github/workflows/nightly.yml

@@ -449,9 +449,9 @@ jobs:
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
           # Build both binaries once; reuse them for pre-warm + suite.
-          # See the e2e-gpu lane in e2e-selfhosted.yml for why the
-          # e2e-test-hooks feature must match what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane in e2e-selfhosted.yml for why this must match
+          # what `cargo xtask e2e` would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -552,9 +552,9 @@ jobs:
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
-          # See the e2e-gpu lane in e2e-selfhosted.yml for why the
-          # e2e-test-hooks feature must match what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane in e2e-selfhosted.yml for why this must match
+          # what `cargo xtask e2e` would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -643,9 +643,9 @@ jobs:
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
-          # See the e2e-gpu lane in e2e-selfhosted.yml for why the
-          # e2e-test-hooks feature must match what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane in e2e-selfhosted.yml for why this must match
+          # what `cargo xtask e2e` would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -743,9 +743,9 @@ jobs:
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
-          # See the e2e-gpu lane in e2e-selfhosted.yml for why the
-          # e2e-test-hooks feature must match what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane in e2e-selfhosted.yml for why this must match
+          # what `cargo xtask e2e` would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 
@@ -849,9 +849,9 @@ jobs:
           # engine's cache lookup didn't fall back to USERPROFILE.
           $env:E2E_SHARED_CACHE_DIR = "$env:RUNNER_WORKSPACE\e2e-shared"
 
-          # See the e2e-gpu lane in e2e-selfhosted.yml for why the
-          # e2e-test-hooks feature must match what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane in e2e-selfhosted.yml for why this must match
+          # what `cargo xtask e2e` would build for itself.
+          cargo build --release -p rocm -p rocmd
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" }
           $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe"
@@ -1091,9 +1091,9 @@ jobs:
           prewarm="/root/work/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
-          # See the e2e-gpu lane in e2e-selfhosted.yml for why the
-          # e2e-test-hooks feature must match what `cargo xtask e2e` would build.
-          cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks
+          # See the e2e-gpu lane in e2e-selfhosted.yml for why this must match
+          # what `cargo xtask e2e` would build for itself.
+          cargo build --release -p rocm -p rocmd
           export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm"
           export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd"
 

apps/rocm/Cargo.toml

@@ -10,9 +10,6 @@ publish.workspace = true
 [lints]
 workspace = true
 
-[features]
-e2e-test-hooks = ["rocm-engine-lemonade/e2e-test-hooks"]
-
 [dependencies]
 anyhow.workspace = true
 clap.workspace = true