From 46bb7043857c9ee3cfdd4ee4d9585f8d66deb65a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Mon, 17 Aug 2026 22:05:07 +0000 Subject: [PATCH 01/54] trident-acl-agent: rewrite as annotation-based ACL update agent Rewrites trident-acl-agent as the on-node half of Trident's Azure Container Linux (ACL) A/B update trigger: a Kubernetes annotation-driven reconcile loop that watches its Node's acl.azure.com/update-request annotation and drives Trident's stage/finalize/rollback/commit operations against tridentd, reporting status back via acl.azure.com/update-status and acl.azure.com/update-commit-status, and reporting update progress/completion/failure to Nebraska (the Omaha-protocol update server, using the nebraska client module from PR #733). Core additions/changes across this branch's history: - Annotation-driven orchestrator: stage/finalize/rollback/commit state machine, persistent state.json bridging the pre/post-reboot halves of an update, status heartbeat, and a historical one-shot omaha-only fallback mode. - Kubernetes client: node_name/api_server/kubeconfig handling, with api_server left unset by default so the kubeconfig's own server is trusted as-is. - --validate-connection diagnostic subcommand for on-node/CI troubleshooting (originally PR #732). - Nebraska/Omaha reachability and update-check integration, including retry logic for post-reboot completion reporting and various correctness fixes (async-context panic, republish loop, review feedback). - update-request annotation gained three per-request Nebraska overrides: server (endpoint), appId, and track - each resolved with the static config as fallback and persisted in state.json so the whole stage->finalize->post-reboot-commit lifecycle of one update consistently targets the same Nebraska server/app/track even if the static default changes in between. - Config converted entirely from a TOML file to TRIDENT_ACL_AGENT_* environment variables (systemd-style: Environment= lines, systemctl edit drop-ins, or any other means of setting the process environment), with DEFAULT_NEBRASKA_APP_ID/TRACK changed to deliberately invalid sentinels so a deployment that forgets to configure them fails loudly. Removed the now-redundant CLI URL override alongside it. - Added crates/trident-acl-agent/README.md documenting every TRIDENT_ACL_AGENT_* environment variable and both operating modes. This branch's own design docs (accepted-design.md, accepted-design-v2.md) are intentionally not carried in this consolidated commit. Rebased onto PR #733 (user/frhuelsz/nebraska-client): this commit's original changes to crates/trident-acl-agent/src/nebraska/{client,error}.rs are dropped here in favor of PR #733's own (superseding) version of those files, which already includes the same completion-retry fix (NebraskaError::CompletionNotAcknowledged) among other improvements. lib.rs is merged to keep both this commit's rewrite and PR #733's AGENT_VERSION addition. --- Cargo.lock | 576 ++++- Cargo.toml | 4 + crates/trident-acl-agent/Cargo.toml | 18 + crates/trident-acl-agent/README.md | 53 + crates/trident-acl-agent/src/annotations.rs | 1195 +++++++++ crates/trident-acl-agent/src/config.rs | 424 +++ crates/trident-acl-agent/src/error.rs | 75 +- crates/trident-acl-agent/src/k8s.rs | 173 ++ crates/trident-acl-agent/src/lib.rs | 182 +- crates/trident-acl-agent/src/main.rs | 724 ++---- crates/trident-acl-agent/src/mock_tridentd.rs | 253 ++ crates/trident-acl-agent/src/omaha/app.rs | 106 - crates/trident-acl-agent/src/omaha/event.rs | 98 - crates/trident-acl-agent/src/omaha/mod.rs | 261 -- crates/trident-acl-agent/src/omaha/request.rs | 363 --- .../trident-acl-agent/src/omaha/response.rs | 282 -- crates/trident-acl-agent/src/omaha/status.rs | 168 -- crates/trident-acl-agent/src/omaha/xml.rs | 18 - crates/trident-acl-agent/src/orchestrator.rs | 2270 +++++++++++++++++ crates/trident-acl-agent/src/state.rs | 354 +++ crates/trident-acl-agent/src/trident.rs | 399 +++ packaging/rpm/trident.spec | 11 + packaging/systemd/trident-acl-agent.service | 12 + 23 files changed, 6136 insertions(+), 1883 deletions(-) create mode 100644 crates/trident-acl-agent/README.md create mode 100644 crates/trident-acl-agent/src/annotations.rs create mode 100644 crates/trident-acl-agent/src/config.rs create mode 100644 crates/trident-acl-agent/src/k8s.rs create mode 100644 crates/trident-acl-agent/src/mock_tridentd.rs delete mode 100644 crates/trident-acl-agent/src/omaha/app.rs delete mode 100644 crates/trident-acl-agent/src/omaha/event.rs delete mode 100644 crates/trident-acl-agent/src/omaha/mod.rs delete mode 100644 crates/trident-acl-agent/src/omaha/request.rs delete mode 100644 crates/trident-acl-agent/src/omaha/response.rs delete mode 100644 crates/trident-acl-agent/src/omaha/status.rs delete mode 100644 crates/trident-acl-agent/src/omaha/xml.rs create mode 100644 crates/trident-acl-agent/src/orchestrator.rs create mode 100644 crates/trident-acl-agent/src/state.rs create mode 100644 crates/trident-acl-agent/src/trident.rs create mode 100644 packaging/systemd/trident-acl-agent.service diff --git a/Cargo.lock b/Cargo.lock index a7fef8faf5..6beb8abf65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,19 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.1", + "once_cell", + "version_check", + "zerocopy 0.8.27", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -26,6 +39,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -109,6 +128,40 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -175,6 +228,17 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.15", + "instant", + "rand 0.8.6", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -210,11 +274,11 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -256,10 +320,11 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -425,6 +490,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -684,6 +759,18 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.13.0" @@ -699,6 +786,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-ordinalize" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "enumflags2" version = "0.7.10" @@ -772,6 +879,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.2.0" @@ -790,6 +917,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -812,6 +945,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1041,6 +1180,35 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "headers" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" +dependencies = [ + "base64 0.22.1", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" +dependencies = [ + "http", +] [[package]] name = "heck" @@ -1188,6 +1356,42 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-http-proxy" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8021e0ae20c08eadc94d0bdafdeda66d4f0858541c146ae6e46b219bfe58497e" +dependencies = [ + "bytes", + "futures-util", + "headers", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -1438,6 +1642,15 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "inventory" version = "0.3.15" @@ -1501,6 +1714,41 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863726d7afb6bc2590eeff7135d923545e5e964f004c2ccf8716c25e70a86f08" +dependencies = [ + "jsonptr", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "jsonpath-rust" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c00ae348f9f8fd2d09f82a98ca381c60df9e0820d8d79fce43e649b4dc3128b" +dependencies = [ + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror 2.0.12", +] + +[[package]] +name = "jsonptr" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dea2b27dd239b2556ed7a25ba842fe47fd602e7fc7433c2a8d6106d4d9edd70" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "jwt" version = "0.16.0" @@ -1516,6 +1764,115 @@ dependencies = [ "sha2", ] +[[package]] +name = "k8s-openapi" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c75b990324f09bef15e791606b7b7a296d02fc88a344f6eba9390970a870ad5" +dependencies = [ + "base64 0.22.1", + "chrono", + "serde", + "serde-value", + "serde_json", +] + +[[package]] +name = "kube" +version = "0.98.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32053dc495efad4d188c7b33cc7c02ef4a6e43038115348348876efd39a53cba" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.98.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d34ad38cdfbd1fa87195d42569f57bb1dda6ba5f260ee32fef9570b7937a0c9" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "either", + "futures", + "home", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-http-proxy", + "hyper-rustls", + "hyper-timeout", + "hyper-util", + "jsonpath-rust", + "k8s-openapi", + "kube-core", + "pem", + "rustls", + "rustls-pemfile", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror 2.0.12", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.98.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97aa830b288a178a90e784d1b0f1539f2d200d2188c7b4a3146d9dc983d596f3" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch", + "k8s-openapi", + "serde", + "serde-value", + "serde_json", + "thiserror 2.0.12", +] + +[[package]] +name = "kube-runtime" +version = "0.98.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a41af186a0fe80c71a13a13994abdc3ebff80859ca6a4b8a6079948328c135b" +dependencies = [ + "ahash", + "async-broadcast", + "async-stream", + "async-trait", + "backoff", + "educe", + "futures", + "hashbrown", + "hostname", + "json-patch", + "jsonptr", + "k8s-openapi", + "kube-client", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "thiserror 2.0.12", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1671,10 +2028,10 @@ dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.1.5", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -1828,6 +2185,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-sys" version = "0.9.116" @@ -1851,6 +2214,15 @@ dependencies = [ "syn", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "os_pipe" version = "1.2.1" @@ -1916,6 +2288,12 @@ dependencies = [ "which", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" @@ -1948,6 +2326,16 @@ dependencies = [ "regex", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2357,7 +2745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.17", + "zerocopy 0.8.27", ] [[package]] @@ -2461,6 +2849,20 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2502,6 +2904,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.7.0", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -2513,9 +2942,23 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] [[package]] name = "rustversion" @@ -2579,6 +3022,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -2586,7 +3038,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2594,9 +3059,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -2621,6 +3086,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -2709,6 +3184,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2741,9 +3227,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook" @@ -3202,6 +3688,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -3223,6 +3719,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", + "slab", "tokio", ] @@ -3325,6 +3822,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "base64 0.22.1", + "bitflags", + "bytes", + "http", + "http-body", + "mime", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -3495,10 +4010,15 @@ name = "trident-acl-agent" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", "env_logger 0.11.5", "futures", + "humantime", + "hyper-util", "indoc", + "k8s-openapi", + "kube", "log", "maplit", "mockito", @@ -3509,12 +4029,16 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", + "serde_yaml", "sha2", "sysdefs", "systemd-journal-logger", + "tempfile", "thiserror 1.0.69", "tokio", + "tokio-stream", "tonic", + "tower", "trident-proto", "url", "uuid", @@ -3685,6 +4209,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.4" @@ -4271,11 +4801,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.17" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ - "zerocopy-derive 0.8.17", + "zerocopy-derive 0.8.27", ] [[package]] @@ -4291,9 +4821,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.17" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", @@ -4321,6 +4851,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerovec" version = "0.10.4" diff --git a/Cargo.toml b/Cargo.toml index d0baaae608..15d0c73504 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,9 +35,12 @@ hex = "0.4.0" hostname = "0.4.0" humantime = "2.3.0" hyper = "1.8.1" +hyper-util = "0.1" indoc = "2.0.5" inventory = "0.3.15" +k8s-openapi = { version = "0.24.0", features = ["v1_32"] } itertools = "0.13.0" +kube = { version = "0.98.0", default-features = false, features = ["client", "rustls-tls", "runtime"] } lazy_static = "1.5.0" libc = "0.2.167" log = "0.4.22" @@ -90,6 +93,7 @@ tar = "0.4.46" tempfile = "3.14.0" tera = "1.20.0" textwrap = "0.16.2" +toml = "0.8.23" thiserror = "1.0.69" tokio = { version = "1.48.0", features = ["full"] } tokio-stream = { version = "0.1.17", features = ["net"] } diff --git a/crates/trident-acl-agent/Cargo.toml b/crates/trident-acl-agent/Cargo.toml index 21c46aaace..dc075e580c 100644 --- a/crates/trident-acl-agent/Cargo.toml +++ b/crates/trident-acl-agent/Cargo.toml @@ -7,18 +7,25 @@ publish = false [dependencies] anyhow = { workspace = true, features = ["backtrace"] } clap = { workspace = true, features = ["derive"] } +chrono = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } +humantime = { workspace = true } +k8s-openapi = { workspace = true } +kube = { workspace = true } log = { workspace = true } quick-xml = { workspace = true, features = ["serialize"] } reqwest = { workspace = true } semver = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_yaml = { workspace = true } serde_path_to_error = { workspace = true } sha2 = { workspace = true } systemd-journal-logger = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-stream = { workspace = true } tonic = { workspace = true } url = { workspace = true, features = ["serde"] } uuid = { workspace = true, features = ["v4", "serde"] } @@ -29,7 +36,18 @@ trident-proto = { path = "../trident-proto" } [dev-dependencies] +tempfile = { workspace = true } indoc = { workspace = true } maplit = { workspace = true } mockito = { workspace = true } serde_json = { workspace = true } +serde_yaml = { workspace = true } +tower = { workspace = true } +# The generated gRPC *server* stubs (UpdateServiceServer/CommitServiceServer) +# are only needed to build an in-process fake tridentd for unit tests -- the +# production trident-acl-agent binary is a gRPC client only. Declaring the +# "server" feature here (dev-dependencies) rather than in [dependencies] +# keeps it out of the production binary's dependency graph; it is only +# pulled in for `cargo test`. +trident-proto = { path = "../trident-proto", features = ["grpc-preview", "server"] } +hyper-util = { workspace = true, features = ["tokio"] } diff --git a/crates/trident-acl-agent/README.md b/crates/trident-acl-agent/README.md new file mode 100644 index 0000000000..3f703fbfd2 --- /dev/null +++ b/crates/trident-acl-agent/README.md @@ -0,0 +1,53 @@ +# trident-acl-agent + +The on-node half of Trident's Azure Container Linux (ACL) A/B update +trigger. Runs in one of two modes, selected by +`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`: + +- **`annotations`** (the default): watches its Node's + `acl.azure.com/update-request` annotation and drives Trident's + stage/finalize/rollback/commit operations against `tridentd` accordingly, + reporting progress and status back to Kubernetes and to Nebraska (the + Omaha-protocol update server). +- **`omaha-only`**: the historical one-shot behavior. Queries Nebraska once, + and if an update is offered, calls tridentd's combined `update()` RPC once + and exits - no Kubernetes or annotation involvement at all. Kept as an + explicit opt-out for nodes that don't participate in the AKS + annotation-driven update protocol. + +## Configuration + +There is no config file. Every setting is an environment variable prefixed +`TRIDENT_ACL_AGENT_`, systemd-style: set it directly in the unit's own +`Environment=` lines, via a drop-in override (`systemctl edit +trident-acl-agent.service`, which creates +`/etc/systemd/system/trident-acl-agent.service.d/override.conf`), or by any +other means that ultimately sets the process's environment before it +starts. + +A variable that is unset, or set to the empty string, falls back to that +setting's default below. A variable that is set to a malformed value (a bad +URL, a bad duration, an unrecognized `goal_source`) causes the agent to +fail to start with an error naming the offending variable. + +| Variable | Default | Description | +|---|---|---| +| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Nebraska/Omaha server URL to poll for updates and report progress/completion events to. Can also be overridden per-update via the `server` field on the `acl.azure.com/update-request` annotation, which takes precedence over this variable for that update's entire lifecycle (stage through post-reboot commit). | +| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Nebraska application ID this node checks in as. Can also be overridden per-update via the `appId` annotation field, same precedence rules as the endpoint. | +| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Nebraska track (channel/group) this node follows. Can also be overridden per-update via the `track` annotation field, same precedence rules as the endpoint. | +| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG`'s own kubeconfig is used as-is (e.g. the real cluster FQDN a node's own `/var/lib/kubelet/kubeconfig` already points at). Only needed when the kubeconfig's own server is wrong for this deployment. | +| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig file used to reach the Kubernetes API server and authenticate as this node. | +| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. Kubernetes Node names must be valid RFC 1123 DNS labels (lowercase), matching how kubelet itself registers the Node - so the default only needs overriding when the agent's environment can't discover the correct hostname on its own. | +| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode: `annotations` or `omaha-only` (see above). | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file, which bridges the pre-reboot `finalize`/`rollback` half of an update and its post-reboot `commit` half across the reboot. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` operation (parsed as a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) is allowed to run before it's considered failed. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` operation is allowed to run before it's considered failed. Parsed the same way as the stage timeout. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat the agent writes while a stage/finalize/rollback operation is running, so AKS-RP and the watchdog can tell a working agent from a stuck one. Parsed the same way as the timeouts. | + +## Diagnostics + +`trident-acl-agent --validate-connection ` +checks connectivity to a single dependency using the current environment +and exits immediately - useful for a systemd `ExecStartPre` check or manual +on-node troubleshooting without running the full orchestrator loop. diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs new file mode 100644 index 0000000000..ea946bb1bc --- /dev/null +++ b/crates/trident-acl-agent/src/annotations.rs @@ -0,0 +1,1195 @@ +//! Request/status annotation protocol types for the Trident ACL agent. +//! +//! This module (schema types, `UpdateRequest::validate()`, and the +//! `#[cfg(test)]` design-doc conformance tests below) implements the +//! `acl.azure.com/update-request`, `acl.azure.com/update-status`, and +//! `acl.azure.com/update-commit-status` node annotation protocol described +//! by the current accepted design (`accepted-design-v2.md`). Keep +//! `UpdateRequest`/`UpdateStatus`/`StatusCode` and `validate()` in sync with +//! that document's formal JSON Schema (its section "Formal JSON Schema") - +//! the `design_doc_*`/`agent_built_*_conform_to_formal_schema` tests in this +//! file's test module pin that JSON Schema in literally and check both the +//! doc's own examples and our constructed annotations against it. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use url::Url; +use uuid::Uuid; + +pub const UPDATE_REQUEST_ANNOTATION: &str = "acl.azure.com/update-request"; +pub const UPDATE_STATUS_ANNOTATION: &str = "acl.azure.com/update-status"; +pub const UPDATE_COMMIT_STATUS_ANNOTATION: &str = "acl.azure.com/update-commit-status"; +pub const SCHEMA_VERSION: &str = "1.0"; +const MAX_MESSAGE_BYTES: usize = 2048; +const TRUNCATION_MARKER: &str = "... (truncated)"; +// TODO(DR-001): current_active_version() now reads /etc/aks-os-version, but +// falls back to this stub if that file isn't present yet (e.g. an image that +// hasn't picked up the file, or a dev/test host). Once the file ships +// unconditionally on every ACL image, this fallback (and this comment) can be +// removed. The stub value below is an explicit sentinel that cannot collide +// with a real AKS/Trident release version string (those look like +// "YYYYMM.N.N"), so it can never accidentally match a real requested target +// version and cause handle_stage/handle_finalize to incorrectly short-circuit +// to AlreadyAtTarget. Do not remove this comment when bumping the stub value; +// keep it (and its non-colliding shape) until the fallback is removed. +pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "camelCase")] +pub enum RequestedOperation { + Stage, + Finalize, + Rollback, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "camelCase")] +pub enum Operation { + Stage, + Finalize, + Rollback, + Commit, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] +pub enum StatusCode { + InProgress, + Success, + AlreadyAtTarget, + NotStaged, + OperationFailed, + TargetBootFailed, + AgentInternalError, + InvalidRequest, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UpdateRequest { + pub schema_version: String, + pub node_update_id: Uuid, + pub operation_id: String, + pub operation: RequestedOperation, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_version: Option, + /// Optional override of the agent's configured Nebraska endpoint + /// (`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` / CLI override) for this + /// update. When present, it takes precedence for every Nebraska call + /// this `nodeUpdateId` makes (`stage`'s update check, and all + /// progress/completion event reports), since Nebraska's per-instance + /// state is tied to one specific server. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub server: Option, + /// Optional override of the agent's configured Nebraska `app_id` + /// (`TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`) for this update. Resolved the + /// same way as [`server`](UpdateRequest::server): takes precedence over + /// the static config for every Nebraska call this `nodeUpdateId` makes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_id: Option, + /// Optional override of the agent's configured Nebraska `track` + /// (`TRIDENT_ACL_AGENT_NEBRASKA_TRACK`) for this update. Resolved and + /// applied the same way as [`server`](UpdateRequest::server) and + /// [`app_id`](UpdateRequest::app_id): takes precedence over the static + /// config for every Nebraska call this `nodeUpdateId` makes. `track` is + /// never optional on the wire itself (Nebraska requires it on every + /// request), only this override is - when absent, the static + /// `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` value is used, exactly as before. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub track: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UpdateStatus { + pub schema_version: String, + pub node_update_id: Uuid, + pub operation_id: String, + pub operation: Operation, + pub code: StatusCode, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub to_version: Option, + pub started_utc: DateTime, + pub last_updated_utc: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_utc: Option>, +} + +impl UpdateRequest { + /// Enforces the same constraints as the request annotation's formal + /// JSON Schema in `accepted-design-v2.md`: schemaVersion match, and + /// targetVersion required for stage/finalize but disallowed for + /// rollback. See this file's module doc. + pub fn validate(self) -> Result { + if self.schema_version != SCHEMA_VERSION { + return Err(format!("unsupported schemaVersion {}", self.schema_version)); + } + match self.operation { + RequestedOperation::Stage | RequestedOperation::Finalize => { + if self.target_version.as_deref().unwrap_or("").is_empty() { + return Err("targetVersion is required for stage/finalize".to_string()); + } + } + RequestedOperation::Rollback => { + if self.target_version.is_some() { + return Err("targetVersion must be omitted for rollback".to_string()); + } + } + } + Ok(self) + } +} + +impl UpdateStatus { + // This constructor mirrors UpdateStatus's wire schema field-for-field + // (see accepted-design-v2.md's two-status-key JSON protocol); splitting + // it into a builder would add ceremony across ~25 call sites in + // orchestrator.rs without making any of them clearer. + #[allow(clippy::too_many_arguments)] + pub fn new( + request: &UpdateRequest, + operation: Operation, + operation_id: String, + code: StatusCode, + message: impl Into, + from_version: Option, + to_version: Option, + started_utc: DateTime, + finished_utc: Option>, + ) -> Self { + let finished_or_started = finished_utc.unwrap_or(started_utc); + Self { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: request.node_update_id, + operation_id, + operation, + code, + message: truncate_message(message.into()), + from_version, + to_version, + started_utc, + last_updated_utc: finished_or_started, + finished_utc, + } + } + + pub fn refreshed_for_write(&self) -> Self { + let mut refreshed = self.clone(); + refreshed.last_updated_utc = Utc::now(); + refreshed.message = truncate_message(refreshed.message); + refreshed + } + + /// Compares two statuses ignoring `last_updated_utc`. + /// + /// `publish_status` stamps a fresh `last_updated_utc` on every write via + /// `refreshed_for_write`, so a straight `PartialEq` between an + /// already-on-the-node status and a cached/completed one to decide + /// whether a re-publish is needed would never be equal after the first + /// publish - triggering another watch event, another "different" + /// comparison, and another publish, forever. Callers that only care + /// whether the *content* already matches (and so a re-publish would be a + /// no-op) must use this instead of `==`/`!=`. + pub fn same_content(&self, other: &Self) -> bool { + self.schema_version == other.schema_version + && self.node_update_id == other.node_update_id + && self.operation_id == other.operation_id + && self.operation == other.operation + && self.code == other.code + && self.message == other.message + && self.from_version == other.from_version + && self.to_version == other.to_version + && self.started_utc == other.started_utc + && self.finished_utc == other.finished_utc + } +} + +fn truncate_message(message: String) -> String { + if message.len() <= MAX_MESSAGE_BYTES { + return message; + } + + let budget = MAX_MESSAGE_BYTES.saturating_sub(TRUNCATION_MARKER.len()); + let mut end = 0; + for (idx, ch) in message.char_indices() { + let next = idx + ch.len_utf8(); + if next > budget { + break; + } + end = next; + } + + let mut truncated = message[..end].to_string(); + truncated.push_str(TRUNCATION_MARKER); + truncated +} + +/// Path to the file the ACL image ships carrying the running OS version. +/// See `CURRENT_VERSION_STUB`'s doc comment above for the stub fallback this +/// probe still uses when the file isn't there yet. +const AKS_OS_VERSION_PATH: &str = "/etc/aks-os-version"; + +pub fn current_active_version() -> String { + read_active_version(AKS_OS_VERSION_PATH).unwrap_or_else(|| { + log::warn!( + "{AKS_OS_VERSION_PATH} not found; falling back to stub current version \ + {CURRENT_VERSION_STUB}" + ); + CURRENT_VERSION_STUB.to_string() + }) +} + +/// Reads and trims the active-version file at `path`. Returns `None` (rather +/// than propagating an error) for any read failure - missing file, permission +/// error, or empty contents - all of which `current_active_version` treats +/// identically: fall back to the stub. Split out from +/// `current_active_version` so tests can point it at a temp file instead of +/// the real `/etc/aks-os-version`. +fn read_active_version(path: &str) -> Option { + let contents = std::fs::read_to_string(path).ok()?; + let trimmed = contents.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } +} + +impl From for Operation { + fn from(value: RequestedOperation) -> Self { + match value { + RequestedOperation::Stage => Operation::Stage, + RequestedOperation::Finalize => Operation::Finalize, + RequestedOperation::Rollback => Operation::Rollback, + } + } +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + use serde_json::Value; + use uuid::Uuid; + + use super::*; + + fn sample_request(operation: RequestedOperation) -> UpdateRequest { + UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), + operation_id: "op-1".to_string(), + operation, + target_version: Some("2.0.0".to_string()), + server: None, + app_id: None, + track: None, + } + } + + fn fixed_time(secs: i64) -> DateTime { + Utc.timestamp_opt(1_700_000_000 + secs, 0).unwrap() + } + + #[test] + fn same_content_ignores_last_updated_utc() { + // Regression test: publish_status() -> refreshed_for_write() stamps a + // fresh last_updated_utc on every write. If the "already completed" + // dedupe check in orchestrator.rs compared statuses with `==`/`!=` + // instead of `same_content`, a cached status would never equal the + // freshly-published one (their last_updated_utc always differs), + // causing an infinite republish loop on every watch event. + let request = sample_request(RequestedOperation::Finalize); + let original = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::Success, + "finalize completed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + let republished = original.refreshed_for_write(); + + assert_ne!( + original.last_updated_utc, republished.last_updated_utc, + "refreshed_for_write should always stamp a new timestamp" + ); + assert_ne!( + original, republished, + "PartialEq must still distinguish them (guards against same_content silently replacing derived Eq)" + ); + assert!( + original.same_content(&republished), + "same_content must ignore last_updated_utc" + ); + } + + #[test] + fn same_content_detects_real_differences() { + let request = sample_request(RequestedOperation::Finalize); + let success = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::Success, + "finalize completed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + let failed = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::OperationFailed, + "finalize failed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + assert!(!success.same_content(&failed)); + } + + #[test] + fn truncates_messages_longer_than_2048_bytes() { + let request = sample_request(RequestedOperation::Stage); + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::OperationFailed, + "x".repeat(3000), + None, + None, + fixed_time(0), + Some(fixed_time(1)), + ); + + assert!(status.message.len() <= MAX_MESSAGE_BYTES); + assert!(status.message.ends_with(TRUNCATION_MARKER)); + } + + /// Round-trips `status` through JSON and returns the parsed `Value`, also + /// asserting the annotation is valid JSON and that deserializing it back + /// produces an identical `UpdateStatus` (guards against any field being + /// silently dropped or renamed by a future schema change). + fn to_annotation_json(status: &UpdateStatus) -> Value { + let text = serde_json::to_string(status).expect("UpdateStatus must serialize to JSON"); + let value: Value = serde_json::from_str(&text).expect("annotation must be valid JSON"); + let round_tripped: UpdateStatus = + serde_json::from_str(&text).expect("annotation must deserialize back to UpdateStatus"); + assert_eq!(&round_tripped, status); + value + } + + #[test] + fn stage_success_annotation_has_expected_shape() { + let request = sample_request(RequestedOperation::Stage); + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::Success, + "stage completed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["schemaVersion"], "1.0"); + assert_eq!(json["operationId"], "op-1"); + assert_eq!(json["operation"], "stage"); + assert_eq!(json["code"], "Success"); + assert_eq!(json["message"], "stage completed"); + assert_eq!(json["fromVersion"], "1.0.0"); + assert_eq!(json["toVersion"], "2.0.0"); + assert!(json.get("startedUtc").is_some()); + assert!(json.get("lastUpdatedUtc").is_some()); + assert!(json.get("finishedUtc").is_some()); + // Confirms camelCase renaming applies to every field, not just a subset. + assert!(json.get("nodeUpdateId").is_some()); + } + + #[test] + fn stage_failure_annotation_has_operation_failed_code() { + let request = sample_request(RequestedOperation::Stage); + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::OperationFailed, + "stage failed: disk full", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["code"], "OperationFailed"); + assert_eq!(json["message"], "stage failed: disk full"); + } + + #[test] + fn finalize_success_annotation_records_operation_and_no_finish_before_reboot() { + let request = sample_request(RequestedOperation::Finalize); + // In-progress finalize status published before reboot has no + // finishedUtc yet - confirm the annotation omits the field entirely + // (skip_serializing_if) rather than emitting `null`. + let in_progress = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::InProgress, + "finalizing update", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + None, + ); + + let json = to_annotation_json(&in_progress); + assert_eq!(json["operation"], "finalize"); + assert_eq!(json["code"], "InProgress"); + assert!( + json.get("finishedUtc").is_none(), + "finishedUtc should be omitted, not null, while in progress" + ); + } + + #[test] + fn finalize_failure_reverted_annotation_has_target_boot_failed_code() { + let request = sample_request(RequestedOperation::Finalize); + let status = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::TargetBootFailed, + "finalize failed: trident reported ab-update-reboot-check failure", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["code"], "TargetBootFailed"); + assert_eq!(json["operationId"], "op-1"); + } + + #[test] + fn commit_success_annotation_reuses_operation_id() { + let request = sample_request(RequestedOperation::Finalize); + let commit_id = request.operation_id.clone(); + let status = UpdateStatus::new( + &request, + Operation::Commit, + commit_id.clone(), + StatusCode::Success, + "commit completed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["operationId"], "op-1"); + assert_eq!(json["operation"], "commit"); + assert_eq!(json["code"], "Success"); + } + + #[test] + fn commit_reboot_required_annotation_uses_agent_internal_error_code() { + let request = sample_request(RequestedOperation::Finalize); + let commit_id = request.operation_id.clone(); + let status = UpdateStatus::new( + &request, + Operation::Commit, + commit_id, + StatusCode::AgentInternalError, + "commit requested another reboot", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["code"], "AgentInternalError"); + assert!(json["message"].as_str().unwrap().contains("another reboot")); + } + + #[test] + fn commit_failure_reverted_annotations_cover_both_reverted_subkinds() { + let request = sample_request(RequestedOperation::Finalize); + let commit_id = request.operation_id.clone(); + + for message in [ + "commit failed: trident reported ab-update-reboot-check failure", + "commit failed: trident reported ab-update-health-check-commit-check failure", + ] { + let status = UpdateStatus::new( + &request, + Operation::Commit, + commit_id.clone(), + StatusCode::TargetBootFailed, + message, + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["code"], "TargetBootFailed"); + assert_eq!(json["operationId"], "op-1"); + } + } + + #[test] + fn commit_failure_generic_annotation_has_operation_failed_code() { + let request = sample_request(RequestedOperation::Finalize); + let commit_id = request.operation_id.clone(); + let status = UpdateStatus::new( + &request, + Operation::Commit, + commit_id, + StatusCode::OperationFailed, + "commit failed: commit rpc failed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + + let json = to_annotation_json(&status); + assert_eq!(json["code"], "OperationFailed"); + assert_eq!(json["operationId"], "op-1"); + } + + #[test] + fn optional_version_fields_are_omitted_not_null_when_absent() { + let request = sample_request(RequestedOperation::Rollback); + let status = UpdateStatus::new( + &request, + Operation::Rollback, + request.operation_id.clone(), + StatusCode::InProgress, + "staging rollback", + Some("1.0.0".to_string()), + None, + fixed_time(0), + None, + ); + + let json = to_annotation_json(&status); + assert!(json.get("toVersion").is_none()); + assert!(json.get("lastUpdatedUtc").is_some()); + assert!(json.get("finishedUtc").is_none()); + assert_eq!(json["fromVersion"], "1.0.0"); + } + + // --- docs/update-trigger-design.md conformance -------------------------- + // + // Pins our annotation (de)serialization/validation code against two + // things lifted verbatim from docs/update-trigger-design.md + // (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md), + // section 2.1 "Trigger mechanism", so a doc/code drift shows up as a + // test failure instead of being discovered against a real AKS-RP: + // 1. The three example JSON payloads (request, finalize status, and + // the derived commit status) parse with our real UpdateRequest / + // UpdateStatus (de)serialization and UpdateRequest::validate(). + // 2. Annotations our own code constructs conform to the two formal + // JSON Schema documents embedded in the same section. + // + // Keep these constants byte-for-byte in sync with the design doc. + + /// docs/update-trigger-design.md 2.1, "Request annotation" example. + const DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE: &str = r#"{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "operation": "finalize", + "targetVersion": "202606.29.0" +}"#; + + /// docs/update-trigger-design.md 2.1, "Status annotation" example. + const DESIGN_DOC_FINALIZE_STATUS_EXAMPLE: &str = r#"{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "operation": "finalize", + "code": "Success", + "message": "boot armed, rebooting, awaiting commit", + "fromVersion": "202605.15.0", + "toVersion": "202606.29.0", + "startedUtc": "2026-06-04T12:00:00Z", + "lastUpdatedUtc": "2026-06-04T12:00:32Z", + "finishedUtc": "2026-06-04T12:00:32Z" +}"#; + + /// docs/update-trigger-design.md 2.1, the derived post-reboot commit status example. + const DESIGN_DOC_COMMIT_STATUS_EXAMPLE: &str = r#"{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "operation": "commit", + "code": "Success", + "message": "booted expected volume, boot order promoted", + "fromVersion": "202605.15.0", + "toVersion": "202606.29.0", + "startedUtc": "2026-06-04T12:01:18Z", + "lastUpdatedUtc": "2026-06-04T12:01:32Z", + "finishedUtc": "2026-06-04T12:01:32Z" +}"#; + + /// The formal JSON Schema for the request annotation, from + /// docs/update-trigger-design.md (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md), + /// section 2.1 "Formal JSON Schema". Keep byte-for-byte in sync with + /// that document. + const DESIGN_DOC_REQUEST_SCHEMA: &str = r#"{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://acl.azure.com/schemas/update-request/1.0.json", + "title": "ACL A/B update request annotation", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "nodeUpdateId", "operationId", "operation"], + "properties": { + "schemaVersion": { "type": "string", "const": "1.0" }, + "nodeUpdateId": { "type": "string", "format": "uuid", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" }, + "operationId": { "type": "string", "format": "uuid", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" }, + "operation": { "type": "string", "enum": ["stage", "finalize", "rollback"] }, + "targetVersion": { "type": "string", "description": "ACL image release version, e.g. 202606.29.0." }, + "server": { "type": "string", "format": "uri", "description": "Optional override of the agent's configured Nebraska endpoint for this update." }, + "appId": { "type": "string", "description": "Optional override of the agent's configured Nebraska app_id for this update." }, + "track": { "type": "string", "description": "Optional override of the agent's configured Nebraska track for this update." } + }, + "allOf": [ + { + "if": { "properties": { "operation": { "enum": ["stage", "finalize"] } }, "required": ["operation"] }, + "then": { "required": ["targetVersion"] } + } + ] +}"#; + + /// The formal JSON Schema for the status annotations, from + /// accepted-design-v2.md section 2.1 "Formal JSON Schema". Keep + /// byte-for-byte in sync with that document. + const DESIGN_DOC_STATUS_SCHEMA: &str = r#"{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://acl.azure.com/schemas/update-status/1.0.json", + "title": "ACL A/B update status annotation", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "nodeUpdateId", "operationId", "operation", "code"], + "properties": { + "schemaVersion": { "type": "string", "const": "1.0" }, + "nodeUpdateId": { "type": "string", "format": "uuid", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" }, + "operationId": { "type": "string", "format": "uuid", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", "description": "The operationId of the request this status reports on. The post-reboot commit status repeats the operationId of the finalize or rollback that caused the reboot." }, + "operation": { "type": "string", "enum": ["stage", "finalize", "rollback", "commit"] }, + "code": { "type": "string", "enum": ["InProgress", "Success", "AlreadyAtTarget", "NotStaged", "OperationFailed", "TargetBootFailed", "AgentInternalError", "InvalidRequest"] }, + "message": { "type": "string", "maxLength": 2048 }, + "fromVersion": { "type": "string" }, + "toVersion": { "type": "string" }, + "startedUtc": { "type": "string", "format": "date-time" }, + "lastUpdatedUtc": { "type": "string", "format": "date-time", "description": "When the agent last wrote this status. The agent refreshes it on every write, including a periodic InProgress heartbeat, so AKS-RP and the watchdog can tell a working agent from a stuck one." }, + "finishedUtc": { "type": "string", "format": "date-time" } + }, + "allOf": [ + { + "if": { "properties": { "code": { "const": "InProgress" } }, "required": ["code"] }, + "then": { "required": ["startedUtc", "lastUpdatedUtc"] }, + "else": { "required": ["startedUtc", "finishedUtc"] } + } + ] +}"#; + + // --- minimal JSON Schema subset validator ------------------------------ + // + // Deliberately not a general-purpose JSON Schema engine: supports only + // the exact vocabulary the two schemas above actually use (type, + // additionalProperties, required, properties.{type,const,enum,format, + // pattern}, and a single-level allOf/if/then/else). Panics loudly on any + // schema keyword/pattern/type/format it doesn't recognize, so if + // accepted-design-v2.md's schemas grow new constraints, this validator's + // blind spots don't silently mask them - the test fails instead, + // prompting an update here. + + fn schema_validate(schema: &Value, instance: &Value) -> Result<(), String> { + let schema_obj = schema.as_object().ok_or("schema is not a JSON object")?; + let obj = instance + .as_object() + .ok_or("instance is not a JSON object")?; + + let properties = schema_obj.get("properties").and_then(Value::as_object); + + if schema_obj + .get("additionalProperties") + .and_then(Value::as_bool) + == Some(false) + { + if let Some(props) = properties { + for key in obj.keys() { + if !props.contains_key(key) { + return Err(format!( + "property {key:?} not declared in schema (additionalProperties: false)" + )); + } + } + } + } + + if let Some(required) = schema_obj.get("required").and_then(Value::as_array) { + for req in required { + let name = req.as_str().ok_or("required entry is not a string")?; + if !obj.contains_key(name) { + return Err(format!("missing required property {name:?}")); + } + } + } + + if let Some(props) = properties { + for (name, prop_schema) in props { + if let Some(value) = obj.get(name) { + schema_validate_property(name, prop_schema, value)?; + } + } + } + + if let Some(all_of) = schema_obj.get("allOf").and_then(Value::as_array) { + for clause in all_of { + let clause_obj = clause.as_object().ok_or("allOf entry is not an object")?; + let condition_met = match clause_obj.get("if") { + Some(if_schema) => schema_if_matches(if_schema, obj), + None => true, + }; + let branch = if condition_met { + clause_obj.get("then") + } else { + clause_obj.get("else") + }; + if let Some(branch) = branch { + schema_validate(branch, instance)?; + } + } + } + + Ok(()) + } + + fn schema_if_matches(if_schema: &Value, obj: &serde_json::Map) -> bool { + let Some(if_obj) = if_schema.as_object() else { + return false; + }; + if let Some(required) = if_obj.get("required").and_then(Value::as_array) { + for req in required { + let Some(name) = req.as_str() else { + return false; + }; + if !obj.contains_key(name) { + return false; + } + } + } + if let Some(props) = if_obj.get("properties").and_then(Value::as_object) { + for (name, prop_schema) in props { + let Some(value) = obj.get(name) else { + return false; + }; + if let Some(enum_values) = prop_schema.get("enum").and_then(Value::as_array) { + if !enum_values.iter().any(|v| v == value) { + return false; + } + } + if let Some(const_value) = prop_schema.get("const") { + if value != const_value { + return false; + } + } + } + } + true + } + + fn schema_validate_property( + name: &str, + prop_schema: &Value, + value: &Value, + ) -> Result<(), String> { + if let Some(expected_type) = prop_schema.get("type").and_then(Value::as_str) { + let matches = match expected_type { + "string" => value.is_string(), + "object" => value.is_object(), + "array" => value.is_array(), + "boolean" => value.is_boolean(), + "number" | "integer" => value.is_number(), + other => panic!( + "test schema validator does not support type {other:?} - extend schema_validate_property" + ), + }; + if !matches { + return Err(format!( + "property {name:?}: expected type {expected_type}, got {value:?}" + )); + } + } + if let Some(const_value) = prop_schema.get("const") { + if value != const_value { + return Err(format!( + "property {name:?}: expected const {const_value:?}, got {value:?}" + )); + } + } + if let Some(enum_values) = prop_schema.get("enum").and_then(Value::as_array) { + if !enum_values.iter().any(|v| v == value) { + return Err(format!( + "property {name:?}: value {value:?} not in enum {enum_values:?}" + )); + } + } + if let Some(format) = prop_schema.get("format").and_then(Value::as_str) { + let s = value.as_str().ok_or_else(|| { + format!("property {name:?}: expected string for format {format:?}") + })?; + match format { + "uuid" => { + Uuid::parse_str(s).map_err(|err| { + format!("property {name:?}: {s:?} is not a valid uuid: {err}") + })?; + } + "date-time" => { + DateTime::parse_from_rfc3339(s).map_err(|err| { + format!("property {name:?}: {s:?} is not a valid date-time: {err}") + })?; + } + "uri" => { + Url::parse(s).map_err(|err| { + format!("property {name:?}: {s:?} is not a valid uri: {err}") + })?; + } + other => panic!( + "test schema validator does not support format {other:?} - extend schema_validate_property" + ), + } + } + if let Some(pattern) = prop_schema.get("pattern").and_then(Value::as_str) { + let s = value + .as_str() + .ok_or_else(|| format!("property {name:?}: expected string to check pattern"))?; + if !schema_pattern_matches(pattern, s) { + return Err(format!( + "property {name:?}: {s:?} does not match pattern {pattern:?}" + )); + } + } + Ok(()) + } + + /// Bespoke stand-in for full regex support: the two schemas above use + /// exactly two distinct patterns, both UUID-shaped, so this matches them + /// by exact pattern text rather than pulling in a regex engine for two + /// known cases. Panics on an unrecognized pattern so a future schema + /// change can't silently pass unchecked. + fn schema_pattern_matches(pattern: &str, value: &str) -> bool { + const BARE_UUID: &str = + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"; + match pattern { + BARE_UUID => Uuid::parse_str(value).is_ok(), + other => panic!( + "test schema validator does not recognize pattern {other:?} - extend schema_pattern_matches" + ), + } + } + + // --- example payload parsing tests ------------------------------------- + + #[test] + fn design_doc_finalize_request_example_parses_and_validates() { + let request: UpdateRequest = serde_json::from_str(DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE) + .expect("design doc's finalize request example must parse as UpdateRequest"); + let request = request + .validate() + .expect("design doc's finalize request example must pass UpdateRequest::validate()"); + assert_eq!(request.schema_version, "1.0"); + assert_eq!( + request.node_update_id, + Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap() + ); + assert_eq!(request.operation_id, "f47ac10b-58cc-4372-a567-0e02b2c3d479"); + assert_eq!(request.operation, RequestedOperation::Finalize); + assert_eq!(request.target_version.as_deref(), Some("202606.29.0")); + } + + #[test] + fn design_doc_finalize_status_example_parses() { + let status: UpdateStatus = serde_json::from_str(DESIGN_DOC_FINALIZE_STATUS_EXAMPLE) + .expect("design doc's finalize status example must parse as UpdateStatus"); + assert_eq!(status.operation, Operation::Finalize); + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.from_version.as_deref(), Some("202605.15.0")); + assert_eq!(status.to_version.as_deref(), Some("202606.29.0")); + assert!(status.finished_utc.is_some()); + } + + #[test] + fn design_doc_commit_status_example_parses() { + let status: UpdateStatus = serde_json::from_str(DESIGN_DOC_COMMIT_STATUS_EXAMPLE) + .expect("design doc's commit status example must parse as UpdateStatus"); + assert_eq!(status.operation, Operation::Commit); + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.operation_id, "f47ac10b-58cc-4372-a567-0e02b2c3d479"); + } + + // --- example payloads validated against the embedded formal schema ---- + + #[test] + fn design_doc_finalize_request_example_matches_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); + let instance: Value = serde_json::from_str(DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE).unwrap(); + schema_validate(&schema, &instance) + .expect("design doc's own finalize request example must satisfy its own schema"); + } + + #[test] + fn design_doc_finalize_status_example_matches_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_STATUS_SCHEMA).unwrap(); + let instance: Value = serde_json::from_str(DESIGN_DOC_FINALIZE_STATUS_EXAMPLE).unwrap(); + schema_validate(&schema, &instance) + .expect("design doc's own finalize status example must satisfy its own schema"); + } + + #[test] + fn design_doc_commit_status_example_matches_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_STATUS_SCHEMA).unwrap(); + let instance: Value = serde_json::from_str(DESIGN_DOC_COMMIT_STATUS_EXAMPLE).unwrap(); + schema_validate(&schema, &instance) + .expect("design doc's own commit status example must satisfy its own schema"); + } + + // --- annotations *we construct* validated against the embedded schema - + + #[test] + fn agent_built_requests_conform_to_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); + let node_update_id = Uuid::new_v4(); + + for (operation, target_version) in [ + (RequestedOperation::Stage, Some("202606.29.0".to_string())), + ( + RequestedOperation::Finalize, + Some("202606.29.0".to_string()), + ), + (RequestedOperation::Rollback, None), + ] { + let request = UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id, + operation_id: Uuid::new_v4().to_string(), + operation, + target_version, + server: None, + app_id: None, + track: None, + }; + let request = request + .validate() + .unwrap_or_else(|err| panic!("{operation:?} request must validate: {err}")); + let instance: Value = serde_json::to_value(&request).unwrap(); + schema_validate(&schema, &instance).unwrap_or_else(|err| { + panic!( + "agent-constructed {operation:?} request must conform to the formal schema: {err}" + ) + }); + } + } + + #[test] + fn agent_built_statuses_conform_to_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_STATUS_SCHEMA).unwrap(); + let request = UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: Uuid::new_v4(), + operation_id: Uuid::new_v4().to_string(), + operation: RequestedOperation::Finalize, + target_version: Some("202606.29.0".to_string()), + server: None, + app_id: None, + track: None, + }; + + // InProgress: startedUtc only, no finishedUtc yet. + let in_progress = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::InProgress, + "finalizing update", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + None, + ); + schema_validate(&schema, &serde_json::to_value(&in_progress).unwrap()) + .expect("agent-constructed InProgress status must conform to the formal schema"); + + // Terminal Success: both startedUtc and finishedUtc present. + let success = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::Success, + "boot armed, rebooting, awaiting commit", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(0), + Some(fixed_time(5)), + ); + schema_validate(&schema, &serde_json::to_value(&success).unwrap()) + .expect("agent-constructed terminal Success status must conform to the formal schema"); + + // Rollback status: no toVersion - still conforms (toVersion is optional). + let rollback_request = UpdateRequest { + operation: RequestedOperation::Rollback, + target_version: None, + ..request.clone() + }; + let rollback_status = UpdateStatus::new( + &rollback_request, + Operation::Rollback, + rollback_request.operation_id.clone(), + StatusCode::Success, + "rollback finalize completed; rebooting for commit", + Some("2.0.0".to_string()), + None, + fixed_time(0), + Some(fixed_time(5)), + ); + schema_validate(&schema, &serde_json::to_value(&rollback_status).unwrap()) + .expect("agent-constructed rollback status must conform to the formal schema"); + + // Derived post-reboot commit status: operationId stays unchanged + // while the separate commit annotation key distinguishes the half. + let commit_status = UpdateStatus::new( + &request, + Operation::Commit, + request.operation_id.clone(), + StatusCode::Success, + "booted expected volume, boot order promoted", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + fixed_time(10), + Some(fixed_time(15)), + ); + schema_validate(&schema, &serde_json::to_value(&commit_status).unwrap()) + .expect("agent-constructed commit status must conform to the formal schema"); + } + + // --- server / appId / track (Nebraska overrides) - + + #[test] + fn server_field_round_trips_and_conforms_to_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); + let mut request = sample_request(RequestedOperation::Stage); + request.operation_id = Uuid::new_v4().to_string(); + request.server = Some(Url::parse("https://nebraska.example/v1/update").unwrap()); + + let json = serde_json::to_value(&request).unwrap(); + assert_eq!(json["server"], "https://nebraska.example/v1/update"); + schema_validate(&schema, &json) + .expect("request with a server override must conform to the formal schema"); + + let round_tripped: UpdateRequest = serde_json::from_value(json).unwrap(); + assert_eq!(round_tripped.server, request.server); + } + + #[test] + fn server_field_absent_when_not_set() { + let request = sample_request(RequestedOperation::Stage); + let json = serde_json::to_value(&request).unwrap(); + assert!(json.get("server").is_none()); + } + + #[test] + fn app_id_field_round_trips_and_conforms_to_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); + let mut request = sample_request(RequestedOperation::Stage); + request.operation_id = Uuid::new_v4().to_string(); + request.app_id = Some("59bbad61-257d-47f4-9730-6848d88e1a6e".to_string()); + + let json = serde_json::to_value(&request).unwrap(); + assert_eq!(json["appId"], "59bbad61-257d-47f4-9730-6848d88e1a6e"); + schema_validate(&schema, &json) + .expect("request with an appId override must conform to the formal schema"); + + let round_tripped: UpdateRequest = serde_json::from_value(json).unwrap(); + assert_eq!(round_tripped.app_id, request.app_id); + } + + #[test] + fn app_id_field_absent_when_not_set() { + let request = sample_request(RequestedOperation::Stage); + let json = serde_json::to_value(&request).unwrap(); + assert!(json.get("appId").is_none()); + } + + #[test] + fn track_field_round_trips_and_conforms_to_formal_schema() { + let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); + let mut request = sample_request(RequestedOperation::Stage); + request.operation_id = Uuid::new_v4().to_string(); + request.track = Some("pin-202608.6.0".to_string()); + + let json = serde_json::to_value(&request).unwrap(); + assert_eq!(json["track"], "pin-202608.6.0"); + schema_validate(&schema, &json) + .expect("request with a track override must conform to the formal schema"); + + let round_tripped: UpdateRequest = serde_json::from_value(json).unwrap(); + assert_eq!(round_tripped.track, request.track); + } + + #[test] + fn track_field_absent_when_not_set() { + let request = sample_request(RequestedOperation::Stage); + let json = serde_json::to_value(&request).unwrap(); + assert!(json.get("track").is_none()); + } + + #[test] + fn read_active_version_returns_none_for_missing_file() { + assert_eq!( + read_active_version("/nonexistent/path/does-not-exist-aks-os-version"), + None + ); + } + + #[test] + fn read_active_version_trims_and_reads_real_file() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("aks-os-version-test-{}", Uuid::new_v4())); + std::fs::write(&path, " 202608.6.0\n").unwrap(); + let result = read_active_version(path.to_str().unwrap()); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + #[test] + fn read_active_version_treats_empty_file_as_absent() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("aks-os-version-test-empty-{}", Uuid::new_v4())); + std::fs::write(&path, " \n").unwrap(); + let result = read_active_version(path.to_str().unwrap()); + std::fs::remove_file(&path).ok(); + assert_eq!(result, None); + } +} diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs new file mode 100644 index 0000000000..dafa7dccdb --- /dev/null +++ b/crates/trident-acl-agent/src/config.rs @@ -0,0 +1,424 @@ +//! Env-var-based config loading for Harpoon. +//! +//! There is no config file. Every setting is an environment variable +//! prefixed `TRIDENT_ACL_AGENT_` (one constant per setting, e.g. +//! [`ENV_NEBRASKA_ENDPOINT`]), systemd-style: set it directly in the unit's +//! own `Environment=` lines, via a drop-in override (`systemctl edit +//! trident-acl-agent.service`, which creates +//! `/etc/systemd/system/trident-acl-agent.service.d/override.conf`), or by +//! any other means that ultimately sets the process's environment before it +//! starts. All are equivalent from the agent's point of view - it just reads +//! `std::env::var`. +//! +//! Annotation mode is the default; `omaha-only` (the historical one-shot +//! behavior) remains available as an explicit opt-out via +//! `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE=omaha-only`. + +use std::{env, path::PathBuf, str::FromStr, time::Duration}; + +use url::Url; + +use crate::{DEFAULT_NEBRASKA_APP_ID, DEFAULT_NEBRASKA_TRACK}; + +/// The environment variables this module reads, one constant per setting. +const ENV_NEBRASKA_ENDPOINT: &str = "TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT"; +const ENV_NEBRASKA_APP_ID: &str = "TRIDENT_ACL_AGENT_NEBRASKA_APP_ID"; +const ENV_NEBRASKA_TRACK: &str = "TRIDENT_ACL_AGENT_NEBRASKA_TRACK"; +const ENV_KUBERNETES_API_SERVER: &str = "TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER"; +const ENV_KUBERNETES_KUBECONFIG: &str = "TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG"; +const ENV_KUBERNETES_NODE_NAME: &str = "TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME"; +const ENV_TRIDENT_SOCKET: &str = "TRIDENT_ACL_AGENT_TRIDENT_SOCKET"; +const ENV_ORCHESTRATION_GOAL_SOURCE: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE"; +const ENV_ORCHESTRATION_STATE_PATH: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH"; +const ENV_ORCHESTRATION_STAGE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT"; +const ENV_ORCHESTRATION_FINALIZE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT"; +const ENV_ORCHESTRATION_HEARTBEAT_INTERVAL: &str = + "TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL"; + +const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); +// TODO: placeholder until the real production Nebraska/Omaha endpoint is +// known. `.invalid` is reserved by RFC 2606 and is guaranteed to never +// resolve, so a deployment that forgets to set +// TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT (or override it per-request via the +// update-request annotation's `server` field) fails loudly at the network +// layer instead of silently querying a real-looking but wrong host. +pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update"; +const DEFAULT_STAGE_TIMEOUT: Duration = Duration::from_secs(20 * 60); +const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); +const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); +pub const DEFAULT_STATE_PATH: &str = "/var/lib/trident-acl-agent/state.json"; +pub const DEFAULT_KUBELET_KUBECONFIG: &str = "/var/lib/kubelet/kubeconfig"; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AgentConfig { + pub nebraska: NebraskaConfig, + pub kubernetes: KubernetesConfig, + pub trident: TridentConfig, + pub orchestration: OrchestrationConfig, +} + +impl AgentConfig { + /// Loads the effective config purely from `TRIDENT_ACL_AGENT_*` + /// environment variables (see the module doc). A merely-absent variable + /// is never an error - it just falls back to that setting's default - + /// but a present-and-malformed value (bad URL, bad duration, unknown + /// `goal_source`, etc.) is. + pub fn from_env() -> Result { + Ok(Self { + nebraska: NebraskaConfig { + endpoint: env_url(ENV_NEBRASKA_ENDPOINT)? + .or_else(|| Some(Url::parse(DEFAULT_NEBRASKA_ENDPOINT).expect("static url"))), + app_id: env_string(ENV_NEBRASKA_APP_ID) + .unwrap_or_else(|| DEFAULT_NEBRASKA_APP_ID.to_string()), + track: env_string(ENV_NEBRASKA_TRACK) + .unwrap_or_else(|| DEFAULT_NEBRASKA_TRACK.to_string()), + }, + kubernetes: KubernetesConfig { + api_server: env_url(ENV_KUBERNETES_API_SERVER)?, + kubeconfig: env_string(ENV_KUBERNETES_KUBECONFIG) + .unwrap_or_else(|| DEFAULT_KUBELET_KUBECONFIG.to_string()), + node_name: env_string(ENV_KUBERNETES_NODE_NAME).unwrap_or_else(default_node_name), + watch_poll_interval: DEFAULT_KUBERNETES_POLL_INTERVAL, + }, + trident: TridentConfig { + socket: env_string(ENV_TRIDENT_SOCKET) + .unwrap_or_else(|| trident_proto::TRIDENT_DEFAULT_SOCKET_URI.to_string()), + }, + orchestration: OrchestrationConfig { + goal_source: env_parse(ENV_ORCHESTRATION_GOAL_SOURCE)?.unwrap_or_default(), + state_path: env_string(ENV_ORCHESTRATION_STATE_PATH) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(DEFAULT_STATE_PATH)), + stage_timeout: env_duration( + ENV_ORCHESTRATION_STAGE_TIMEOUT, + DEFAULT_STAGE_TIMEOUT, + )?, + finalize_timeout: env_duration( + ENV_ORCHESTRATION_FINALIZE_TIMEOUT, + DEFAULT_FINALIZE_TIMEOUT, + )?, + heartbeat_interval: env_duration( + ENV_ORCHESTRATION_HEARTBEAT_INTERVAL, + DEFAULT_HEARTBEAT_INTERVAL, + )?, + }, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NebraskaConfig { + pub endpoint: Option, + pub app_id: String, + pub track: String, +} + +impl Default for NebraskaConfig { + fn default() -> Self { + Self { + endpoint: Some(Url::parse(DEFAULT_NEBRASKA_ENDPOINT).expect("static url")), + app_id: DEFAULT_NEBRASKA_APP_ID.to_string(), + track: DEFAULT_NEBRASKA_TRACK.to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KubernetesConfig { + /// Explicit override for the Kubernetes API server URL. When unset, the + /// server embedded in `kubeconfig` is used as-is (e.g. the real cluster + /// FQDN a node's own `/var/lib/kubelet/kubeconfig` already points at). + /// Only needed when the kubeconfig's own server is wrong for this + /// deployment - e.g. a pod deployment wanting the in-cluster + /// `https://kubernetes.default.svc` name, which a plain node-level + /// kubeconfig has no reason to contain. + pub api_server: Option, + pub kubeconfig: String, + pub node_name: String, + pub watch_poll_interval: Duration, +} + +impl Default for KubernetesConfig { + fn default() -> Self { + Self { + api_server: None, + kubeconfig: DEFAULT_KUBELET_KUBECONFIG.to_string(), + node_name: default_node_name(), + watch_poll_interval: DEFAULT_KUBERNETES_POLL_INTERVAL, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TridentConfig { + pub socket: String, +} + +impl Default for TridentConfig { + fn default() -> Self { + Self { + socket: trident_proto::TRIDENT_DEFAULT_SOCKET_URI.to_string(), + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum GoalSource { + /// Historical one-shot behavior: query Nebraska/Omaha once, and if an + /// update is offered, call tridentd's combined `update()` RPC once and + /// exit. No Kubernetes involvement at all - no annotations, no watch, + /// no Node access. Kept as an explicit opt-out for nodes that don't + /// participate in the AKS annotation-driven update protocol. + OmahaOnly, + /// The annotation-driven reconcile loop: watches the Node's + /// `acl.azure.com/update-request` annotation and drives Trident's + /// stage/finalize/rollback/commit operations against tridentd + /// accordingly, writing progress back to `acl.azure.com/update-status` + /// and `acl.azure.com/update-commit-status` (see accepted-design-v2.md). + /// This is the default mode. + #[default] + Annotations, +} + +impl FromStr for GoalSource { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "omaha-only" => Ok(GoalSource::OmahaOnly), + "annotations" => Ok(GoalSource::Annotations), + other => Err(anyhow::anyhow!( + "unknown goal_source {other:?} (expected \"annotations\" or \"omaha-only\")" + )), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OrchestrationConfig { + pub goal_source: GoalSource, + pub state_path: PathBuf, + /// Placeholder default pending real data from storm aclagent scenario runs. + pub stage_timeout: Duration, + /// Placeholder default pending real data from storm aclagent scenario runs. + pub finalize_timeout: Duration, + /// Refresh cadence for in-flight InProgress heartbeats. Default is well + /// below the ~10 minute watchdog staleness target proposed in + /// accepted-design-v2.md. + pub heartbeat_interval: Duration, +} + +impl Default for OrchestrationConfig { + fn default() -> Self { + Self { + goal_source: GoalSource::Annotations, + state_path: PathBuf::from(DEFAULT_STATE_PATH), + stage_timeout: DEFAULT_STAGE_TIMEOUT, + finalize_timeout: DEFAULT_FINALIZE_TIMEOUT, + heartbeat_interval: DEFAULT_HEARTBEAT_INTERVAL, + } + } +} + +/// Reads `name`, treating both "unset" and "set to the empty string" as +/// absent - a drop-in override that clears a variable to `""` should fall +/// back to the default, not try to parse an empty value. +fn env_raw(name: &str) -> Option { + env::var(name).ok().filter(|v| !v.is_empty()) +} + +fn env_string(name: &str) -> Option { + env_raw(name) +} + +fn env_url(name: &str) -> Result, anyhow::Error> { + env_raw(name) + .map(|v| Url::parse(&v).map_err(|err| anyhow::anyhow!("invalid URL for {name}: {err}"))) + .transpose() +} + +fn env_duration(name: &str, default: Duration) -> Result { + env_raw(name) + .map(|v| { + humantime::parse_duration(&v) + .map_err(|err| anyhow::anyhow!("invalid duration for {name}: {err}")) + }) + .transpose() + .map(|parsed| parsed.unwrap_or(default)) +} + +fn env_parse(name: &str) -> Result, anyhow::Error> +where + T: FromStr, +{ + env_raw(name).map(|v| v.parse::()).transpose() +} + +fn default_node_name() -> String { + // Kubernetes Node names must be valid RFC 1123 DNS labels, which are + // lowercase-only; kubelet itself lowercases the hostname when it + // registers the Node object. Match that behavior here so a mixed-case + // hostname doesn't produce a node_name that can never match the actual + // Node the agent is supposed to reconcile against. + osutils::hostname::read() + .unwrap_or_else(|_| "localhost".to_string()) + .to_lowercase() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Clears every var this module reads. Environment mutation is process- + /// global and `std::env::remove_var`/`set_var` are `unsafe` (not + /// thread-safe against concurrent reads elsewhere in the process), so + /// all of the defaults/overrides/empty-value/malformed-value cases below + /// are intentionally folded into one sequential `#[test]` rather than + /// several separate ones that `cargo test` could run in parallel against + /// the same variables. + fn clear_env() { + // SAFETY: single-threaded within this test function; no other test + // in this crate reads or writes these TRIDENT_ACL_AGENT_* variables. + unsafe { + env::remove_var(ENV_NEBRASKA_ENDPOINT); + env::remove_var(ENV_NEBRASKA_APP_ID); + env::remove_var(ENV_NEBRASKA_TRACK); + env::remove_var(ENV_KUBERNETES_API_SERVER); + env::remove_var(ENV_KUBERNETES_KUBECONFIG); + env::remove_var(ENV_KUBERNETES_NODE_NAME); + env::remove_var(ENV_TRIDENT_SOCKET); + env::remove_var(ENV_ORCHESTRATION_GOAL_SOURCE); + env::remove_var(ENV_ORCHESTRATION_STATE_PATH); + env::remove_var(ENV_ORCHESTRATION_STAGE_TIMEOUT); + env::remove_var(ENV_ORCHESTRATION_FINALIZE_TIMEOUT); + env::remove_var(ENV_ORCHESTRATION_HEARTBEAT_INTERVAL); + } + } + + #[test] + fn env_config_defaults_then_overrides() { + clear_env(); + + let config = AgentConfig::from_env().unwrap(); + assert_eq!( + config.nebraska.endpoint.unwrap().as_str(), + DEFAULT_NEBRASKA_ENDPOINT + ); + assert_eq!(config.nebraska.app_id, DEFAULT_NEBRASKA_APP_ID); + assert_eq!(config.nebraska.track, DEFAULT_NEBRASKA_TRACK); + assert_eq!( + config.kubernetes.api_server, None, + "api_server should default to unset so the kubeconfig's own server is used as-is" + ); + assert_eq!( + config.kubernetes.kubeconfig, + DEFAULT_KUBELET_KUBECONFIG.to_string() + ); + assert_eq!( + config.trident.socket, + trident_proto::TRIDENT_DEFAULT_SOCKET_URI + ); + assert_eq!(config.orchestration.goal_source, GoalSource::Annotations); + assert_eq!( + config.orchestration.state_path, + PathBuf::from(DEFAULT_STATE_PATH) + ); + assert_eq!(config.orchestration.stage_timeout, DEFAULT_STAGE_TIMEOUT); + assert_eq!( + config.orchestration.finalize_timeout, + DEFAULT_FINALIZE_TIMEOUT + ); + assert_eq!( + config.orchestration.heartbeat_interval, + DEFAULT_HEARTBEAT_INTERVAL + ); + + // SAFETY: see clear_env's doc comment. + unsafe { + env::set_var( + ENV_NEBRASKA_ENDPOINT, + "https://custom-nebraska.example.invalid/v1/update", + ); + env::set_var(ENV_NEBRASKA_APP_ID, "custom-app"); + env::set_var(ENV_NEBRASKA_TRACK, "custom-track"); + env::set_var(ENV_KUBERNETES_API_SERVER, "https://cluster.example.invalid"); + env::set_var(ENV_KUBERNETES_KUBECONFIG, "/etc/harpoon/kubeconfig"); + env::set_var(ENV_KUBERNETES_NODE_NAME, "node-42"); + env::set_var(ENV_TRIDENT_SOCKET, "unix:///custom/trident.sock"); + env::set_var(ENV_ORCHESTRATION_GOAL_SOURCE, "omaha-only"); + env::set_var( + ENV_ORCHESTRATION_STATE_PATH, + "/var/lib/trident-acl-agent/custom-state.json", + ); + env::set_var(ENV_ORCHESTRATION_STAGE_TIMEOUT, "21m"); + env::set_var(ENV_ORCHESTRATION_FINALIZE_TIMEOUT, "11m"); + env::set_var(ENV_ORCHESTRATION_HEARTBEAT_INTERVAL, "45s"); + } + + let config = AgentConfig::from_env().unwrap(); + clear_env(); + + assert_eq!( + config.nebraska.endpoint.unwrap().as_str(), + "https://custom-nebraska.example.invalid/v1/update" + ); + assert_eq!(config.nebraska.app_id, "custom-app"); + assert_eq!(config.nebraska.track, "custom-track"); + assert_eq!( + config.kubernetes.api_server.unwrap().as_str(), + "https://cluster.example.invalid/" + ); + assert_eq!( + config.kubernetes.kubeconfig.as_str(), + "/etc/harpoon/kubeconfig" + ); + assert_eq!(config.kubernetes.node_name, "node-42"); + assert_eq!(config.trident.socket, "unix:///custom/trident.sock"); + assert_eq!(config.orchestration.goal_source, GoalSource::OmahaOnly); + assert_eq!( + config.orchestration.state_path, + PathBuf::from("/var/lib/trident-acl-agent/custom-state.json") + ); + assert_eq!( + config.orchestration.stage_timeout, + Duration::from_secs(21 * 60) + ); + assert_eq!( + config.orchestration.finalize_timeout, + Duration::from_secs(11 * 60) + ); + assert_eq!( + config.orchestration.heartbeat_interval, + Duration::from_secs(45) + ); + + // --- empty value falls back to default, same as unset ------------- + clear_env(); + // SAFETY: see clear_env's doc comment. + unsafe { + env::set_var(ENV_NEBRASKA_APP_ID, ""); + } + let config = AgentConfig::from_env().unwrap(); + assert_eq!(config.nebraska.app_id, DEFAULT_NEBRASKA_APP_ID); + + // --- a present-but-malformed URL is a parse error ------------------ + clear_env(); + // SAFETY: see clear_env's doc comment. + unsafe { + env::set_var(ENV_NEBRASKA_ENDPOINT, "not a url"); + } + let err = AgentConfig::from_env().unwrap_err(); + assert!(err.to_string().contains(ENV_NEBRASKA_ENDPOINT), "{err}"); + + // --- a present-but-unknown goal_source is a parse error ------------ + clear_env(); + // SAFETY: see clear_env's doc comment. + unsafe { + env::set_var(ENV_ORCHESTRATION_GOAL_SOURCE, "bogus"); + } + let err = AgentConfig::from_env().unwrap_err(); + assert!(err.to_string().contains("bogus"), "{err}"); + + clear_env(); + } +} diff --git a/crates/trident-acl-agent/src/error.rs b/crates/trident-acl-agent/src/error.rs index 177e936222..fa87674879 100644 --- a/crates/trident-acl-agent/src/error.rs +++ b/crates/trident-acl-agent/src/error.rs @@ -1,48 +1,27 @@ -use serde::{Deserialize, Serialize}; - -use crate::omaha::event::{EventResult, OmahaEventType}; - -#[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "kebab-case")] -pub enum HarpoonError { - #[error("Failed to initialize the Harpoon client: {0}")] - InitializationError(String), - - #[error("The version provided '{version}' is not valid semver: {inner}")] - InvalidVersion { version: String, inner: String }, - - #[error("Failed to read machine-id: {0}")] - MachineIdRead(String), - - #[error("Failed to read hostname: {0}")] - HostnameRead(String), - - #[error("Internal error: {0}")] - Internal(String), - - #[error("Failed to send request: {0}")] - SendRequest(String), - - #[error("Received an HTTP error: {0}")] - HttpError(String), - - #[error("Failed to parse response: {0}")] - ParseResponse(String), - - #[error("Received an invalid response from the server: {0}")] - InvalidResponse(String), - - #[error("Failed to query for updates: {0}")] - QueryError(String), - - #[error("Failed to fetch the updated document: {0}")] - FetchError(String), - - #[error( - "Expected a yaml document, but the provided URL does not have a .yaml extension '{0}'" - )] - ExpectedYamlDocument(String), - - #[error("Event '{0:?}:{1:?}' was not acknowledged by server.")] - EventNotAcknowledged(OmahaEventType, EventResult), -} +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum HarpoonError { + #[error("Failed to initialize the Harpoon client: {0}")] + InitializationError(String), + + #[error("The version provided '{version}' is not valid semver: {inner}")] + InvalidVersion { version: String, inner: String }, + + #[error("Failed to read machine-id: {0}")] + MachineIdRead(String), + + #[error("Failed to read hostname: {0}")] + HostnameRead(String), + + #[error("Internal error: {0}")] + Internal(String), + + /// Wraps a [`nebraska::NebraskaError`](crate::nebraska::NebraskaError). + /// Stored as a string rather than `#[from]` because `NebraskaError` + /// doesn't derive `Serialize`/`Deserialize`/`PartialEq`, which + /// `HarpoonError` requires for annotation-status round-tripping. + #[error("Nebraska request failed: {0}")] + Nebraska(String), +} diff --git a/crates/trident-acl-agent/src/k8s.rs b/crates/trident-acl-agent/src/k8s.rs new file mode 100644 index 0000000000..6017f19f33 --- /dev/null +++ b/crates/trident-acl-agent/src/k8s.rs @@ -0,0 +1,173 @@ +//! Thin Kubernetes client wrapper for Harpoon's node self-patching protocol. +//! +//! Implements the Node get/watch/patch access described in the current +//! accepted design (`accepted-design-v2.md`). +//! +//! The design calls for get/patch access to exactly one Node object (§2.2–§2.6). +//! Node changes are observed via the Kubernetes watch API (`kube::runtime::watcher`) +//! rather than polling, so annotation updates are delivered promptly and without +//! placing repeated load on the API server. `watch_poll_interval` still bounds +//! how quickly the watcher notices a dropped/re-established connection (used +//! as the watcher's backoff ceiling) and how often the fake test API server +//! needs to support being polled if it does not support real watches. + +use std::{collections::BTreeMap, path::Path}; + +use anyhow::Context; +use futures::{stream::BoxStream, StreamExt, TryStreamExt}; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Patch, PatchParams}, + config::{KubeConfigOptions, Kubeconfig}, + error::ErrorResponse, + runtime::{watcher, WatchStreamExt}, + Api, Client, Config, +}; +use serde_json::json; + +use crate::config::KubernetesConfig; + +#[derive(Debug, thiserror::Error)] +pub enum K8sClientError { + #[error("failed to build Kubernetes client config: {0}")] + Config(#[from] anyhow::Error), + #[error("node object no longer exists")] + NodeGone, + #[error("failed Kubernetes API call: {0}")] + Api(#[source] kube::Error), + #[error("Kubernetes watch stream failed: {0}")] + Watch(#[from] kube::runtime::watcher::Error), +} + +#[derive(Clone)] +pub struct NodeClient { + api: Api, + poll_interval: std::time::Duration, + cluster_url: String, +} + +impl NodeClient { + pub async fn new(config: &KubernetesConfig) -> Result { + let client_config = load_client_config(config).await?; + let cluster_url = client_config.cluster_url.to_string(); + let client = Client::try_from(client_config).map_err(anyhow::Error::new)?; + Ok(Self { + api: Api::all(client), + poll_interval: config.watch_poll_interval, + cluster_url, + }) + } + + pub fn cluster_url(&self) -> &str { + &self.cluster_url + } + + pub async fn get_node(&self, name: &str) -> Result { + self.api.get(name).await.map_err(map_kube_error) + } + + pub async fn patch_node_labels( + &self, + name: &str, + labels: BTreeMap, + ) -> Result { + let patch = json!({ "metadata": { "labels": labels } }); + self.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map_err(map_kube_error) + } + + pub async fn patch_node_annotations( + &self, + name: &str, + annotations: BTreeMap, + ) -> Result { + let patch = json!({ "metadata": { "annotations": annotations } }); + self.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map_err(map_kube_error) + } + + pub async fn patch_node_metadata( + &self, + name: &str, + labels: BTreeMap>, + annotations: BTreeMap>, + ) -> Result { + let patch = json!({ + "metadata": { + "labels": labels, + "annotations": annotations, + } + }); + self.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map_err(map_kube_error) + } + + pub fn watch_node(&self, name: String) -> BoxStream<'static, Result> { + let watcher_config = watcher::Config::default() + .fields(&format!("metadata.name={name}")) + .timeout(self.poll_interval.as_secs().max(1) as u32); + + watcher(self.api.clone(), watcher_config) + .default_backoff() + .touched_objects() + .map_err(K8sClientError::from) + .boxed() + } +} + +fn map_kube_error(err: kube::Error) -> K8sClientError { + if matches!(&err, kube::Error::Api(ErrorResponse { code: 404, .. })) { + K8sClientError::NodeGone + } else { + K8sClientError::Api(err) + } +} + +async fn load_client_config(config: &KubernetesConfig) -> Result { + let path = Path::new(&config.kubeconfig); + let kubeconfig = Kubeconfig::read_from(path) + .with_context(|| format!("failed to read kubeconfig {}", path.display()))?; + let mut client_config = + Config::from_custom_kubeconfig(kubeconfig, &KubeConfigOptions::default()).await?; + if let Some(api_server) = &config.api_server { + client_config.cluster_url = api_server.as_str().parse()?; + } + Ok(client_config) +} + +#[cfg(test)] +mod tests { + use kube::error::ErrorResponse; + + use super::*; + + #[test] + fn maps_404_to_node_gone() { + let err = kube::Error::Api(ErrorResponse { + status: "Failure".to_string(), + message: "nodes \"n\" not found".to_string(), + reason: "NotFound".to_string(), + code: 404, + }); + + assert!(matches!(map_kube_error(err), K8sClientError::NodeGone)); + } + + #[test] + fn leaves_other_api_errors_as_api() { + let err = kube::Error::Api(ErrorResponse { + status: "Failure".to_string(), + message: "forbidden".to_string(), + reason: "Forbidden".to_string(), + code: 403, + }); + + assert!(matches!(map_kube_error(err), K8sClientError::Api(_))); + } +} diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index af91fcae10..31761886da 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -1,10 +1,26 @@ -//! Library surface for the `trident-acl-agent` crate. +//! # Harpoon //! -//! Currently this exposes the [`nebraska`] client module, a self-contained, -//! reusable implementation of the Nebraska/Omaha update protocol. It is usable -//! both by this crate's agent binary and by a future Trident ACL Agent that +//! Harpoon is Trident's ACL update sidecar. Historically it was a one-shot +//! Omaha client that called Trident's combined `Update()` RPC once and exited. +//! This crate now defaults to the AKS annotation protocol described in the +//! local design doc (`aks-rp ↔ trident-acl-agent`, especially §3–§6 and +//! §12–§13), while preserving the original `omaha-only` mode as an explicit +//! opt-out (see `config::GoalSource`). +//! +//! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) +//! goes through the [`nebraska`] client module, a self-contained, reusable +//! implementation of the Nebraska/Omaha update protocol. It is usable both by +//! this crate's agent binary and by a future Trident ACL Agent that //! orchestrates updates differently. +use anyhow::Context; +use semver::Version; + +pub mod annotations; +pub mod config; +pub mod error; +pub mod id; +pub mod k8s; pub mod nebraska; /// The version this agent reports to Nebraska as the updater's own version, for @@ -20,3 +36,161 @@ pub const AGENT_VERSION: &str = match option_env!("TRIDENT_VERSION") { Some(version) => version, None => env!("CARGO_PKG_VERSION"), }; + +pub mod orchestrator; +pub mod state; +pub mod trident; + +/// Only built for `cargo test` (relies on trident-proto's `server` feature, +/// which is only enabled via trident-acl-agent's dev-dependencies - see +/// mock_tridentd.rs's module docs). +#[cfg(test)] +pub mod mock_tridentd; + +use error::HarpoonError; +use nebraska::{CheckOutcome, Client, MachineId, NebraskaError}; +use trident::TridentClient; + +pub use id::IdSource; + +// Deliberately invalid sentinels, mirroring DEFAULT_NEBRASKA_ENDPOINT's +// `.invalid` domain trick: a deployment that forgets to configure (or +// override via the update-request annotation's `appId`/`track` fields) a +// real app_id/track fails loudly against Nebraska instead of silently +// querying a real-looking but wrong app/group. +pub const DEFAULT_NEBRASKA_APP_ID: &str = "00000000-0000-0000-0000-000000000000"; +pub const DEFAULT_NEBRASKA_TRACK: &str = "unspecified"; + +/// Builds a validated [`MachineId`] from an [`IdSource`], translating the +/// crate's own machine-id/hostname read errors into a single [`HarpoonError`]. +fn build_machine_id(source: IdSource) -> Result { + MachineId::new(source.produce_id()?).map_err(|err| HarpoonError::Nebraska(err.to_string())) +} + +/// Historical one-shot flow: query the Nebraska/Omaha server at +/// `config.nebraska.endpoint` once, and if an update is offered, call +/// tridentd's combined `Update()` RPC once and exit. No Kubernetes/annotation +/// involvement. +pub async fn run_omaha_only(config: &config::AgentConfig) -> Result<(), anyhow::Error> { + let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { + anyhow::anyhow!( + "no Nebraska endpoint configured: pass on the CLI or set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT" + ) + })?; + + // Client::check_for_update() is a blocking call (reqwest::blocking under + // the hood, see nebraska::transport) - calling it directly from this + // async fn can panic ("Cannot drop a runtime in a context where blocking + // is not allowed") because reqwest::blocking spins up its own inner + // Tokio runtime per call, which isn't safe to tear down from inside an + // already-running async task. Run it on a dedicated blocking thread. + let app_id = config.nebraska.app_id.clone(); + let track = config.nebraska.track.clone(); + let machine_id = build_machine_id(IdSource::MachineIdHashed)?; + let outcome = tokio::task::spawn_blocking(move || { + let client = Client::new(endpoint, app_id, track, machine_id); + client.check_for_update(&Version::new(0, 0, 0)) + }) + .await + .context("Nebraska query task panicked")? + .map_err(|err| anyhow::anyhow!("Nebraska query failed: {err}"))?; + + match outcome { + CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { + log::debug!("No update available from Nebraska"); + Ok(()) + } + CheckOutcome::UpdateAvailable(offer) => { + log::info!("Triggering one-shot Omaha update to {}", offer.version); + let mut client = TridentClient::connect(&config.trident.socket).await?; + let combined_timeout = + config.orchestration.stage_timeout + config.orchestration.finalize_timeout; + // Integrity of the downloaded image is verified by Trident itself + // via the image's own COSI metadata, so the Nebraska-reported hash + // (offer.primary.hash) is not passed here. + client + .update(&offer.primary.url, None, combined_timeout) + .await?; + Ok(()) + } + } +} + +/// Checks that the Omaha/Nebraska server at `url` is reachable and speaking +/// the Omaha protocol, without treating any app-level result (including a +/// non-OK app/update-check status) as a failure. Unlike +/// [`Client::check_for_update`], this only fails on network/transport +/// problems or a response that isn't well-formed Omaha XML -- it's meant for +/// a pure "can we talk to this server at all" check (e.g. +/// `--validate-connection nebraska`), not for deciding whether an update is +/// available. +pub fn check_nebraska_reachable( + url: &url::Url, + app_id: &str, + track: &str, + machine_id_source: IdSource, +) -> Result<(), HarpoonError> { + let machine_id = build_machine_id(machine_id_source)?; + let client = Client::new(url.clone(), app_id, track, machine_id); + match client.check_for_update(&Version::new(0, 0, 0)) { + Ok(_) => Ok(()), + // A well-formed response reporting a non-OK app/update-check status + // still proves the server is reachable and speaking Omaha; only a + // transport/parse-level failure means it is not. + Err(NebraskaError::ServerError(_)) => Ok(()), + Err(err) => Err(HarpoonError::Nebraska(err.to_string())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_check_nebraska_reachable_succeeds_on_error_app_status() { + // check_nebraska_reachable() is meant to be a pure "can we reach this + // server and does it speak Omaha" check, unlike check_for_update() + // which also validates app-level semantics. A well-formed response + // with a non-OK app status should still count as "reachable" here, + // even though check_for_update() would reject the same response as a + // NebraskaError::ServerError. + let mut server = mockito::Server::new(); + + let omaha_mock = server + .mock("POST", "/") + .with_status(200) + .with_body(indoc::indoc! {r#" + + + + + + + "#}) + .expect(1) + .create(); + + check_nebraska_reachable( + &url::Url::parse(&server.url()).unwrap(), + "test", + "track", + IdSource::MachineIdHashed, + ) + .unwrap(); + + omaha_mock.assert(); + } + + #[test] + fn test_check_nebraska_reachable_fails_on_transport_error() { + let err = check_nebraska_reachable( + // Port 0 never accepts a connection. + &url::Url::parse("http://127.0.0.1:0/").unwrap(), + "test", + "track", + IdSource::MachineIdHashed, + ) + .unwrap_err(); + assert!(matches!(err, HarpoonError::Nebraska(_))); + } +} diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 75677af298..60aa2bb8b1 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -1,549 +1,243 @@ -//! # Harpoon -//! -//! Harpoon is a lightweight Omaha protocol client for documents. It queries a -//! server at a given address for a specific app and track to fetch an updated -//! document. -//! -//! This crate is specifically meant to function as an Omaha client for Trident -//! to fetch updated Host Configuration documents. -//! -//! -//! - +use anyhow::Context; use clap::Parser; -use futures::StreamExt; -use log::{debug, error, info, trace, warn, LevelFilter}; -use semver::Version; -use sha2::{Digest, Sha256}; -use tonic::{transport::Endpoint, Streaming}; -use trident_proto::v1::{ - servicing_response::Response as ResponseBody, update_service_client::UpdateServiceClient, - FinalizeUpdateRequest, HostConfiguration, LogLevel, RebootHandling, RebootManagement, - ServicingResponse, StageUpdateRequest, StatusCode, UpdateRequest, +use log::{LevelFilter, Log, Metadata, Record}; + +use trident_acl_agent::{ + check_nebraska_reachable, + config::{AgentConfig, GoalSource}, + k8s::NodeClient, + orchestrator::Orchestrator, + run_omaha_only, + trident::TridentClient, + IdSource, }; -use url::Url; -use uuid::Uuid; -pub mod error; -pub mod id; -pub mod omaha; +/// Module/target prefixes for the underlying HTTP/gRPC/watch client stack. +/// These crates emit very verbose `log`-facade tracing (connection setup, +/// per-frame HTTP2 detail, watch reconnect churn) that is rarely useful at +/// the same verbosity as the agent's own orchestration logic, so it's +/// filtered independently via `--network-verbosity`. +const NETWORK_LOG_TARGETS: &[&str] = &[ + "hyper", + "h2", + "tower", + "tonic", + "reqwest", + "rustls", + "kube", + "kube_client", + "kube_runtime", +]; + +/// A `log::Log` wrapper that applies a separate level filter to the noisy +/// HTTP/gRPC/watch client crates (see [`NETWORK_LOG_TARGETS`]) while leaving +/// every other target (the agent's own code) at the main `--verbosity` +/// level. +struct FilteredLogger { + inner: L, + verbosity: LevelFilter, + network_verbosity: LevelFilter, +} -use error::HarpoonError; -use omaha::{ - event::{OmahaEvent, OmahaEventType}, - request::{AppRequest, Request}, - response::Package, -}; +impl Log for FilteredLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + let level = if is_network_target(metadata.target()) { + self.network_verbosity + } else { + self.verbosity + }; + metadata.level() <= level + } -pub use id::IdSource; -pub use omaha::event::EventResult; + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + self.inner.log(record); + } + } -#[derive(Debug, PartialEq, Eq)] -pub struct HarpoonQueryResponse { - pub session_id: Uuid, - pub result: QueryResult, + fn flush(&self) { + self.inner.flush(); + } } -#[derive(Debug, PartialEq, Eq)] -pub enum QueryResult { - NoUpdate, - NewDocument { url: Url, version: Version }, +fn is_network_target(target: &str) -> bool { + NETWORK_LOG_TARGETS + .iter() + .any(|prefix| target == *prefix || target.starts_with(&format!("{prefix}::"))) } +/// Harpoon can either run the annotation-driven orchestrator (the default) +/// or fall back to its original one-shot Omaha flow. Mode selection is +/// environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`): +/// shipping defaults enable the AKS annotation protocol, while a VM +/// extension, systemd drop-in, or AgentBaker-set environment can opt a node +/// out to `omaha-only` if needed. #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { /// Logging verbosity [OFF, ERROR, WARN, INFO, DEBUG, TRACE] #[arg(global = true, short, long, default_value_t = LevelFilter::Debug)] - pub verbosity: LevelFilter, - - /// The URL of the Nebraska server to use. Likely should end in `/v1/update` - #[arg()] - pub url: Url, + verbosity: LevelFilter, + + /// Logging verbosity for the underlying HTTP/gRPC/watch client stack + /// (hyper, h2, tower, tonic, reqwest, rustls, kube). Kept separate from + /// `--verbosity` because it can be extremely noisy (per-frame HTTP2 + /// detail, watch reconnect churn) [OFF, ERROR, WARN, INFO, DEBUG, TRACE]. + #[arg(global = true, long, default_value_t = LevelFilter::Warn)] + network_verbosity: LevelFilter, + + /// Validate connectivity to a single dependency and exit immediately, + /// instead of running the agent. Useful for troubleshooting one + /// connection in isolation (e.g. a systemd ExecStartPre check, or manual + /// diagnostics on-node) without running the full orchestrator loop. + /// Exits with status 0 if the connection could be established, non-zero + /// (with an error message) otherwise. + #[arg(long, value_enum)] + validate_connection: Option, } -fn main() { - let args = Args::parse(); - - if let Some(Ok(journal_logger)) = - systemd_journal_logger::connected_to_journal().then(systemd_journal_logger::JournalLog::new) - { - journal_logger - .install() - .expect("Failed to install systemd journal logger"); - log::set_max_level(args.verbosity); - } else { - env_logger::builder() - .format_timestamp(None) - .filter_level(args.verbosity) - .init(); - } - - let r = query_and_fetch_yaml_document( - &args.url, - "b0ec8f0d-1c13-4bf4-9efd-ea54464a7098", - "west-us", - &Version::new(0, 0, 0), - IdSource::MachineIdHashed, - ) - .expect("Failed to query Omaha server"); - - match r.result { - QueryResult::NoUpdate => { - debug!("No update available"); - } - QueryResult::NewDocument { url, version } => { - debug!("Updating to version {version}"); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("Failed to create tokio runtime"); - - rt.block_on(trigger(&url, None)) - .expect("Failed to run update"); - } - } +/// A single dependency `--validate-connection` can check. +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +enum ConnectionTarget { + /// Validates reachability of the Kubernetes API server by fetching this + /// node's own Node object (the same access the agent's reconcile loop + /// already requires). + Kubernetes, + /// Validates reachability of tridentd by connecting to its gRPC Unix + /// socket. Connecting is sufficient - no RPC call is needed, since the + /// connection itself fails immediately if nothing is listening. + Tridentd, + /// Validates reachability of the Nebraska/Omaha server by issuing a real + /// update-check query. Any well-formed Omaha response (including "no + /// update available") counts as success - only a network/transport + /// failure is treated as unreachable. + Nebraska, } -async fn trigger(url: &Url, hash: Option) -> Result<(), anyhow::Error> { - // For now, we will just log the trigger. In the future, this function can be - // used to trigger an update check on the server side, for example by sending - // a specific event or making a specific API call to the server. - debug!("Triggering update with URL: {url} and hash: {hash:?}"); - - let channel = Endpoint::new(trident_proto::TRIDENT_DEFAULT_SOCKET_URI)? - .connect() - .await?; - let mut client = UpdateServiceClient::new(channel); - - let response = client - .update(tonic::Request::new(UpdateRequest { - stage: Some(StageUpdateRequest { - config: Some(HostConfiguration { - // TODO: Handle escaping of URL and hash. - config: match hash { - Some(hash) => format!("image:\n url: {url}\n sha384: {hash}"), - None => { - format!("image:\n url: {url}\n sha384: ignored") - } - }, - }), - }), - finalize: Some(FinalizeUpdateRequest { - reboot: Some(RebootManagement { - handling: RebootHandling::CallerHandlesReboot.into(), - }), - }), - })) - .await?; - - handle_servicing_stream(response.into_inner()).await -} - -async fn handle_servicing_stream( - mut stream: Streaming, +/// Checks connectivity to exactly one of `target`'s dependencies and returns +/// `Ok(())` on success. The caller (`main`) surfaces any `Err` the normal way +/// (`anyhow`'s `Termination` impl prints the error and exits non-zero), so +/// this function only needs to produce a descriptive error on failure - no +/// explicit `process::exit` is required. +async fn validate_connection( + target: ConnectionTarget, + config: &AgentConfig, ) -> Result<(), anyhow::Error> { - // Iterate through the stream until we get a Completed message - loop { - match stream.next().await { - Some(Ok(response)) => match response.response { - Some(ResponseBody::Started(_)) => { - info!("[Trident] Install started"); - // Continue to next message - } - Some(ResponseBody::Log(log)) => { - let msg = format!("[Trident] {}", log.message); - match log.level() { - LogLevel::Unspecified | LogLevel::Trace => trace!("{msg}"), - LogLevel::Debug => debug!("{msg}"), - LogLevel::Info => info!("{msg}"), - LogLevel::Warn => warn!("{msg}"), - LogLevel::Error => error!("{msg}"), - } - } - Some(ResponseBody::Completed(final_status)) => { - if final_status.status() == StatusCode::Success { - info!( - "Trident install succeeded: status={:?}", - final_status.status() - ); - break Ok(()); - } else { - error!("Trident install failed: status={:?}", final_status.status()); - match final_status.error { - Some(err) => { - error!("Trident reported error: {}", err.message); - break Err(anyhow::anyhow!(err.message)); - } - None => { - break Err(anyhow::anyhow!("Trident install failed")); - } - } - } - } - None => { - // Empty response, continue - continue; - } - }, - Some(Err(e)) => { - break Err(anyhow::anyhow!("Error reading from Trident stream: {e}")); - } - None => { - break Err(anyhow::anyhow!( - "Trident install stream ended without control message" - )); - } + match target { + ConnectionTarget::Kubernetes => { + let client = NodeClient::new(&config.kubernetes) + .await + .context("failed to build Kubernetes client")?; + // Report the actually-resolved server (kubeconfig's own server, + // unless overridden by kubernetes.api_server), not a value + // guessed from config - the two only match when an override is + // set. + let cluster_url = client.cluster_url(); + client + .get_node(&config.kubernetes.node_name) + .await + .with_context(|| { + format!( + "failed to reach Kubernetes API server at {} (get Node {:?})", + cluster_url, config.kubernetes.node_name + ) + })?; + log::info!( + "kubernetes: reached API server at {} and fetched Node {:?}", + cluster_url, + config.kubernetes.node_name + ); } - } -} - -/// Query the Omaha server at the given URL for the given app and track to fetch -/// an updated YAML document. -/// -/// Returns the session ID and the result of the query. If an update is -/// available, the new version and the updated document are returned. -/// -/// This function should ONLY be used for querying YAML documents (i.e YAML text -/// files) because the whole file will be downloaded, and the function will only -/// look at the first package returned by the omaha server to fetch the -/// document. The function expects the document to be a single file with `.yaml` -/// extension. -pub fn query_and_fetch_yaml_document( - url: &Url, - app_id: &str, - track: &str, - document_version: &Version, - machine_id_source: IdSource, -) -> Result { - let request = Request::default().with_app( - AppRequest::new(app_id, document_version, track, machine_id_source)?.with_update_check(), - ); - - let response = omaha::send(url, &request)?; - - debug!( - "Received response from Omaha server at '{url}' for app '{app_id}' on track '{track}': {response:#?}", - url = url, - app_id = app_id, - track = track, - response = response - ); - if response.apps().len() != 1 { - return Err(HarpoonError::InvalidResponse( - "Expected exactly one app in response".to_string(), - )); - } - - let app = response.apps().first().unwrap(); - - if app.app_id() != app_id { - return Err(HarpoonError::InvalidResponse( - "Unexpected app ID in response".to_string(), - )); - } - - if app.status().is_error() { - return Err(HarpoonError::QueryError(format!( - "Received a non-OK app status: {0}", - app.status() - ))); - } - - let update_check = app.update_check().ok_or_else(|| { - HarpoonError::InvalidResponse("Missing update check in response".to_string()) - })?; - debug!("Received update check response: {update_check:#?}"); - - if update_check.status().is_error() { - return Err(HarpoonError::QueryError(format!( - "Received an error status in update check: {0}", - update_check.status() - ))); - } - - if update_check.status().is_no_update() { - // Successfully checked that there is no update available! - debug!( - "No update available for app '{}' v{}", - app_id, document_version - ); - return Ok(HarpoonQueryResponse { - session_id: request.session_id(), - result: QueryResult::NoUpdate, - }); - } - - // If we got here, an update is available! - let new_version = update_check.version().ok_or_else(|| { - HarpoonError::InvalidResponse("Missing new version in update check response".to_string()) - })?; - - let update_base_url = update_check.urls().next().ok_or_else(|| { - HarpoonError::InvalidResponse("Missing URL in update check response".to_string()) - })?; - - if update_check.packages().len() != 1 { - return Err(HarpoonError::InvalidResponse( - "Expected exactly one package in update check response".to_string(), - )); - } - - let package_url = update_base_url - .join(&update_check.packages().first().unwrap().name) - .map_err(|err| { - HarpoonError::InvalidResponse(format!("Failed to join URL with package name: {err}")) - })?; - - debug!( - "Downloaded update for app '{}' v{} to v{}", - app_id, document_version, new_version - ); - debug!("Document URL: {package_url}"); - - Ok(HarpoonQueryResponse { - session_id: request.session_id(), - result: QueryResult::NewDocument { - url: package_url, - version: new_version.as_version().clone(), - }, - }) -} - -/// Downloads an update package provided by the Omaha server at the given base -/// URL. -/// -/// On success, returns the document as a string and the URL from which it was -/// downloaded. -/// -/// The function takes care of validating the size and hash of the downloaded -/// document. -#[allow(unused)] -fn download_document( - update_base_url: &Url, - package: &Package, - file_extension: &str, -) -> Result<(String, Url), HarpoonError> { - if !package.name.ends_with(file_extension) { - return Err(HarpoonError::ExpectedYamlDocument(package.name.clone())); - } - - // If the package size is larger than 1MB, log a warning. This may mean that - // we are not downloading the correct document. - if package.size >= 1024 * 1024 { - warn!( - "Reported document size is larger than 1MB ({}). This may NOT be a '{}' text document.", - package.size, file_extension - ); - } - - let package_url = update_base_url.join(&package.name).map_err(|err| { - HarpoonError::InvalidResponse(format!("Failed to join URL with package name: {err}")) - })?; - - let document = reqwest::blocking::Client::new() - .get(package_url.clone()) - .send() - .map_err(|err| HarpoonError::FetchError(err.to_string()))? - .text() - .map_err(|err| HarpoonError::FetchError(err.to_string()))?; - - // Check that the downloaded document size matches the package size. - trace!( - "Validating document size: actual [{}] == expected [{}]", - document.len(), - package.size - ); - if package.size != document.len() as u64 { - return Err(HarpoonError::FetchError(format!( - "Downloaded document size does not match package size: {} != {}", - document.len(), - package.size - ))); - } - - // If we have a hash, validate it. - if !package.hash.is_empty() { - let actual = format!("{:x}", Sha256::digest(document.as_bytes())); - let expected = package.hash.to_lowercase(); - trace!( - "Validating document hash: actual [{}] == expected [{}]", - actual, - expected - ); - if actual != expected { - return Err(HarpoonError::FetchError(format!( - "Downloaded document hash does not match package hash: {actual} != {expected}" - ))); + ConnectionTarget::Tridentd => { + TridentClient::connect(&config.trident.socket) + .await + .with_context(|| { + format!("failed to reach tridentd at {}", config.trident.socket) + })?; + log::info!("tridentd: connected to {}", config.trident.socket); } - } - - Ok((document, package_url)) -} - -/// A wrapper to hide away the details of what Omaha events are actually -/// relevant. Trident only needs to know about Install and Update events. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EventType { - Install, - Update, -} - -impl From for OmahaEventType { - fn from(event_type: EventType) -> Self { - match event_type { - EventType::Install => OmahaEventType::EventUpdateInstalled, - EventType::Update => OmahaEventType::UpdateComplete, + ConnectionTarget::Nebraska => { + let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { + anyhow::anyhow!( + "nebraska.endpoint is not configured (set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT)" + ) + })?; + let app_id = config.nebraska.app_id.clone(); + // check_nebraska_reachable() is a blocking call (reqwest::blocking + // under the hood, see omaha::send) - calling it directly from this + // async fn can panic ("Cannot drop a runtime in a context where + // blocking is not allowed") because reqwest::blocking spins up + // its own inner Tokio runtime per call, which isn't safe to tear + // down from inside an already-running async task. Run it on a + // dedicated blocking thread instead. + // + // Deliberately uses check_nebraska_reachable() rather than + // query_for_update(): the latter also validates app-level + // semantics (app ID match, non-error app/update-check status), + // which would make this a "can we get a valid update check" test + // rather than the pure reachability check documented on + // ConnectionTarget::Nebraska above. + let endpoint_for_task = endpoint.clone(); + let track = config.nebraska.track.clone(); + tokio::task::spawn_blocking(move || { + check_nebraska_reachable( + &endpoint_for_task, + &app_id, + &track, + IdSource::MachineIdHashed, + ) + }) + .await + .context("Nebraska connectivity check task panicked")? + .with_context(|| format!("failed to reach Nebraska server at {endpoint}"))?; + log::info!("nebraska: reached server at {endpoint}"); } } -} - -/// Reports an Omaha event to the server at the given URL for the given app and -/// track. -fn report_omaha_event( - url: &Url, - app_id: &str, - track: &str, - event: OmahaEventType, - result: EventResult, - machine_id_source: IdSource, -) -> Result<(), HarpoonError> { - omaha::send_event( - url, - &Request::default().with_app( - AppRequest::new_event(app_id, track, machine_id_source)? - .with_event(OmahaEvent::new(event, result)), - ), - )?; Ok(()) } -/// Reports a generic event to the Omaha server at the given URL for the given -/// app and track. -pub fn report_event( - url: &Url, - app_id: &str, - track: &str, - event: EventType, - result: EventResult, - machine_id_source: IdSource, -) -> Result<(), HarpoonError> { - report_omaha_event(url, app_id, track, event.into(), result, machine_id_source) -} - -#[cfg(test)] -mod tests { - use mockito::Matcher; - - use super::*; - - #[test] - fn test_download_document() { - let mut server = mockito::Server::new(); - - let data = "test document"; - - let document_mock = server - .mock("GET", "/test.yaml") - .with_body(data) - .with_header("content-length", &data.len().to_string()) - .with_header("content-type", "text/plain") - .with_status(200) - .expect(1) - .create(); - - let url = Url::parse(&server.url()).unwrap(); - let package = Package { - name: "test.yaml".to_string(), - size: 13, - hash: format!("{:x}", Sha256::digest(data.as_bytes())), - hash_sha256: None, - required: true, - }; - - let (document, package_url) = download_document(&url, &package, ".yaml").unwrap(); - - document_mock.assert(); - - assert_eq!(document, data); +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + let args = Args::parse(); - assert_eq!( - package_url, - Url::parse(&format!("{}/test.yaml", server.url())).unwrap() - ); + let max_level = args.verbosity.max(args.network_verbosity); + if let Some(Ok(journal_logger)) = + systemd_journal_logger::connected_to_journal().then(systemd_journal_logger::JournalLog::new) + { + log::set_boxed_logger(Box::new(FilteredLogger { + inner: journal_logger, + verbosity: args.verbosity, + network_verbosity: args.network_verbosity, + })) + .expect("Failed to install systemd journal logger"); + log::set_max_level(max_level); + } else { + let inner = env_logger::builder() + .format_timestamp(None) + .filter_level(max_level) + .build(); + log::set_boxed_logger(Box::new(FilteredLogger { + inner, + verbosity: args.verbosity, + network_verbosity: args.network_verbosity, + })) + .expect("Failed to install env logger"); + log::set_max_level(max_level); } - #[test] - fn test_query_and_fetch_document() { - let mut server = mockito::Server::new(); + let config = AgentConfig::from_env()?; - let data = "test document"; - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .match_body(Matcher::Regex(".* - - - - - - - - - - - - - - - "#}, - server.url(), - Sha256::digest(data.as_bytes()), - data.len() - )) - .expect(1) - .create(); - - // let omaha_event_mock = server - // .mock("POST", "/") - // .with_status(200) - // .match_body(Matcher::Regex(".* - // - // - // - // - // - // "#}) - // .expect(1) - // .create(); - - let response = query_and_fetch_yaml_document( - &Url::parse(&server.url()).unwrap(), - "test", - "track", - &Version::new(0, 1, 0), - IdSource::MachineIdHashed, - ) - .unwrap(); - - omaha_mock.assert(); - // omaha_event_mock.assert(); + if let Some(target) = args.validate_connection { + return validate_connection(target, &config).await; + } - assert_eq!( - response, - HarpoonQueryResponse { - session_id: response.session_id, - result: QueryResult::NewDocument { - url: Url::parse(&format!("{}/test.yaml", server.url())).unwrap(), - version: Version::new(1, 0, 0), - } - } - ); + match config.orchestration.goal_source { + // Historical one-shot flow: query Nebraska once, apply an update if + // offered, and exit. No Kubernetes/annotation involvement. + GoalSource::OmahaOnly => run_omaha_only(&config).await, + // Default: the annotation-driven reconcile loop (watches + // acl.azure.com/update-request, drives stage/finalize/rollback/ + // commit against tridentd, writes acl.azure.com/update-status). + GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, } } diff --git a/crates/trident-acl-agent/src/mock_tridentd.rs b/crates/trident-acl-agent/src/mock_tridentd.rs new file mode 100644 index 0000000000..d0e60670f0 --- /dev/null +++ b/crates/trident-acl-agent/src/mock_tridentd.rs @@ -0,0 +1,253 @@ +//! In-process mock tridentd used only by unit tests. +//! +//! Implements the real generated `UpdateService`/`CommitService` server +//! traits (gated behind trident-proto's `server` feature, enabled only in +//! `[dev-dependencies]` - see trident-acl-agent/Cargo.toml) so tests can +//! exercise the *real* `TridentClient` request/response/error-mapping code +//! against canned stage/finalize/commit outcomes, without a real tridentd +//! process or unix socket. +//! +//! Tests wire a `TridentClient` to this mock server over an in-memory +//! `tokio::io::duplex` transport via `Endpoint::connect_with_connector` + +//! `TridentClient::from_channel` - see `connect_mock_client` below. + +use std::sync::{Arc, Mutex}; + +use hyper_util::rt::TokioIo; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Endpoint, Request, Response, Status}; +use trident_proto::v1::{ + commit_service_server::{CommitService, CommitServiceServer}, + rollback_service_server::{RollbackService, RollbackServiceServer}, + servicing_response::Response as ResponseBody, + update_service_server::{UpdateService, UpdateServiceServer}, + CommitRequest, Completed, FinalizeUpdateRequest, RebootStatus, RollbackFinalizeRequest, + RollbackRequest, RollbackStageRequest, ServicingKind, ServicingResponse, StageUpdateRequest, + StatusCode as ProtoStatusCode, TridentError, UpdateRequest, +}; + +use crate::trident::TridentClient; + +/// Canned outcome a `MockTridentd` should return for a given RPC call. +#[derive(Clone, Debug)] +pub enum Outcome { + /// Respond with a successful `Completed` message. `servicing_kind` + /// mirrors what a real tridentd populates on every servicing RPC + /// (`ServicingKind::NoneRequired` for a no-op, the real kind + /// otherwise) - tests that care about the no-op-detection path (see + /// orchestrator.rs's `handle_rollback`) set this explicitly; other + /// tests that don't inspect it can pass `None`. + Success { + reboot_status: RebootStatus, + servicing_kind: Option, + }, + /// Respond with a failed `Completed` message carrying the given error + /// subkind (e.g. "ab-update-reboot-check"). + Failure { + subkind: &'static str, + message: &'static str, + }, +} + +impl Outcome { + fn into_servicing_response(self) -> ServicingResponse { + let completed = match self { + Outcome::Success { + reboot_status, + servicing_kind, + } => Completed { + status: ProtoStatusCode::Success as i32, + error: None, + reboot_status: reboot_status as i32, + image_hash: None, + servicing_kind: servicing_kind.map(|k| k as i32), + }, + Outcome::Failure { subkind, message } => Completed { + status: ProtoStatusCode::Failure as i32, + error: Some(TridentError { + kind: 0, + subkind: subkind.to_string(), + message: message.to_string(), + error_message: message.to_string(), + location: None, + }), + reboot_status: RebootStatus::Unspecified as i32, + image_hash: None, + servicing_kind: None, + }, + }; + ServicingResponse { + timestamp: None, + response: Some(ResponseBody::Completed(completed)), + } + } +} + +/// Configurable canned responses for the three RPCs `TridentClient` calls. +/// Each field defaults to `None`; a test sets only the outcome(s) it cares +/// about, and the mock server panics if a call arrives with no outcome +/// configured (surfacing test-setup bugs immediately rather than silently +/// hanging or defaulting). +#[derive(Default)] +pub struct MockTridentdConfig { + pub stage: Option, + pub finalize: Option, + pub commit: Option, + pub rollback_stage: Option, + pub rollback_finalize: Option, +} + +#[derive(Clone)] +struct MockTridentd { + config: Arc>, +} + +async fn respond_with( + outcome: Outcome, +) -> Result>>, Status> { + let (tx, rx) = tokio::sync::mpsc::channel(4); + tx.send(Ok(outcome.into_servicing_response())) + .await + .expect("mock tridentd channel send should not fail"); + Ok(Response::new(ReceiverStream::new(rx))) +} + +#[tonic::async_trait] +impl UpdateService for MockTridentd { + type UpdateStream = ReceiverStream>; + type UpdateStageStream = ReceiverStream>; + type UpdateFinalizeStream = ReceiverStream>; + + async fn update( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "update() is not used by trident-acl-agent", + )) + } + + async fn update_stage( + &self, + _request: Request, + ) -> Result, Status> { + let outcome = + self.config.lock().unwrap().stage.clone().expect( + "test must configure MockTridentdConfig::stage before calling update_stage", + ); + respond_with(outcome).await + } + + async fn update_finalize( + &self, + _request: Request, + ) -> Result, Status> { + let outcome = self.config.lock().unwrap().finalize.clone().expect( + "test must configure MockTridentdConfig::finalize before calling update_finalize", + ); + respond_with(outcome).await + } +} + +#[tonic::async_trait] +impl CommitService for MockTridentd { + type CommitStream = ReceiverStream>; + + async fn commit( + &self, + _request: Request, + ) -> Result, Status> { + let outcome = self + .config + .lock() + .unwrap() + .commit + .clone() + .expect("test must configure MockTridentdConfig::commit before calling commit"); + respond_with(outcome).await + } +} + +#[tonic::async_trait] +impl RollbackService for MockTridentd { + type RollbackStream = ReceiverStream>; + type RollbackStageStream = ReceiverStream>; + type RollbackFinalizeStream = ReceiverStream>; + + // check_rollback is no longer part of the stable v1 RollbackService + // trait (demoted back to trident.v1preview - trident-acl-agent detects + // a no-op rollback via RollbackStage's servicing_kind now instead, see + // orchestrator.rs's handle_rollback), so this mock no longer needs to + // implement it. + + async fn rollback( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "rollback() is not used by trident-acl-agent", + )) + } + + async fn rollback_stage( + &self, + _request: Request, + ) -> Result, Status> { + let outcome = self.config.lock().unwrap().rollback_stage.clone().expect( + "test must configure MockTridentdConfig::rollback_stage before calling rollback_stage", + ); + respond_with(outcome).await + } + + async fn rollback_finalize( + &self, + _request: Request, + ) -> Result, Status> { + let outcome = self + .config + .lock() + .unwrap() + .rollback_finalize + .clone() + .expect( + "test must configure MockTridentdConfig::rollback_finalize before calling rollback_finalize", + ); + respond_with(outcome).await + } +} + +/// Starts an in-process mock tridentd wired to `client` over an in-memory +/// duplex transport (no real socket/subprocess), and returns a +/// `TridentClient` connected to it. `config` is shared (`Arc>`) +/// so the caller can reconfigure outcomes between calls if a test needs to +/// simulate stage-then-finalize-then-commit in one session. +pub async fn connect_mock_client(config: Arc>) -> TridentClient { + let (client_io, server_io) = tokio::io::duplex(64 * 1024); + + let mock = MockTridentd { config }; + tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(UpdateServiceServer::new(mock.clone())) + .add_service(CommitServiceServer::new(mock.clone())) + .add_service(RollbackServiceServer::new(mock)) + .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server_io))) + .await + .expect("mock tridentd server should not fail"); + }); + + let mut client_io = Some(client_io); + let channel = Endpoint::try_from("http://[::]:50051") + .expect("static endpoint URI should always parse") + .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { + let client_io = client_io.take(); + async move { + client_io.map(TokioIo::new).ok_or_else(|| { + std::io::Error::other("mock client connector called more than once") + }) + } + })) + .await + .expect("in-memory duplex connection should succeed"); + + TridentClient::from_channel(channel) +} diff --git a/crates/trident-acl-agent/src/omaha/app.rs b/crates/trident-acl-agent/src/omaha/app.rs deleted file mode 100644 index 67ae13f172..0000000000 --- a/crates/trident-acl-agent/src/omaha/app.rs +++ /dev/null @@ -1,106 +0,0 @@ -use std::fmt::Display; - -use semver::Version; -use serde::{Deserialize, Serialize}; - -/// A thin wrapper around `semver::Version` to provide serialization and -/// deserialization. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct AppVersion(Version); - -impl Display for AppVersion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -impl From for AppVersion { - fn from(version: Version) -> Self { - Self(version) - } -} - -impl From<&Version> for AppVersion { - fn from(version: &Version) -> Self { - Self(version.clone()) - } -} - -impl AppVersion { - /// Returns a reference to the inner `Version`. - pub(crate) fn as_version(&self) -> &Version { - &self.0 - } - - #[allow(dead_code)] - /// Returns a new version with the given major, minor, and patch. - pub(crate) fn new(major: u64, minor: u64, patch: u64) -> Self { - Self(Version::new(major, minor, patch)) - } -} - -impl Default for AppVersion { - fn default() -> Self { - Self(Version::new(0, 0, 0)) - } -} - -/// Implement serialization for AppVersion -impl Serialize for AppVersion { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.0.to_string()) - } -} - -/// Implement deserialization for AppVersion -impl<'de> Deserialize<'de> for AppVersion { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - Ok(Self( - Version::parse(&String::deserialize(deserializer)?) - .map_err(serde::de::Error::custom)?, - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json; - - #[test] - fn test_app_version_serialization() { - (0..100).for_each(|major| { - (0..100).for_each(|minor| { - (0..100).for_each(|patch| { - let version = AppVersion::from(Version::new(major, minor, patch)); - let json = serde_json::to_string(&version).unwrap(); - assert_eq!(json, format!("\"{major}.{minor}.{patch}\"")); - let deserialized: AppVersion = serde_json::from_str(&json).unwrap(); - assert_eq!(deserialized, version); - }); - }); - }); - } - - #[test] - fn test_app_version_serialization_invalid() { - fn deserialize(v: &str) { - serde_json::from_str::(&format!("\"{v}\"")).unwrap_err(); - } - - // Incomplete semver - deserialize("1.2"); - - // Invalid semver - deserialize("1.2.3.4"); - - // Outright invalid stuff - deserialize("aa"); - } -} diff --git a/crates/trident-acl-agent/src/omaha/event.rs b/crates/trident-acl-agent/src/omaha/event.rs deleted file mode 100644 index 1a6ffa57fa..0000000000 --- a/crates/trident-acl-agent/src/omaha/event.rs +++ /dev/null @@ -1,98 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, PartialEq, Eq)] -pub(crate) struct OmahaEvent { - #[serde(rename = "@eventtype")] - pub(crate) event_type: OmahaEventType, - - #[serde(rename = "@eventresult")] - pub(crate) event_result: EventResult, -} - -impl OmahaEvent { - pub fn new(event_type: OmahaEventType, event_result: EventResult) -> Self { - Self { - event_type, - event_result, - } - } -} - -/// Event types for Omaha events. -#[allow(dead_code)] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] -pub enum OmahaEventType { - #[serde(rename = "0")] - Unknown, - - #[serde(rename = "1")] - DownloadComplete, - - #[serde(rename = "2")] - InstallComplete, - - #[serde(rename = "3")] - UpdateComplete, - - #[serde(rename = "4")] - Uninstall, - - #[serde(rename = "5")] - DownloadStarted, - - #[serde(rename = "6")] - InstallStarted, - - #[serde(rename = "10")] - SetupStarted, - - #[serde(rename = "11")] - SetupFinished, - - #[serde(rename = "12")] - UpdateApplicationStarted, - - #[serde(rename = "13")] - UpdateDownloadStarted, - - #[serde(rename = "14")] - UpdateDownloadFinished, - - /// Custom value defined by Nebraska. - #[serde(rename = "800")] - EventUpdateInstalled, -} - -#[allow(dead_code)] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] -pub enum EventResult { - #[serde(rename = "0")] - Error, - - #[serde(rename = "1")] - Success, - - #[serde(rename = "2")] - SuccessReboot, -} - -#[derive(Debug, Deserialize)] -pub(crate) struct EventAcknowledge { - #[serde(rename = "@status")] - event: EventAcknowledgeStatus, -} - -impl EventAcknowledge { - pub(crate) fn is_ok(&self) -> bool { - matches!(self.event, EventAcknowledgeStatus::Ok) - } -} - -#[derive(Debug, Deserialize)] -pub enum EventAcknowledgeStatus { - #[serde(rename = "ok")] - Ok, - - #[serde(other)] - Unknown, -} diff --git a/crates/trident-acl-agent/src/omaha/mod.rs b/crates/trident-acl-agent/src/omaha/mod.rs deleted file mode 100644 index ee28ab5dfb..0000000000 --- a/crates/trident-acl-agent/src/omaha/mod.rs +++ /dev/null @@ -1,261 +0,0 @@ -//! Super basic implementation of the Omaha protocol as defined in -//! https://github.com/google/omaha/blob/main/doc/ServerProtocol.md. -//! -//! # Superseded by the `nebraska` module -//! -//! This module predates, and is superseded by, the crate's `nebraska` client -//! module (`crate::nebraska`, exposed via the library target), which encodes the -//! protocol's silently-failing invariants (whitelisted events, mandatory -//! `track`, unbraced machine id, `error-updateInProgressOnInstance` tolerance) -//! in the type system. New code should use `nebraska`; this module remains only -//! until the agent's control flow is migrated to it (a deliberate follow-up, to -//! keep that diff separate). It is not `#[deprecated]` because the agent binary -//! still depends on it and the crate is built with `-D warnings`. Migration -//! mapping: -//! -//! - `query_and_fetch_*` / `omaha::send` with an `` → -//! `nebraska::Client::check_for_update` -//! - `report_event` for a progress event → `nebraska::Client::report_progress` -//! - the batched post-reboot completion → `nebraska::Client::complete_after_reboot` -//! - a failure/reset event → `nebraska::Client::report_failure` - -use log::{debug, trace}; -use url::Url; - -pub(crate) mod app; -pub(crate) mod event; -pub(crate) mod request; -pub(crate) mod response; -pub(crate) mod status; -mod xml; - -use request::Request; -use response::Response; - -use crate::error::HarpoonError; - -const OMAHA_VERSION: &str = "3.0"; -const XML_HEADER_VERSION: &str = "1.0"; -const XML_HEADER_ENCODING: &str = "UTF-8"; - -/// Sends a generic request to the Omaha server at the given URL and returns the -/// resulting response. -pub(crate) fn send(url: &Url, req: &Request) -> Result { - let body = req - .to_xml() - .map_err(|e| HarpoonError::Internal(format!("Failed to serialize request XML: {e}")))?; - - debug!("Sending Omaha request to '{url}'",); - trace!("Omaha request body:\n{}", String::from_utf8_lossy(&body)); - let client = reqwest::blocking::Client::new(); - let response = client - .post(url.as_str()) - .header("Content-Type", "application/xml") - .body(body) - .send() - .map_err(|e| HarpoonError::SendRequest(e.to_string()))? - .error_for_status() - .map_err(|e| HarpoonError::HttpError(e.to_string()))?; - - let text = response - .text() - .map_err(|e| HarpoonError::HttpError(e.to_string()))?; - - trace!("Omaha response body:\n{}", text); - - let xmld = &mut quick_xml::de::Deserializer::from_str(&text); - let response: Response = serde_path_to_error::deserialize(xmld) - .map_err(|e| HarpoonError::ParseResponse(e.to_string()))?; - - trace!("Parsed response body:\n{:#?}", response); - - response.validate()?; - - Ok(response) -} - -/// Sends an event request to the Omaha server at the given URL and returns the -/// resulting response. -pub(crate) fn send_event(url: &Url, req: &Request) -> Result { - // Send the response and get the response - let response: Response = send(url, req)?; - - // Validate that all the events of the request were acknowledged. - for (app, events) in req.apps().iter().map(|app| (app.app_id(), app.events())) { - let resp_app = response - .apps() - .iter() - .find(|a| a.app_id() == app) - .ok_or_else(|| { - HarpoonError::InvalidResponse(format!("Missing app '{app}' in response")) - })?; - - if events.len() != resp_app.events().len() { - return Err(HarpoonError::InvalidResponse(format!( - "Expected {} events for app '{}', got {}", - events.len(), - app, - resp_app.events().len() - ))); - } - - for (request_event, response_event) in events.iter().zip(resp_app.events().iter()) { - if !response_event.is_ok() { - return Err(HarpoonError::EventNotAcknowledged( - request_event.event_type, - request_event.event_result, - )); - } - } - } - - Ok(response) -} - -#[cfg(test)] -mod tests { - use super::*; - - use event::{OmahaEvent, OmahaEventType}; - use request::AppRequest; - - use crate::{EventResult, IdSource}; - - #[test] - fn test_send() { - // Request a new server from the pool - let mut server = mockito::Server::new(); - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .with_body(indoc::indoc! {r#" - - - - "#}) - .expect(1) - .create(); - - let url = Url::parse(&server.url()).unwrap(); - let request = Request::default(); - - let response = send(&url, &request).unwrap(); - assert_eq!(response.apps().len(), 0); - - omaha_mock.assert(); - } - - #[test] - fn test_send_event() { - // Request a new server from the pool - let mut server = mockito::Server::new(); - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .with_body(indoc::indoc! {r#" - - - - - - - "#}) - .expect(1) - .create(); - - let url = Url::parse(&server.url()).unwrap(); - let request = Request::default().with_app( - AppRequest::new_event("app_id", "track", IdSource::MachineIdHashed) - .unwrap() - .with_event(OmahaEvent::new( - OmahaEventType::EventUpdateInstalled, - EventResult::Success, - )), - ); - - let response = send_event(&url, &request).unwrap(); - assert_eq!(response.apps().len(), 1); - - omaha_mock.assert(); - } - - #[test] - fn test_send_event_reply_missing() { - // Request a new server from the pool - let mut server = mockito::Server::new(); - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .with_body(indoc::indoc! {r#" - - - - - - "#}) - .expect(1) - .create(); - - let url = Url::parse(&server.url()).unwrap(); - let request = Request::default().with_app( - AppRequest::new_event("app_id", "track", IdSource::MachineIdHashed) - .unwrap() - .with_event(OmahaEvent::new( - OmahaEventType::EventUpdateInstalled, - EventResult::Success, - )), - ); - - let err = send_event(&url, &request).unwrap_err(); - assert_eq!( - err, - HarpoonError::InvalidResponse("Expected 1 events for app 'app_id', got 0".into()) - ); - - omaha_mock.assert(); - } - - #[test] - fn test_send_event_unacknowledged() { - // Request a new server from the pool - let mut server = mockito::Server::new(); - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .with_body(indoc::indoc! {r#" - - - - - - - "#}) - .expect(1) - .create(); - - let url = Url::parse(&server.url()).unwrap(); - let request = Request::default().with_app( - AppRequest::new_event("app_id", "track", IdSource::MachineIdHashed) - .unwrap() - .with_event(OmahaEvent::new( - OmahaEventType::EventUpdateInstalled, - EventResult::Success, - )), - ); - - let err = send_event(&url, &request).unwrap_err(); - assert_eq!( - err, - HarpoonError::EventNotAcknowledged( - OmahaEventType::EventUpdateInstalled, - EventResult::Success - ) - ); - - omaha_mock.assert(); - } -} diff --git a/crates/trident-acl-agent/src/omaha/request.rs b/crates/trident-acl-agent/src/omaha/request.rs deleted file mode 100644 index 508d115b0a..0000000000 --- a/crates/trident-acl-agent/src/omaha/request.rs +++ /dev/null @@ -1,363 +0,0 @@ -use quick_xml::{ - events::{BytesDecl, Event}, - Writer, -}; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use osutils::osrelease::OsRelease; -use sysdefs::arch::SystemArchitecture; - -use crate::{error::HarpoonError, IdSource}; - -use super::{ - app::AppVersion, event::OmahaEvent, OMAHA_VERSION, XML_HEADER_ENCODING, XML_HEADER_VERSION, -}; - -#[derive(Debug, Serialize)] -pub(crate) struct Request { - #[serde(rename = "@protocol")] - protocol: &'static str, - - #[serde(rename = "@version")] - version: &'static str, - - #[serde(rename = "@ismachine", serialize_with = "bool2num")] - is_machine: bool, - - #[serde(rename = "@sessionid")] - session_id: Uuid, - - #[serde(rename = "hw")] - hw: HwData, - - #[serde(rename = "os")] - os: OsData, - - #[serde(rename = "app")] - apps: Vec, -} - -fn bool2num(value: &bool, serializer: S) -> Result -where - S: serde::Serializer, -{ - serializer.serialize_str(if *value { "1" } else { "0" }) -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct HwData {} - -#[derive(Debug, Serialize, Deserialize)] -pub(crate) struct OsData { - #[serde(rename = "@arch")] - architecture: &'static str, - - #[serde(rename = "@version", skip_serializing_if = "Option::is_none")] - version: Option, - - #[serde(rename = "@platform")] - platform: &'static str, -} - -impl Default for Request { - fn default() -> Self { - Self { - protocol: OMAHA_VERSION, - version: env!("CARGO_PKG_VERSION"), - is_machine: true, - session_id: Uuid::new_v4(), - hw: HwData {}, - os: OsData { - platform: "linux", - version: OsRelease::read().unwrap_or_default().version, - architecture: match SystemArchitecture::current() { - SystemArchitecture::Amd64 => "amd64", - SystemArchitecture::Aarch64 => "arm64", - }, - }, - apps: Vec::new(), - } - } -} - -impl Request { - #[allow(dead_code)] - pub(crate) fn new_with_session_id(session_id: Uuid) -> Self { - Self { - session_id, - ..Default::default() - } - } - - pub(crate) fn to_xml(&self) -> Result, quick_xml::SeError> { - let mut data = Vec::new(); - let mut writer = Writer::new(&mut data); - writer.write_event(Event::Decl(BytesDecl::new( - XML_HEADER_VERSION, - Some(XML_HEADER_ENCODING), - None, - )))?; - writer.write_serializable("request", self)?; - Ok(data) - } - - pub(crate) fn session_id(&self) -> Uuid { - self.session_id - } - - pub(crate) fn with_app(mut self, app: AppRequest) -> Self { - self.apps.push(app); - self - } - - pub(crate) fn apps(&self) -> &[AppRequest] { - &self.apps - } -} - -#[derive(Debug, Serialize, PartialEq, Eq)] -pub(crate) struct AppRequest { - #[serde(rename = "@appid")] - app_id: String, - - #[serde(rename = "@version")] - version: AppVersion, - - #[serde(rename = "@nextversion", skip_serializing_if = "Option::is_none")] - next_version: Option, - - #[serde(rename = "@track")] - track: String, - - #[serde(rename = "@machineid")] - machine_id: String, - - #[serde(rename = "updatecheck", skip_serializing_if = "Option::is_none")] - update_check: Option, - - #[serde(rename = "event", skip_serializing_if = "Vec::is_empty")] - events: Vec, -} - -impl AppRequest { - /// Creates a new `AppRequest` with the given `app_id` to be used to send - /// update events to the server, and the given `machine_id_source` to - /// determine the machine ID. - pub(crate) fn new_event( - app_id: impl Into, - track: impl Into, - machine_id_source: IdSource, - ) -> Result { - Self::new(app_id, AppVersion::default(), track, machine_id_source) - } - - pub(crate) fn new( - app_id: impl Into, - version: impl Into, - track: impl Into, - machine_id_source: IdSource, - ) -> Result { - Ok(Self::new_with_machine_id( - app_id, - version, - track, - machine_id_source.produce_id()?, - )) - } - - pub(crate) fn new_with_machine_id( - app_id: impl Into, - version: impl Into, - track: impl Into, - machine_id: String, - ) -> Self { - Self { - app_id: app_id.into(), - version: version.into(), - next_version: None, - track: track.into(), - machine_id, - update_check: None, - events: Vec::new(), - } - } - - #[allow(dead_code)] - pub(crate) fn with_next_version(mut self, next_version: impl Into) -> Self { - self.next_version = Some(next_version.into()); - self - } - - pub(crate) fn with_update_check(mut self) -> Self { - self.update_check = Some(UpdateCheckRequest); - self - } - - pub(crate) fn with_event(mut self, event: OmahaEvent) -> Self { - self.events.push(event); - self - } - - pub(crate) fn events(&self) -> &[OmahaEvent] { - &self.events - } - - pub(crate) fn app_id(&self) -> &str { - &self.app_id - } -} - -#[derive(Debug, Serialize, PartialEq, Eq)] -pub(crate) struct UpdateCheckRequest; - -#[cfg(test)] -mod tests { - use super::*; - - use osutils::machine_id::MachineId; - - use crate::{omaha::event::OmahaEventType, EventResult}; - - #[test] - fn test_bool2num() { - let mut serializer = serde_json::Serializer::new(Vec::new()); - bool2num(&true, &mut serializer).unwrap(); - assert_eq!(serializer.into_inner(), "\"1\"".as_bytes()); - - let mut serializer = serde_json::Serializer::new(Vec::new()); - bool2num(&false, &mut serializer).unwrap(); - assert_eq!(serializer.into_inner(), "\"0\"".as_bytes()); - } - - #[test] - fn test_request_default() { - let request = Request::default(); - assert_eq!(request.protocol, OMAHA_VERSION); - assert_eq!(request.version, env!("CARGO_PKG_VERSION")); - assert!(request.is_machine); - assert_eq!(request.hw, HwData {}); - assert_eq!(request.os.platform, "linux"); - assert_eq!( - request.os.version, - OsRelease::read().unwrap_or_default().version - ); - assert_eq!( - request.os.architecture, - match SystemArchitecture::current() { - SystemArchitecture::Amd64 => "amd64", - SystemArchitecture::Aarch64 => "arm64", - } - ); - assert_eq!(request.apps(), &[]); - } - - #[test] - fn test_request_new_with_session_id() { - let session_id = Uuid::new_v4(); - let request = Request::new_with_session_id(session_id); - assert_eq!(request.session_id(), session_id); - } - - #[test] - fn test_request_with_app() { - let app = AppRequest::new( - "app_id", - AppVersion::default(), - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - let request = Request::default().with_app(app); - assert_eq!(request.apps().len(), 1); - } - - #[test] - fn test_app_request_new() { - let app = AppRequest::new( - "app_id", - AppVersion::default(), - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - assert_eq!(app.app_id(), "app_id"); - assert_eq!(app.version, AppVersion::default()); - assert_eq!(app.next_version, None); - assert_eq!(app.track, "track"); - assert_eq!( - app.machine_id, - MachineId::read().unwrap().hashed_uuid().to_string() - ); - assert_eq!(app.update_check, None); - assert_eq!(app.events, Vec::new()); - } - - #[test] - fn test_app_request_new_with_machine_id() { - let machine_id = Uuid::new_v4().to_string(); - let app = AppRequest::new_with_machine_id( - "app_id", - AppVersion::default(), - "track", - machine_id.clone(), - ); - assert_eq!(app.machine_id, machine_id); - } - - #[test] - fn test_app_request_with_next_version() { - let app = AppRequest::new( - "app_id", - AppVersion::default(), - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - let next_version = AppVersion::default(); - let app = app.with_next_version(next_version.clone()); - assert_eq!(app.next_version, Some(next_version)); - } - - #[test] - fn test_app_request_with_update_check() { - let app = AppRequest::new( - "app_id", - AppVersion::default(), - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - let app = app.with_update_check(); - assert_eq!(app.update_check, Some(UpdateCheckRequest)); - } - - #[test] - fn test_app_request_with_event() { - let app = AppRequest::new( - "app_id", - AppVersion::default(), - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - let event = OmahaEvent::new(OmahaEventType::Unknown, EventResult::Error); - let app = app.with_event(event); - assert_eq!(app.events().len(), 1); - } - - #[test] - fn test_app_new_event() { - let app = AppRequest::new_event("app_id", "track", IdSource::MachineIdHashed).unwrap(); - assert_eq!(app.app_id(), "app_id"); - assert_eq!(app.version, AppVersion::default()); - assert_eq!(app.next_version, None); - assert_eq!(app.track, "track"); - assert_eq!( - app.machine_id, - MachineId::read().unwrap().hashed_uuid().to_string() - ); - assert_eq!(app.update_check, None); - assert_eq!(app.events, Vec::new()); - } -} diff --git a/crates/trident-acl-agent/src/omaha/response.rs b/crates/trident-acl-agent/src/omaha/response.rs deleted file mode 100644 index ef171b2f48..0000000000 --- a/crates/trident-acl-agent/src/omaha/response.rs +++ /dev/null @@ -1,282 +0,0 @@ -use serde::Deserialize; -use url::Url; - -use crate::{def_unwrap_list, error::HarpoonError}; - -use super::{ - app::AppVersion, - event::EventAcknowledge, - status::{AppStatus, UpdateCheckStatus}, - OMAHA_VERSION, -}; - -#[derive(Debug, Deserialize)] -pub(crate) struct Response { - #[serde(rename = "@protocol")] - protocol: String, - - #[serde(rename = "@server")] - _server: String, - - #[serde(rename = "daystart")] - _daystart: Daystart, - - #[serde(default, rename = "app")] - apps: Vec, -} - -impl Response { - pub(crate) fn validate(&self) -> Result<(), HarpoonError> { - if self.protocol != OMAHA_VERSION { - return Err(HarpoonError::InvalidResponse(format!( - "Invalid Omaha version '{}', expected '{}'", - self.protocol, OMAHA_VERSION - ))); - } - - Ok(()) - } - - pub(crate) fn apps(&self) -> &[AppResponse] { - &self.apps - } -} - -#[derive(Debug, Deserialize)] -pub(crate) struct AppResponse { - #[serde(rename = "@appid")] - app_id: String, - - #[serde(rename = "@status")] - status: AppStatus, - - #[serde(default, rename = "updatecheck")] - update_check: Option, - - #[serde(default, rename = "event")] - events: Vec, -} - -impl AppResponse { - pub(crate) fn app_id(&self) -> &str { - &self.app_id - } - - pub(crate) fn status(&self) -> &AppStatus { - &self.status - } - - pub(crate) fn update_check(&self) -> Option<&UpdateCheckResponse> { - self.update_check.as_ref() - } - - pub(crate) fn events(&self) -> &[EventAcknowledge] { - &self.events - } -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Daystart { - #[serde(rename = "@elapsed_seconds")] - pub(crate) elapsed_seconds: u64, -} - -#[derive(Debug, Deserialize)] -pub(crate) struct UpdateCheckResponse { - #[serde(rename = "@status")] - status: UpdateCheckStatus, - - #[serde(rename = "urls", deserialize_with = "unwrap_urls")] - urls: Vec, - - #[serde(rename = "manifest")] - manifest: Option, -} - -impl UpdateCheckResponse { - pub(crate) fn status(&self) -> &UpdateCheckStatus { - &self.status - } - - pub(crate) fn urls(&self) -> impl Iterator { - self.urls.iter().map(|url| &url.codebase) - } - - pub(crate) fn version(&self) -> Option<&AppVersion> { - self.manifest.as_ref().map(|m| &m.version) - } - - pub(crate) fn packages(&self) -> &[Package] { - self.manifest.as_ref().map_or(&[], |m| &m.packages) - } -} - -#[derive(Debug, Deserialize)] -struct DownloadUrl { - #[serde(rename = "@codebase")] - codebase: Url, -} - -def_unwrap_list!(unwrap_urls, DownloadUrl, "url"); - -#[derive(Debug, Deserialize)] -struct Manifest { - #[serde(rename = "@version")] - version: AppVersion, - #[serde(default, rename = "packages", deserialize_with = "unwrap_packages")] - packages: Vec, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Package { - #[serde(rename = "@hash")] - pub(crate) hash: String, - - #[serde(rename = "@hash_sha256")] - pub(crate) hash_sha256: Option, - - #[serde(rename = "@name")] - pub(crate) name: String, - - #[serde(rename = "@size")] - pub(crate) size: u64, - - #[serde(rename = "@required")] - pub(crate) required: bool, -} - -def_unwrap_list!(unwrap_packages, Package, "package"); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_validate_response() { - let mut response = Response { - protocol: OMAHA_VERSION.to_string(), - _server: "server".to_string(), - _daystart: Daystart { elapsed_seconds: 0 }, - apps: vec![], - }; - - response.validate().unwrap(); - - response.protocol = "invalid".to_string(); - response.validate().unwrap_err(); - } - - #[test] - fn test_parse_simple() { - let response = indoc::indoc! {r#" - - - - - - - "# - }; - - let response: Response = quick_xml::de::from_str(response).unwrap(); - assert_eq!(response.protocol, OMAHA_VERSION); - assert_eq!(response._server, "nebraska"); - assert_eq!(response._daystart.elapsed_seconds, 0); - assert_eq!(response.apps.len(), 1); - assert_eq!(response.apps[0].app_id(), "com.microsoft.azurelinux"); - assert_eq!(response.apps[0].status(), &AppStatus::Ok); - } - - #[test] - fn test_parse_update_check_noupdate() { - let response = indoc::indoc! {r#" - - - - - - - - - - "# - }; - let response: Response = quick_xml::de::from_str(response).unwrap(); - let app = &response.apps[0]; - let update_check = app.update_check().unwrap(); - assert_eq!(update_check.status(), &UpdateCheckStatus::NoUpdate); - assert_eq!(update_check.urls().count(), 0); - assert!(update_check.manifest.is_none()); - } - - #[test] - fn test_parse_update_check_update() { - let response = indoc::indoc! {r#" - - - - - - - - - - - - - - - - - "#}; - let response: Response = quick_xml::de::from_str(response).unwrap(); - let app = &response.apps[0]; - let update_check = app.update_check().unwrap(); - assert_eq!(update_check.status, UpdateCheckStatus::Ok); - assert_eq!(update_check.urls.len(), 1); - assert_eq!( - update_check.urls[0].codebase, - Url::parse("https://example.com/").unwrap() - ); - assert_eq!(update_check.version().unwrap(), &AppVersion::new(2, 0, 2)); - assert_eq!(update_check.packages().len(), 1); - let package = &update_check.packages()[0]; - assert_eq!(package.hash, "hash"); - assert_eq!(package.hash_sha256.as_deref(), Some("hash_sha256")); - assert_eq!(package.name, "package"); - assert_eq!(package.size, 123); - assert!(package.required); - } - - #[test] - fn test_parse_event_response() { - let response = indoc::indoc! {r#" - - - - - - - - "#}; - let response: Response = quick_xml::de::from_str(response).unwrap(); - let app = &response.apps()[0]; - let events = app.events(); - assert_eq!(events.len(), 1); - assert!(events[0].is_ok()); - } -} diff --git a/crates/trident-acl-agent/src/omaha/status.rs b/crates/trident-acl-agent/src/omaha/status.rs deleted file mode 100644 index b8e1c576c0..0000000000 --- a/crates/trident-acl-agent/src/omaha/status.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::fmt::{self, Display, Formatter}; - -use serde::Deserialize; - -#[derive(Debug, Deserialize, PartialEq, Eq)] -pub(crate) enum AppStatus { - #[serde(rename = "ok")] - Ok, - - #[serde(rename = "restricted")] - Restricted, - - #[serde(rename = "error-unknownApplication")] - ErrorUnknownApplication, - - #[serde(rename = "error-invalidAppId")] - ErrorInvalidAppId, - - #[serde(untagged)] - Other(String), -} - -impl AppStatus { - pub(crate) fn is_error(&self) -> bool { - !matches!(self, AppStatus::Ok) - } -} - -impl Display for AppStatus { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - AppStatus::Ok => write!(f, "ok"), - AppStatus::Restricted => write!(f, "restricted"), - AppStatus::ErrorUnknownApplication => write!(f, "error-unknownApplication"), - AppStatus::ErrorInvalidAppId => write!(f, "error-invalidAppId"), - AppStatus::Other(other) => write!(f, "other: {other}"), - } - } -} - -#[derive(Debug, Deserialize, PartialEq, Eq)] -pub(crate) enum UpdateCheckStatus { - #[serde(rename = "noupdate")] - NoUpdate, - - #[serde(rename = "ok")] - Ok, - - #[serde(rename = "error-osnotsupported")] - ErrorOsNotSupported, - - #[serde(rename = "error-unsupportedProtocol")] - ErrorUnsupportedProtocol, - - #[serde(rename = "error-pluginRestrictedHost")] - ErrorPluginRestrictedHost, - - #[serde(rename = "error-hash")] - ErrorHash, - - #[serde(rename = "error-internal")] - ErrorInternal, - - #[serde(untagged)] - Other(String), -} - -impl UpdateCheckStatus { - pub(crate) fn is_error(&self) -> bool { - !matches!(self, UpdateCheckStatus::NoUpdate | UpdateCheckStatus::Ok) - } - - pub(crate) fn is_no_update(&self) -> bool { - matches!(self, UpdateCheckStatus::NoUpdate) - } -} - -impl Display for UpdateCheckStatus { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - UpdateCheckStatus::NoUpdate => write!(f, "noupdate"), - UpdateCheckStatus::Ok => write!(f, "ok"), - UpdateCheckStatus::ErrorOsNotSupported => write!(f, "error-osnotsupported"), - UpdateCheckStatus::ErrorUnsupportedProtocol => write!(f, "error-unsupportedProtocol"), - UpdateCheckStatus::ErrorPluginRestrictedHost => write!(f, "error-pluginRestrictedHost"), - UpdateCheckStatus::ErrorHash => write!(f, "error-hash"), - UpdateCheckStatus::ErrorInternal => write!(f, "error-internal"), - UpdateCheckStatus::Other(other) => write!(f, "other: {other}"), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_app_status_display() { - assert_eq!(AppStatus::Ok.to_string(), "ok"); - assert_eq!(AppStatus::Restricted.to_string(), "restricted"); - assert_eq!( - AppStatus::ErrorUnknownApplication.to_string(), - "error-unknownApplication" - ); - assert_eq!( - AppStatus::ErrorInvalidAppId.to_string(), - "error-invalidAppId" - ); - assert_eq!( - AppStatus::Other("other".to_string()).to_string(), - "other: other" - ); - } - - #[test] - fn test_update_check_status_display() { - assert_eq!(UpdateCheckStatus::NoUpdate.to_string(), "noupdate"); - assert_eq!(UpdateCheckStatus::Ok.to_string(), "ok"); - assert_eq!( - UpdateCheckStatus::ErrorOsNotSupported.to_string(), - "error-osnotsupported" - ); - assert_eq!( - UpdateCheckStatus::ErrorUnsupportedProtocol.to_string(), - "error-unsupportedProtocol" - ); - assert_eq!( - UpdateCheckStatus::ErrorPluginRestrictedHost.to_string(), - "error-pluginRestrictedHost" - ); - assert_eq!(UpdateCheckStatus::ErrorHash.to_string(), "error-hash"); - assert_eq!( - UpdateCheckStatus::ErrorInternal.to_string(), - "error-internal" - ); - assert_eq!( - UpdateCheckStatus::Other("other".to_string()).to_string(), - "other: other" - ); - } - - #[test] - fn test_flag_methods() { - assert!(!AppStatus::Ok.is_error()); - assert!(AppStatus::Restricted.is_error()); - assert!(AppStatus::ErrorUnknownApplication.is_error()); - assert!(AppStatus::ErrorInvalidAppId.is_error()); - assert!(AppStatus::Other("other".to_string()).is_error()); - - assert!(UpdateCheckStatus::NoUpdate.is_no_update()); - assert!(!UpdateCheckStatus::Ok.is_no_update()); - assert!(!UpdateCheckStatus::ErrorOsNotSupported.is_no_update()); - assert!(!UpdateCheckStatus::ErrorUnsupportedProtocol.is_no_update()); - assert!(!UpdateCheckStatus::ErrorPluginRestrictedHost.is_no_update()); - assert!(!UpdateCheckStatus::ErrorHash.is_no_update()); - assert!(!UpdateCheckStatus::ErrorInternal.is_no_update()); - assert!(!UpdateCheckStatus::Other("other".to_string()).is_no_update()); - - assert!(!UpdateCheckStatus::NoUpdate.is_error()); - assert!(!UpdateCheckStatus::Ok.is_error()); - assert!(UpdateCheckStatus::ErrorOsNotSupported.is_error()); - assert!(UpdateCheckStatus::ErrorUnsupportedProtocol.is_error()); - assert!(UpdateCheckStatus::ErrorPluginRestrictedHost.is_error()); - assert!(UpdateCheckStatus::ErrorHash.is_error()); - assert!(UpdateCheckStatus::ErrorInternal.is_error()); - assert!(UpdateCheckStatus::Other("other".to_string()).is_error()); - } -} diff --git a/crates/trident-acl-agent/src/omaha/xml.rs b/crates/trident-acl-agent/src/omaha/xml.rs deleted file mode 100644 index 4f1c1a7488..0000000000 --- a/crates/trident-acl-agent/src/omaha/xml.rs +++ /dev/null @@ -1,18 +0,0 @@ -#[macro_export] -macro_rules! def_unwrap_list { - ($fnname:ident, $name:ident, $xmlname:expr) => { - fn $fnname<'de, D>(deserializer: D) -> Result, D::Error> - where - D: serde::Deserializer<'de>, - { - /// Represents ... - #[derive(serde::Deserialize)] - struct List { - // default allows empty list - #[serde(default, rename = $xmlname)] - element: Vec<$name>, - } - Ok(List::deserialize(deserializer)?.element) - } - }; -} diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs new file mode 100644 index 0000000000..86d0fe5d7a --- /dev/null +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -0,0 +1,2270 @@ +//! The Trident ACL agent's reconcile loop: watches the Node's request +//! annotation, drives Trident (stage/finalize/rollback/commit) over gRPC, +//! and writes the status annotation back, including post-reboot. +//! +//! Implements the node-side control flow from `docs/update-trigger-design.md`: +//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md +//! (sections 2.1 "Trigger mechanism", 2.3 "Stage/finalize/rollback split +//! and post-reboot commit", and 2.5 "Rollback"). See that document for the +//! full state-machine rationale; keep it in sync with this file if the +//! design changes. + +use std::{collections::BTreeMap, future::Future}; + +use anyhow::Context; +use chrono::{DateTime, Utc}; +use futures::StreamExt; +use k8s_openapi::api::core::v1::Node; +use semver::Version; +use trident_proto::v1::{RebootStatus, ServicingKind}; +use url::Url; +use uuid::Uuid; + +use osutils::dependencies::Dependency; + +use crate::{ + annotations::{ + current_active_version, Operation, RequestedOperation, StatusCode, UpdateRequest, + UpdateStatus, SCHEMA_VERSION, UPDATE_COMMIT_STATUS_ANNOTATION, UPDATE_REQUEST_ANNOTATION, + UPDATE_STATUS_ANNOTATION, + }, + config::AgentConfig, + k8s::{K8sClientError, NodeClient}, + nebraska::{CheckOutcome, Client as NebraskaClient, ProgressEvent}, + state::{PendingCommit, StateStore}, + trident::{CompletedResponse, TridentClient, TridentClientError}, + IdSource, +}; + +const FINAL_STATUS_PATCH_RETRIES: usize = 3; +const FINAL_STATUS_PATCH_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); + +/// The machine-id source used for every Nebraska request this module makes, +/// event reports included. Must match the source used by `handle_stage`'s +/// initial `check_for_update` so all requests for a given node present the +/// same instance identity to Nebraska. +const NEBRASKA_MACHINE_ID_SOURCE: IdSource = IdSource::MachineIdHashed; + +/// A single Nebraska event report to send, decoupled from the async +/// machinery that sends it (see `Orchestrator::report_nebraska_event`) so +/// the "what to report" decision can be made by plain, unit-testable +/// functions (`stage_nebraska_report`, `finalize_nebraska_report`, +/// `commit_nebraska_report` below). +#[derive(Debug, Clone, PartialEq, Eq)] +enum NebraskaReport { + /// An in-flight progress event; see `nebraska::ProgressEvent`. Sending + /// one commits the instance to eventually reporting a terminal event + /// (`Completed` or `Failed`) too. + Progress { + version: Version, + event: ProgressEvent, + }, + /// The terminal "success" event, sent after a reboot onto the new + /// version. + Completed { previous: Version, current: Version }, + /// The terminal "failure" event. Clears Nebraska's `update_in_progress` + /// for the instance so a later check can grant an update again - + /// required to avoid permanently wedging the instance after a progress + /// event was already sent. + Failed { previous: Version, current: Version }, +} + +impl NebraskaReport { + /// A short label for logging. + fn label(&self) -> &'static str { + match self { + NebraskaReport::Progress { event, .. } => event.label(), + NebraskaReport::Completed { .. } => "completed", + NebraskaReport::Failed { .. } => "failed", + } + } +} + +#[derive(Clone, Default)] +pub struct SystemRebooter; + +pub trait RebootHandle: Clone + Send + Sync + 'static { + fn reboot(&self) -> Result<(), anyhow::Error>; +} + +impl RebootHandle for SystemRebooter { + fn reboot(&self) -> Result<(), anyhow::Error> { + // Route through the repo's centralized dependency runner so a + // missing systemctl binary or non-zero exit produces the same + // uniform, actionable error type used everywhere else in the + // codebase (see crates/trident/src/reboot.rs for the same pattern). + Dependency::Systemctl + .cmd() + .arg("reboot") + .run_and_check() + .map_err(|err| anyhow::anyhow!("failed to issue systemctl reboot: {err}")) + } +} + +pub struct Orchestrator { + config: AgentConfig, + k8s: NodeClient, + rebooter: R, + state: StateStore, +} + +impl Orchestrator { + pub async fn from_config(config: AgentConfig) -> Result { + let k8s = NodeClient::new(&config.kubernetes).await?; + Ok(Self { + state: StateStore::new(config.orchestration.state_path.clone()), + config, + k8s, + rebooter: SystemRebooter, + }) + } +} + +impl Orchestrator +where + R: RebootHandle, +{ + pub async fn run(&self) -> Result<(), anyhow::Error> { + if let Err(err) = self.recover_from_trident_state().await { + if self.log_and_swallow_node_gone(&err, "recovering persisted state") { + return Ok(()); + } + return Err(err); + } + let mut stream = self + .k8s + .watch_node(self.config.kubernetes.node_name.clone()); + while let Some(node) = stream.next().await { + let node = node?; + match self.reconcile_node(&node).await { + Ok(LoopControl::Continue) => {} + Ok(LoopControl::ExitForReboot) => return Ok(()), + Err(err) => { + if self.log_and_swallow_node_gone(&err, "reconciling node") { + return Ok(()); + } + return Err(err); + } + } + } + Ok(()) + } + + async fn recover_from_trident_state(&self) -> Result<(), anyhow::Error> { + let node = self.k8s.get_node(&self.config.kubernetes.node_name).await?; + let snapshot = Snapshot::from_node(&node); + let persisted = self.state.load()?; + + if let Some(pending) = persisted.pending_commit.clone() { + return self.resume_pending_commit(pending).await; + } + + if let Some(request) = snapshot.request.clone() { + if let Some(entry) = persisted.completed.get(&request.operation_id) { + if let Some(commit) = entry.commit.clone() { + let matches = snapshot + .commit_status + .as_ref() + .is_some_and(|current| current.same_content(&commit)); + if !matches { + self.publish_status(&commit).await?; + } + } + if let Some(operation) = entry.operation.clone() { + let matches = snapshot + .operation_status + .as_ref() + .is_some_and(|current| current.same_content(&operation)); + if !matches { + self.publish_status(&operation).await?; + } + return Ok(()); + } + } + } + + if let Some(request) = snapshot.request { + if matches!( + request.operation, + RequestedOperation::Finalize | RequestedOperation::Rollback + ) { + let status = self.reconstruct_without_state(&request, None, None).await; + self.record_and_publish(status).await?; + } + } + Ok(()) + } + + async fn reconcile_node(&self, node: &Node) -> Result { + let snapshot = Snapshot::from_node(node); + log::debug!( + "received node update: request={:?} operation_status={:?} commit_status={:?}", + snapshot.request, + snapshot.operation_status, + snapshot.commit_status + ); + let persisted = self.state.load()?; + + if let Some(invalid) = snapshot.invalid_request.clone() { + // Dedupe the same way the completed-status cache above does: + // only publish once per operationId, so a persistently invalid + // annotation doesn't re-PATCH on every reconcile. + if !persisted.completed.contains_key(&invalid.operation_id) { + let now = Utc::now(); + let status = UpdateStatus { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: invalid.node_update_id, + operation_id: invalid.operation_id, + operation: invalid.operation, + code: StatusCode::InvalidRequest, + message: invalid.reason, + from_version: None, + to_version: None, + started_utc: now, + last_updated_utc: now, + finished_utc: Some(now), + }; + self.record_and_publish(status).await?; + } + return Ok(LoopControl::Continue); + } + + let Some(request) = snapshot.request.clone() else { + return Ok(LoopControl::Continue); + }; + + let cached = persisted + .completed + .get(&request.operation_id) + .and_then(|entry| entry.operation.clone()); + if let Some(status) = cached { + let matches = snapshot + .operation_status + .as_ref() + .is_some_and(|current| current.same_content(&status)); + if !matches { + self.publish_status(&status).await?; + } + return Ok(LoopControl::Continue); + } + + if let Some(pending) = persisted.pending_commit.as_ref() { + // Reject on operationId, not nodeUpdateId: the actual conflict + // this guard exists to prevent is "a second finalize/rollback + // starts while one is still waiting for its post-reboot + // commit" (accepted-design-v2.md's in-flight conflict rule). + // Keying on nodeUpdateId alone let a retried/re-issued request + // that reused the same nodeUpdateId but a new operationId slip + // through this guard entirely and re-enter handle_finalize/ + // handle_rollback concurrently with the still-outstanding + // original operation. + if request.operation_id != pending.request.operation_id { + let started = Utc::now(); + let status = UpdateStatus::new( + &request, + request.operation.into(), + request.operation_id.clone(), + StatusCode::InvalidRequest, + format!( + "another finalize/rollback (operationId {}) is waiting for post-reboot commit", + pending.request.operation_id + ), + pending.from_version.clone(), + pending.to_version.clone(), + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + } + + match request.operation { + RequestedOperation::Stage => { + self.handle_stage(request).await?; + Ok(LoopControl::Continue) + } + RequestedOperation::Finalize => self.handle_finalize(request).await, + RequestedOperation::Rollback => self.handle_rollback(request).await, + } + } + + /// Resolves which Nebraska endpoint to use for `request`: the request + /// annotation's own `server` override, if present, otherwise the + /// agent's configured `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` (or CLI + /// override). Every Nebraska call this `nodeUpdateId` makes - stage's + /// update check plus every progress/completion event report - must go + /// through this resolver rather than reading `self.config.nebraska.endpoint` + /// directly, since Nebraska's per-instance state is tied to one specific + /// server: mixing endpoints across one update's lifecycle would split + /// that state across two servers. + fn resolve_nebraska_endpoint(&self, request: &UpdateRequest) -> Option { + request + .server + .clone() + .or_else(|| self.config.nebraska.endpoint.clone()) + } + + /// Resolves which Nebraska app id to use for `request`: the request + /// annotation's own `appId` override, if present, otherwise the agent's + /// configured `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`. Unlike + /// [`resolve_nebraska_endpoint`], this always resolves to a value - + /// `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` always has one (defaulting to + /// [`crate::DEFAULT_NEBRASKA_APP_ID`]) - so there is no error case to + /// handle at call sites. + fn resolve_nebraska_app_id(&self, request: &UpdateRequest) -> String { + request + .app_id + .clone() + .unwrap_or_else(|| self.config.nebraska.app_id.clone()) + } + + /// Resolves which Nebraska track to use for `request`: the request + /// annotation's own `track` override, if present, otherwise the agent's + /// configured `TRIDENT_ACL_AGENT_NEBRASKA_TRACK`. Same always-resolves + /// behavior as [`resolve_nebraska_app_id`] - + /// `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` always has a default + /// ([`crate::DEFAULT_NEBRASKA_TRACK`]) - so there is no error case here + /// either. + fn resolve_nebraska_track(&self, request: &UpdateRequest) -> String { + request + .track + .clone() + .unwrap_or_else(|| self.config.nebraska.track.clone()) + } + + async fn handle_stage(&self, request: UpdateRequest) -> Result<(), anyhow::Error> { + let started = Utc::now(); + let from_version = Some(current_active_version()); + let to_version = request.target_version.clone(); + if from_version == to_version { + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::AlreadyAtTarget, + "node already running requested target version", + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(()); + } + + let in_progress = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::InProgress, + "staging update", + from_version.clone(), + to_version.clone(), + started, + None, + ); + self.publish_status(&in_progress).await?; + let endpoint = self.resolve_nebraska_endpoint(&request).ok_or_else(|| { + anyhow::anyhow!( + "annotation mode requires request.server, TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT, or CLI override" + ) + })?; + let app_id = self.resolve_nebraska_app_id(&request); + let track = self.resolve_nebraska_track(&request); + let machine_id = crate::build_machine_id(IdSource::MachineIdHashed)?; + let outcome = tokio::task::spawn_blocking(move || { + let client = NebraskaClient::new(endpoint, app_id, track, machine_id); + client.check_for_update(&Version::new(0, 0, 0)) + }) + .await + .context("Nebraska query task panicked")? + .map_err(|err| anyhow::anyhow!("Nebraska query failed: {err}"))?; + let offered = match outcome { + CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::OperationFailed, + "Nebraska currently offers no update for the requested target", + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(()); + } + CheckOutcome::UpdateAvailable(offer) => offer, + }; + if request.target_version.as_deref() != Some(offered.version.to_string().as_str()) { + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::InvalidRequest, + format!( + "requested target version {:?} but Nebraska offers {}", + request.target_version, offered.version + ), + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(()); + } + + let current_ver = parse_nebraska_version(&from_version, "stage"); + if let Some(ref v) = current_ver { + self.report_nebraska_event( + &request, + NebraskaReport::Progress { + version: v.clone(), + event: ProgressEvent::DownloadStarted, + }, + ) + .await; + } + + let mut client = TridentClient::connect(&self.config.trident.socket).await?; + // Integrity of the downloaded image is verified by Trident itself + // via the image's own COSI metadata, so the Nebraska-reported hash + // (offered.primary.hash) is not passed here. + let result = self + .run_with_status_heartbeat( + in_progress, + client.update_stage( + &offered.primary.url, + None, + self.config.orchestration.stage_timeout, + ), + ) + .await; + if let Some(ref v) = current_ver { + self.report_nebraska_event(&request, stage_nebraska_report(v, &result)) + .await; + } + let status = stage_result_to_status(&request, from_version, to_version, started, result); + self.record_and_publish(status).await + } + + async fn handle_finalize(&self, request: UpdateRequest) -> Result { + let started = Utc::now(); + let from_version = Some(current_active_version()); + let to_version = request.target_version.clone(); + if from_version == to_version { + let status = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::AlreadyAtTarget, + "node already running requested target version", + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + + let completed = self.state.load()?.completed; + let staged = completed + .values() + .filter_map(|entry| entry.operation.as_ref()) + .find(|status| { + status.node_update_id == request.node_update_id + && status.operation == Operation::Stage + && matches!( + status.code, + StatusCode::Success | StatusCode::AlreadyAtTarget + ) + }); + let staged = match staged { + None => { + let status = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::NotStaged, + "finalize requested without prior successful stage for nodeUpdateId", + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + Some(staged) => staged, + }; + if staged.to_version != to_version { + let status = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::InvalidRequest, + format!( + "finalize targetVersion {:?} does not match the version staged for this nodeUpdateId ({:?})", + to_version, staged.to_version + ), + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + + let in_progress = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::InProgress, + "finalizing update", + from_version.clone(), + to_version.clone(), + started, + None, + ); + self.publish_status(&in_progress).await?; + let current_ver = parse_nebraska_version(&from_version, "finalize"); + let mut client = TridentClient::connect(&self.config.trident.socket).await?; + let result = self + .run_with_status_heartbeat( + in_progress, + client.update_finalize(self.config.orchestration.finalize_timeout), + ) + .await; + if let Some(ref v) = current_ver { + self.report_nebraska_event(&request, finalize_nebraska_report(v, &result)) + .await; + } + match result { + Ok(_) => { + let boot_marker = current_boot_marker()?; + self.state.set_pending_commit(PendingCommit { + request: request.clone(), + operation_id: request.operation_id.clone(), + operation: Operation::Finalize, + from_version: from_version.clone(), + to_version: to_version.clone(), + started_utc: started, + boot_marker, + })?; + let terminal = finalize_success_status( + &request, + from_version.clone(), + to_version.clone(), + started, + ); + if let Err(err) = self.state.remember_completed(terminal.clone()) { + log::warn!("failed to record finalize completion in state.json: {err}"); + } + self.best_effort_publish_terminal(&terminal).await; + match self.rebooter.reboot() { + Ok(()) => Ok(LoopControl::ExitForReboot), + Err(err) => { + self.state.clear_pending_commit()?; + if let Some(ref v) = current_ver { + self.report_nebraska_event( + &request, + NebraskaReport::Failed { + previous: v.clone(), + current: v.clone(), + }, + ) + .await; + } + let status = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::AgentInternalError, + format!("finalize succeeded but reboot failed: {err}"), + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + Ok(LoopControl::Continue) + } + } + } + Err(err) => { + self.state.clear_pending_commit()?; + let status = + finalize_failure_status(&request, from_version, to_version, started, &err); + self.record_and_publish(status).await?; + Ok(LoopControl::Continue) + } + } + } + + async fn handle_rollback(&self, request: UpdateRequest) -> Result { + let started = Utc::now(); + let from_version = Some(current_active_version()); + + let mut client = TridentClient::connect(&self.config.trident.socket).await?; + + let staging = UpdateStatus::new( + &request, + Operation::Rollback, + request.operation_id.clone(), + StatusCode::InProgress, + "staging rollback", + from_version.clone(), + None, + started, + None, + ); + self.publish_status(&staging).await?; + + let stage_response = match self + .run_with_status_heartbeat( + staging, + client.rollback_stage(self.config.orchestration.stage_timeout), + ) + .await + { + Ok(response) => response, + Err(err) => { + let status = rollback_stage_failure_status(&request, from_version, started, &err); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + }; + + if !matches!( + stage_response.servicing_kind, + Some(ServicingKind::ManualRollbackAb) + ) { + let status = UpdateStatus::new( + &request, + Operation::Rollback, + request.operation_id.clone(), + StatusCode::OperationFailed, + "no AB rollback available to perform for this node", + from_version, + None, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + + let finalizing = UpdateStatus::new( + &request, + Operation::Rollback, + request.operation_id.clone(), + StatusCode::InProgress, + "finalizing rollback", + from_version.clone(), + None, + started, + None, + ); + self.publish_status(&finalizing).await?; + match self + .run_with_status_heartbeat( + finalizing, + client.rollback_finalize(self.config.orchestration.finalize_timeout), + ) + .await + { + Ok(_) => { + let boot_marker = current_boot_marker()?; + self.state.set_pending_commit(PendingCommit { + request: request.clone(), + operation_id: request.operation_id.clone(), + operation: Operation::Rollback, + from_version: from_version.clone(), + to_version: None, + started_utc: started, + boot_marker, + })?; + let terminal = + rollback_finalize_success_status(&request, from_version.clone(), started); + if let Err(err) = self.state.remember_completed(terminal.clone()) { + log::warn!("failed to record rollback completion in state.json: {err}"); + } + self.best_effort_publish_terminal(&terminal).await; + match self.rebooter.reboot() { + Ok(()) => Ok(LoopControl::ExitForReboot), + Err(err) => { + self.state.clear_pending_commit()?; + let status = UpdateStatus::new( + &request, + Operation::Rollback, + request.operation_id.clone(), + StatusCode::AgentInternalError, + format!("rollback finalize succeeded but reboot failed: {err}"), + from_version, + None, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + Ok(LoopControl::Continue) + } + } + } + Err(err) => { + self.state.clear_pending_commit()?; + let status = + rollback_finalize_failure_status(&request, from_version, started, &err); + self.record_and_publish(status).await?; + Ok(LoopControl::Continue) + } + } + } + + async fn resume_pending_commit(&self, pending: PendingCommit) -> Result<(), anyhow::Error> { + let current_boot = current_boot_marker()?; + if current_boot == pending.boot_marker { + log::info!( + "pending commit {} is still waiting for the reboot to happen", + pending.operation_id + ); + return Ok(()); + } + + let mut client = match TridentClient::connect(&self.config.trident.socket).await { + Ok(client) => client, + Err(err) => { + let status = self + .reconstruct_without_state( + &pending.request, + pending.from_version.clone(), + Some(err.to_string()), + ) + .await; + self.state.clear_pending_commit()?; + self.record_and_publish(status).await?; + return Ok(()); + } + }; + let in_progress = UpdateStatus::new( + &pending.request, + Operation::Commit, + pending.operation_id.clone(), + StatusCode::InProgress, + "committing post-reboot state", + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + None, + ); + self.publish_status(&in_progress).await?; + let result = client + .commit(self.config.orchestration.finalize_timeout) + .await; + if pending.operation == Operation::Finalize { + if let (Some(previous), Some(current)) = ( + parse_nebraska_version(&pending.from_version, "post-reboot commit"), + parse_nebraska_version(&pending.to_version, "post-reboot commit"), + ) { + self.report_nebraska_event( + &pending.request, + commit_nebraska_report(&previous, ¤t, &result), + ) + .await; + } + } + let status = self.map_commit_result(&pending, result); + self.state.clear_pending_commit()?; + self.record_and_publish(status).await + } + + fn map_commit_result( + &self, + pending: &PendingCommit, + result: Result, + ) -> UpdateStatus { + commit_result_to_status(pending, result) + } + + async fn reconstruct_without_state( + &self, + request: &UpdateRequest, + from_version: Option, + connect_error: Option, + ) -> UpdateStatus { + // state.json did not survive the reboot (or was never written, e.g. + // the agent crashed before persisting pendingCommit). Per + // accepted-design-v2.md §2.3's degraded path, reconstruct the answer by + // calling commit() unconditionally rather than guessing from labels + // or the target version alone - tridentd's commit() is self-checking + // and its own (ServicingKind/RebootStatus/Result) response already + // distinguishes "swap happened, run commit" from "reboot hasn't + // happened yet" from "target armed but firmware fell back" far more + // reliably than a bare version-string comparison could. + if let Some(status) = + reconstruct_precheck_status(request, from_version.clone(), connect_error.as_deref()) + { + return status; + } + + let mut client = match TridentClient::connect(&self.config.trident.socket).await { + Ok(client) => client, + Err(err) => { + return UpdateStatus::new( + request, + request.operation.into(), + request.operation_id.clone(), + StatusCode::AgentInternalError, + format!("state.json missing after reboot and tridentd unreachable: {err}"), + from_version, + request.target_version.clone(), + Utc::now(), + Some(Utc::now()), + ); + } + }; + + let started = Utc::now(); + let result = client + .commit(self.config.orchestration.finalize_timeout) + .await; + // Same rationale as resume_pending_commit: only Finalize is a + // Nebraska-tracked update; this degraded path is only reached for + // Finalize|Rollback (reconstruct_precheck_status above), so + // Rollback is implicitly excluded here too. + if matches!(request.operation, RequestedOperation::Finalize) { + if let (Some(previous), Some(current)) = ( + parse_nebraska_version(&from_version, "reconstructed post-reboot commit"), + parse_nebraska_version(&request.target_version, "reconstructed post-reboot commit"), + ) { + self.report_nebraska_event( + request, + commit_nebraska_report(&previous, ¤t, &result), + ) + .await; + } + } + reconstruct_commit_result_to_status(request, from_version, started, result) + } + + async fn record_and_publish(&self, status: UpdateStatus) -> Result<(), anyhow::Error> { + let status = status.refreshed_for_write(); + self.state.remember_completed(status.clone())?; + self.best_effort_publish_terminal(&status).await; + Ok(()) + } + + async fn publish_status(&self, status: &UpdateStatus) -> Result<(), anyhow::Error> { + let status = status.refreshed_for_write(); + let mut annotations = BTreeMap::new(); + let annotation_key = match status.operation { + Operation::Commit => UPDATE_COMMIT_STATUS_ANNOTATION, + _ => UPDATE_STATUS_ANNOTATION, + }; + annotations.insert( + annotation_key.to_string(), + Some(serde_json::to_string(&status)?), + ); + log::info!( + "sending {annotation_key} annotation to node {}: {status:?}", + self.config.kubernetes.node_name + ); + self.k8s + .patch_node_metadata( + &self.config.kubernetes.node_name, + BTreeMap::new(), + annotations, + ) + .await?; + Ok(()) + } + + async fn best_effort_publish_terminal(&self, status: &UpdateStatus) { + for _ in 0..FINAL_STATUS_PATCH_RETRIES { + match self.publish_status(status).await { + Ok(()) => return, + Err(err) if self.is_node_gone_error(&err) => { + log::info!( + "stopping terminal status publish because node {} no longer exists", + self.config.kubernetes.node_name + ); + return; + } + Err(_) => tokio::time::sleep(FINAL_STATUS_PATCH_BACKOFF).await, + } + } + } + + fn is_node_gone_error(&self, err: &anyhow::Error) -> bool { + matches!( + err.downcast_ref::(), + Some(K8sClientError::NodeGone) + ) + } + + fn log_and_swallow_node_gone(&self, err: &anyhow::Error, context: &str) -> bool { + if self.is_node_gone_error(err) { + log::info!( + "stopping trident-acl-agent while {}: node {} no longer exists", + context, + self.config.kubernetes.node_name + ); + true + } else { + false + } + } + + async fn run_with_status_heartbeat(&self, status: UpdateStatus, future: F) -> F::Output + where + F: Future, + { + tokio::pin!(future); + let mut interval = tokio::time::interval(self.config.orchestration.heartbeat_interval); + interval.tick().await; + let mut stop_heartbeats = false; + loop { + tokio::select! { + result = &mut future => return result, + _ = interval.tick(), if !stop_heartbeats => { + if let Err(err) = self.publish_status(&status).await { + if self.is_node_gone_error(&err) { + log::info!( + "stopping heartbeats because node {} no longer exists", + self.config.kubernetes.node_name + ); + stop_heartbeats = true; + } else { + log::warn!("failed to refresh in-progress status heartbeat: {err}"); + } + } + } + } + } + } + + /// Sends a single Nebraska event report, logging (but never + /// propagating) failure. + /// + /// This is deliberately best-effort: per the `nebraska` module's own + /// docs, an instance that sends **no** events at all is always safe + /// (Nebraska self-heals to Complete on the instance's next check at the + /// new version), so a failed report here must never fail - or even + /// delay past its own retries - the underlying Trident operation it + /// describes. `complete_after_reboot` reports already retry internally + /// (see `nebraska::Client::complete_after_reboot`); everything else is a + /// single attempt, on the theory that the next stage/finalize/commit + /// step (or self-heal) will re-establish correct Nebraska state anyway. + /// + /// Runs on a dedicated blocking thread: the nebraska client's + /// `reqwest::blocking`-based transport cannot safely be dropped from + /// inside an already-running async task (see the same rationale on + /// `handle_stage`'s `check_for_update` call). + async fn report_nebraska_event(&self, request: &UpdateRequest, report: NebraskaReport) { + let Some(endpoint) = self.resolve_nebraska_endpoint(request) else { + // Should not happen in practice: every call site only reaches + // here after handle_stage has already required an endpoint for + // this node update. Guard anyway since this is best-effort + // telemetry, not something worth panicking over. + log::warn!( + "skipping Nebraska '{}' report: no Nebraska endpoint configured (no request.server override and no [nebraska].endpoint)", + report.label() + ); + return; + }; + let app_id = self.resolve_nebraska_app_id(request); + let track = self.resolve_nebraska_track(request); + let machine_id = match crate::build_machine_id(NEBRASKA_MACHINE_ID_SOURCE) { + Ok(id) => id, + Err(err) => { + log::warn!( + "skipping Nebraska '{}' report: failed to build machine id: {err}", + report.label() + ); + return; + } + }; + let label = report.label(); + let result = tokio::task::spawn_blocking(move || { + let client = NebraskaClient::new(endpoint, app_id, track, machine_id); + match report { + NebraskaReport::Progress { version, event } => { + client.report_progress(&version, event) + } + NebraskaReport::Completed { previous, current } => client + .complete_after_reboot(&previous, ¤t) + .map(|_| ()), + NebraskaReport::Failed { previous, current } => { + client.report_failure(&previous, ¤t) + } + } + }) + .await; + match result { + Ok(Ok(())) => log::debug!("reported Nebraska '{label}' event"), + Ok(Err(err)) => log::warn!("Nebraska '{label}' event report failed: {err}"), + Err(err) => log::warn!("Nebraska '{label}' event report task panicked: {err}"), + } + } +} + +/// Parses `version` (e.g. an `UpdateStatus::from_version`/`to_version` +/// field) as a semver [`Version`] for use in a Nebraska event report, +/// logging and returning `None` rather than failing if it's absent or not +/// valid semver. Nebraska event reporting is best-effort telemetry (see +/// `Orchestrator::report_nebraska_event`), so a malformed/missing version +/// string must only skip the report, never the Trident operation it +/// describes. +fn current_boot_marker() -> Result { + let raw = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .context("failed to read /proc/sys/kernel/random/boot_id")?; + let marker = raw.trim().to_string(); + if marker.is_empty() { + anyhow::bail!("/proc/sys/kernel/random/boot_id was empty"); + } + Ok(marker) +} + +fn parse_nebraska_version(version: &Option, context: &str) -> Option { + let raw = version.as_deref()?; + match Version::parse(raw) { + Ok(v) => Some(v), + Err(err) => { + log::warn!( + "skipping Nebraska event report for {context}: {raw:?} is not valid semver: {err}" + ); + None + } + } +} + +/// A request annotation that parsed as JSON but failed schema/semantic +/// validation (e.g. wrong schemaVersion, missing targetVersion for +/// stage/finalize, or a targetVersion present on a rollback request). Kept +/// distinct from "no request at all" so reconcile_node can surface an +/// InvalidRequest status instead of silently ignoring the annotation. +#[derive(Debug, Clone)] +struct InvalidRequest { + node_update_id: Uuid, + operation_id: String, + operation: Operation, + reason: String, +} + +#[derive(Debug, Clone, Default)] +struct Snapshot { + request: Option, + invalid_request: Option, + operation_status: Option, + commit_status: Option, +} + +impl Snapshot { + fn from_node(node: &Node) -> Self { + let annotations = node.metadata.annotations.as_ref(); + let raw_request = annotations.and_then(|a| a.get(UPDATE_REQUEST_ANNOTATION)); + let (request, invalid_request) = match raw_request + .map(|v| serde_json::from_str::(v)) + { + None => (None, None), + Some(Ok(candidate)) => match candidate.clone().validate() { + Ok(valid) => (Some(valid), None), + Err(reason) => ( + None, + Some(InvalidRequest { + node_update_id: candidate.node_update_id, + operation_id: candidate.operation_id, + operation: candidate.operation.into(), + reason, + }), + ), + }, + Some(Err(err)) => { + // Cannot attribute a status to an operationId we couldn't + // even parse out of the annotation - log loudly instead so + // this doesn't fail silently, but there's no request to + // surface an InvalidRequest status against. + log::warn!( + "ignoring malformed {UPDATE_REQUEST_ANNOTATION} annotation (JSON parse failed): {err}" + ); + (None, None) + } + }; + let operation_status = annotations + .and_then(|a| a.get(UPDATE_STATUS_ANNOTATION)) + .and_then(|v| serde_json::from_str::(v).ok()); + let commit_status = annotations + .and_then(|a| a.get(UPDATE_COMMIT_STATUS_ANNOTATION)) + .and_then(|v| serde_json::from_str::(v).ok()); + Self { + request, + invalid_request, + operation_status, + commit_status, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoopControl { + Continue, + ExitForReboot, +} + +/// Decides which Nebraska event a stage() result should produce, given the +/// instance's currently-running version (which - for stage - is the same +/// version on both success and failure; only the post-reboot commit ever +/// changes it). Pure function - see `stage_result_to_status` for rationale. +fn stage_nebraska_report( + current_version: &Version, + result: &Result, +) -> NebraskaReport { + match result { + Ok(_) => NebraskaReport::Progress { + version: current_version.clone(), + event: ProgressEvent::DownloadFinished, + }, + Err(_) => NebraskaReport::Failed { + previous: current_version.clone(), + current: current_version.clone(), + }, + } +} + +/// Decides which Nebraska event a finalize() result should produce. Pure +/// function - see `stage_result_to_status` for rationale. (Used at the +/// finalize *RPC* call site only; the `Installed` progress event for a +/// success is sent directly at the call site since it doesn't depend on the +/// result shape.) +fn finalize_nebraska_report( + current_version: &Version, + result: &Result, +) -> NebraskaReport { + match result { + Ok(_) => NebraskaReport::Progress { + version: current_version.clone(), + event: ProgressEvent::Installed, + }, + Err(_) => NebraskaReport::Failed { + previous: current_version.clone(), + current: current_version.clone(), + }, + } +} + +/// Decides which Nebraska event a post-reboot commit() result should +/// produce, discharging the commitment made by the progress events sent +/// during stage/finalize. Pure function - see `stage_result_to_status` for +/// rationale. +/// +/// - A clean commit success reports `Completed`, moving the instance to the +/// new version. +/// - A commit asking for *another* reboot is not a state Nebraska has any +/// representation for; treat it as not-yet-complete and report `Failed` +/// (previous == current, since the instance is still effectively on the +/// old version from Nebraska's point of view) so a later check can grant +/// again rather than leaving the instance wedged in progress forever. +/// - A commit result indicating the update was reverted (health/reboot +/// check failure) or any other failure both report `Failed` for the same +/// reason: the instance did not end up on the new version. +fn commit_nebraska_report( + previous_version: &Version, + new_version: &Version, + result: &Result, +) -> NebraskaReport { + match result { + Ok(response) if response.reboot_status == RebootStatus::RebootRequired => { + NebraskaReport::Failed { + previous: previous_version.clone(), + current: previous_version.clone(), + } + } + Ok(_) => NebraskaReport::Completed { + previous: previous_version.clone(), + current: new_version.clone(), + }, + Err(_) => NebraskaReport::Failed { + previous: previous_version.clone(), + current: previous_version.clone(), + }, + } +} + +/// Maps a stage() result to the terminal `UpdateStatus` for that stage +/// attempt. Pure function: no I/O, no side effects - exists so tests can +/// exercise the full success/failure matrix against a fake tridentd without +/// needing a real Kubernetes API or state store. +fn stage_result_to_status( + request: &UpdateRequest, + from_version: Option, + to_version: Option, + started: chrono::DateTime, + result: Result, +) -> UpdateStatus { + match result { + Ok(_) => UpdateStatus::new( + request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::Success, + "stage completed", + from_version, + to_version, + started, + Some(Utc::now()), + ), + Err(err) => UpdateStatus::new( + request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::OperationFailed, + format!("stage failed: {err}"), + from_version, + to_version, + started, + Some(Utc::now()), + ), + } +} + +/// Builds the terminal `UpdateStatus` for a successful finalize() call. Pure +/// function - see `stage_result_to_status` for rationale. +fn finalize_success_status( + request: &UpdateRequest, + from_version: Option, + to_version: Option, + started: chrono::DateTime, +) -> UpdateStatus { + UpdateStatus::new( + request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::Success, + "finalize completed; rebooting for commit", + from_version, + to_version, + started, + Some(Utc::now()), + ) +} + +/// Builds the terminal `UpdateStatus` for a failed finalize() call. Pure +/// function - see `stage_result_to_status` for rationale. +fn finalize_failure_status( + request: &UpdateRequest, + from_version: Option, + to_version: Option, + started: chrono::DateTime, + err: &TridentClientError, +) -> UpdateStatus { + UpdateStatus::new( + request, + Operation::Finalize, + request.operation_id.clone(), + map_trident_failure(err), + format!("finalize failed: {err}"), + from_version, + to_version, + started, + Some(Utc::now()), + ) +} + +fn map_trident_failure(error: &TridentClientError) -> StatusCode { + if indicates_target_boot_failed(error) { + StatusCode::TargetBootFailed + } else { + StatusCode::OperationFailed + } +} + +fn indicates_target_boot_failed(error: &TridentClientError) -> bool { + error + .remote() + .map(|remote| { + // "ab-update-reboot-check"/"ab-update-health-check-commit-check" + // are the forward-update (finalize/commit) reboot-check + // subkinds (ServicingError::AbUpdateRebootCheck / + // HealthChecksError::AbUpdateHealthCheckCommitCheck). + // "manual-rollback-reboot-check" is the *rollback*-specific + // sibling (ServicingError::ManualRollbackRebootCheck), emitted + // when a post-rollback reboot's firmware A/B fallback lands on + // the wrong slot. It is a distinct enum variant with its own + // kebab-case serde subkind, not a copy of the forward-update + // one - both must be checked here, or a real rollback + // boot-fallback silently reports as generic OperationFailed + // instead of TargetBootFailed. + remote.subkind == "ab-update-reboot-check" + || remote.subkind == "ab-update-health-check-commit-check" + || remote.subkind == "manual-rollback-reboot-check" + }) + .unwrap_or(false) +} + +/// Pure function extracted from `Orchestrator::map_commit_result` so tests +/// can exercise it directly (with a mock-tridentd-driven `Result`) without +/// needing a full `Orchestrator` instance. See `stage_result_to_status` for +/// rationale. +/// Pre-flight checks for the state.json-missing degraded reconstruction +/// path (accepted-design-v2.md §2.3). Returns `Some(status)` when reconstruction +/// cannot proceed (tridentd already known-unreachable, or the outstanding +/// request isn't a finalize/rollback), or `None` when the caller should go +/// on to call tridentd's commit() to determine the real outcome. +fn reconstruct_precheck_status( + request: &UpdateRequest, + from_version: Option, + connect_error: Option<&str>, +) -> Option { + if let Some(err) = connect_error { + return Some(UpdateStatus::new( + request, + request.operation.into(), + request.operation_id.clone(), + StatusCode::AgentInternalError, + format!("state.json missing after reboot and tridentd unreachable: {err}"), + from_version, + request.target_version.clone(), + Utc::now(), + Some(Utc::now()), + )); + } + + if !matches!( + request.operation, + RequestedOperation::Finalize | RequestedOperation::Rollback + ) { + return Some(UpdateStatus::new( + request, + request.operation.into(), + request.operation_id.clone(), + StatusCode::AgentInternalError, + "unable to reconstruct operation without state.json", + from_version, + request.target_version.clone(), + Utc::now(), + Some(Utc::now()), + )); + } + + None +} + +/// Maps tridentd's commit() result to the terminal status for the +/// state.json-missing degraded reconstruction path (accepted-design-v2.md +/// §2.3). Always reports under the original operationId, mirroring the +/// normal post-reboot commit path in `commit_result_to_status`. +fn reconstruct_commit_result_to_status( + request: &UpdateRequest, + from_version: Option, + started: DateTime, + result: Result, +) -> UpdateStatus { + match result { + Ok(response) if response.reboot_status == RebootStatus::RebootRequired => UpdateStatus::new( + request, + Operation::Commit, + request.operation_id.clone(), + StatusCode::AgentInternalError, + "state.json missing after reboot; commit requested another reboot", + from_version, + request.target_version.clone(), + started, + Some(Utc::now()), + ), + Ok(_) => UpdateStatus::new( + request, + Operation::Commit, + request.operation_id.clone(), + StatusCode::Success, + "state.json missing after reboot; commit() confirmed the swap and completed", + from_version, + request.target_version.clone(), + started, + Some(Utc::now()), + ), + Err(err) if indicates_target_boot_failed(&err) => UpdateStatus::new( + request, + Operation::Commit, + request.operation_id.clone(), + StatusCode::TargetBootFailed, + format!( + "state.json missing after reboot; commit detected rollback to previous version: {err}" + ), + from_version, + request.target_version.clone(), + started, + Some(Utc::now()), + ), + Err(err) => UpdateStatus::new( + request, + Operation::Commit, + request.operation_id.clone(), + map_trident_failure(&err), + format!("state.json missing after reboot; commit failed: {err}"), + from_version, + request.target_version.clone(), + started, + Some(Utc::now()), + ), + } +} + +fn commit_result_to_status( + pending: &PendingCommit, + result: Result, +) -> UpdateStatus { + match result { + Ok(response) if response.reboot_status == RebootStatus::RebootRequired => { + UpdateStatus::new( + &pending.request, + Operation::Commit, + pending.operation_id.clone(), + StatusCode::AgentInternalError, + "commit requested another reboot", + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + Some(Utc::now()), + ) + } + Ok(_) => UpdateStatus::new( + &pending.request, + Operation::Commit, + pending.operation_id.clone(), + StatusCode::Success, + "commit completed", + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + Some(Utc::now()), + ), + Err(err) if indicates_target_boot_failed(&err) => UpdateStatus::new( + &pending.request, + Operation::Commit, + pending.operation_id.clone(), + StatusCode::TargetBootFailed, + format!("commit detected rollback to previous version: {err}"), + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + Some(Utc::now()), + ), + Err(err) => UpdateStatus::new( + &pending.request, + Operation::Commit, + pending.operation_id.clone(), + map_trident_failure(&err), + format!("commit failed: {err}"), + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + Some(Utc::now()), + ), + } +} + +/// Builds the terminal `UpdateStatus` for a failed rollback_stage() call. +/// Pure function - see `stage_result_to_status` for rationale. +fn rollback_stage_failure_status( + request: &UpdateRequest, + from_version: Option, + started: chrono::DateTime, + err: &TridentClientError, +) -> UpdateStatus { + UpdateStatus::new( + request, + Operation::Rollback, + request.operation_id.clone(), + map_trident_failure(err), + format!("rollback stage failed: {err}"), + from_version, + None, + started, + Some(Utc::now()), + ) +} + +/// Builds the terminal `UpdateStatus` for a successful rollback_finalize() +/// call. Pure function - see `stage_result_to_status` for rationale. +fn rollback_finalize_success_status( + request: &UpdateRequest, + from_version: Option, + started: chrono::DateTime, +) -> UpdateStatus { + UpdateStatus::new( + request, + Operation::Rollback, + request.operation_id.clone(), + StatusCode::Success, + "rollback finalize completed; rebooting for commit", + from_version, + None, + started, + Some(Utc::now()), + ) +} + +/// Builds the terminal `UpdateStatus` for a failed rollback_finalize() call. +/// Pure function - see `stage_result_to_status` for rationale. +fn rollback_finalize_failure_status( + request: &UpdateRequest, + from_version: Option, + started: chrono::DateTime, + err: &TridentClientError, +) -> UpdateStatus { + UpdateStatus::new( + request, + Operation::Rollback, + request.operation_id.clone(), + map_trident_failure(err), + format!("rollback finalize failed: {err}"), + from_version, + None, + started, + Some(Utc::now()), + ) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use chrono::Utc; + use uuid::Uuid; + + use super::*; + use crate::{ + annotations::{RequestedOperation, SCHEMA_VERSION}, + mock_tridentd::{connect_mock_client, MockTridentdConfig, Outcome}, + }; + + fn request(operation: RequestedOperation) -> UpdateRequest { + UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: Uuid::new_v4(), + operation_id: "op-1".to_string(), + operation, + target_version: Some("2.0.0".to_string()), + server: None, + app_id: None, + track: None, + } + } + + fn pending(operation: Operation) -> PendingCommit { + PendingCommit { + request: request(RequestedOperation::Finalize), + operation_id: "op-1".to_string(), + operation, + from_version: Some("1.0.0".to_string()), + to_version: Some("2.0.0".to_string()), + started_utc: Utc::now(), + boot_marker: "boot-1".to_string(), + } + } + + // --- rollback --- + + #[tokio::test] + async fn rollback_stage_failure_maps_to_operation_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + rollback_stage: Some(Outcome::Failure { + subkind: "some-rollback-stage-error", + message: "disk full", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .rollback_stage(std::time::Duration::from_secs(5)) + .await; + + let request = request(RequestedOperation::Rollback); + let status = rollback_stage_failure_status( + &request, + Some("2.0.0".to_string()), + Utc::now(), + &result.unwrap_err(), + ); + + assert_eq!(status.code, StatusCode::OperationFailed); + assert_eq!(status.operation, Operation::Rollback); + assert!(status.message.contains("rollback stage failed")); + } + + /// Regression coverage for the "rollback with nothing to roll back" + /// fix: RollbackStage's response must carry the real ServicingKind + /// (ManualRollbackAb for a real rollback, NoneRequired for a no-op) so + /// handle_rollback() in this module can distinguish the two before + /// finalizing/rebooting - see the `matches!(stage_response.servicing_kind, ..)` + /// check there. This test pins the wire plumbing `TridentClient` + /// depends on: a mocked RollbackStage response's servicing_kind must + /// survive unchanged into `CompletedResponse`. + #[tokio::test] + async fn rollback_stage_success_reports_servicing_kind() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + rollback_stage: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: Some(ServicingKind::ManualRollbackAb), + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let response = client + .rollback_stage(std::time::Duration::from_secs(5)) + .await + .expect("mocked rollback_stage should succeed"); + assert_eq!( + response.servicing_kind, + Some(ServicingKind::ManualRollbackAb) + ); + } + + #[tokio::test] + async fn rollback_stage_noop_reports_none_required_servicing_kind() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + rollback_stage: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: Some(ServicingKind::NoneRequired), + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let response = client + .rollback_stage(std::time::Duration::from_secs(5)) + .await + .expect("mocked rollback_stage should succeed"); + assert_eq!(response.servicing_kind, Some(ServicingKind::NoneRequired)); + } + + #[tokio::test] + async fn rollback_finalize_success_maps_to_success_status() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + rollback_finalize: Some(Outcome::Success { + reboot_status: RebootStatus::RebootRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .rollback_finalize(std::time::Duration::from_secs(5)) + .await; + assert!(result.is_ok()); + + let request = request(RequestedOperation::Rollback); + let status = + rollback_finalize_success_status(&request, Some("2.0.0".to_string()), Utc::now()); + + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Rollback); + assert_eq!(status.to_version, None); + } + + #[tokio::test] + async fn rollback_finalize_failure_maps_to_operation_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + rollback_finalize: Some(Outcome::Failure { + subkind: "some-rollback-finalize-error", + message: "boom", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .rollback_finalize(std::time::Duration::from_secs(5)) + .await; + + let request = request(RequestedOperation::Rollback); + let status = rollback_finalize_failure_status( + &request, + Some("2.0.0".to_string()), + Utc::now(), + &result.unwrap_err(), + ); + + assert_eq!(status.code, StatusCode::OperationFailed); + assert_eq!(status.operation, Operation::Rollback); + assert!(status.message.contains("rollback finalize failed")); + } + + #[tokio::test] + async fn rollback_finalize_reverted_maps_to_reverted_to_previous() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + rollback_finalize: Some(Outcome::Failure { + subkind: "ab-update-reboot-check", + message: "reverted", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .rollback_finalize(std::time::Duration::from_secs(5)) + .await; + + let request = request(RequestedOperation::Rollback); + let status = rollback_finalize_failure_status( + &request, + Some("2.0.0".to_string()), + Utc::now(), + &result.unwrap_err(), + ); + + assert_eq!(status.code, StatusCode::TargetBootFailed); + } + + // --- stage --- + + #[tokio::test] + async fn stage_success_maps_to_success_status() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + stage: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .update_stage( + &"http://example.test/image".parse().unwrap(), + None, + std::time::Duration::from_secs(5), + ) + .await; + + let request = request(RequestedOperation::Stage); + let status = stage_result_to_status( + &request, + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + Utc::now(), + result, + ); + + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Stage); + assert_eq!(status.from_version, Some("1.0.0".to_string())); + assert_eq!(status.to_version, Some("2.0.0".to_string())); + } + + #[tokio::test] + async fn stage_failure_maps_to_operation_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + stage: Some(Outcome::Failure { + subkind: "some-stage-error", + message: "disk full", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .update_stage( + &"http://example.test/image".parse().unwrap(), + None, + std::time::Duration::from_secs(5), + ) + .await; + + let request = request(RequestedOperation::Stage); + let status = stage_result_to_status(&request, None, None, Utc::now(), result); + + assert_eq!(status.code, StatusCode::OperationFailed); + assert!(status.message.contains("stage failed")); + assert!(status.message.contains("disk full")); + } + + #[tokio::test] + async fn stage_nebraska_report_success_is_download_finished() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + stage: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .update_stage( + &"http://example.test/image".parse().unwrap(), + None, + std::time::Duration::from_secs(5), + ) + .await; + + let version = semver::Version::new(1, 0, 0); + let report = stage_nebraska_report(&version, &result); + assert_eq!( + report, + NebraskaReport::Progress { + version, + event: ProgressEvent::DownloadFinished, + } + ); + } + + #[tokio::test] + async fn stage_nebraska_report_failure_releases_wedge() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + stage: Some(Outcome::Failure { + subkind: "some-stage-error", + message: "disk full", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .update_stage( + &"http://example.test/image".parse().unwrap(), + None, + std::time::Duration::from_secs(5), + ) + .await; + + let version = semver::Version::new(1, 0, 0); + let report = stage_nebraska_report(&version, &result); + assert_eq!( + report, + NebraskaReport::Failed { + previous: version.clone(), + current: version, + } + ); + } + + // --- finalize --- + + #[tokio::test] + async fn finalize_success_maps_to_success_status() { + let status = finalize_success_status( + &request(RequestedOperation::Finalize), + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + Utc::now(), + ); + + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Finalize); + assert!(status.message.contains("rebooting")); + } + + #[tokio::test] + async fn finalize_failure_with_generic_error_maps_to_operation_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + finalize: Some(Outcome::Failure { + subkind: "some-finalize-error", + message: "partition swap failed", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let err = client + .update_finalize(std::time::Duration::from_secs(5)) + .await + .unwrap_err(); + + let status = finalize_failure_status( + &request(RequestedOperation::Finalize), + None, + None, + Utc::now(), + &err, + ); + + assert_eq!(status.code, StatusCode::OperationFailed); + assert!(status.message.contains("finalize failed")); + assert!(status.message.contains("partition swap failed")); + } + + #[tokio::test] + async fn finalize_failure_with_reboot_check_subkind_maps_to_reverted() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + finalize: Some(Outcome::Failure { + subkind: "ab-update-reboot-check", + message: "boot did not land on target partition", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let err = client + .update_finalize(std::time::Duration::from_secs(5)) + .await + .unwrap_err(); + + let status = finalize_failure_status( + &request(RequestedOperation::Finalize), + None, + None, + Utc::now(), + &err, + ); + + assert_eq!(status.code, StatusCode::TargetBootFailed); + } + + #[tokio::test] + async fn finalize_nebraska_report_success_is_installed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + finalize: Some(Outcome::Success { + reboot_status: RebootStatus::RebootRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .update_finalize(std::time::Duration::from_secs(5)) + .await; + + let version = semver::Version::new(1, 0, 0); + let report = finalize_nebraska_report(&version, &result); + assert_eq!( + report, + NebraskaReport::Progress { + version, + event: ProgressEvent::Installed, + } + ); + } + + #[tokio::test] + async fn finalize_nebraska_report_failure_releases_wedge() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + finalize: Some(Outcome::Failure { + subkind: "some-finalize-error", + message: "partition swap failed", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client + .update_finalize(std::time::Duration::from_secs(5)) + .await; + + let version = semver::Version::new(1, 0, 0); + let report = finalize_nebraska_report(&version, &result); + assert_eq!( + report, + NebraskaReport::Failed { + previous: version.clone(), + current: version, + } + ); + } + + // --- commit --- + + #[tokio::test] + async fn commit_success_maps_to_success_status() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let status = commit_result_to_status(&pending(Operation::Finalize), result); + + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Commit); + } + + #[tokio::test] + async fn commit_success_but_reboot_required_maps_to_agent_internal_error() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let status = commit_result_to_status(&pending(Operation::Finalize), result); + + assert_eq!(status.code, StatusCode::AgentInternalError); + assert!(status.message.contains("another reboot")); + } + + #[tokio::test] + async fn commit_failure_with_generic_error_maps_to_operation_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Failure { + subkind: "some-commit-error", + message: "commit rpc failed", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let status = commit_result_to_status(&pending(Operation::Finalize), result); + + assert_eq!(status.code, StatusCode::OperationFailed); + assert!(status.message.contains("commit failed")); + } + + #[tokio::test] + async fn commit_failure_with_reboot_check_subkind_maps_to_reverted() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Failure { + subkind: "ab-update-reboot-check", + message: "boot did not land on target partition", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let status = commit_result_to_status(&pending(Operation::Finalize), result); + + assert_eq!(status.code, StatusCode::TargetBootFailed); + } + + #[tokio::test] + async fn commit_failure_with_health_check_subkind_maps_to_reverted() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Failure { + subkind: "ab-update-health-check-commit-check", + message: "post-commit health check failed", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let status = commit_result_to_status(&pending(Operation::Finalize), result); + + assert_eq!(status.code, StatusCode::TargetBootFailed); + } + + #[tokio::test] + async fn commit_nebraska_report_success_is_completed_with_new_version() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let previous = semver::Version::new(1, 0, 0); + let current = semver::Version::new(2, 0, 0); + let report = commit_nebraska_report(&previous, ¤t, &result); + assert_eq!(report, NebraskaReport::Completed { previous, current }); + } + + #[tokio::test] + async fn commit_nebraska_report_reboot_required_reports_failed_not_completed() { + // A commit asking for another reboot is not really "complete" from + // Nebraska's point of view: the instance hasn't landed on the new + // version, so this must not report Completed (which would tell + // Nebraska the fleet's instance count moved when it hasn't). + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let previous = semver::Version::new(1, 0, 0); + let current = semver::Version::new(2, 0, 0); + let report = commit_nebraska_report(&previous, ¤t, &result); + assert_eq!( + report, + NebraskaReport::Failed { + previous: previous.clone(), + current: previous, + } + ); + } + + #[tokio::test] + async fn commit_nebraska_report_reverted_reports_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Failure { + subkind: "ab-update-reboot-check", + message: "boot did not land on target partition", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let previous = semver::Version::new(1, 0, 0); + let current = semver::Version::new(2, 0, 0); + let report = commit_nebraska_report(&previous, ¤t, &result); + assert_eq!( + report, + NebraskaReport::Failed { + previous: previous.clone(), + current: previous, + } + ); + } + + // --- reconstruct_without_state (state.json missing after reboot) --- + + #[test] + fn reconstruct_precheck_reports_agent_internal_error_when_tridentd_unreachable() { + let request = request(RequestedOperation::Finalize); + let status = reconstruct_precheck_status( + &request, + Some("1.0.0".to_string()), + Some("connection refused"), + ) + .expect("connect error should short-circuit reconstruction"); + + assert_eq!(status.code, StatusCode::AgentInternalError); + assert!(status.message.contains("tridentd unreachable")); + assert_eq!(status.operation_id, request.operation_id); + } + + #[test] + fn reconstruct_precheck_reports_agent_internal_error_for_non_finalize_rollback_operation() { + let request = request(RequestedOperation::Stage); + let status = reconstruct_precheck_status(&request, None, None) + .expect("stage requests cannot be reconstructed without state.json"); + + assert_eq!(status.code, StatusCode::AgentInternalError); + assert!(status + .message + .contains("unable to reconstruct operation without state.json")); + } + + #[test] + fn reconstruct_precheck_allows_finalize_and_rollback_through() { + for operation in [RequestedOperation::Finalize, RequestedOperation::Rollback] { + let request = request(operation); + assert!( + reconstruct_precheck_status(&request, None, None).is_none(), + "expected {operation:?} to proceed to commit() reconstruction" + ); + } + } + + #[tokio::test] + async fn reconstruct_commit_result_success_maps_to_success_status() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let request = request(RequestedOperation::Finalize); + let status = reconstruct_commit_result_to_status( + &request, + Some("1.0.0".to_string()), + Utc::now(), + result, + ); + + assert_eq!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Commit); + assert_eq!(status.operation_id, request.operation_id.clone()); + assert!(status.message.contains("commit() confirmed the swap")); + } + + #[tokio::test] + async fn reconstruct_commit_result_reboot_required_maps_to_agent_internal_error() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootRequired, + servicing_kind: None, + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let request = request(RequestedOperation::Finalize); + let status = reconstruct_commit_result_to_status(&request, None, Utc::now(), result); + + assert_eq!(status.code, StatusCode::AgentInternalError); + assert!(status.message.contains("requested another reboot")); + } + + #[tokio::test] + async fn reconstruct_commit_result_reverted_subkind_maps_to_reverted_to_previous() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Failure { + subkind: "ab-update-reboot-check", + message: "boot did not land on target partition", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let request = request(RequestedOperation::Rollback); + let status = reconstruct_commit_result_to_status(&request, None, Utc::now(), result); + + assert_eq!(status.code, StatusCode::TargetBootFailed); + assert!(status.message.contains("detected rollback")); + } + + #[tokio::test] + async fn reconstruct_commit_result_generic_failure_maps_to_operation_failed() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Failure { + subkind: "some-commit-error", + message: "commit rpc failed", + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(std::time::Duration::from_secs(5)).await; + + let request = request(RequestedOperation::Finalize); + let status = reconstruct_commit_result_to_status(&request, None, Utc::now(), result); + + assert_eq!(status.code, StatusCode::OperationFailed); + assert!(status.message.contains("commit failed")); + // Regression check for DR-003: the generic-failure branch must use the + // same Operation::Commit / shared operationId as every other + // branch of this function, matching commit_result_to_status and the + // doc comment above reconstruct_commit_result_to_status. + assert_eq!(status.operation, Operation::Commit); + assert_eq!(status.operation_id, request.operation_id.clone()); + } + + // --- parse_nebraska_version --- + + #[test] + fn parse_nebraska_version_parses_valid_semver() { + assert_eq!( + parse_nebraska_version(&Some("1.2.3".to_string()), "test"), + Some(semver::Version::new(1, 2, 3)) + ); + } + + #[test] + fn parse_nebraska_version_returns_none_for_missing_version() { + assert_eq!(parse_nebraska_version(&None, "test"), None); + } + + #[test] + fn parse_nebraska_version_returns_none_for_invalid_semver() { + assert_eq!( + parse_nebraska_version(&Some("not-a-version".to_string()), "test"), + None + ); + } +} diff --git a/crates/trident-acl-agent/src/state.rs b/crates/trident-acl-agent/src/state.rs new file mode 100644 index 0000000000..ed58060dd2 --- /dev/null +++ b/crates/trident-acl-agent/src/state.rs @@ -0,0 +1,354 @@ +//! Persistent agent state (`/var/lib/trident-acl-agent/state.json`): +//! completed-operation cache and the pending post-reboot commit record. +//! +//! Implements the `state.json` mechanism from the current accepted design +//! (`accepted-design-v2.md`, section 2.3), which bridges the pre-reboot +//! finalize/rollback half and the post-reboot commit half of an operation +//! across the reboot. + +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; + +use crate::annotations::{Operation, UpdateRequest, UpdateStatus}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PersistentState { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_commit: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub completed: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CompletedEntry { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PendingCommit { + pub request: UpdateRequest, + pub operation_id: String, + pub operation: Operation, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub to_version: Option, + pub started_utc: chrono::DateTime, + pub boot_marker: String, +} + +#[derive(Debug, Clone)] +pub struct StateStore { + path: PathBuf, +} + +impl StateStore { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn load(&self) -> Result { + match fs::read_to_string(&self.path) { + Ok(raw) => Ok(serde_json::from_str(&raw).context("failed to parse state.json")?), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Ok(PersistentState::default()) + } + Err(err) => { + Err(anyhow::Error::new(err) + .context(format!("failed to read {}", self.path.display()))) + } + } + } + + pub fn save(&self, state: &PersistentState) -> Result<(), anyhow::Error> { + let parent = match self.path.parent() { + Some(parent) => { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + parent + } + None => Path::new("."), + }; + + let temp_path = parent.join(format!( + "{}.tmp-{}", + self.path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("state.json"), + std::process::id() + )); + fs::write(&temp_path, serde_json::to_string_pretty(state)?) + .with_context(|| format!("failed to write {}", temp_path.display()))?; + fs::rename(&temp_path, &self.path).with_context(|| { + format!( + "failed to atomically replace {} with {}", + self.path.display(), + temp_path.display() + ) + }) + } + + pub fn remember_completed(&self, status: UpdateStatus) -> Result<(), anyhow::Error> { + let mut state = self.load()?; + let entry = state + .completed + .entry(status.operation_id.clone()) + .or_default(); + match status.operation { + Operation::Commit => entry.commit = Some(status), + _ => entry.operation = Some(status), + } + self.save(&state) + } + + pub fn set_pending_commit(&self, pending: PendingCommit) -> Result<(), anyhow::Error> { + let mut state = self.load()?; + state.pending_commit = Some(pending); + self.save(&state) + } + + pub fn clear_pending_commit(&self) -> Result<(), anyhow::Error> { + let mut state = self.load()?; + state.pending_commit = None; + self.save(&state) + } +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use url::Url; + use uuid::Uuid; + + use super::*; + use crate::annotations::{RequestedOperation, StatusCode, SCHEMA_VERSION}; + + fn store() -> (tempfile::TempDir, StateStore) { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("state.json"); + let store = StateStore::new(path); + (dir, store) + } + + fn sample_request() -> UpdateRequest { + UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: Uuid::new_v4(), + operation_id: "op-1".to_string(), + operation: RequestedOperation::Finalize, + target_version: Some("2.0.0".to_string()), + server: None, + app_id: None, + track: None, + } + } + + fn sample_status(operation: Operation) -> UpdateStatus { + UpdateStatus::new( + &sample_request(), + operation, + "op-1".to_string(), + StatusCode::Success, + format!("{operation:?} completed"), + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + Utc::now(), + Some(Utc::now()), + ) + } + + fn sample_pending() -> PendingCommit { + PendingCommit { + request: sample_request(), + operation_id: "op-1".to_string(), + operation: Operation::Finalize, + from_version: Some("1.0.0".to_string()), + to_version: Some("2.0.0".to_string()), + started_utc: Utc::now(), + boot_marker: "boot-1".to_string(), + } + } + + #[test] + fn load_returns_default_when_file_missing() { + let (_dir, store) = store(); + let state = store + .load() + .expect("load should not fail when file is absent"); + assert_eq!(state, PersistentState::default()); + assert!(state.pending_commit.is_none()); + assert!(state.completed.is_empty()); + } + + #[test] + fn save_then_load_round_trips_full_state() { + let (_dir, store) = store(); + let mut completed = std::collections::BTreeMap::new(); + completed.insert( + "op-1".to_string(), + CompletedEntry { + operation: Some(sample_status(Operation::Finalize)), + commit: Some(sample_status(Operation::Commit)), + }, + ); + let state = PersistentState { + pending_commit: Some(sample_pending()), + completed, + }; + store.save(&state).expect("save should succeed"); + let loaded = store.load().expect("load should succeed after save"); + + assert_eq!(loaded, state); + } + + #[test] + fn save_creates_parent_directories() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let nested_path = dir.path().join("nested").join("deeper").join("state.json"); + let store = StateStore::new(nested_path.clone()); + + store + .save(&PersistentState::default()) + .expect("save should create missing parent directories"); + + assert!(nested_path.exists()); + } + + #[test] + fn remember_completed_tracks_operation_and_commit_separately_under_same_operation_id() { + let (_dir, store) = store(); + store + .remember_completed(sample_status(Operation::Finalize)) + .expect("remember_completed should succeed"); + store + .remember_completed(sample_status(Operation::Commit)) + .expect("remember_completed should succeed"); + + let state = store.load().expect("load should succeed"); + let entry = state.completed.get("op-1").expect("entry should exist"); + assert!(entry.operation.is_some()); + assert!(entry.commit.is_some()); + assert_eq!(state.completed.len(), 1); + } + + #[test] + fn remember_completed_overwrites_same_half_only() { + let (_dir, store) = store(); + store + .remember_completed(sample_status(Operation::Finalize)) + .expect("first remember_completed should succeed"); + + let mut updated = sample_status(Operation::Finalize); + updated.message = "updated message".to_string(); + store + .remember_completed(updated) + .expect("second remember_completed should succeed"); + + let state = store.load().expect("load should succeed"); + let entry = state.completed.get("op-1").expect("entry should exist"); + assert_eq!(entry.operation.as_ref().unwrap().message, "updated message"); + assert!(entry.commit.is_none()); + } + + #[test] + fn set_and_clear_pending_commit_round_trip() { + let (_dir, store) = store(); + assert!(store.load().unwrap().pending_commit.is_none()); + + let pending = sample_pending(); + store + .set_pending_commit(pending.clone()) + .expect("set_pending_commit should succeed"); + let state = store.load().expect("load should succeed"); + assert_eq!(state.pending_commit, Some(pending)); + + store + .clear_pending_commit() + .expect("clear_pending_commit should succeed"); + let state = store.load().expect("load should succeed"); + assert!(state.pending_commit.is_none()); + } + + #[test] + fn pending_commit_persists_server_app_id_and_track_overrides() { + // PendingCommit.request carries the whole UpdateRequest, so a + // server/appId/track override present at finalize time must survive + // the reboot unchanged, ready for the post-reboot commit's Nebraska + // event report (see Orchestrator::resolve_nebraska_endpoint, + // Orchestrator::resolve_nebraska_app_id, and + // Orchestrator::resolve_nebraska_track). + let (_dir, store) = store(); + let mut pending = sample_pending(); + pending.request.server = Some(Url::parse("https://nebraska.example/v1/update").unwrap()); + pending.request.app_id = Some("59bbad61-257d-47f4-9730-6848d88e1a6e".to_string()); + pending.request.track = Some("pin-202608.6.0".to_string()); + + store + .set_pending_commit(pending.clone()) + .expect("set_pending_commit should succeed"); + let state = store.load().expect("load should succeed"); + assert_eq!(state.pending_commit, Some(pending)); + } + + #[test] + fn set_pending_commit_preserves_existing_completed_entries() { + let (_dir, store) = store(); + store + .remember_completed(sample_status(Operation::Finalize)) + .expect("remember_completed should succeed"); + store + .set_pending_commit(sample_pending()) + .expect("set_pending_commit should succeed"); + + let state = store.load().expect("load should succeed"); + assert!(state.pending_commit.is_some()); + assert_eq!(state.completed.len(), 1); + assert!(state.completed.contains_key("op-1")); + } + + #[test] + fn save_is_atomic_replace() { + let (_dir, store) = store(); + store + .save(&PersistentState::default()) + .expect("initial save should succeed"); + + let metadata_before = std::fs::metadata(store.path()).expect("state file should exist"); + let state = PersistentState { + pending_commit: Some(sample_pending()), + completed: BTreeMap::new(), + }; + store.save(&state).expect("second save should succeed"); + + let metadata_after = std::fs::metadata(store.path()).expect("state file should exist"); + assert!(metadata_after.len() > 0); + assert!(metadata_before.modified().is_ok()); + } + + #[test] + fn deserialize_rejects_unknown_top_level_fields() { + let err = serde_json::from_str::( + r#"{"pendingCommit": null, "completed": {}, "unexpectedField": true}"#, + ) + .unwrap_err(); + + assert!(err.to_string().contains("unexpectedField")); + } +} diff --git a/crates/trident-acl-agent/src/trident.rs b/crates/trident-acl-agent/src/trident.rs new file mode 100644 index 0000000000..e12236dfab --- /dev/null +++ b/crates/trident-acl-agent/src/trident.rs @@ -0,0 +1,399 @@ +//! gRPC helpers for talking to `tridentd`. +//! +//! Implements the Trident-invocation half of `docs/update-trigger-design.md`: +//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md +//! (the "Trident invocation" column of section 2.1's operations table, +//! and the stage/finalize/rollback-finalize CallerHandlesReboot split in +//! section 2.3). +//! +//! The annotation protocol drives stage/finalize/commit directly against +//! tridentd's stable v1 API (§4–§5). Startup recovery no longer pre-queries +//! the preview `StatusService::GetServicingState`: commit() is self-checking +//! (tridentd only commits from a valid servicing_state and otherwise returns +//! ServicingKind::NoneRequired as a harmless no-op), so the orchestrator +//! always calls commit() unconditionally and falls back to annotation-based +//! progress for anything commit() reports nothing to do for. See +//! orchestrator.rs's recover_from_trident_state for the full rationale. + +use std::time::Duration; + +use anyhow::anyhow; +use futures::StreamExt; +use tonic::{transport::Endpoint, Request, Streaming}; +use trident_proto::v1::{ + commit_service_client::CommitServiceClient, rollback_service_client::RollbackServiceClient, + servicing_response::Response as ResponseBody, update_service_client::UpdateServiceClient, + CommitRequest, FinalizeUpdateRequest, HostConfiguration, LogLevel, ManualRollbackKind, + RebootHandling, RebootManagement, RebootStatus, RollbackFinalizeRequest, RollbackStageRequest, + ServicingKind, ServicingResponse, StageUpdateRequest, StatusCode, TridentErrorKind, + UpdateRequest, +}; +use url::Url; + +#[derive(Debug, Clone)] +pub struct CompletedResponse { + pub reboot_status: RebootStatus, + pub servicing_kind: Option, +} + +#[derive(Debug, Clone)] +pub struct RemoteError { + pub kind: Option, + pub subkind: String, + pub message: String, + pub error_message: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum TridentClientError { + #[error("failed to connect to trident socket {socket}: {source}")] + Connect { + socket: String, + #[source] + source: tonic::transport::Error, + }, + #[error("failed to start trident request {operation}: {source}")] + Request { + operation: &'static str, + #[source] + source: tonic::Status, + }, + #[error("trident stream for {operation} ended before a Completed message")] + MissingCompletion { operation: &'static str }, + #[error("trident stream for {operation} failed: {source}")] + Stream { + operation: &'static str, + #[source] + source: anyhow::Error, + }, + #[error("trident reported {operation} failure: {details:?}")] + Remote { + operation: &'static str, + details: RemoteError, + }, + #[error("{operation} timed out after {timeout:?}")] + Timeout { + operation: &'static str, + timeout: Duration, + }, +} + +impl TridentClientError { + pub fn remote(&self) -> Option<&RemoteError> { + match self { + Self::Remote { details, .. } => Some(details), + _ => None, + } + } +} + +pub struct TridentClient { + update_client: UpdateServiceClient, + commit_client: CommitServiceClient, + rollback_client: RollbackServiceClient, +} + +impl TridentClient { + pub async fn connect(socket: &str) -> Result { + let endpoint = + Endpoint::new(socket.to_string()).map_err(|source| TridentClientError::Connect { + socket: socket.to_string(), + source, + })?; + let channel = endpoint + .connect() + .await + .map_err(|source| TridentClientError::Connect { + socket: socket.to_string(), + source, + })?; + + Ok(Self::from_channel(channel)) + } + + /// Builds a client directly from an existing tonic Channel, bypassing + /// socket/URI resolution entirely. Production code always goes through + /// connect(); this exists so tests can hand the client a channel wired + /// to an in-process fake tridentd (e.g. via Endpoint::connect_with_connector + /// over an in-memory duplex stream) and exercise the exact same + /// request/response/error-mapping code as production, without a real + /// unix socket or subprocess. + pub fn from_channel(channel: tonic::transport::Channel) -> Self { + Self { + update_client: UpdateServiceClient::new(channel.clone()), + commit_client: CommitServiceClient::new(channel.clone()), + rollback_client: RollbackServiceClient::new(channel), + } + } + + pub async fn update( + &mut self, + url: &Url, + hash: Option<&str>, + timeout: Duration, + ) -> Result { + let response = self + .update_client + .update(Request::new(UpdateRequest { + stage: Some(StageUpdateRequest { + config: Some(host_configuration_from_image(url, hash)), + }), + finalize: Some(FinalizeUpdateRequest { + reboot: Some(RebootManagement { + handling: RebootHandling::CallerHandlesReboot.into(), + }), + }), + })) + .await + .map_err(|source| TridentClientError::Request { + operation: "update", + source, + })? + .into_inner(); + + run_with_timeout( + "update", + timeout, + consume_servicing_stream("update", response), + ) + .await + } + + pub async fn update_stage( + &mut self, + url: &Url, + hash: Option<&str>, + timeout: Duration, + ) -> Result { + let response = self + .update_client + .update_stage(Request::new(StageUpdateRequest { + config: Some(host_configuration_from_image(url, hash)), + })) + .await + .map_err(|source| TridentClientError::Request { + operation: "update_stage", + source, + })? + .into_inner(); + + run_with_timeout( + "update_stage", + timeout, + consume_servicing_stream("update_stage", response), + ) + .await + } + + pub async fn update_finalize( + &mut self, + timeout: Duration, + ) -> Result { + let response = self + .update_client + .update_finalize(Request::new(FinalizeUpdateRequest { + reboot: Some(RebootManagement { + handling: RebootHandling::CallerHandlesReboot.into(), + }), + })) + .await + .map_err(|source| TridentClientError::Request { + operation: "update_finalize", + source, + })? + .into_inner(); + + run_with_timeout( + "update_finalize", + timeout, + consume_servicing_stream("update_finalize", response), + ) + .await + } + + pub async fn commit( + &mut self, + timeout: Duration, + ) -> Result { + let response = self + .commit_client + .commit(Request::new(CommitRequest { + reboot: Some(RebootManagement { + // The agent, not tridentd, must own every reboot + // decision: AKS-RP is the sole authority over + // reboot/rollback (accepted-design-v2.md §2.5). If commit() + // ever reports NeedsReboot (e.g. a health-check failure, + // were health checks ever re-enabled), the agent needs + // to see that as a RebootRequired response it controls + // and reports via labels, not have tridentd reboot out + // from under it. + handling: RebootHandling::CallerHandlesReboot.into(), + }), + })) + .await + .map_err(|source| TridentClientError::Request { + operation: "commit", + source, + })? + .into_inner(); + + run_with_timeout( + "commit", + timeout, + consume_servicing_stream("commit", response), + ) + .await + } + + /// Stages an A/B rollback. Only `AbRollbackRequested` is used - per the + /// accepted design, trident-acl-agent only ever drives AB-kind manual + /// rollback; runtime-kind and "any" rollback are out of scope for the + /// annotation-driven protocol. + pub async fn rollback_stage( + &mut self, + timeout: Duration, + ) -> Result { + let response = self + .rollback_client + .rollback_stage(Request::new(RollbackStageRequest { + kind: ManualRollbackKind::AbRollbackRequested.into(), + })) + .await + .map_err(|source| TridentClientError::Request { + operation: "rollback_stage", + source, + })? + .into_inner(); + + run_with_timeout( + "rollback_stage", + timeout, + consume_servicing_stream("rollback_stage", response), + ) + .await + } + + pub async fn rollback_finalize( + &mut self, + timeout: Duration, + ) -> Result { + let response = self + .rollback_client + .rollback_finalize(Request::new(RollbackFinalizeRequest { + reboot: Some(RebootManagement { + // Same rationale as commit()/update_finalize(): AKS-RP, + // via the agent, is the sole authority over reboot + // timing (accepted-design-v2.md §2.5). + handling: RebootHandling::CallerHandlesReboot.into(), + }), + })) + .await + .map_err(|source| TridentClientError::Request { + operation: "rollback_finalize", + source, + })? + .into_inner(); + + run_with_timeout( + "rollback_finalize", + timeout, + consume_servicing_stream("rollback_finalize", response), + ) + .await + } +} + +#[derive(serde::Serialize)] +struct ImageSpec<'a> { + url: &'a str, + sha384: &'a str, +} + +#[derive(serde::Serialize)] +struct HostConfigurationYaml<'a> { + image: ImageSpec<'a>, +} + +pub fn host_configuration_from_image(url: &Url, hash: Option<&str>) -> HostConfiguration { + // Build via serde_yaml rather than raw string formatting so a URL or + // hash containing YAML-special characters (e.g. ':' or '#') can't + // produce invalid YAML or silently change the parsed structure fed to + // tridentd as configuration. + let spec = HostConfigurationYaml { + image: ImageSpec { + url: url.as_str(), + sha384: hash.unwrap_or("ignored"), + }, + }; + HostConfiguration { + config: serde_yaml::to_string(&spec) + .expect("serializing a simple struct to YAML cannot fail"), + } +} + +async fn run_with_timeout( + operation: &'static str, + timeout: Duration, + future: impl std::future::Future>, +) -> Result { + tokio::time::timeout(timeout, future) + .await + .map_err(|_| TridentClientError::Timeout { operation, timeout })? +} + +async fn consume_servicing_stream( + operation: &'static str, + mut stream: Streaming, +) -> Result { + while let Some(item) = stream.next().await { + let response = item.map_err(|source| TridentClientError::Stream { + operation, + source: anyhow!(source), + })?; + + match response.response { + Some(ResponseBody::Started(_)) => { + log::info!("[Trident:{operation}] started"); + } + Some(ResponseBody::Log(log_record)) => { + let msg = format!("[Trident:{operation}] {}", log_record.message); + match log_record.level() { + LogLevel::Unspecified | LogLevel::Trace => log::trace!("{msg}"), + LogLevel::Debug => log::debug!("{msg}"), + LogLevel::Info => log::info!("{msg}"), + LogLevel::Warn => log::warn!("{msg}"), + LogLevel::Error => log::error!("{msg}"), + } + } + Some(ResponseBody::Completed(completed)) => { + if completed.status() == StatusCode::Success { + return Ok(CompletedResponse { + reboot_status: completed.reboot_status(), + servicing_kind: completed + .servicing_kind + .and_then(|value| ServicingKind::try_from(value).ok()), + }); + } + + let details = completed + .error + .map(|error| RemoteError { + kind: TridentErrorKind::try_from(error.kind).ok(), + subkind: error.subkind, + message: error.message, + error_message: error.error_message, + }) + .unwrap_or(RemoteError { + kind: None, + subkind: "unknown".to_string(), + message: format!("Trident {operation} failed without structured error"), + error_message: String::new(), + }); + return Err(TridentClientError::Remote { operation, details }); + } + None => continue, + } + } + + Err(TridentClientError::MissingCompletion { operation }) +} diff --git a/packaging/rpm/trident.spec b/packaging/rpm/trident.spec index eda6ba4f8a..62a1d35ec7 100644 --- a/packaging/rpm/trident.spec +++ b/packaging/rpm/trident.spec @@ -237,6 +237,16 @@ The Trident ACL Agent triggers updates of ACL images. %files acl-agent %license LICENSE NOTICE %{_bindir}/%{name}-acl-agent +%{_unitdir}/%{name}-acl-agent.service + +%post acl-agent +%systemd_post %{name}-acl-agent.service + +%preun acl-agent +%systemd_preun %{name}-acl-agent.service + +%postun acl-agent +%systemd_postun_with_restart %{name}-acl-agent.service %endif # ------------------------------------------------------------------------------ @@ -305,6 +315,7 @@ cargo test --all --no-fail-fast -- --skip test_run_systemd_check --skip test_pre install -D -m 755 target/release/%{name} %{buildroot}/%{_bindir}/%{name} %if %{defined rpm_ver} install -D -m 755 target/release/%{name}-acl-agent %{buildroot}/%{_bindir}/%{name}-acl-agent +install -D -m 644 packaging/systemd/%{name}-acl-agent.service %{buildroot}%{_unitdir}/%{name}-acl-agent.service %endif # Copy Trident SELinux policy module to /usr/share/selinux/packages diff --git a/packaging/systemd/trident-acl-agent.service b/packaging/systemd/trident-acl-agent.service new file mode 100644 index 0000000000..eee90451e2 --- /dev/null +++ b/packaging/systemd/trident-acl-agent.service @@ -0,0 +1,12 @@ +[Unit] +Description=Trident ACL Agent +After=network-online.target tridentd.socket +Wants=network-online.target tridentd.socket + +[Service] +ExecStart=trident-acl-agent +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target From b2b0d45a8bc26c1fb0619d47a7d04d63e1c784a4 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Mon, 17 Aug 2026 22:51:10 +0000 Subject: [PATCH 02/54] deny.toml: allow ISC and Zlib licenses kube-client dependency (added for the ACL agent rewrite) pulls in ring, rustls-webpki, untrusted (ISC) and foldhash (Zlib), none of which were on the license allow-list, failing check-licenses. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- deny.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deny.toml b/deny.toml index ceeae94c3b..f6a2f8091f 100644 --- a/deny.toml +++ b/deny.toml @@ -52,6 +52,8 @@ allow = [ "Apache-2.0", "BSD-3-Clause", "Unicode-3.0", + "ISC", + "Zlib", ] # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the From f50d8d088c946973b515c74a6f1396f47df25b45 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 19 Aug 2026 16:58:38 +0000 Subject: [PATCH 03/54] notice: allow ISC in about.toml, regenerate NOTICE deny.toml was updated to allow ISC/Zlib for the new kube-client transitive deps (ring, rustls-webpki, untrusted, foldhash) but packaging/notice/about.toml accepted list was missed, failing make validate-notice in CI. Add ISC and regenerate NOTICE. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- NOTICE | 1898 ++++++++++++++++++++++++++++++++++- packaging/notice/about.toml | 1 + 2 files changed, 1882 insertions(+), 17 deletions(-) diff --git a/NOTICE b/NOTICE index ab768eb0b1..07901dd517 100644 --- a/NOTICE +++ b/NOTICE @@ -10,6 +10,494 @@ Run `make update-notice` to regenerate it after changing dependencies. -------------------------------------------------------------------------------- Apache License 2.0 (Apache-2.0) +Used by: + - k8s-openapi 0.24.0 + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- +Apache License 2.0 (Apache-2.0) + +Used by: + - ring 0.17.14 + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +BoringSSL uses the Chromium test infrastructure to run a continuous build, +trybots etc. The scripts which manage this, and the script for generating build +metadata, are under the Chromium license. Distributing code linked against +BoringSSL does not trigger this license. + +Copyright 2015 The Chromium Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- +Apache License 2.0 (Apache-2.0) + Used by: - pulldown-cmark-to-cmark 21.1.0 @@ -847,6 +1335,7 @@ Used by: - prost-build 0.14.1 - prost-derive 0.14.1 - prost-types 0.14.1 + - ring 0.17.14 Apache License Version 2.0, January 2004 @@ -1262,6 +1751,10 @@ limitations under the License. Apache License 2.0 (Apache-2.0) Used by: + - kube 0.98.0 + - kube-client 0.98.0 + - kube-core 0.98.0 + - kube-runtime 0.98.0 - ryu 1.0.18 - sync_wrapper 1.0.2 @@ -1342,6 +1835,67 @@ limitations under the License. -------------------------------------------------------------------------------- BSD 3-Clause "New" or "Revised" License (BSD-3-Clause) +Used by: + - encoding_rs 0.8.35 + +// Copyright © WHATWG (Apple, Google, Mozilla, Microsoft). +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// The PUA code points special-cased in the GB18030 encoder. +pub(crate) static GB18030_2022_OVERRIDE_PUA: [u16; 18] = [ + 0xE78D, 0xE78E, 0xE78F, 0xE790, 0xE791, 0xE792, 0xE793, 0xE794, 0xE795, 0xE796, 0xE81E, 0xE826, + 0xE82B, 0xE82C, 0xE832, 0xE843, 0xE854, 0xE864, +]; + +/// The bytes corresponding to the PUA code points special-cased in the GB18030 encoder. +pub(crate) static GB18030_2022_OVERRIDE_BYTES: [[u8; 2]; 18] = [ + [0xA6, 0xD9], + [0xA6, 0xDA], + [0xA6, 0xDB], + [0xA6, 0xDC], + [0xA6, 0xDD], + [0xA6, 0xDE], + [0xA6, 0xDF], + [0xA6, 0xEC], + [0xA6, 0xED], + [0xA6, 0xF3], + [0xFE, 0x59], + [0xFE, 0x61], + [0xFE, 0x66], + [0xFE, 0x67], + [0xFE, 0x6D], + [0xFE, 0x7E], + [0xFE, 0x90], + [0xFE, 0xA0], +]; + +-------------------------------------------------------------------------------- +BSD 3-Clause "New" or "Revised" License (BSD-3-Clause) + Used by: - matchit 0.8.4 @@ -1447,6 +2001,40 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- BSD 3-Clause "New" or "Revised" License (BSD-3-Clause) +Used by: + - instant 0.1.13 + +Copyright (c) 2019, Sébastien Crozet +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the author nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- +BSD 3-Clause "New" or "Revised" License (BSD-3-Clause) + Used by: - encoding_rs 0.8.35 @@ -1477,6 +2065,649 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +/* Copyright (c) 2014, Intel Corporation. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +#ifndef OPENSSL_HEADER_EC_ECP_NISTZ384_H +#define OPENSSL_HEADER_EC_ECP_NISTZ384_H + +#include "../../limbs/limbs.h" + +#define P384_LIMBS (384u / LIMB_BITS) + +typedef struct { + Limb X[P384_LIMBS]; + Limb Y[P384_LIMBS]; + Limb Z[P384_LIMBS]; +} P384_POINT; + +typedef struct { + Limb X[P384_LIMBS]; + Limb Y[P384_LIMBS]; +} P384_POINT_AFFINE; + + +#endif // OPENSSL_HEADER_EC_ECP_NISTZ384_H + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2015-2016 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +//! EdDSA Signatures. + +use super::ops::ELEM_LEN; +use crate::digest; + +pub mod signing; +pub mod verification; + +/// The length of an Ed25519 public key. +pub const ED25519_PUBLIC_KEY_LEN: usize = ELEM_LEN; + +pub fn eddsa_digest(signature_r: &[u8], public_key: &[u8], msg: &[u8]) -> digest::Digest { + let mut ctx = digest::Context::new(&digest::SHA512); + ctx.update(signature_r); + ctx.update(public_key); + ctx.update(msg); + ctx.finish() +} + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - untrusted 0.9.0 + +// Copyright 2015-2016 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2015-2022 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +use crate::limb::Limb; + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct N0([Limb; 2]); + +impl N0 { + #[cfg(feature = "alloc")] + pub(super) const LIMBS_USED: usize = 64 / crate::limb::LIMB_BITS; + + #[inline] + pub const fn precalculated(n0: u64) -> Self { + #[cfg(target_pointer_width = "64")] + { + Self([n0, 0]) + } + + #[cfg(target_pointer_width = "32")] + { + Self([n0 as Limb, (n0 >> crate::limb::LIMB_BITS) as Limb]) + } + } +} + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2015-2025 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +use crate::{bb, error}; + +#[deprecated( + note = "To be removed. Internal function not intended for external use with no promises regarding side channels." +)] +pub fn verify_slices_are_equal(a: &[u8], b: &[u8]) -> Result<(), error::Unspecified> { + bb::verify_slices_are_equal(a, b) +} + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2016 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +//! Elliptic curve operations and schemes using Curve25519. + +pub mod ed25519; + +pub mod x25519; + +mod ops; +mod scalar; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2016-2024 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +use crate::error::{KeyRejected, Unspecified}; + +impl From for Unspecified { + fn from(source: untrusted::EndOfInput) -> Self { + super::erase(source) + } +} + +impl From for Unspecified { + fn from(source: core::array::TryFromSliceError) -> Self { + super::erase(source) + } +} + +impl From for Unspecified { + fn from(source: KeyRejected) -> Self { + super::erase(source) + } +} + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2018 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +//! Serialization and deserialization. + +#[doc(hidden)] +pub mod der; + +#[cfg(feature = "alloc")] +mod writer; + +#[cfg(feature = "alloc")] +pub(crate) mod der_writer; + +pub(crate) mod positive; + +pub use self::positive::Positive; + +#[cfg(feature = "alloc")] +pub(crate) use self::writer::TooLongError; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2019-2024 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +use super::BlockLen; + +pub(super) use self::{ + sha2_32::{block_data_order_32, State32, SHA256_BLOCK_LEN}, + sha2_64::{block_data_order_64, State64, SHA512_BLOCK_LEN}, +}; + +pub(super) const CHAINING_WORDS: usize = 8; + +#[cfg(any( + all(target_arch = "aarch64", target_endian = "little"), + all(target_arch = "arm", target_endian = "little"), + target_arch = "x86_64" +))] +#[macro_use] +mod ffi; + +pub(super) mod fallback; +mod sha2_32; +mod sha2_64; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2024 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +// TODO(MSRV 1.76): Replace with `core::ptr::from_mut`. +#[allow(dead_code)] +#[inline(always)] +pub fn from_mut(r: &mut T) -> *mut T { + r +} + +// TODO(MSRV 1.76): Replace with `core::ptr::from_ref`. +#[allow(dead_code)] +#[inline(always)] +pub const fn from_ref(r: &T) -> *const T { + r +} + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2024 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +//! Integration tests for non-public APIs. + +mod bits_tests; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2024 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +pub use self::{ + array::Array, + base::{IndexError, Overlapping}, + partial_block::PartialBlock, +}; + +mod array; +mod base; +mod partial_block; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2025 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +#![cfg(all(target_arch = "aarch64", target_endian = "little"))] + +pub(in super::super) mod mont; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2025 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +#![cfg(target_arch = "x86_64")] + +pub(in super::super::super) mod mont; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2025 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +mod storage; + +pub(super) use self::storage::{AlignedStorage, LIMBS_PER_CHUNK}; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +// Copyright 2025 Brian Smith. +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +pub(super) mod aarch64; +pub(super) mod x86_64; + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - ring 0.17.14 + +Copyright 2015-2025 Brian Smith. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +-------------------------------------------------------------------------------- +ISC License (ISC) + +Used by: + - rustls-webpki 0.103.13 + +Except as otherwise noted, this project is licensed under the following +(ISC-style) terms: + +Copyright 2015 Brian Smith. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +The files under third-party/chromium are licensed as described in +third-party/chromium/LICENSE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + +Used by: + - cfg_aliases 0.2.1 + +# 3rd Party Notices + +The `cfg_aliases!` macro uses a lot of the code from [`tectonic_cfg_support::target_cfg!`] macro which is under the following license: + +[`tectonic_cfg_support::target_cfg!`]: https://github.com/tectonic-typesetting/tectonic/blob/f2439b936470ad27bdf92882064bc4702ee01899/cfg_support/src/lib.rs#L166 + + tectonic_cfg_support is licensed under the MIT License. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the “Software”), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +--- + +-------------------------------------------------------------------------------- +MIT License (MIT) + +Used by: + - schemars_derive 0.8.21 + +#![allow(clippy::all)] +// Copied from regex_syntax crate to avoid pulling in the whole crate just for a utility function +// https://github.com/rust-lang/regex/blob/431c4e4867e1eb33eb39b23ed47c9934b2672f8f/regex-syntax/src/lib.rs +// +// Copyright (c) 2014 The Rust Project Developers +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +pub fn escape(text: &str) -> String { + let mut quoted = String::new(); + escape_into(text, &mut quoted); + quoted +} + +fn escape_into(text: &str, buf: &mut String) { + buf.reserve(text.len()); + for c in text.chars() { + if is_meta_character(c) { + buf.push('\\'); + } + buf.push(c); + } +} + +fn is_meta_character(c: char) -> bool { + match c { + '\\' | '.' | '+' | '*' | '?' | '(' | ')' | '|' | '[' | ']' | '{' | '}' | '^' | '$' + | '#' | '&' | '-' | '~' => true, + _ => false, + } +} + -------------------------------------------------------------------------------- MIT License (MIT) @@ -1495,16 +2726,108 @@ distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + +Used by: + - globwalk 0.9.1 + +// Copyright (c) 2018 Gilad Naaman +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +use globwalk::GlobWalkerBuilder; +use std::env::args; + +fn main() { + let patterns = args().skip(1).collect::>(); + + for f in GlobWalkerBuilder::from_patterns(".", &patterns[..]) + .build() + .unwrap() + { + println!("{:?}", f.unwrap().path()); + } +} + +-------------------------------------------------------------------------------- +MIT License (MIT) + +Used by: + - atomic-waker 1.1.2 + +=============================================================================== + +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +=============================================================================== + +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. -THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) @@ -1672,12 +2995,14 @@ MIT License (MIT) Used by: - backtrace 0.3.74 - - cc 1.2.2 + - cc 1.4.0 - cfg-if 1.0.0 - filetime 0.2.25 + - find-msvc-tools 0.1.9 - flate2 1.0.35 - jobserver 0.1.32 - openssl-probe 0.1.5 + - openssl-probe 0.2.1 - openssl-sys 0.9.116 - pkg-config 0.3.31 - rustc-demangle 0.1.24 @@ -1797,7 +3122,7 @@ THE SOFTWARE. MIT License (MIT) Used by: - - bitflags 2.6.0 + - bitflags 2.13.1 - glob 0.3.1 - log 0.4.22 - num-traits 0.2.19 @@ -1954,6 +3279,38 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - ordered-float 2.10.1 + +Copyright (c) 2015 Jonathan Reem + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - slug 0.1.6 - tempfile 3.14.0 @@ -2365,6 +3722,9 @@ DEALINGS IN THE SOFTWARE. MIT License (MIT) Used by: + - hyper-rustls 0.27.9 + - rustls 0.23.43 + - rustls-native-certs 0.8.4 - rustls-pemfile 2.2.0 Copyright (c) 2016 Joseph Birr-Pixton @@ -2543,6 +3903,64 @@ SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - backoff 0.4.0 + +Copyright (c) 2016 Tibor Benke + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + +Used by: + - serde-value 0.7.0 + +Copyright (c) 2016 arcnmx + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - multimap 0.10.1 @@ -3010,6 +4428,38 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - tokio-rustls 0.26.4 + +Copyright (c) 2017 quininer kel + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - signal-hook 0.4.3 - signal-hook-registry 1.4.7 @@ -3203,6 +4653,38 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - ahash 0.8.12 + +Copyright (c) 2018 Tom Kaitchuck + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - want 0.3.1 @@ -3322,6 +4804,38 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - zeroize 1.9.0 + +Copyright (c) 2018-2026 The RustCrypto Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - slab 0.4.9 @@ -3549,6 +5063,38 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - tower-http 0.6.11 + +Copyright (c) 2019-2021 Tower Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - http-body 1.0.1 @@ -3756,7 +5302,7 @@ SOFTWARE. MIT License (MIT) Used by: - - rustls-pki-types 1.10.0 + - rustls-pki-types 1.15.1 Copyright (c) 2023 Dirkjan Ochtman @@ -4065,7 +5611,7 @@ MIT License (MIT) Used by: - zerocopy 0.7.35 - - zerocopy 0.8.17 + - zerocopy 0.8.27 - zerocopy-derive 0.7.35 Copyright 2023 The Fuchsia Authors @@ -4194,7 +5740,35 @@ Used by: MIT License -Copyright (c) 2017 Frommi +Copyright (c) 2017 Frommi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + +Used by: + - smart-default 0.7.1 + +MIT License + +Copyright (c) 2017 Idan Arye Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -4218,11 +5792,11 @@ SOFTWARE. MIT License (MIT) Used by: - - smart-default 0.7.1 + - json-patch 3.0.1 MIT License -Copyright (c) 2017 Idan Arye +Copyright (c) 2017 Ivan Dubrov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -4561,6 +6135,34 @@ SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - secrecy 0.10.3 + +MIT License + +Copyright (c) 2019-2024 iqlusion + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - convert_case 0.6.0 @@ -4661,6 +6263,34 @@ SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - jsonptr 0.6.3 + +MIT License + +Copyright (c) 2022 Chance Dinkins + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - matchit 0.8.4 @@ -4689,6 +6319,36 @@ SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - educe 0.6.0 + - enum-ordinalize 4.3.2 + - enum-ordinalize-derive 4.3.2 + +MIT License + +Copyright (c) 2023 magiclen.org (Ron Li) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - serde_variant 0.1.3 @@ -4780,6 +6440,8 @@ SOFTWARE. MIT License (MIT) Used by: + - async-stream 0.3.6 + - async-stream-impl 0.3.6 - chrono 0.4.38 - chrono-tz 0.9.0 - chrono-tz-build 0.3.0 @@ -4924,6 +6586,34 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - jsonpath-rust 0.7.5 + +MIT License + +Copyright (c) [2021] [Boris Zhguchev] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - adler2 2.0.0 - anyhow 1.0.94 @@ -4931,6 +6621,8 @@ Used by: - atomic-waker 1.1.2 - displaydoc 0.2.5 - dyn-clone 1.0.17 + - event-listener 5.4.2 + - event-listener-strategy 0.5.4 - fastrand 2.2.0 - home 0.5.9 - indoc 2.0.5 @@ -4940,6 +6632,7 @@ Used by: - linux-raw-sys 0.11.0 - linux-raw-sys 0.4.14 - once_cell 1.20.2 + - parking 2.2.1 - pest 2.7.14 - pest_derive 2.7.14 - pest_generator 2.7.14 @@ -4968,6 +6661,7 @@ Used by: - thiserror-impl 2.0.12 - unicode-ident 1.0.14 - unsafe-libyaml 0.2.11 + - zerocopy 0.7.35 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated @@ -4996,6 +6690,36 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - allocator-api2 0.2.21 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - systemd-journal-logger 2.2.2 @@ -5033,6 +6757,36 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - is-terminal 0.4.13 + +Portions of this project are derived from atty, which bears the following +copyright notice and permission notice: + +Copyright (c) 2015-2019 Doug Tangren + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - pulldown-cmark 0.13.0 @@ -5061,6 +6815,34 @@ THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - tracing-core 0.1.36 + +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - typenum 1.17.0 @@ -5271,7 +7053,7 @@ SOFTWARE. MIT License (MIT) Used by: - - shlex 1.3.0 + - shlex 2.0.1 The MIT License (MIT) @@ -5384,6 +7166,34 @@ SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - pem 3.0.6 + +The MIT License (MIT) + +Copyright (c) 2016 Jonathan Creekmore + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - hyper-timeout 0.5.2 @@ -5534,6 +7344,34 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - async-broadcast 0.7.2 + +The MIT License (MIT) + +Copyright (c) 2020 Yoshua Wuyts + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - duct 0.13.7 - os_pipe 1.2.1 @@ -5859,3 +7697,29 @@ freely, subject to the following restrictions: 3. This notice may not be removed or altered from any source distribution. -------------------------------------------------------------------------------- +zlib License (Zlib) + +Used by: + - foldhash 0.1.5 + +Copyright (c) 2024 Orson Peters + +This software is provided 'as-is', without any express or implied warranty. In +no event will the authors be held liable for any damages arising from the use of +this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim + that you wrote the original software. If you use this software in a product, + an acknowledgment in the product documentation would be appreciated but is + not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + +-------------------------------------------------------------------------------- diff --git a/packaging/notice/about.toml b/packaging/notice/about.toml index 924e836e9e..2ee0852d3d 100644 --- a/packaging/notice/about.toml +++ b/packaging/notice/about.toml @@ -21,6 +21,7 @@ accepted = [ "BSD-3-Clause", "Unicode-3.0", "Zlib", + "ISC", ] # Pin the target triples so the generated dependency graph — and therefore the From 4083120e3b238537c857b4619c2a3d31a33aafe0 Mon Sep 17 00:00:00 2001 From: bfjelds Date: Wed, 19 Aug 2026 10:03:20 -0700 Subject: [PATCH 04/54] use explicit path for TAA.service ExecStart Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packaging/systemd/trident-acl-agent.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/systemd/trident-acl-agent.service b/packaging/systemd/trident-acl-agent.service index eee90451e2..24679de68b 100644 --- a/packaging/systemd/trident-acl-agent.service +++ b/packaging/systemd/trident-acl-agent.service @@ -4,7 +4,7 @@ After=network-online.target tridentd.socket Wants=network-online.target tridentd.socket [Service] -ExecStart=trident-acl-agent +ExecStart=/usr/bin/trident-acl-agent Restart=on-failure RestartSec=5 From 2b0ac9694ad757713e674b52ccb879c8d6015ed1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 19 Aug 2026 17:56:59 +0000 Subject: [PATCH 05/54] ci: re-trigger checks (transient license-mirror failure on prior run) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> From 4005d77f23b09372b862c6e4ca70bd6dfaf0ec8d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 19 Aug 2026 18:43:48 +0000 Subject: [PATCH 06/54] notice: run cargo fetch --locked before cargo-about to avoid flaky drops cargo-about --offline silently omits a crate license section (rather than erroring) when that crate source isn ' t already unpacked in the local registry cache. On cache-cold CI agents this intermittently produced an incomplete NOTICE for newly-added deps (seen with ring/rustls-webpki/ untrusted), failing validate-notice non-deterministically. Running cargo fetch --locked first guarantees every crate ' s source is present before cargo-about reads it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: af8c8d5a-40b2-443a-9122-01fc83040ea3 --- Makefile | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 04f2a5fe3c..8638d018f4 100644 --- a/Makefile +++ b/Makefile @@ -405,16 +405,22 @@ NOTICE_CHECKED_IN := NOTICE NOTICE_GENERATED := target/NOTICE.generated NOTICE_JSON := target/notice.json NOTICE_RENDERER := packaging/notice/render_notice.py -# --locked pins Cargo.lock; --offline keeps generation deterministic and matches -# the network-isolated CI (no clearlydefined.io lookups); --workspace matches the -# scope of the cargo-deny check. Output is JSON so render_notice.py can group by -# license content (cargo-about's own section grouping is not host-stable). +# --locked pins Cargo.lock; --offline keeps generation deterministic (no +# clearlydefined.io lookups, so results don't depend on network state); +# --workspace matches the scope of the cargo-deny check. `cargo fetch --locked` +# runs first so every crate's source (and thus its license file) is present on +# disk before cargo-about reads it -- otherwise cargo-about silently omits the +# license section for any crate whose source isn't already cached, rather than +# erroring, which caused intermittent NOTICE-validation failures on cache-cold +# CI agents. Output is JSON so render_notice.py can group by license content +# (cargo-about's own section grouping is not host-stable). CARGO_ABOUT_ARGS := generate --workspace --locked --offline -c packaging/notice/about.toml --format json # Regenerate the checked-in NOTICE locally. Run after changing dependencies. .PHONY: update-notice update-notice: @mkdir -p target + cargo fetch --locked cargo about $(CARGO_ABOUT_ARGS) -o $(NOTICE_JSON) python3 $(NOTICE_RENDERER) $(NOTICE_JSON) > $(NOTICE_CHECKED_IN) @echo Updated $(NOTICE_CHECKED_IN) @@ -427,6 +433,7 @@ validate-notice: @echo "" @echo "Validating third-party NOTICE..." @mkdir -p target + cargo fetch --locked cargo about $(CARGO_ABOUT_ARGS) -o $(NOTICE_JSON) python3 $(NOTICE_RENDERER) $(NOTICE_JSON) > $(NOTICE_GENERATED) @diff $(NOTICE_CHECKED_IN) $(NOTICE_GENERATED) || { \ From 9c9424919a4c7e6ac39b9420dd178573e9b4ef4c Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 19 Aug 2026 19:14:10 +0000 Subject: [PATCH 07/54] ci: pin cargo-about to 0.9.2 to fix flaky NOTICE validation 0.9.1 (the previously pinned version) could silently discard a crates detected license files instead of including them in output, rather than erroring. This intermittently dropped whole license sections (observed for ring, and previously for is-terminal/atty/tracing-core) from the generated NOTICE, causing non-deterministic validate-notice failures depending on which cargo-about run happened to hit the bug. Fixed upstream in cargo-about 0.9.2 (PR EmbarkStudios/cargo-about#312, resolving issue #309: detected license files are never discarded). Reproduced locally: cargo-about 0.9.1 drops rings license section from a fresh cargo about generate; 0.9.2 does not. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: af8c8d5a-40b2-443a-9122-01fc83040ea3 --- .../templates/stages/validate_makefile/dev-build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/validate_makefile/dev-build.yml b/.pipelines/templates/stages/validate_makefile/dev-build.yml index f14cb1bbcd..89bc45ce78 100644 --- a/.pipelines/templates/stages/validate_makefile/dev-build.yml +++ b/.pipelines/templates/stages/validate_makefile/dev-build.yml @@ -67,7 +67,12 @@ stages: - script: | # Use a lower optimization level to speed up cargo-about installation. # cargo-about's binary is gated behind the non-default `cli` feature. - CARGO_PROFILE_RELEASE_OPT_LEVEL=0 cargo install --locked cargo-about@0.9.1 --features cli + # Pinned >=0.9.2: 0.9.1 could silently discard a crate's detected + # license files instead of including them (fixed upstream by + # https://github.com/EmbarkStudios/cargo-about/pull/312), which + # intermittently dropped whole license sections (e.g. for `ring`) + # from the generated NOTICE and failed validate-notice. + CARGO_PROFILE_RELEASE_OPT_LEVEL=0 cargo install --locked cargo-about@0.9.2 --features cli displayName: Install cargo-about workingDirectory: $(TRIDENT_SOURCE_DIR) From ff724e91ff8e02c7303172646d0cff0660e02672 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 19 Aug 2026 21:50:48 +0000 Subject: [PATCH 08/54] docs: update design-doc link hash to eb7e534b2415ad52b37ef22fd49685e81e56c8aa Repoint the pinned Compute-ACL-Update-Service commit link in the module docs (trident.rs, orchestrator.rs, annotations.rs x2) from 1cfe79ec53bfc6936771e2433cba3dec0906b4fd to eb7e534b2415ad52b37ef22fd49685e81e56c8aa, per review. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321246a2-c976-405e-956a-d9e68008e0c7 --- crates/trident-acl-agent/src/annotations.rs | 4 ++-- crates/trident-acl-agent/src/orchestrator.rs | 2 +- crates/trident-acl-agent/src/trident.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index ea946bb1bc..b86f8d0b11 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -603,7 +603,7 @@ mod tests { // // Pins our annotation (de)serialization/validation code against two // things lifted verbatim from docs/update-trigger-design.md - // (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md), + // (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md), // section 2.1 "Trigger mechanism", so a doc/code drift shows up as a // test failure instead of being discovered against a real AKS-RP: // 1. The three example JSON payloads (request, finalize status, and @@ -654,7 +654,7 @@ mod tests { }"#; /// The formal JSON Schema for the request annotation, from - /// docs/update-trigger-design.md (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md), + /// docs/update-trigger-design.md (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md), /// section 2.1 "Formal JSON Schema". Keep byte-for-byte in sync with /// that document. const DESIGN_DOC_REQUEST_SCHEMA: &str = r#"{ diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs index 86d0fe5d7a..a5aea3dc41 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -3,7 +3,7 @@ //! and writes the status annotation back, including post-reboot. //! //! Implements the node-side control flow from `docs/update-trigger-design.md`: -//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md +//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md //! (sections 2.1 "Trigger mechanism", 2.3 "Stage/finalize/rollback split //! and post-reboot commit", and 2.5 "Rollback"). See that document for the //! full state-machine rationale; keep it in sync with this file if the diff --git a/crates/trident-acl-agent/src/trident.rs b/crates/trident-acl-agent/src/trident.rs index e12236dfab..87a7ba7fb4 100644 --- a/crates/trident-acl-agent/src/trident.rs +++ b/crates/trident-acl-agent/src/trident.rs @@ -1,7 +1,7 @@ //! gRPC helpers for talking to `tridentd`. //! //! Implements the Trident-invocation half of `docs/update-trigger-design.md`: -//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GC1cfe79ec53bfc6936771e2433cba3dec0906b4fd&path=/docs/update-trigger-design.md +//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md //! (the "Trident invocation" column of section 2.1's operations table, //! and the stage/finalize/rollback-finalize CallerHandlesReboot split in //! section 2.3). From e6b0cf6dd2ea635cf68efc848979c13fcd5d4354 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 19 Aug 2026 22:20:40 +0000 Subject: [PATCH 09/54] trident-acl-agent: require server/appId/track on stage/finalize per accepted-design-v3 accepted-design-v3.md tightens the update-request annotation contract: stage/finalize must carry server/appId/track (the formal JSON Schema's allOf now requires them alongside targetVersion), and the agent "must not fall back to a built-in endpoint, because a fallback lets a node update from a source AKS-RP did not choose" (2.1). Bring the code in line with that - it previously made all three optional-with-fallback, never rejected a stage/finalize missing them, and the pinned formal schema literal was stale (missing the new allOf rule and the server pattern). crates/trident-acl-agent/src/annotations.rs: - UpdateRequest::validate() now rejects a stage/finalize request missing server, appId, or track with InvalidRequest (unchanged for rollback, which reports no Nebraska event and carries no update source). - Updated DESIGN_DOC_REQUEST_SCHEMA to match accepted-design-v3.md byte-for-byte: added the second allOf rule and the server property's uri pattern; extended the test schema validator's tiny built-in pattern matcher to recognize it. - Updated server/appId/track field doc comments to describe the required-on-stage/finalize, no-fallback semantics instead of optional-override-with-fallback. - New validate() unit tests (accepts with all three set, rejects each one missing on stage/finalize, allows rollback without them) and fixed existing tests that exercised the old optional/fallback behavior (agent_built_requests_conform_to_formal_schema and the server/appId/track field round-trip tests now use a fully-populated request when checking schema conformance for stage/finalize). crates/trident-acl-agent/src/orchestrator.rs: - resolve_nebraska_endpoint/app_id/track no longer fall back to self.config.nebraska.* - they resolve purely from the request's own fields (appId/track now return Option instead of always resolving to a value). UpdateRequest::validate() already guarantees these are present for stage/finalize before the orchestrator is ever reached, so a None here (handle_stage, report_nebraska_event) is treated as an agent-internal error / skipped best-effort report, same defense-in-depth posture as before, just without a silent fallback. crates/trident-acl-agent/README.md, config.rs: clarified that TRIDENT_ACL_AGENT_NEBRASKA_* env vars only apply to omaha-only mode now - annotations mode has no config fallback for these three fields. Also bumped every remaining accepted-design-v2.md reference in this crate to accepted-design-v3.md (state.rs, trident.rs, orchestrator.rs, k8s.rs, config.rs, annotations.rs) - those sections (2.3, 2.5, the status schema, etc.) are otherwise unchanged between v2 and v3. cargo build --workspace, cargo test/clippy/fmt for trident-acl-agent all pass clean (143 tests). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321246a2-c976-405e-956a-d9e68008e0c7 --- crates/trident-acl-agent/README.md | 106 ++++----- crates/trident-acl-agent/src/annotations.rs | 214 ++++++++++++++----- crates/trident-acl-agent/src/config.rs | 16 +- crates/trident-acl-agent/src/k8s.rs | 2 +- crates/trident-acl-agent/src/orchestrator.rs | 136 +++++++----- crates/trident-acl-agent/src/state.rs | 2 +- crates/trident-acl-agent/src/trident.rs | 4 +- 7 files changed, 312 insertions(+), 168 deletions(-) diff --git a/crates/trident-acl-agent/README.md b/crates/trident-acl-agent/README.md index 3f703fbfd2..552a72e173 100644 --- a/crates/trident-acl-agent/README.md +++ b/crates/trident-acl-agent/README.md @@ -1,53 +1,53 @@ -# trident-acl-agent - -The on-node half of Trident's Azure Container Linux (ACL) A/B update -trigger. Runs in one of two modes, selected by -`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`: - -- **`annotations`** (the default): watches its Node's - `acl.azure.com/update-request` annotation and drives Trident's - stage/finalize/rollback/commit operations against `tridentd` accordingly, - reporting progress and status back to Kubernetes and to Nebraska (the - Omaha-protocol update server). -- **`omaha-only`**: the historical one-shot behavior. Queries Nebraska once, - and if an update is offered, calls tridentd's combined `update()` RPC once - and exits - no Kubernetes or annotation involvement at all. Kept as an - explicit opt-out for nodes that don't participate in the AKS - annotation-driven update protocol. - -## Configuration - -There is no config file. Every setting is an environment variable prefixed -`TRIDENT_ACL_AGENT_`, systemd-style: set it directly in the unit's own -`Environment=` lines, via a drop-in override (`systemctl edit -trident-acl-agent.service`, which creates -`/etc/systemd/system/trident-acl-agent.service.d/override.conf`), or by any -other means that ultimately sets the process's environment before it -starts. - -A variable that is unset, or set to the empty string, falls back to that -setting's default below. A variable that is set to a malformed value (a bad -URL, a bad duration, an unrecognized `goal_source`) causes the agent to -fail to start with an error naming the offending variable. - -| Variable | Default | Description | -|---|---|---| -| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Nebraska/Omaha server URL to poll for updates and report progress/completion events to. Can also be overridden per-update via the `server` field on the `acl.azure.com/update-request` annotation, which takes precedence over this variable for that update's entire lifecycle (stage through post-reboot commit). | -| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Nebraska application ID this node checks in as. Can also be overridden per-update via the `appId` annotation field, same precedence rules as the endpoint. | -| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Nebraska track (channel/group) this node follows. Can also be overridden per-update via the `track` annotation field, same precedence rules as the endpoint. | -| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG`'s own kubeconfig is used as-is (e.g. the real cluster FQDN a node's own `/var/lib/kubelet/kubeconfig` already points at). Only needed when the kubeconfig's own server is wrong for this deployment. | -| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig file used to reach the Kubernetes API server and authenticate as this node. | -| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. Kubernetes Node names must be valid RFC 1123 DNS labels (lowercase), matching how kubelet itself registers the Node - so the default only needs overriding when the agent's environment can't discover the correct hostname on its own. | -| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode: `annotations` or `omaha-only` (see above). | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file, which bridges the pre-reboot `finalize`/`rollback` half of an update and its post-reboot `commit` half across the reboot. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` operation (parsed as a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) is allowed to run before it's considered failed. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` operation is allowed to run before it's considered failed. Parsed the same way as the stage timeout. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat the agent writes while a stage/finalize/rollback operation is running, so AKS-RP and the watchdog can tell a working agent from a stuck one. Parsed the same way as the timeouts. | - -## Diagnostics - -`trident-acl-agent --validate-connection ` -checks connectivity to a single dependency using the current environment -and exits immediately - useful for a systemd `ExecStartPre` check or manual -on-node troubleshooting without running the full orchestrator loop. +# trident-acl-agent + +The on-node half of Trident's Azure Container Linux (ACL) A/B update +trigger. Runs in one of two modes, selected by +`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`: + +- **`annotations`** (the default): watches its Node's + `acl.azure.com/update-request` annotation and drives Trident's + stage/finalize/rollback/commit operations against `tridentd` accordingly, + reporting progress and status back to Kubernetes and to Nebraska (the + Omaha-protocol update server). +- **`omaha-only`**: the historical one-shot behavior. Queries Nebraska once, + and if an update is offered, calls tridentd's combined `update()` RPC once + and exits - no Kubernetes or annotation involvement at all. Kept as an + explicit opt-out for nodes that don't participate in the AKS + annotation-driven update protocol. + +## Configuration + +There is no config file. Every setting is an environment variable prefixed +`TRIDENT_ACL_AGENT_`, systemd-style: set it directly in the unit's own +`Environment=` lines, via a drop-in override (`systemctl edit +trident-acl-agent.service`, which creates +`/etc/systemd/system/trident-acl-agent.service.d/override.conf`), or by any +other means that ultimately sets the process's environment before it +starts. + +A variable that is unset, or set to the empty string, falls back to that +setting's default below. A variable that is set to a malformed value (a bad +URL, a bad duration, an unrecognized `goal_source`) causes the agent to +fail to start with an error naming the offending variable. + +| Variable | Default | Description | +|---|---|---| +| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Nebraska/Omaha server URL to poll for updates and report progress/completion events to, for `omaha-only` mode. In `annotations` mode, `stage`/`finalize` requests must instead carry their own `server` field on the `acl.azure.com/update-request` annotation - there is deliberately no fallback to this variable, since a fallback would let a node update from a source AKS-RP did not choose; a request missing it is rejected with `InvalidRequest`. | +| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Nebraska application ID this node checks in as, for `omaha-only` mode. In `annotations` mode, required on the `acl.azure.com/update-request` annotation's `appId` field instead, same no-fallback rule as the endpoint. | +| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Nebraska track (channel/group) this node follows, for `omaha-only` mode. In `annotations` mode, required on the `acl.azure.com/update-request` annotation's `track` field instead, same no-fallback rule as the endpoint. | +| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG`'s own kubeconfig is used as-is (e.g. the real cluster FQDN a node's own `/var/lib/kubelet/kubeconfig` already points at). Only needed when the kubeconfig's own server is wrong for this deployment. | +| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig file used to reach the Kubernetes API server and authenticate as this node. | +| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. Kubernetes Node names must be valid RFC 1123 DNS labels (lowercase), matching how kubelet itself registers the Node - so the default only needs overriding when the agent's environment can't discover the correct hostname on its own. | +| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode: `annotations` or `omaha-only` (see above). | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file, which bridges the pre-reboot `finalize`/`rollback` half of an update and its post-reboot `commit` half across the reboot. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` operation (parsed as a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) is allowed to run before it's considered failed. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` operation is allowed to run before it's considered failed. Parsed the same way as the stage timeout. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat the agent writes while a stage/finalize/rollback operation is running, so AKS-RP and the watchdog can tell a working agent from a stuck one. Parsed the same way as the timeouts. | + +## Diagnostics + +`trident-acl-agent --validate-connection ` +checks connectivity to a single dependency using the current environment +and exits immediately - useful for a systemd `ExecStartPre` check or manual +on-node troubleshooting without running the full orchestrator loop. diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index b86f8d0b11..829e5a7564 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -4,7 +4,7 @@ //! `#[cfg(test)]` design-doc conformance tests below) implements the //! `acl.azure.com/update-request`, `acl.azure.com/update-status`, and //! `acl.azure.com/update-commit-status` node annotation protocol described -//! by the current accepted design (`accepted-design-v2.md`). Keep +//! by the current accepted design (`accepted-design-v3.md`). Keep //! `UpdateRequest`/`UpdateStatus`/`StatusCode` and `validate()` in sync with //! that document's formal JSON Schema (its section "Formal JSON Schema") - //! the `design_doc_*`/`agent_built_*_conform_to_formal_schema` tests in this @@ -72,28 +72,27 @@ pub struct UpdateRequest { pub operation: RequestedOperation, #[serde(default, skip_serializing_if = "Option::is_none")] pub target_version: Option, - /// Optional override of the agent's configured Nebraska endpoint - /// (`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` / CLI override) for this - /// update. When present, it takes precedence for every Nebraska call - /// this `nodeUpdateId` makes (`stage`'s update check, and all - /// progress/completion event reports), since Nebraska's per-instance - /// state is tied to one specific server. + /// The Omaha endpoint that serves the target image, with the path (e.g. + /// `https:///v1/update`). Required for `stage`/`finalize` (see + /// [`UpdateRequest::validate`]) - AKS-RP holds this constant across one + /// update's `stage` -> `finalize` -> `commit` lifecycle, in the same way + /// it holds `nodeUpdateId` constant, since Nebraska's per-instance state + /// is tied to one specific server. Omitted for `rollback`, which reports + /// no Nebraska event. There is deliberately no static/config-file + /// fallback if this is absent on a `stage`/`finalize` request: a + /// fallback would let a node update from a source AKS-RP did not + /// choose, so the agent rejects such a request with `InvalidRequest` + /// instead. #[serde(default, skip_serializing_if = "Option::is_none")] pub server: Option, - /// Optional override of the agent's configured Nebraska `app_id` - /// (`TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`) for this update. Resolved the - /// same way as [`server`](UpdateRequest::server): takes precedence over - /// the static config for every Nebraska call this `nodeUpdateId` makes. + /// The Omaha application id of the ACL image on `server`. Same + /// requirement/lifecycle/no-fallback rules as + /// [`server`](UpdateRequest::server) - see its docs. #[serde(default, skip_serializing_if = "Option::is_none")] pub app_id: Option, - /// Optional override of the agent's configured Nebraska `track` - /// (`TRIDENT_ACL_AGENT_NEBRASKA_TRACK`) for this update. Resolved and - /// applied the same way as [`server`](UpdateRequest::server) and - /// [`app_id`](UpdateRequest::app_id): takes precedence over the static - /// config for every Nebraska call this `nodeUpdateId` makes. `track` is - /// never optional on the wire itself (Nebraska requires it on every - /// request), only this override is - when absent, the static - /// `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` value is used, exactly as before. + /// The Omaha track that `server` resolves to the group serving this + /// node. Same requirement/lifecycle/no-fallback rules as + /// [`server`](UpdateRequest::server) - see its docs. #[serde(default, skip_serializing_if = "Option::is_none")] pub track: Option, } @@ -119,9 +118,10 @@ pub struct UpdateStatus { impl UpdateRequest { /// Enforces the same constraints as the request annotation's formal - /// JSON Schema in `accepted-design-v2.md`: schemaVersion match, and + /// JSON Schema in `accepted-design-v3.md`: schemaVersion match, /// targetVersion required for stage/finalize but disallowed for - /// rollback. See this file's module doc. + /// rollback, and server/appId/track required for stage/finalize. See + /// this file's module doc. pub fn validate(self) -> Result { if self.schema_version != SCHEMA_VERSION { return Err(format!("unsupported schemaVersion {}", self.schema_version)); @@ -131,6 +131,15 @@ impl UpdateRequest { if self.target_version.as_deref().unwrap_or("").is_empty() { return Err("targetVersion is required for stage/finalize".to_string()); } + if self.server.is_none() { + return Err("server is required for stage/finalize".to_string()); + } + if self.app_id.as_deref().unwrap_or("").is_empty() { + return Err("appId is required for stage/finalize".to_string()); + } + if self.track.as_deref().unwrap_or("").is_empty() { + return Err("track is required for stage/finalize".to_string()); + } } RequestedOperation::Rollback => { if self.target_version.is_some() { @@ -144,7 +153,7 @@ impl UpdateRequest { impl UpdateStatus { // This constructor mirrors UpdateStatus's wire schema field-for-field - // (see accepted-design-v2.md's two-status-key JSON protocol); splitting + // (see accepted-design-v3.md's two-status-key JSON protocol); splitting // it into a builder would add ceremony across ~25 call sites in // orchestrator.rs without making any of them clearer. #[allow(clippy::too_many_arguments)] @@ -614,13 +623,20 @@ mod tests { // // Keep these constants byte-for-byte in sync with the design doc. - /// docs/update-trigger-design.md 2.1, "Request annotation" example. + /// docs/update-trigger-design.md 2.1, "Request annotation" example + /// (adapted to `finalize` to pair with the status/commit examples + /// below, which also share this `finalize`; server/appId/track values + /// are the doc's own example values for those fields, required on + /// stage/finalize per `accepted-design-v3.md`). const DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE: &str = r#"{ "schemaVersion": "1.0", "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", "operationId": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "operation": "finalize", - "targetVersion": "202606.29.0" + "targetVersion": "202606.29.0", + "server": "https://nebraska.example.com/v1/update", + "appId": "11111111-2222-3333-4444-555555555555", + "track": "pin-202606.29.0" }"#; /// docs/update-trigger-design.md 2.1, "Status annotation" example. @@ -654,9 +670,8 @@ mod tests { }"#; /// The formal JSON Schema for the request annotation, from - /// docs/update-trigger-design.md (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md), - /// section 2.1 "Formal JSON Schema". Keep byte-for-byte in sync with - /// that document. + /// `accepted-design-v3.md` section 2.1 "Formal JSON Schema". Keep + /// byte-for-byte in sync with that document. const DESIGN_DOC_REQUEST_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://acl.azure.com/schemas/update-request/1.0.json", @@ -670,20 +685,24 @@ mod tests { "operationId": { "type": "string", "format": "uuid", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" }, "operation": { "type": "string", "enum": ["stage", "finalize", "rollback"] }, "targetVersion": { "type": "string", "description": "ACL image release version, e.g. 202606.29.0." }, - "server": { "type": "string", "format": "uri", "description": "Optional override of the agent's configured Nebraska endpoint for this update." }, - "appId": { "type": "string", "description": "Optional override of the agent's configured Nebraska app_id for this update." }, - "track": { "type": "string", "description": "Optional override of the agent's configured Nebraska track for this update." } + "server": { "type": "string", "format": "uri", "pattern": "^https://[^/]", "description": "Omaha endpoint that serves the target image, with the path, e.g. https:///v1/update." }, + "appId": { "type": "string", "description": "Omaha application id of the ACL image on that endpoint." }, + "track": { "type": "string", "description": "Omaha track that the update server resolves to the group serving the node." } }, "allOf": [ { "if": { "properties": { "operation": { "enum": ["stage", "finalize"] } }, "required": ["operation"] }, "then": { "required": ["targetVersion"] } + }, + { + "if": { "properties": { "operation": { "enum": ["stage", "finalize"] } }, "required": ["operation"] }, + "then": { "required": ["server", "appId", "track"] } } ] }"#; /// The formal JSON Schema for the status annotations, from - /// accepted-design-v2.md section 2.1 "Formal JSON Schema". Keep + /// `accepted-design-v3.md` section 2.1 "Formal JSON Schema". Keep /// byte-for-byte in sync with that document. const DESIGN_DOC_STATUS_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -721,7 +740,7 @@ mod tests { // additionalProperties, required, properties.{type,const,enum,format, // pattern}, and a single-level allOf/if/then/else). Panics loudly on any // schema keyword/pattern/type/format it doesn't recognize, so if - // accepted-design-v2.md's schemas grow new constraints, this validator's + // accepted-design-v3.md's schemas grow new constraints, this validator's // blind spots don't silently mask them - the test fails instead, // prompting an update here. @@ -896,21 +915,107 @@ mod tests { } /// Bespoke stand-in for full regex support: the two schemas above use - /// exactly two distinct patterns, both UUID-shaped, so this matches them - /// by exact pattern text rather than pulling in a regex engine for two - /// known cases. Panics on an unrecognized pattern so a future schema + /// only a few distinct patterns - two UUID-shaped ones and the + /// `server` https-with-host one - so this matches them by exact pattern + /// text rather than pulling in a regex engine for such a small, + /// enumerable set. Panics on an unrecognized pattern so a future schema /// change can't silently pass unchecked. fn schema_pattern_matches(pattern: &str, value: &str) -> bool { const BARE_UUID: &str = r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"; + const HTTPS_WITH_HOST: &str = r"^https://[^/]"; match pattern { BARE_UUID => Uuid::parse_str(value).is_ok(), + HTTPS_WITH_HOST => value + .strip_prefix("https://") + .and_then(|rest| rest.chars().next()) + .is_some_and(|c| c != '/'), other => panic!( "test schema validator does not recognize pattern {other:?} - extend schema_pattern_matches" ), } } + // --- UpdateRequest::validate() ------------------------------------------ + + fn valid_nebraska_request(operation: RequestedOperation) -> UpdateRequest { + UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: Uuid::new_v4(), + operation_id: "op-1".to_string(), + operation, + target_version: Some("202606.29.0".to_string()), + server: Some(Url::parse("https://nebraska.example/v1/update").unwrap()), + app_id: Some("app-id".to_string()), + track: Some("track".to_string()), + } + } + + #[test] + fn validate_accepts_stage_and_finalize_with_nebraska_fields() { + for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { + valid_nebraska_request(operation) + .validate() + .unwrap_or_else(|err| panic!("{operation:?} with server/appId/track: {err}")); + } + } + + #[test] + fn validate_rejects_stage_and_finalize_missing_server() { + for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { + let mut request = valid_nebraska_request(operation); + request.server = None; + let err = request + .validate() + .expect_err("missing server must be rejected for stage/finalize"); + assert!(err.contains("server"), "{err}"); + } + } + + #[test] + fn validate_rejects_stage_and_finalize_missing_app_id() { + for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { + let mut request = valid_nebraska_request(operation); + request.app_id = None; + let err = request + .validate() + .expect_err("missing appId must be rejected for stage/finalize"); + assert!(err.contains("appId"), "{err}"); + } + } + + #[test] + fn validate_rejects_stage_and_finalize_missing_track() { + for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { + let mut request = valid_nebraska_request(operation); + request.track = None; + let err = request + .validate() + .expect_err("missing track must be rejected for stage/finalize"); + assert!(err.contains("track"), "{err}"); + } + } + + #[test] + fn validate_allows_rollback_without_nebraska_fields() { + // Rollback reports no Nebraska event, so it carries no update + // source (accepted-design-v3.md 2.1): server/appId/track are not + // required, and validate() must not reject their absence. + let request = UpdateRequest { + schema_version: SCHEMA_VERSION.to_string(), + node_update_id: Uuid::new_v4(), + operation_id: "op-1".to_string(), + operation: RequestedOperation::Rollback, + target_version: None, + server: None, + app_id: None, + track: None, + }; + request + .validate() + .expect("rollback without server/appId/track must validate"); + } + // --- example payload parsing tests ------------------------------------- #[test] @@ -983,13 +1088,26 @@ mod tests { let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); let node_update_id = Uuid::new_v4(); - for (operation, target_version) in [ - (RequestedOperation::Stage, Some("202606.29.0".to_string())), + for (operation, target_version, nebraska) in [ + ( + RequestedOperation::Stage, + Some("202606.29.0".to_string()), + Some(( + "https://nebraska.example.com/v1/update", + "11111111-2222-3333-4444-555555555555", + "pin-202606.29.0", + )), + ), ( RequestedOperation::Finalize, Some("202606.29.0".to_string()), + Some(( + "https://nebraska.example.com/v1/update", + "11111111-2222-3333-4444-555555555555", + "pin-202606.29.0", + )), ), - (RequestedOperation::Rollback, None), + (RequestedOperation::Rollback, None, None), ] { let request = UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), @@ -997,9 +1115,9 @@ mod tests { operation_id: Uuid::new_v4().to_string(), operation, target_version, - server: None, - app_id: None, - track: None, + server: nebraska.map(|(server, ..)| Url::parse(server).unwrap()), + app_id: nebraska.map(|(_, app_id, _)| app_id.to_string()), + track: nebraska.map(|(_, _, track)| track.to_string()), }; let request = request .validate() @@ -1094,19 +1212,19 @@ mod tests { .expect("agent-constructed commit status must conform to the formal schema"); } - // --- server / appId / track (Nebraska overrides) - + // --- server / appId / track (Nebraska fields, required for stage/finalize) - #[test] fn server_field_round_trips_and_conforms_to_formal_schema() { let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); - let mut request = sample_request(RequestedOperation::Stage); + let mut request = valid_nebraska_request(RequestedOperation::Stage); request.operation_id = Uuid::new_v4().to_string(); request.server = Some(Url::parse("https://nebraska.example/v1/update").unwrap()); let json = serde_json::to_value(&request).unwrap(); assert_eq!(json["server"], "https://nebraska.example/v1/update"); schema_validate(&schema, &json) - .expect("request with a server override must conform to the formal schema"); + .expect("request with server/appId/track must conform to the formal schema"); let round_tripped: UpdateRequest = serde_json::from_value(json).unwrap(); assert_eq!(round_tripped.server, request.server); @@ -1122,14 +1240,14 @@ mod tests { #[test] fn app_id_field_round_trips_and_conforms_to_formal_schema() { let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); - let mut request = sample_request(RequestedOperation::Stage); + let mut request = valid_nebraska_request(RequestedOperation::Stage); request.operation_id = Uuid::new_v4().to_string(); request.app_id = Some("59bbad61-257d-47f4-9730-6848d88e1a6e".to_string()); let json = serde_json::to_value(&request).unwrap(); assert_eq!(json["appId"], "59bbad61-257d-47f4-9730-6848d88e1a6e"); schema_validate(&schema, &json) - .expect("request with an appId override must conform to the formal schema"); + .expect("request with server/appId/track must conform to the formal schema"); let round_tripped: UpdateRequest = serde_json::from_value(json).unwrap(); assert_eq!(round_tripped.app_id, request.app_id); @@ -1145,14 +1263,14 @@ mod tests { #[test] fn track_field_round_trips_and_conforms_to_formal_schema() { let schema: Value = serde_json::from_str(DESIGN_DOC_REQUEST_SCHEMA).unwrap(); - let mut request = sample_request(RequestedOperation::Stage); + let mut request = valid_nebraska_request(RequestedOperation::Stage); request.operation_id = Uuid::new_v4().to_string(); request.track = Some("pin-202608.6.0".to_string()); let json = serde_json::to_value(&request).unwrap(); assert_eq!(json["track"], "pin-202608.6.0"); schema_validate(&schema, &json) - .expect("request with a track override must conform to the formal schema"); + .expect("request with server/appId/track must conform to the formal schema"); let round_tripped: UpdateRequest = serde_json::from_value(json).unwrap(); assert_eq!(round_tripped.track, request.track); diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs index dafa7dccdb..d3f8c8c3e2 100644 --- a/crates/trident-acl-agent/src/config.rs +++ b/crates/trident-acl-agent/src/config.rs @@ -37,11 +37,13 @@ const ENV_ORCHESTRATION_HEARTBEAT_INTERVAL: &str = const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); // TODO: placeholder until the real production Nebraska/Omaha endpoint is -// known. `.invalid` is reserved by RFC 2606 and is guaranteed to never -// resolve, so a deployment that forgets to set -// TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT (or override it per-request via the -// update-request annotation's `server` field) fails loudly at the network -// layer instead of silently querying a real-looking but wrong host. +// known, for omaha-only mode. `.invalid` is reserved by RFC 2606 and is +// guaranteed to never resolve, so a deployment that forgets to set +// TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT fails loudly at the network layer +// instead of silently querying a real-looking but wrong host. Annotation +// mode does not use this default at all: stage/finalize requests must +// carry their own `server` field, with no fallback to this config (see +// Orchestrator::resolve_nebraska_endpoint). pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update"; const DEFAULT_STAGE_TIMEOUT: Duration = Duration::from_secs(20 * 60); const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); @@ -174,7 +176,7 @@ pub enum GoalSource { /// `acl.azure.com/update-request` annotation and drives Trident's /// stage/finalize/rollback/commit operations against tridentd /// accordingly, writing progress back to `acl.azure.com/update-status` - /// and `acl.azure.com/update-commit-status` (see accepted-design-v2.md). + /// and `acl.azure.com/update-commit-status` (see accepted-design-v3.md). /// This is the default mode. #[default] Annotations, @@ -204,7 +206,7 @@ pub struct OrchestrationConfig { pub finalize_timeout: Duration, /// Refresh cadence for in-flight InProgress heartbeats. Default is well /// below the ~10 minute watchdog staleness target proposed in - /// accepted-design-v2.md. + /// accepted-design-v3.md. pub heartbeat_interval: Duration, } diff --git a/crates/trident-acl-agent/src/k8s.rs b/crates/trident-acl-agent/src/k8s.rs index 6017f19f33..4e64eb51bb 100644 --- a/crates/trident-acl-agent/src/k8s.rs +++ b/crates/trident-acl-agent/src/k8s.rs @@ -1,7 +1,7 @@ //! Thin Kubernetes client wrapper for Harpoon's node self-patching protocol. //! //! Implements the Node get/watch/patch access described in the current -//! accepted design (`accepted-design-v2.md`). +//! accepted design (`accepted-design-v3.md`). //! //! The design calls for get/patch access to exactly one Node object (§2.2–§2.6). //! Node changes are observed via the Kubernetes watch API (`kube::runtime::watcher`) diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs index a5aea3dc41..c263991aed 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -252,7 +252,7 @@ where // Reject on operationId, not nodeUpdateId: the actual conflict // this guard exists to prevent is "a second finalize/rollback // starts while one is still waiting for its post-reboot - // commit" (accepted-design-v2.md's in-flight conflict rule). + // commit" (accepted-design-v3.md's in-flight conflict rule). // Keying on nodeUpdateId alone let a retried/re-issued request // that reused the same nodeUpdateId but a new operationId slip // through this guard entirely and re-enter handle_finalize/ @@ -289,48 +289,39 @@ where } } - /// Resolves which Nebraska endpoint to use for `request`: the request - /// annotation's own `server` override, if present, otherwise the - /// agent's configured `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` (or CLI - /// override). Every Nebraska call this `nodeUpdateId` makes - stage's - /// update check plus every progress/completion event report - must go - /// through this resolver rather than reading `self.config.nebraska.endpoint` - /// directly, since Nebraska's per-instance state is tied to one specific - /// server: mixing endpoints across one update's lifecycle would split - /// that state across two servers. + /// Resolves which Nebraska endpoint to use for `request`: its own + /// `server` field. Every Nebraska call this `nodeUpdateId` makes - + /// stage's update check plus every progress/completion event report - + /// must go through this resolver rather than reading + /// `self.config.nebraska.endpoint` directly, since Nebraska's + /// per-instance state is tied to one specific server: mixing endpoints + /// across one update's lifecycle would split that state across two + /// servers. + /// + /// Per `accepted-design-v3.md` 2.1, `stage`/`finalize` requests must + /// carry `server` and there is deliberately no static-config fallback + /// here: a fallback would let a node update from a source AKS-RP did + /// not choose. `UpdateRequest::validate()` already rejects a + /// stage/finalize missing it with `InvalidRequest` before the + /// orchestrator ever reaches this resolver, so `None` here should not + /// happen in practice; callers still treat it as absent (rather than + /// panicking) as defense in depth. fn resolve_nebraska_endpoint(&self, request: &UpdateRequest) -> Option { - request - .server - .clone() - .or_else(|| self.config.nebraska.endpoint.clone()) - } - - /// Resolves which Nebraska app id to use for `request`: the request - /// annotation's own `appId` override, if present, otherwise the agent's - /// configured `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`. Unlike - /// [`resolve_nebraska_endpoint`], this always resolves to a value - - /// `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` always has one (defaulting to - /// [`crate::DEFAULT_NEBRASKA_APP_ID`]) - so there is no error case to - /// handle at call sites. - fn resolve_nebraska_app_id(&self, request: &UpdateRequest) -> String { - request - .app_id - .clone() - .unwrap_or_else(|| self.config.nebraska.app_id.clone()) - } - - /// Resolves which Nebraska track to use for `request`: the request - /// annotation's own `track` override, if present, otherwise the agent's - /// configured `TRIDENT_ACL_AGENT_NEBRASKA_TRACK`. Same always-resolves - /// behavior as [`resolve_nebraska_app_id`] - - /// `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` always has a default - /// ([`crate::DEFAULT_NEBRASKA_TRACK`]) - so there is no error case here - /// either. - fn resolve_nebraska_track(&self, request: &UpdateRequest) -> String { - request - .track - .clone() - .unwrap_or_else(|| self.config.nebraska.track.clone()) + request.server.clone() + } + + /// Resolves which Nebraska app id to use for `request`: its own `appId` + /// field. Same stage/finalize-required, no-fallback rules as + /// [`resolve_nebraska_endpoint`] - see its docs. + fn resolve_nebraska_app_id(&self, request: &UpdateRequest) -> Option { + request.app_id.clone() + } + + /// Resolves which Nebraska track to use for `request`: its own `track` + /// field. Same stage/finalize-required, no-fallback rules as + /// [`resolve_nebraska_endpoint`] - see its docs. + fn resolve_nebraska_track(&self, request: &UpdateRequest) -> Option { + request.track.clone() } async fn handle_stage(&self, request: UpdateRequest) -> Result<(), anyhow::Error> { @@ -365,13 +356,29 @@ where None, ); self.publish_status(&in_progress).await?; + // UpdateRequest::validate() already requires server/appId/track for + // stage/finalize before this handler is ever reached (see + // resolve_nebraska_endpoint's docs), so treat a missing one here as + // an agent-internal error rather than silently defaulting - there + // is deliberately no static-config fallback for the annotation flow. let endpoint = self.resolve_nebraska_endpoint(&request).ok_or_else(|| { anyhow::anyhow!( - "annotation mode requires request.server, TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT, or CLI override" + "stage request has no request.server despite passing validation (nodeUpdateId {})", + request.node_update_id + ) + })?; + let app_id = self.resolve_nebraska_app_id(&request).ok_or_else(|| { + anyhow::anyhow!( + "stage request has no request.appId despite passing validation (nodeUpdateId {})", + request.node_update_id + ) + })?; + let track = self.resolve_nebraska_track(&request).ok_or_else(|| { + anyhow::anyhow!( + "stage request has no request.track despite passing validation (nodeUpdateId {})", + request.node_update_id ) })?; - let app_id = self.resolve_nebraska_app_id(&request); - let track = self.resolve_nebraska_track(&request); let machine_id = crate::build_machine_id(IdSource::MachineIdHashed)?; let outcome = tokio::task::spawn_blocking(move || { let client = NebraskaClient::new(endpoint, app_id, track, machine_id); @@ -798,7 +805,7 @@ where ) -> UpdateStatus { // state.json did not survive the reboot (or was never written, e.g. // the agent crashed before persisting pendingCommit). Per - // accepted-design-v2.md §2.3's degraded path, reconstruct the answer by + // accepted-design-v3.md §2.3's degraded path, reconstruct the answer by // calling commit() unconditionally rather than guessing from labels // or the target version alone - tridentd's commit() is self-checking // and its own (ServicingKind/RebootStatus/Result) response already @@ -966,18 +973,35 @@ where /// `handle_stage`'s `check_for_update` call). async fn report_nebraska_event(&self, request: &UpdateRequest, report: NebraskaReport) { let Some(endpoint) = self.resolve_nebraska_endpoint(request) else { - // Should not happen in practice: every call site only reaches - // here after handle_stage has already required an endpoint for - // this node update. Guard anyway since this is best-effort - // telemetry, not something worth panicking over. + // Should not happen in practice: UpdateRequest::validate() + // already requires request.server for stage/finalize before + // the orchestrator ever reaches here, and there is + // deliberately no static-config fallback (see + // resolve_nebraska_endpoint's docs). Guard anyway since this is + // best-effort telemetry, not something worth panicking over. + log::warn!( + "skipping Nebraska '{}' report: request has no server (nodeUpdateId {})", + report.label(), + request.node_update_id + ); + return; + }; + let Some(app_id) = self.resolve_nebraska_app_id(request) else { + log::warn!( + "skipping Nebraska '{}' report: request has no appId (nodeUpdateId {})", + report.label(), + request.node_update_id + ); + return; + }; + let Some(track) = self.resolve_nebraska_track(request) else { log::warn!( - "skipping Nebraska '{}' report: no Nebraska endpoint configured (no request.server override and no [nebraska].endpoint)", - report.label() + "skipping Nebraska '{}' report: request has no track (nodeUpdateId {})", + report.label(), + request.node_update_id ); return; }; - let app_id = self.resolve_nebraska_app_id(request); - let track = self.resolve_nebraska_track(request); let machine_id = match crate::build_machine_id(NEBRASKA_MACHINE_ID_SOURCE) { Ok(id) => id, Err(err) => { @@ -1310,7 +1334,7 @@ fn indicates_target_boot_failed(error: &TridentClientError) -> bool { /// needing a full `Orchestrator` instance. See `stage_result_to_status` for /// rationale. /// Pre-flight checks for the state.json-missing degraded reconstruction -/// path (accepted-design-v2.md §2.3). Returns `Some(status)` when reconstruction +/// path (accepted-design-v3.md §2.3). Returns `Some(status)` when reconstruction /// cannot proceed (tridentd already known-unreachable, or the outstanding /// request isn't a finalize/rollback), or `None` when the caller should go /// on to call tridentd's commit() to determine the real outcome. @@ -1354,7 +1378,7 @@ fn reconstruct_precheck_status( } /// Maps tridentd's commit() result to the terminal status for the -/// state.json-missing degraded reconstruction path (accepted-design-v2.md +/// state.json-missing degraded reconstruction path (accepted-design-v3.md /// §2.3). Always reports under the original operationId, mirroring the /// normal post-reboot commit path in `commit_result_to_status`. fn reconstruct_commit_result_to_status( diff --git a/crates/trident-acl-agent/src/state.rs b/crates/trident-acl-agent/src/state.rs index ed58060dd2..025f9f1607 100644 --- a/crates/trident-acl-agent/src/state.rs +++ b/crates/trident-acl-agent/src/state.rs @@ -2,7 +2,7 @@ //! completed-operation cache and the pending post-reboot commit record. //! //! Implements the `state.json` mechanism from the current accepted design -//! (`accepted-design-v2.md`, section 2.3), which bridges the pre-reboot +//! (`accepted-design-v3.md`, section 2.3), which bridges the pre-reboot //! finalize/rollback half and the post-reboot commit half of an operation //! across the reboot. diff --git a/crates/trident-acl-agent/src/trident.rs b/crates/trident-acl-agent/src/trident.rs index 87a7ba7fb4..982489a830 100644 --- a/crates/trident-acl-agent/src/trident.rs +++ b/crates/trident-acl-agent/src/trident.rs @@ -221,7 +221,7 @@ impl TridentClient { reboot: Some(RebootManagement { // The agent, not tridentd, must own every reboot // decision: AKS-RP is the sole authority over - // reboot/rollback (accepted-design-v2.md §2.5). If commit() + // reboot/rollback (accepted-design-v3.md §2.5). If commit() // ever reports NeedsReboot (e.g. a health-check failure, // were health checks ever re-enabled), the agent needs // to see that as a RebootRequired response it controls @@ -283,7 +283,7 @@ impl TridentClient { reboot: Some(RebootManagement { // Same rationale as commit()/update_finalize(): AKS-RP, // via the agent, is the sole authority over reboot - // timing (accepted-design-v2.md §2.5). + // timing (accepted-design-v3.md §2.5). handling: RebootHandling::CallerHandlesReboot.into(), }), })) From fbdc2411e7654983859068935965faebe6356092 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 19:16:04 +0000 Subject: [PATCH 10/54] trident-acl-agent: make annotation prefix configurable via env var Add TRIDENT_ACL_AGENT_ANNOTATION_PREFIX, defaulting to acl.azure.com. Introduces AnnotationKeys (replaces the hardcoded acl.azure.com/* consts) built once in Orchestrator::from_config from KubernetesConfig::annotation_prefix and threaded through Snapshot::from_node/publish_status, so the agent is not hardcoded to the AKS annotation namespace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/src/annotations.rs | 77 ++++++++++++++++--- crates/trident-acl-agent/src/config.rs | 37 +++++++++- crates/trident-acl-agent/src/main.rs | 6 +- crates/trident-acl-agent/src/orchestrator.rs | 78 ++++++++++---------- 4 files changed, 145 insertions(+), 53 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index 829e5a7564..e935dc46dc 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -2,12 +2,16 @@ //! //! This module (schema types, `UpdateRequest::validate()`, and the //! `#[cfg(test)]` design-doc conformance tests below) implements the -//! `acl.azure.com/update-request`, `acl.azure.com/update-status`, and -//! `acl.azure.com/update-commit-status` node annotation protocol described -//! by the current accepted design (`accepted-design-v3.md`). Keep -//! `UpdateRequest`/`UpdateStatus`/`StatusCode` and `validate()` in sync with -//! that document's formal JSON Schema (its section "Formal JSON Schema") - -//! the `design_doc_*`/`agent_built_*_conform_to_formal_schema` tests in this +//! `/update-request`, `/update-status`, and +//! `/update-commit-status` node annotation protocol described +//! by the current accepted design (`accepted-design-v3.md`), where +//! `` defaults to `acl.azure.com` (see +//! [`AnnotationKeys`]/[`crate::config::DEFAULT_ANNOTATION_PREFIX`]) and is +//! overridable via the `TRIDENT_ACL_AGENT_ANNOTATION_PREFIX` environment +//! variable. Keep `UpdateRequest`/`UpdateStatus`/`StatusCode` and +//! `validate()` in sync with that document's formal JSON Schema (its +//! section "Formal JSON Schema") - the +//! `design_doc_*`/`agent_built_*_conform_to_formal_schema` tests in this //! file's test module pin that JSON Schema in literally and check both the //! doc's own examples and our constructed annotations against it. @@ -16,9 +20,45 @@ use serde::{Deserialize, Serialize}; use url::Url; use uuid::Uuid; -pub const UPDATE_REQUEST_ANNOTATION: &str = "acl.azure.com/update-request"; -pub const UPDATE_STATUS_ANNOTATION: &str = "acl.azure.com/update-status"; -pub const UPDATE_COMMIT_STATUS_ANNOTATION: &str = "acl.azure.com/update-commit-status"; +use crate::config::DEFAULT_ANNOTATION_PREFIX; + +/// Suffix (appended to the configured annotation prefix) for the request +/// annotation, e.g. `acl.azure.com/update-request`. +pub const UPDATE_REQUEST_SUFFIX: &str = "update-request"; +/// Suffix for the operation-status annotation, e.g. +/// `acl.azure.com/update-status`. +pub const UPDATE_STATUS_SUFFIX: &str = "update-status"; +/// Suffix for the post-reboot commit-status annotation, e.g. +/// `acl.azure.com/update-commit-status`. +pub const UPDATE_COMMIT_STATUS_SUFFIX: &str = "update-commit-status"; + +/// The full annotation keys for one deployment's configured annotation +/// prefix. Built once from [`crate::config::KubernetesConfig::annotation_prefix`] +/// and threaded through instead of hardcoding the AKS-specific +/// `acl.azure.com` prefix. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnnotationKeys { + pub request: String, + pub status: String, + pub commit_status: String, +} + +impl AnnotationKeys { + pub fn new(prefix: &str) -> Self { + Self { + request: format!("{prefix}/{UPDATE_REQUEST_SUFFIX}"), + status: format!("{prefix}/{UPDATE_STATUS_SUFFIX}"), + commit_status: format!("{prefix}/{UPDATE_COMMIT_STATUS_SUFFIX}"), + } + } +} + +impl Default for AnnotationKeys { + fn default() -> Self { + Self::new(DEFAULT_ANNOTATION_PREFIX) + } +} + pub const SCHEMA_VERSION: &str = "1.0"; const MAX_MESSAGE_BYTES: usize = 2048; const TRUNCATION_MARKER: &str = "... (truncated)"; @@ -284,6 +324,25 @@ mod tests { use super::*; + #[test] + fn annotation_keys_default_uses_acl_azure_com_prefix() { + let keys = AnnotationKeys::default(); + assert_eq!(keys.request, "acl.azure.com/update-request"); + assert_eq!(keys.status, "acl.azure.com/update-status"); + assert_eq!(keys.commit_status, "acl.azure.com/update-commit-status"); + } + + #[test] + fn annotation_keys_new_applies_custom_prefix() { + let keys = AnnotationKeys::new("contoso.example.com"); + assert_eq!(keys.request, "contoso.example.com/update-request"); + assert_eq!(keys.status, "contoso.example.com/update-status"); + assert_eq!( + keys.commit_status, + "contoso.example.com/update-commit-status" + ); + } + fn sample_request(operation: RequestedOperation) -> UpdateRequest { UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs index d3f8c8c3e2..5a01f75206 100644 --- a/crates/trident-acl-agent/src/config.rs +++ b/crates/trident-acl-agent/src/config.rs @@ -34,6 +34,12 @@ const ENV_ORCHESTRATION_STAGE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_S const ENV_ORCHESTRATION_FINALIZE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT"; const ENV_ORCHESTRATION_HEARTBEAT_INTERVAL: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL"; +/// Overrides the annotation-key prefix (e.g. `acl.azure.com` in +/// `acl.azure.com/update-request`). Defaults to +/// [`DEFAULT_ANNOTATION_PREFIX`] so a deployment not tied to AKS's +/// `acl.azure.com` domain can point the agent at its own annotation +/// namespace without a code change. +const ENV_KUBERNETES_ANNOTATION_PREFIX: &str = "TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX"; const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); // TODO: placeholder until the real production Nebraska/Omaha endpoint is @@ -50,6 +56,9 @@ const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); pub const DEFAULT_STATE_PATH: &str = "/var/lib/trident-acl-agent/state.json"; pub const DEFAULT_KUBELET_KUBECONFIG: &str = "/var/lib/kubelet/kubeconfig"; +/// Default annotation-key prefix, matching AKS's `acl.azure.com` namespace. +/// Override with `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. +pub const DEFAULT_ANNOTATION_PREFIX: &str = "acl.azure.com"; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct AgentConfig { @@ -81,6 +90,8 @@ impl AgentConfig { .unwrap_or_else(|| DEFAULT_KUBELET_KUBECONFIG.to_string()), node_name: env_string(ENV_KUBERNETES_NODE_NAME).unwrap_or_else(default_node_name), watch_poll_interval: DEFAULT_KUBERNETES_POLL_INTERVAL, + annotation_prefix: env_string(ENV_KUBERNETES_ANNOTATION_PREFIX) + .unwrap_or_else(|| DEFAULT_ANNOTATION_PREFIX.to_string()), }, trident: TridentConfig { socket: env_string(ENV_TRIDENT_SOCKET) @@ -138,6 +149,12 @@ pub struct KubernetesConfig { pub kubeconfig: String, pub node_name: String, pub watch_poll_interval: Duration, + /// Annotation-key prefix used for the request/status/commit-status + /// annotations (e.g. `acl.azure.com` in `acl.azure.com/update-request`). + /// Defaults to [`DEFAULT_ANNOTATION_PREFIX`], overridable via + /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` so the annotation + /// namespace isn't hardcoded to AKS. + pub annotation_prefix: String, } impl Default for KubernetesConfig { @@ -147,6 +164,7 @@ impl Default for KubernetesConfig { kubeconfig: DEFAULT_KUBELET_KUBECONFIG.to_string(), node_name: default_node_name(), watch_poll_interval: DEFAULT_KUBERNETES_POLL_INTERVAL, + annotation_prefix: DEFAULT_ANNOTATION_PREFIX.to_string(), } } } @@ -173,11 +191,15 @@ pub enum GoalSource { /// participate in the AKS annotation-driven update protocol. OmahaOnly, /// The annotation-driven reconcile loop: watches the Node's - /// `acl.azure.com/update-request` annotation and drives Trident's + /// `/update-request` annotation and drives Trident's /// stage/finalize/rollback/commit operations against tridentd - /// accordingly, writing progress back to `acl.azure.com/update-status` - /// and `acl.azure.com/update-commit-status` (see accepted-design-v3.md). - /// This is the default mode. + /// accordingly, writing progress back to + /// `/update-status` and + /// `/update-commit-status` (see + /// accepted-design-v3.md). `` defaults to + /// [`DEFAULT_ANNOTATION_PREFIX`] (`acl.azure.com`), overridable via + /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. This is the default + /// mode. #[default] Annotations, } @@ -294,6 +316,7 @@ mod tests { env::remove_var(ENV_ORCHESTRATION_STAGE_TIMEOUT); env::remove_var(ENV_ORCHESTRATION_FINALIZE_TIMEOUT); env::remove_var(ENV_ORCHESTRATION_HEARTBEAT_INTERVAL); + env::remove_var(ENV_KUBERNETES_ANNOTATION_PREFIX); } } @@ -334,6 +357,10 @@ mod tests { config.orchestration.heartbeat_interval, DEFAULT_HEARTBEAT_INTERVAL ); + assert_eq!( + config.kubernetes.annotation_prefix, + DEFAULT_ANNOTATION_PREFIX + ); // SAFETY: see clear_env's doc comment. unsafe { @@ -355,6 +382,7 @@ mod tests { env::set_var(ENV_ORCHESTRATION_STAGE_TIMEOUT, "21m"); env::set_var(ENV_ORCHESTRATION_FINALIZE_TIMEOUT, "11m"); env::set_var(ENV_ORCHESTRATION_HEARTBEAT_INTERVAL, "45s"); + env::set_var(ENV_KUBERNETES_ANNOTATION_PREFIX, "contoso.example.com"); } let config = AgentConfig::from_env().unwrap(); @@ -393,6 +421,7 @@ mod tests { config.orchestration.heartbeat_interval, Duration::from_secs(45) ); + assert_eq!(config.kubernetes.annotation_prefix, "contoso.example.com"); // --- empty value falls back to default, same as unset ------------- clear_env(); diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 60aa2bb8b1..2f5476f442 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -236,8 +236,10 @@ async fn main() -> Result<(), anyhow::Error> { // offered, and exit. No Kubernetes/annotation involvement. GoalSource::OmahaOnly => run_omaha_only(&config).await, // Default: the annotation-driven reconcile loop (watches - // acl.azure.com/update-request, drives stage/finalize/rollback/ - // commit against tridentd, writes acl.azure.com/update-status). + // /update-request, drives stage/finalize/rollback/ + // commit against tridentd, writes /update-status; prefix + // defaults to acl.azure.com, overridable via + // TRIDENT_ACL_AGENT_ANNOTATION_PREFIX). GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, } } diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs index c263991aed..c78c587fd8 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -24,9 +24,8 @@ use osutils::dependencies::Dependency; use crate::{ annotations::{ - current_active_version, Operation, RequestedOperation, StatusCode, UpdateRequest, - UpdateStatus, SCHEMA_VERSION, UPDATE_COMMIT_STATUS_ANNOTATION, UPDATE_REQUEST_ANNOTATION, - UPDATE_STATUS_ANNOTATION, + current_active_version, AnnotationKeys, Operation, RequestedOperation, StatusCode, + UpdateRequest, UpdateStatus, SCHEMA_VERSION, }, config::AgentConfig, k8s::{K8sClientError, NodeClient}, @@ -106,16 +105,19 @@ pub struct Orchestrator { k8s: NodeClient, rebooter: R, state: StateStore, + annotation_keys: AnnotationKeys, } impl Orchestrator { pub async fn from_config(config: AgentConfig) -> Result { let k8s = NodeClient::new(&config.kubernetes).await?; + let annotation_keys = AnnotationKeys::new(&config.kubernetes.annotation_prefix); Ok(Self { state: StateStore::new(config.orchestration.state_path.clone()), config, k8s, rebooter: SystemRebooter, + annotation_keys, }) } } @@ -152,7 +154,7 @@ where async fn recover_from_trident_state(&self) -> Result<(), anyhow::Error> { let node = self.k8s.get_node(&self.config.kubernetes.node_name).await?; - let snapshot = Snapshot::from_node(&node); + let snapshot = Snapshot::from_node(&node, &self.annotation_keys); let persisted = self.state.load()?; if let Some(pending) = persisted.pending_commit.clone() { @@ -196,7 +198,7 @@ where } async fn reconcile_node(&self, node: &Node) -> Result { - let snapshot = Snapshot::from_node(node); + let snapshot = Snapshot::from_node(node, &self.annotation_keys); log::debug!( "received node update: request={:?} operation_status={:?} commit_status={:?}", snapshot.request, @@ -869,8 +871,8 @@ where let status = status.refreshed_for_write(); let mut annotations = BTreeMap::new(); let annotation_key = match status.operation { - Operation::Commit => UPDATE_COMMIT_STATUS_ANNOTATION, - _ => UPDATE_STATUS_ANNOTATION, + Operation::Commit => &self.annotation_keys.commit_status, + _ => &self.annotation_keys.status, }; annotations.insert( annotation_key.to_string(), @@ -1088,41 +1090,41 @@ struct Snapshot { } impl Snapshot { - fn from_node(node: &Node) -> Self { + fn from_node(node: &Node, keys: &AnnotationKeys) -> Self { let annotations = node.metadata.annotations.as_ref(); - let raw_request = annotations.and_then(|a| a.get(UPDATE_REQUEST_ANNOTATION)); - let (request, invalid_request) = match raw_request - .map(|v| serde_json::from_str::(v)) - { - None => (None, None), - Some(Ok(candidate)) => match candidate.clone().validate() { - Ok(valid) => (Some(valid), None), - Err(reason) => ( - None, - Some(InvalidRequest { - node_update_id: candidate.node_update_id, - operation_id: candidate.operation_id, - operation: candidate.operation.into(), - reason, - }), - ), - }, - Some(Err(err)) => { - // Cannot attribute a status to an operationId we couldn't - // even parse out of the annotation - log loudly instead so - // this doesn't fail silently, but there's no request to - // surface an InvalidRequest status against. - log::warn!( - "ignoring malformed {UPDATE_REQUEST_ANNOTATION} annotation (JSON parse failed): {err}" - ); - (None, None) - } - }; + let raw_request = annotations.and_then(|a| a.get(&keys.request)); + let (request, invalid_request) = + match raw_request.map(|v| serde_json::from_str::(v)) { + None => (None, None), + Some(Ok(candidate)) => match candidate.clone().validate() { + Ok(valid) => (Some(valid), None), + Err(reason) => ( + None, + Some(InvalidRequest { + node_update_id: candidate.node_update_id, + operation_id: candidate.operation_id, + operation: candidate.operation.into(), + reason, + }), + ), + }, + Some(Err(err)) => { + // Cannot attribute a status to an operationId we couldn't + // even parse out of the annotation - log loudly instead so + // this doesn't fail silently, but there's no request to + // surface an InvalidRequest status against. + log::warn!( + "ignoring malformed {} annotation (JSON parse failed): {err}", + keys.request + ); + (None, None) + } + }; let operation_status = annotations - .and_then(|a| a.get(UPDATE_STATUS_ANNOTATION)) + .and_then(|a| a.get(&keys.status)) .and_then(|v| serde_json::from_str::(v).ok()); let commit_status = annotations - .and_then(|a| a.get(UPDATE_COMMIT_STATUS_ANNOTATION)) + .and_then(|a| a.get(&keys.commit_status)) .and_then(|v| serde_json::from_str::(v).ok()); Self { request, From d1ab40bdb91b64faf8b71ed5e816001124008f2d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 19:28:05 +0000 Subject: [PATCH 11/54] trident-acl-agent: read current version from os-release, not aks-os-version current_active_version() now looks up a configurable key (default IMAGE_VERSION, override via TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY) in /etc/os-release instead of the AKS-specific /etc/aks-os-version file. If the key is absent, falls back to a configurable stub value (default CURRENT_VERSION_STUB, override via TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB). read_active_version() -> read_os_release_value() now parses generic os-release KEY=VALUE syntax (quotes, comments, blank lines) instead of treating the whole file as one value. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/src/annotations.rs | 190 ++++++++++++++++---- 1 file changed, 156 insertions(+), 34 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index e935dc46dc..68dec9982b 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -62,17 +62,27 @@ impl Default for AnnotationKeys { pub const SCHEMA_VERSION: &str = "1.0"; const MAX_MESSAGE_BYTES: usize = 2048; const TRUNCATION_MARKER: &str = "... (truncated)"; -// TODO(DR-001): current_active_version() now reads /etc/aks-os-version, but -// falls back to this stub if that file isn't present yet (e.g. an image that -// hasn't picked up the file, or a dev/test host). Once the file ships +// TODO(DR-001): current_active_version() now reads the `IMAGE_VERSION` key +// (overridable via TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY) out of os-release, +// but falls back to this stub if that key isn't present yet (e.g. an image +// whose os-release doesn't carry it, or a dev/test host). Once the key ships // unconditionally on every ACL image, this fallback (and this comment) can be // removed. The stub value below is an explicit sentinel that cannot collide // with a real AKS/Trident release version string (those look like // "YYYYMM.N.N"), so it can never accidentally match a real requested target // version and cause handle_stage/handle_finalize to incorrectly short-circuit // to AlreadyAtTarget. Do not remove this comment when bumping the stub value; -// keep it (and its non-colliding shape) until the fallback is removed. +// keep it (and its non-colliding shape) until the fallback is removed. The +// stub itself is overridable via TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for +// dev/test hosts that want a specific sentinel. pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; +/// Default os-release key `current_active_version` looks up for the running +/// image's version. Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` +/// so a deployment that doesn't set `IMAGE_VERSION` can point at whatever +/// key its os-release does carry. +pub const DEFAULT_CURRENT_VERSION_KEY: &str = "IMAGE_VERSION"; +const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; +const ENV_CURRENT_VERSION_STUB: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB"; #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] @@ -275,35 +285,57 @@ fn truncate_message(message: String) -> String { truncated } -/// Path to the file the ACL image ships carrying the running OS version. -/// See `CURRENT_VERSION_STUB`'s doc comment above for the stub fallback this -/// probe still uses when the file isn't there yet. -const AKS_OS_VERSION_PATH: &str = "/etc/aks-os-version"; +/// Reads `name`, treating both "unset" and "set to the empty string" as +/// absent, matching `config::env_raw`'s convention: a drop-in override that +/// clears a variable to `""` should fall back to the default, not try to use +/// an empty value. +fn env_override(name: &str) -> Option { + std::env::var(name).ok().filter(|v| !v.is_empty()) +} pub fn current_active_version() -> String { - read_active_version(AKS_OS_VERSION_PATH).unwrap_or_else(|| { + let key = env_override(ENV_CURRENT_VERSION_KEY) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_KEY.to_string()); + read_os_release_value(osutils::osrelease::OS_RELEASE_PATH, &key).unwrap_or_else(|| { + let stub = env_override(ENV_CURRENT_VERSION_STUB) + .unwrap_or_else(|| CURRENT_VERSION_STUB.to_string()); log::warn!( - "{AKS_OS_VERSION_PATH} not found; falling back to stub current version \ - {CURRENT_VERSION_STUB}" + "{key} not found in {}; falling back to stub current version {stub}", + osutils::osrelease::OS_RELEASE_PATH ); - CURRENT_VERSION_STUB.to_string() + stub }) } -/// Reads and trims the active-version file at `path`. Returns `None` (rather -/// than propagating an error) for any read failure - missing file, permission -/// error, or empty contents - all of which `current_active_version` treats -/// identically: fall back to the stub. Split out from -/// `current_active_version` so tests can point it at a temp file instead of -/// the real `/etc/aks-os-version`. -fn read_active_version(path: &str) -> Option { +/// Reads `path` (an os-release-formatted file: `KEY=VALUE` lines, blank +/// lines and `#` comments ignored, values optionally single- or +/// double-quoted - see +/// ) +/// and returns the trimmed, unquoted value for `key`, or `None` if the file +/// can't be read, `key` isn't present, or its value is empty - all of which +/// `current_active_version` treats identically: fall back to the stub. +/// Split out from `current_active_version` so tests can point it at a temp +/// file instead of the real os-release. +fn read_os_release_value(path: &str, key: &str) -> Option { let contents = std::fs::read_to_string(path).ok()?; - let trimmed = contents.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_string()) + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((line_key, raw_value)) = line.split_once('=') else { + continue; + }; + if line_key.trim() != key { + continue; + } + let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); + if value.is_empty() { + return None; + } + return Some(value.to_string()); } + None } impl From for Operation { @@ -1343,30 +1375,120 @@ mod tests { } #[test] - fn read_active_version_returns_none_for_missing_file() { + fn read_os_release_value_returns_none_for_missing_file() { assert_eq!( - read_active_version("/nonexistent/path/does-not-exist-aks-os-version"), + read_os_release_value( + "/nonexistent/path/does-not-exist-os-release", + DEFAULT_CURRENT_VERSION_KEY + ), None ); } #[test] - fn read_active_version_trims_and_reads_real_file() { + fn read_os_release_value_finds_requested_key() { let dir = std::env::temp_dir(); - let path = dir.join(format!("aks-os-version-test-{}", Uuid::new_v4())); - std::fs::write(&path, " 202608.6.0\n").unwrap(); - let result = read_active_version(path.to_str().unwrap()); + let path = dir.join(format!("os-release-test-{}", Uuid::new_v4())); + std::fs::write( + &path, + "NAME=\"Azure Linux\"\nIMAGE_VERSION=202608.6.0\nVERSION_ID=3.0\n", + ) + .unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); std::fs::remove_file(&path).ok(); assert_eq!(result.as_deref(), Some("202608.6.0")); } #[test] - fn read_active_version_treats_empty_file_as_absent() { + fn read_os_release_value_trims_quotes_and_whitespace() { let dir = std::env::temp_dir(); - let path = dir.join(format!("aks-os-version-test-empty-{}", Uuid::new_v4())); - std::fs::write(&path, " \n").unwrap(); - let result = read_active_version(path.to_str().unwrap()); + let path = dir.join(format!("os-release-test-quoted-{}", Uuid::new_v4())); + std::fs::write(&path, " IMAGE_VERSION = \"202608.6.0\" \n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + #[test] + fn read_os_release_value_returns_none_for_missing_key() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-missing-key-{}", Uuid::new_v4())); + std::fs::write(&path, "NAME=\"Azure Linux\"\nVERSION_ID=3.0\n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result, None); + } + + #[test] + fn read_os_release_value_returns_none_for_empty_value() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-empty-value-{}", Uuid::new_v4())); + std::fs::write(&path, "IMAGE_VERSION=\n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); std::fs::remove_file(&path).ok(); assert_eq!(result, None); } + + #[test] + fn read_os_release_value_skips_comments_and_blank_lines() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-comments-{}", Uuid::new_v4())); + std::fs::write( + &path, + "# a comment\n\n# IMAGE_VERSION=should-be-ignored\nIMAGE_VERSION=202608.6.0\n", + ) + .unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + /// Clears both env vars `current_active_version` reads. Environment + /// mutation is process-global and `std::env::remove_var`/`set_var` are + /// `unsafe` (not thread-safe against concurrent reads elsewhere in the + /// process), so the defaults/overrides cases below are intentionally + /// folded into one sequential `#[test]` rather than several separate + /// ones that `cargo test` could run in parallel against the same + /// variables. + fn clear_current_version_env() { + // SAFETY: single-threaded within this test function; no other test + // in this crate reads or writes these two variables. + unsafe { + std::env::remove_var(ENV_CURRENT_VERSION_KEY); + std::env::remove_var(ENV_CURRENT_VERSION_STUB); + } + } + + #[test] + fn current_active_version_key_and_stub_are_overridable_via_env() { + clear_current_version_env(); + assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_KEY, "CUSTOM_VERSION_KEY"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_KEY).as_deref(), + Some("CUSTOM_VERSION_KEY") + ); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_STUB).as_deref(), + Some("custom-stub") + ); + + // An empty override is treated the same as unset. + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_KEY, ""); + } + assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + + clear_current_version_env(); + } } From 254efdee90538954a0fa8e14e223e33ec1786e78 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 20:53:32 +0000 Subject: [PATCH 12/54] docs: add Trident ACL Agent explanation doc Describes the annotation-driven update contract, request/status/ commit-status schema, sequence flow, and the TRIDENT_ACL_AGENT_* env-var configuration (annotation prefix, Nebraska server, os-release version key/stub) via systemd drop-in overrides. Framed as a generic Kubernetes orchestration mechanism rather than an AKS-specific document. --- docs/Explanation/Trident-ACL-Agent.md | 250 ++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 docs/Explanation/Trident-ACL-Agent.md diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md new file mode 100644 index 0000000000..af88445aec --- /dev/null +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -0,0 +1,250 @@ +# Trident ACL Agent + +`trident-acl-agent` is an on-node daemon that drives Trident +[A/B updates](./AB-Update.md) from a Kubernetes control plane, using node +annotations instead of a direct API call as the trigger. It is the on-node +half of Azure Container Linux (ACL)'s update mechanism, but the mechanism +itself is not AKS-specific: any Kubernetes control-plane component (a +custom controller, an operator, or a script driven by `kubectl patch`) can +orchestrate updates across a fleet of nodes by writing to the annotation +contract described below, provided it is willing to speak the +[Omaha](https://github.com/omaha-consortium/omaha) protocol for image +distribution and honors the agent's per-node protocol. + +## Modes + +The agent runs in one of two modes, selected by +`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`: + +- **`annotations`** (the default) — watches this Node's annotations for an + update request, and drives Trident's stage/finalize/rollback/commit + operations against `tridentd` accordingly, reporting progress and status + back to Kubernetes and to the configured Omaha server. This is the mode + described in the rest of this document. +- **`omaha-only`** — a one-shot mode with no Kubernetes involvement at all: + the agent queries its Omaha server once, and if an update is offered, + calls `tridentd`'s combined `update()` RPC once and exits. Useful for a + node that isn't part of a Kubernetes-orchestrated fleet. + +## The annotation contract + +In `annotations` mode, an orchestrator (a Kubernetes controller with RBAC +permission to PATCH the target Node object) triggers an update by writing a +JSON payload to a request annotation on the Node. The agent watches that +annotation, drives the requested operation against `tridentd`, and writes +its progress and result back to two status annotations on the same Node. + +Three annotation keys make up the contract, all sharing one configurable +prefix (`acl.azure.com` by default — see [Configuration](#configuration) +below): + +| Annotation | Written by | Purpose | +|---|---|---| +| `/update-request` | Orchestrator | Requests `stage`, `finalize`, or `rollback` for this node. | +| `/update-status` | Agent | Reports the status of the requested operation. | +| `/update-commit-status` | Agent | Reports the status of the implicit post-reboot `commit` that follows a `finalize` or `rollback`. | + +A request annotation looks like: + +```json +{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "c9d6f0a2-3b41-4e8d-9f27-1a5b6c7d8e90", + "operation": "stage", + "targetVersion": "202606.29.0", + "server": "https://nebraska.example.com/v1/update", + "appId": "11111111-2222-3333-4444-555555555555", + "track": "pin-202606.29.0" +} +``` + +- `nodeUpdateId` identifies one node's update sequence and is held constant + across `stage` → `finalize` → `commit`. +- `operationId` identifies this specific step; the agent uses it to decide + whether to start new work, resume in-flight work, or re-emit a cached + terminal status as a no-op on a duplicate PATCH. +- `targetVersion` is the image release version to update to. Required for + `stage`/`finalize`; omitted for `rollback`, whose target (the previous + partition) is implicit. +- `server`, `appId`, and `track` name the Omaha instance that serves the + target image and receives progress events for it. They are required on + `stage`/`finalize` requests, with **no static fallback** — a request + missing them is rejected with `InvalidRequest` rather than falling back to + a built-in endpoint, so a node can never update from a source the + orchestrator did not explicitly choose. + +`operation` maps to Trident invocations as follows: + +| `operation` | Trident invocation | Effect | +|---|---|---| +| `stage` | `trident update --allowed-operations=stage` | Streams the target image to the inactive partition. No reboot. | +| `finalize` | `trident update --allowed-operations=finalize` (gRPC `UpdateFinalize`, caller-handled reboot) | Arms boot for the staged target, writes a terminal `finalize` status, then triggers the reboot. | +| `rollback` | `trident rollback --ab` (gRPC `RollbackStage`/`RollbackFinalize`, caller-handled reboot) | Swaps back to the previous partition, mirroring `finalize` on the return path. Only the last update can be undone this way. | + +A fourth phase, `commit`, runs implicitly after the post-`finalize`/ +`rollback` reboot: the agent runs `trident commit` on the new partition and +writes a `commit` status without needing a separate annotation request. The +orchestrator watches `/update-commit-status` as the terminal signal +that the reboot half of the update succeeded. + +`stage` end-to-end: + +```mermaid +sequenceDiagram + actor Orchestrator + participant API as K8s API Server + participant Agent as Trident ACL Agent
(on the node) + participant Trident + + Orchestrator->>API: 1. PATCH request: stage (opId A) + Note over Agent: agent picks up the request
on its next poll + Agent->>API: 2. read request, PATCH status: stage InProgress + Agent->>Trident: 3. Stage (image to inactive partition) + Trident-->>Agent: staged | error + Agent->>API: 4. PATCH status: stage Success | + API-->>Orchestrator: terminal stage code +``` + +`finalize` / `rollback`, spanning the reboot: + +```mermaid +sequenceDiagram + actor Orchestrator + participant API as K8s API Server + participant Agent as Trident ACL Agent
(on the node) + participant Trident + + Note over Orchestrator,Trident: pre-reboot half + Orchestrator->>API: 1. PATCH request: finalize (opId A) + Agent->>API: 2. read request, PATCH status: finalize InProgress + Agent->>Trident: 3. UpdateFinalize (caller-handled reboot) + Trident-->>Agent: boot armed, reboot required + Agent->>Agent: 4. persist pendingCommit + boot marker to state.json + Agent->>API: 5. PATCH status: finalize Success + API-->>Orchestrator: finalize Success (reboot pending) + Note over Agent,Trident: 6. agent triggers reboot, boots new partition + Note over Orchestrator,Trident: post-reboot half + Agent->>Agent: 7. read state.json, confirm a boot happened since the marker + Agent->>Trident: 8. Commit (validate volume, promote boot order) + Trident-->>Agent: committed | reverted to previous + Agent->>API: 9. PATCH status: commit Success | TargetBootFailed + API-->>Orchestrator: terminal commit code +``` + +A status annotation's `code` is one of `InProgress`, `Success`, +`AlreadyAtTarget`, `NotStaged`, `OperationFailed`, `TargetBootFailed`, +`AgentInternalError`, or `InvalidRequest` — see the request/status schema +types in `crates/trident-acl-agent/src/annotations.rs` for the full +contract, including the formal JSON Schema both sides validate against. + +## Pre/post-reboot state and the watchdog + +Because `finalize`/`rollback` spans a reboot, the agent persists a small +state file (`TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH`) recording that a +commit is pending and a marker for "a boot happened after this point". On +restart, the agent checks this state to resume the post-reboot `commit` +step rather than re-running `finalize` from scratch. + +While an operation is in flight, the agent refreshes the `InProgress` +status's `lastUpdatedUtc` on a heartbeat cadence +(`TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL`), so an external +watchdog can distinguish a working agent from a stuck one and reprovision a +node that never reports a terminal `commit` status within its SLA. + +## Configuration + +There is no config file. Every setting is an environment variable prefixed +`TRIDENT_ACL_AGENT_`, systemd-style: set it in the unit's own +`Environment=` lines, via a drop-in override, or by any other means that +sets the process's environment before it starts. + +A variable that is unset, or set to the empty string, falls back to its +default. A variable set to a malformed value (a bad URL, a bad duration, an +unrecognized `goal_source`) causes the agent to fail to start with an error +naming the offending variable. + +| Variable | Default | Description | +|---|---|---| +| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.azure.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.azure.com` in `acl.azure.com/update-request`). Not tied to AKS or Azure — any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `IMAGE_VERSION` | The `/etc/os-release` key the agent reads to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `IMAGE_VERSION` is what ACL images carry; a deployment building its own images can point this at whatever key its own `os-release` provides instead — see [below](#configuring-the-on-disk-version). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `0.0.0-unprobed-trident-acl-agent-stub` | Sentinel value used as the node's "current version" when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present in `/etc/os-release` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). Deliberately shaped so it can never collide with a real release version and cause a false `AlreadyAtTarget`. | +| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Omaha server URL to poll and report events to, for `omaha-only` mode only. In `annotations` mode this is never used as a fallback — `stage`/`finalize` requests must carry their own `server` field (see [above](#the-annotation-contract)). | +| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Omaha application id this node checks in as, for `omaha-only` mode. In `annotations` mode, required on the request's `appId` field instead. | +| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Omaha track this node follows, for `omaha-only` mode. In `annotations` mode, required on the request's `track` field instead. | +| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in the kubeconfig is used as-is. | +| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig used to reach the Kubernetes API server and authenticate as this node. | +| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. | +| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode: `annotations` or `omaha-only`. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file bridging the pre-reboot and post-reboot halves of `finalize`/`rollback` across the reboot. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` is allowed to run (a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) before it's considered failed. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` is allowed to run before it's considered failed. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat. | + +### Setting env vars via a systemd drop-in + +The agent ships as `trident-acl-agent.service`, with no `Environment=` +lines of its own beyond `ExecStart`. Any setting is overridden with a +drop-in file, without editing the packaged unit: + +```console +$ sudo systemctl edit trident-acl-agent.service +``` + +This opens `/etc/systemd/system/trident-acl-agent.service.d/override.conf` +in an editor. For example, to point the agent at a custom annotation +namespace and Omaha server for a non-AKS Kubernetes deployment: + +```ini +[Service] +Environment=TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX=acl.contoso.com +Environment=TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update +Environment=TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee +Environment=TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable +``` + +With the prefix above, the orchestrator now reads/writes +`acl.contoso.com/update-request`, `acl.contoso.com/update-status`, and +`acl.contoso.com/update-commit-status` instead of the `acl.azure.com/*` +defaults. Reload and restart to apply: + +```console +$ sudo systemctl daemon-reload +$ sudo systemctl restart trident-acl-agent.service +``` + +`systemctl cat trident-acl-agent.service` shows the merged unit (packaged +unit plus drop-in), useful for confirming the override took effect. + +### Configuring the on-disk version + +The agent determines the node's current version by reading a key out of +`/etc/os-release`, defaulting to `IMAGE_VERSION` — the key ACL images +stamp. A deployment building its own images, not derived from ACL, may not +carry `IMAGE_VERSION` at all; rather than requiring every image build to +add an ACL-specific field, point the agent at the standard +[`os-release`](https://www.freedesktop.org/software/systemd/man/latest/os-release.html) +field `VERSION_ID` instead: + +```ini +[Service] +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=VERSION_ID +``` + +With this set, the agent reads `VERSION_ID` from `/etc/os-release` (e.g. +`VERSION_ID=202606.29.0`) as the node's current version, and compares it +against a request's `targetVersion` the same way it would for +`IMAGE_VERSION` — including short-circuiting to `AlreadyAtTarget` when they +already match. If the configured key is absent from `/etc/os-release` +(for example, on a dev/test host with a minimal `os-release`), the agent +falls back to `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` +(`0.0.0-unprobed-trident-acl-agent-stub` by default), a sentinel value +that can never accidentally match a real requested version. + +## Diagnostics + +`trident-acl-agent --validate-connection ` +checks connectivity to a single dependency using the current environment +and exits immediately — useful for a systemd `ExecStartPre` check or manual +on-node troubleshooting without running the full orchestrator loop. From de9fb3a509651e52c7150d750b7b49f0e54bf262 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 20:58:41 +0000 Subject: [PATCH 13/54] docs: clarify stage queries Nebraska for the image; trim AKS wording Note in the operations table and stage sequence diagram that Nebraska/ Omaha is queried for the target image during stage, not just streamed from a location already known. Also drop an unnecessary AKS-specific caveat from the intro paragraph. --- docs/Explanation/Trident-ACL-Agent.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index af88445aec..f63c0f5050 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -3,9 +3,9 @@ `trident-acl-agent` is an on-node daemon that drives Trident [A/B updates](./AB-Update.md) from a Kubernetes control plane, using node annotations instead of a direct API call as the trigger. It is the on-node -half of Azure Container Linux (ACL)'s update mechanism, but the mechanism -itself is not AKS-specific: any Kubernetes control-plane component (a -custom controller, an operator, or a script driven by `kubectl patch`) can +half of Azure Container Linux (ACL)'s update mechanism. Any Kubernetes +control-plane component (a custom controller, an operator, or a script +driven by `kubectl patch`) can orchestrate updates across a fleet of nodes by writing to the annotation contract described below, provided it is willing to speak the [Omaha](https://github.com/omaha-consortium/omaha) protocol for image @@ -78,7 +78,7 @@ A request annotation looks like: | `operation` | Trident invocation | Effect | |---|---|---| -| `stage` | `trident update --allowed-operations=stage` | Streams the target image to the inactive partition. No reboot. | +| `stage` | `trident update --allowed-operations=stage` | Queries the `server`/`appId`/`track` Omaha endpoint for `targetVersion`, then streams the resulting image to the inactive partition. No reboot. | | `finalize` | `trident update --allowed-operations=finalize` (gRPC `UpdateFinalize`, caller-handled reboot) | Arms boot for the staged target, writes a terminal `finalize` status, then triggers the reboot. | | `rollback` | `trident rollback --ab` (gRPC `RollbackStage`/`RollbackFinalize`, caller-handled reboot) | Swaps back to the previous partition, mirroring `finalize` on the return path. Only the last update can be undone this way. | @@ -95,14 +95,17 @@ sequenceDiagram actor Orchestrator participant API as K8s API Server participant Agent as Trident ACL Agent
(on the node) + participant Nebraska as Omaha server participant Trident Orchestrator->>API: 1. PATCH request: stage (opId A) Note over Agent: agent picks up the request
on its next poll Agent->>API: 2. read request, PATCH status: stage InProgress - Agent->>Trident: 3. Stage (image to inactive partition) + Agent->>Nebraska: 3. query targetVersion (server/appId/track) + Nebraska-->>Agent: image location + Agent->>Trident: 4. Stage (image to inactive partition) Trident-->>Agent: staged | error - Agent->>API: 4. PATCH status: stage Success | + Agent->>API: 5. PATCH status: stage Success | API-->>Orchestrator: terminal stage code ``` From b55802f301ab4292e17fda1b504b22f4593221b0 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 21:19:50 +0000 Subject: [PATCH 14/54] trident-acl-agent: make os-release path configurable Adds TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH so a deployment can point current_active_version() at any os-release-formatted file instead of the real /etc/os-release, with the same key/stub-fallback rules. Updates the ACL agent doc accordingly. --- crates/trident-acl-agent/src/annotations.rs | 89 ++++++++++++++++++--- docs/Explanation/Trident-ACL-Agent.md | 5 +- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index 68dec9982b..c34b7fddd1 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -76,11 +76,22 @@ const TRUNCATION_MARKER: &str = "... (truncated)"; // stub itself is overridable via TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for // dev/test hosts that want a specific sentinel. pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; +/// Default path `current_active_version` reads. Overridable via +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` so a deployment can point the +/// agent at any file that follows the os-release format (`KEY=VALUE` lines, +/// optionally quoted, blank lines and `#` comments ignored - see +/// ) +/// instead of the real `/etc/os-release`, e.g. a vendor-specific file that +/// carries the running image's version under a key `/etc/os-release` +/// doesn't have room for. +pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; /// Default os-release key `current_active_version` looks up for the running /// image's version. Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` /// so a deployment that doesn't set `IMAGE_VERSION` can point at whatever -/// key its os-release does carry. +/// key its os-release (or `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` +/// override) does carry. pub const DEFAULT_CURRENT_VERSION_KEY: &str = "IMAGE_VERSION"; +const ENV_CURRENT_VERSION_PATH: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH"; const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; const ENV_CURRENT_VERSION_STUB: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB"; @@ -294,15 +305,14 @@ fn env_override(name: &str) -> Option { } pub fn current_active_version() -> String { + let path = env_override(ENV_CURRENT_VERSION_PATH) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); let key = env_override(ENV_CURRENT_VERSION_KEY) .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_KEY.to_string()); - read_os_release_value(osutils::osrelease::OS_RELEASE_PATH, &key).unwrap_or_else(|| { + read_os_release_value(&path, &key).unwrap_or_else(|| { let stub = env_override(ENV_CURRENT_VERSION_STUB) .unwrap_or_else(|| CURRENT_VERSION_STUB.to_string()); - log::warn!( - "{key} not found in {}; falling back to stub current version {stub}", - osutils::osrelease::OS_RELEASE_PATH - ); + log::warn!("{key} not found in {path}; falling back to stub current version {stub}"); stub }) } @@ -1443,27 +1453,38 @@ mod tests { assert_eq!(result.as_deref(), Some("202608.6.0")); } - /// Clears both env vars `current_active_version` reads. Environment + /// Clears all three env vars `current_active_version` reads. Environment /// mutation is process-global and `std::env::remove_var`/`set_var` are /// `unsafe` (not thread-safe against concurrent reads elsewhere in the - /// process), so the defaults/overrides cases below are intentionally - /// folded into one sequential `#[test]` rather than several separate - /// ones that `cargo test` could run in parallel against the same - /// variables. + /// process), so the defaults/overrides/read-path cases below are + /// intentionally folded into one sequential `#[test]` rather than + /// several separate ones that `cargo test` could run in parallel + /// against the same variables. fn clear_current_version_env() { // SAFETY: single-threaded within this test function; no other test - // in this crate reads or writes these two variables. + // in this crate reads or writes these three variables. unsafe { + std::env::remove_var(ENV_CURRENT_VERSION_PATH); std::env::remove_var(ENV_CURRENT_VERSION_KEY); std::env::remove_var(ENV_CURRENT_VERSION_STUB); } } #[test] - fn current_active_version_key_and_stub_are_overridable_via_env() { + fn current_active_version_path_key_and_stub_are_overridable_via_env() { clear_current_version_env(); + assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, "/custom/os-release"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_PATH).as_deref(), + Some("/custom/os-release") + ); + // SAFETY: see clear_current_version_env's doc comment. unsafe { std::env::set_var(ENV_CURRENT_VERSION_KEY, "CUSTOM_VERSION_KEY"); @@ -1485,10 +1506,52 @@ mod tests { // An empty override is treated the same as unset. // SAFETY: see clear_current_version_env's doc comment. unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, ""); std::env::set_var(ENV_CURRENT_VERSION_KEY, ""); } + assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); clear_current_version_env(); + + // current_active_version() itself honors TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH, + // pointing it at an arbitrary os-release-formatted file instead of the + // real /etc/os-release. + let dir = std::env::temp_dir(); + let found_path = dir.join(format!( + "os-release-test-current-version-{}", + Uuid::new_v4() + )); + std::fs::write( + &found_path, + "NAME=\"Contoso Linux\"\nVERSION_ID=202608.6.0\n", + ) + .unwrap(); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, found_path.to_str().unwrap()); + std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); + } + assert_eq!(current_active_version(), "202608.6.0"); + std::fs::remove_file(&found_path).ok(); + clear_current_version_env(); + + // When the configured key isn't present at the configured path, it + // still falls back to the (possibly also-overridden) stub, exactly + // as it does for the real /etc/os-release. + let missing_path = dir.join(format!( + "os-release-test-current-version-missing-{}", + Uuid::new_v4() + )); + std::fs::write(&missing_path, "NAME=\"Contoso Linux\"\n").unwrap(); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, missing_path.to_str().unwrap()); + std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); + std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub-for-missing-key"); + } + assert_eq!(current_active_version(), "custom-stub-for-missing-key"); + std::fs::remove_file(&missing_path).ok(); + clear_current_version_env(); } } diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index f63c0f5050..d28d933ad6 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -170,8 +170,9 @@ naming the offending variable. | Variable | Default | Description | |---|---|---| | `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.azure.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.azure.com` in `acl.azure.com/update-request`). Not tied to AKS or Azure — any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `IMAGE_VERSION` | The `/etc/os-release` key the agent reads to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `IMAGE_VERSION` is what ACL images carry; a deployment building its own images can point this at whatever key its own `os-release` provides instead — see [below](#configuring-the-on-disk-version). | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `0.0.0-unprobed-trident-acl-agent-stub` | Sentinel value used as the node's "current version" when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present in `/etc/os-release` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). Deliberately shaped so it can never collide with a real release version and cause a false `AlreadyAtTarget`. | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | The file the agent reads to determine the node's currently running version. Any file works, as long as it follows the os-release format (`KEY=VALUE` lines, optionally single- or double-quoted, blank lines and `#` comments ignored) — see [below](#configuring-the-on-disk-version). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `IMAGE_VERSION` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `IMAGE_VERSION` is what ACL images carry; a deployment building its own images can point this at whatever key its own `os-release`-formatted file provides instead — see [below](#configuring-the-on-disk-version). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `0.0.0-unprobed-trident-acl-agent-stub` | Sentinel value used as the node's "current version" when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). Deliberately shaped so it can never collide with a real release version and cause a false `AlreadyAtTarget`. | | `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Omaha server URL to poll and report events to, for `omaha-only` mode only. In `annotations` mode this is never used as a fallback — `stage`/`finalize` requests must carry their own `server` field (see [above](#the-annotation-contract)). | | `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Omaha application id this node checks in as, for `omaha-only` mode. In `annotations` mode, required on the request's `appId` field instead. | | `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Omaha track this node follows, for `omaha-only` mode. In `annotations` mode, required on the request's `track` field instead. | From 93569f9a9a90adf1c81eed1b5ee6baadbbc05c86 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 21:27:24 +0000 Subject: [PATCH 15/54] trident-acl-agent: default annotation prefix to acl.microsoft.com, version key to VERSION_ID Hide omaha-only from user-facing docs (#[doc(hidden)], reworded comments); it remains a functional internal escape hatch but is no longer presented as a supported deployment option. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/trident-acl-agent/README.md | 35 +++++---- crates/trident-acl-agent/src/annotations.rs | 58 ++++++++------- crates/trident-acl-agent/src/config.rs | 35 +++++---- crates/trident-acl-agent/src/main.rs | 11 +-- docs/Explanation/Trident-ACL-Agent.md | 82 ++++++++++----------- 5 files changed, 112 insertions(+), 109 deletions(-) diff --git a/crates/trident-acl-agent/README.md b/crates/trident-acl-agent/README.md index 552a72e173..8303e00286 100644 --- a/crates/trident-acl-agent/README.md +++ b/crates/trident-acl-agent/README.md @@ -1,19 +1,14 @@ # trident-acl-agent -The on-node half of Trident's Azure Container Linux (ACL) A/B update -trigger. Runs in one of two modes, selected by -`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`: - -- **`annotations`** (the default): watches its Node's - `acl.azure.com/update-request` annotation and drives Trident's - stage/finalize/rollback/commit operations against `tridentd` accordingly, - reporting progress and status back to Kubernetes and to Nebraska (the - Omaha-protocol update server). -- **`omaha-only`**: the historical one-shot behavior. Queries Nebraska once, - and if an update is offered, calls tridentd's combined `update()` RPC once - and exits - no Kubernetes or annotation involvement at all. Kept as an - explicit opt-out for nodes that don't participate in the AKS - annotation-driven update protocol. +The on-node half of Trident's annotation-driven Kubernetes A/B update +trigger. It watches its Node's `acl.microsoft.com/update-request` +annotation (the prefix is configurable, see below) and drives Trident's +stage/finalize/rollback/commit operations against `tridentd` accordingly, +reporting progress and status back to Kubernetes and to Nebraska (the +Omaha-protocol update server). + +See [`docs/Explanation/Trident-ACL-Agent.md`](../../docs/Explanation/Trident-ACL-Agent.md) +for a full description of the annotation contract and the reconcile flow. ## Configuration @@ -32,14 +27,18 @@ fail to start with an error naming the offending variable. | Variable | Default | Description | |---|---|---| -| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Nebraska/Omaha server URL to poll for updates and report progress/completion events to, for `omaha-only` mode. In `annotations` mode, `stage`/`finalize` requests must instead carry their own `server` field on the `acl.azure.com/update-request` annotation - there is deliberately no fallback to this variable, since a fallback would let a node update from a source AKS-RP did not choose; a request missing it is rejected with `InvalidRequest`. | -| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Nebraska application ID this node checks in as, for `omaha-only` mode. In `annotations` mode, required on the `acl.azure.com/update-request` annotation's `appId` field instead, same no-fallback rule as the endpoint. | -| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Nebraska track (channel/group) this node follows, for `omaha-only` mode. In `annotations` mode, required on the `acl.azure.com/update-request` annotation's `track` field instead, same no-fallback rule as the endpoint. | +| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Nebraska/Omaha server URL to poll for updates and report progress/completion events to. `stage`/`finalize` requests must instead carry their own `server` field on the `acl.microsoft.com/update-request` annotation - there is deliberately no fallback to this variable, since a fallback would let a node update from a source the annotation's author did not choose; a request missing it is rejected with `InvalidRequest`. | +| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Nebraska application ID this node checks in as. Required on the `acl.microsoft.com/update-request` annotation's `appId` field instead, same no-fallback rule as the endpoint. | +| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Nebraska track (channel/group) this node follows. Required on the `acl.microsoft.com/update-request` annotation's `track` field instead, same no-fallback rule as the endpoint. | +| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.microsoft.com` | Prefix for the `update-request`/`update-status`/`update-commit-status` Node annotations this agent watches and writes. | | `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG`'s own kubeconfig is used as-is (e.g. the real cluster FQDN a node's own `/var/lib/kubelet/kubeconfig` already points at). Only needed when the kubeconfig's own server is wrong for this deployment. | | `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig file used to reach the Kubernetes API server and authenticate as this node. | | `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. Kubernetes Node names must be valid RFC 1123 DNS labels (lowercase), matching how kubelet itself registers the Node - so the default only needs overriding when the agent's environment can't discover the correct hostname on its own. | | `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode: `annotations` or `omaha-only` (see above). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | Path to the key-value file (must follow the `os-release` schema) this agent reads to determine the node's currently-running version. | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key read from `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` to determine the node's currently-running version. | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `CURRENT_VERSION_STUB` | Fallback value reported as the current version when the configured key is missing from the configured file. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode. `annotations` is the only supported mode; other values are internal/undocumented. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file, which bridges the pre-reboot `finalize`/`rollback` half of an update and its post-reboot `commit` half across the reboot. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` operation (parsed as a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) is allowed to run before it's considered failed. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` operation is allowed to run before it's considered failed. Parsed the same way as the stage timeout. | diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index c34b7fddd1..a37134ed2b 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -5,7 +5,7 @@ //! `/update-request`, `/update-status`, and //! `/update-commit-status` node annotation protocol described //! by the current accepted design (`accepted-design-v3.md`), where -//! `` defaults to `acl.azure.com` (see +//! `` defaults to `acl.microsoft.com` (see //! [`AnnotationKeys`]/[`crate::config::DEFAULT_ANNOTATION_PREFIX`]) and is //! overridable via the `TRIDENT_ACL_AGENT_ANNOTATION_PREFIX` environment //! variable. Keep `UpdateRequest`/`UpdateStatus`/`StatusCode` and @@ -23,19 +23,19 @@ use uuid::Uuid; use crate::config::DEFAULT_ANNOTATION_PREFIX; /// Suffix (appended to the configured annotation prefix) for the request -/// annotation, e.g. `acl.azure.com/update-request`. +/// annotation, e.g. `acl.microsoft.com/update-request`. pub const UPDATE_REQUEST_SUFFIX: &str = "update-request"; /// Suffix for the operation-status annotation, e.g. -/// `acl.azure.com/update-status`. +/// `acl.microsoft.com/update-status`. pub const UPDATE_STATUS_SUFFIX: &str = "update-status"; /// Suffix for the post-reboot commit-status annotation, e.g. -/// `acl.azure.com/update-commit-status`. +/// `acl.microsoft.com/update-commit-status`. pub const UPDATE_COMMIT_STATUS_SUFFIX: &str = "update-commit-status"; /// The full annotation keys for one deployment's configured annotation /// prefix. Built once from [`crate::config::KubernetesConfig::annotation_prefix`] -/// and threaded through instead of hardcoding the AKS-specific -/// `acl.azure.com` prefix. +/// and threaded through instead of hardcoding a fixed +/// `acl.microsoft.com` prefix. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AnnotationKeys { pub request: String, @@ -62,19 +62,18 @@ impl Default for AnnotationKeys { pub const SCHEMA_VERSION: &str = "1.0"; const MAX_MESSAGE_BYTES: usize = 2048; const TRUNCATION_MARKER: &str = "... (truncated)"; -// TODO(DR-001): current_active_version() now reads the `IMAGE_VERSION` key -// (overridable via TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY) out of os-release, -// but falls back to this stub if that key isn't present yet (e.g. an image -// whose os-release doesn't carry it, or a dev/test host). Once the key ships -// unconditionally on every ACL image, this fallback (and this comment) can be -// removed. The stub value below is an explicit sentinel that cannot collide -// with a real AKS/Trident release version string (those look like -// "YYYYMM.N.N"), so it can never accidentally match a real requested target -// version and cause handle_stage/handle_finalize to incorrectly short-circuit -// to AlreadyAtTarget. Do not remove this comment when bumping the stub value; -// keep it (and its non-colliding shape) until the fallback is removed. The -// stub itself is overridable via TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for -// dev/test hosts that want a specific sentinel. +// current_active_version() reads the `VERSION_ID` key (overridable via +// TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY, e.g. to `IMAGE_VERSION` for an ACL +// image that stamps its own per-build version there) out of os-release, but +// falls back to this stub if that key isn't present (e.g. a minimal +// dev/test os-release). The stub value below is an explicit sentinel that +// cannot collide with a real release version string, so it can never +// accidentally match a real requested target version and cause +// handle_stage/handle_finalize to incorrectly short-circuit to +// AlreadyAtTarget. Do not remove this comment when bumping the stub value; +// keep it (and its non-colliding shape). The stub itself is overridable via +// TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for dev/test hosts that want a +// specific sentinel. pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; /// Default path `current_active_version` reads. Overridable via /// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` so a deployment can point the @@ -86,11 +85,14 @@ pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; /// doesn't have room for. pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; /// Default os-release key `current_active_version` looks up for the running -/// image's version. Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` -/// so a deployment that doesn't set `IMAGE_VERSION` can point at whatever -/// key its os-release (or `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` -/// override) does carry. -pub const DEFAULT_CURRENT_VERSION_KEY: &str = "IMAGE_VERSION"; +/// image's version: `VERSION_ID`, a standard key every os-release carries +/// (see +/// ). +/// Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` - e.g. to +/// `IMAGE_VERSION` for an ACL image that stamps its own per-build version +/// under that key instead - or point +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` at a different file entirely. +pub const DEFAULT_CURRENT_VERSION_KEY: &str = "VERSION_ID"; const ENV_CURRENT_VERSION_PATH: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH"; const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; const ENV_CURRENT_VERSION_STUB: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB"; @@ -367,11 +369,11 @@ mod tests { use super::*; #[test] - fn annotation_keys_default_uses_acl_azure_com_prefix() { + fn annotation_keys_default_uses_acl_microsoft_com_prefix() { let keys = AnnotationKeys::default(); - assert_eq!(keys.request, "acl.azure.com/update-request"); - assert_eq!(keys.status, "acl.azure.com/update-status"); - assert_eq!(keys.commit_status, "acl.azure.com/update-commit-status"); + assert_eq!(keys.request, "acl.microsoft.com/update-request"); + assert_eq!(keys.status, "acl.microsoft.com/update-status"); + assert_eq!(keys.commit_status, "acl.microsoft.com/update-commit-status"); } #[test] diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs index 5a01f75206..3b1878b35a 100644 --- a/crates/trident-acl-agent/src/config.rs +++ b/crates/trident-acl-agent/src/config.rs @@ -34,11 +34,10 @@ const ENV_ORCHESTRATION_STAGE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_S const ENV_ORCHESTRATION_FINALIZE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT"; const ENV_ORCHESTRATION_HEARTBEAT_INTERVAL: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL"; -/// Overrides the annotation-key prefix (e.g. `acl.azure.com` in -/// `acl.azure.com/update-request`). Defaults to -/// [`DEFAULT_ANNOTATION_PREFIX`] so a deployment not tied to AKS's -/// `acl.azure.com` domain can point the agent at its own annotation -/// namespace without a code change. +/// Overrides the annotation-key prefix (e.g. `acl.microsoft.com` in +/// `acl.microsoft.com/update-request`). Defaults to +/// [`DEFAULT_ANNOTATION_PREFIX`] so a deployment can point the agent at its +/// own annotation namespace without a code change. const ENV_KUBERNETES_ANNOTATION_PREFIX: &str = "TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX"; const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); @@ -56,9 +55,9 @@ const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); pub const DEFAULT_STATE_PATH: &str = "/var/lib/trident-acl-agent/state.json"; pub const DEFAULT_KUBELET_KUBECONFIG: &str = "/var/lib/kubelet/kubeconfig"; -/// Default annotation-key prefix, matching AKS's `acl.azure.com` namespace. +/// Default annotation-key prefix. /// Override with `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. -pub const DEFAULT_ANNOTATION_PREFIX: &str = "acl.azure.com"; +pub const DEFAULT_ANNOTATION_PREFIX: &str = "acl.microsoft.com"; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct AgentConfig { @@ -150,10 +149,11 @@ pub struct KubernetesConfig { pub node_name: String, pub watch_poll_interval: Duration, /// Annotation-key prefix used for the request/status/commit-status - /// annotations (e.g. `acl.azure.com` in `acl.azure.com/update-request`). - /// Defaults to [`DEFAULT_ANNOTATION_PREFIX`], overridable via - /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` so the annotation - /// namespace isn't hardcoded to AKS. + /// annotations (e.g. `acl.microsoft.com` in + /// `acl.microsoft.com/update-request`). Defaults to + /// [`DEFAULT_ANNOTATION_PREFIX`], overridable via + /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` so a deployment can + /// pick its own namespace instead. pub annotation_prefix: String, } @@ -187,8 +187,11 @@ pub enum GoalSource { /// Historical one-shot behavior: query Nebraska/Omaha once, and if an /// update is offered, call tridentd's combined `update()` RPC once and /// exit. No Kubernetes involvement at all - no annotations, no watch, - /// no Node access. Kept as an explicit opt-out for nodes that don't - /// participate in the AKS annotation-driven update protocol. + /// no Node access. Not fully designed and not a supported deployment + /// option - kept only as an internal escape hatch, and deliberately + /// left out of user-facing docs. `Annotations` is the only documented, + /// supported mode. + #[doc(hidden)] OmahaOnly, /// The annotation-driven reconcile loop: watches the Node's /// `/update-request` annotation and drives Trident's @@ -197,9 +200,9 @@ pub enum GoalSource { /// `/update-status` and /// `/update-commit-status` (see /// accepted-design-v3.md). `` defaults to - /// [`DEFAULT_ANNOTATION_PREFIX`] (`acl.azure.com`), overridable via - /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. This is the default - /// mode. + /// [`DEFAULT_ANNOTATION_PREFIX`] (`acl.microsoft.com`), overridable via + /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. This is the only + /// supported mode. #[default] Annotations, } diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 2f5476f442..a81f6bc087 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -233,13 +233,14 @@ async fn main() -> Result<(), anyhow::Error> { match config.orchestration.goal_source { // Historical one-shot flow: query Nebraska once, apply an update if - // offered, and exit. No Kubernetes/annotation involvement. + // offered, and exit. No Kubernetes/annotation involvement. Not a + // documented/supported deployment option (see config::GoalSource). GoalSource::OmahaOnly => run_omaha_only(&config).await, - // Default: the annotation-driven reconcile loop (watches - // /update-request, drives stage/finalize/rollback/ + // The only supported mode: the annotation-driven reconcile loop + // (watches /update-request, drives stage/finalize/rollback/ // commit against tridentd, writes /update-status; prefix - // defaults to acl.azure.com, overridable via - // TRIDENT_ACL_AGENT_ANNOTATION_PREFIX). + // defaults to acl.microsoft.com, overridable via + // TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX). GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, } } diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index d28d933ad6..794331903a 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -11,31 +11,16 @@ contract described below, provided it is willing to speak the [Omaha](https://github.com/omaha-consortium/omaha) protocol for image distribution and honors the agent's per-node protocol. -## Modes - -The agent runs in one of two modes, selected by -`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`: - -- **`annotations`** (the default) — watches this Node's annotations for an - update request, and drives Trident's stage/finalize/rollback/commit - operations against `tridentd` accordingly, reporting progress and status - back to Kubernetes and to the configured Omaha server. This is the mode - described in the rest of this document. -- **`omaha-only`** — a one-shot mode with no Kubernetes involvement at all: - the agent queries its Omaha server once, and if an update is offered, - calls `tridentd`'s combined `update()` RPC once and exits. Useful for a - node that isn't part of a Kubernetes-orchestrated fleet. - ## The annotation contract -In `annotations` mode, an orchestrator (a Kubernetes controller with RBAC -permission to PATCH the target Node object) triggers an update by writing a -JSON payload to a request annotation on the Node. The agent watches that -annotation, drives the requested operation against `tridentd`, and writes -its progress and result back to two status annotations on the same Node. +An orchestrator (a Kubernetes controller with RBAC permission to PATCH the +target Node object) triggers an update by writing a JSON payload to a +request annotation on the Node. The agent watches that annotation, drives +the requested operation against `tridentd`, and writes its progress and +result back to two status annotations on the same Node. Three annotation keys make up the contract, all sharing one configurable -prefix (`acl.azure.com` by default — see [Configuration](#configuration) +prefix (`acl.microsoft.com` by default — see [Configuration](#configuration) below): | Annotation | Written by | Purpose | @@ -169,18 +154,17 @@ naming the offending variable. | Variable | Default | Description | |---|---|---| -| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.azure.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.azure.com` in `acl.azure.com/update-request`). Not tied to AKS or Azure — any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | +| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.microsoft.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.microsoft.com` in `acl.microsoft.com/update-request`). Any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | | `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | The file the agent reads to determine the node's currently running version. Any file works, as long as it follows the os-release format (`KEY=VALUE` lines, optionally single- or double-quoted, blank lines and `#` comments ignored) — see [below](#configuring-the-on-disk-version). | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `IMAGE_VERSION` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `IMAGE_VERSION` is what ACL images carry; a deployment building its own images can point this at whatever key its own `os-release`-formatted file provides instead — see [below](#configuring-the-on-disk-version). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `VERSION_ID` is the standard `os-release` field most images already stamp; a deployment that instead carries an ACL-specific `IMAGE_VERSION` field can point this variable at that key instead — see [below](#configuring-the-on-disk-version). | | `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `0.0.0-unprobed-trident-acl-agent-stub` | Sentinel value used as the node's "current version" when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). Deliberately shaped so it can never collide with a real release version and cause a false `AlreadyAtTarget`. | -| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Omaha server URL to poll and report events to, for `omaha-only` mode only. In `annotations` mode this is never used as a fallback — `stage`/`finalize` requests must carry their own `server` field (see [above](#the-annotation-contract)). | -| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Omaha application id this node checks in as, for `omaha-only` mode. In `annotations` mode, required on the request's `appId` field instead. | -| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Omaha track this node follows, for `omaha-only` mode. In `annotations` mode, required on the request's `track` field instead. | +| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Omaha server URL, never used as a fallback — `stage`/`finalize` requests must carry their own `server` field (see [above](#the-annotation-contract)). | +| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Omaha application id this node checks in as, never used as a fallback — required on the request's `appId` field instead. | +| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Omaha track this node follows, never used as a fallback — required on the request's `track` field instead. | | `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in the kubeconfig is used as-is. | | `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig used to reach the Kubernetes API server and authenticate as this node. | | `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. | | `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode: `annotations` or `omaha-only`. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file bridging the pre-reboot and post-reboot halves of `finalize`/`rollback` across the reboot. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` is allowed to run (a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) before it's considered failed. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` is allowed to run before it's considered failed. | @@ -198,7 +182,7 @@ $ sudo systemctl edit trident-acl-agent.service This opens `/etc/systemd/system/trident-acl-agent.service.d/override.conf` in an editor. For example, to point the agent at a custom annotation -namespace and Omaha server for a non-AKS Kubernetes deployment: +namespace and Omaha server: ```ini [Service] @@ -210,7 +194,7 @@ Environment=TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable With the prefix above, the orchestrator now reads/writes `acl.contoso.com/update-request`, `acl.contoso.com/update-status`, and -`acl.contoso.com/update-commit-status` instead of the `acl.azure.com/*` +`acl.contoso.com/update-commit-status` instead of the `acl.microsoft.com/*` defaults. Reload and restart to apply: ```console @@ -224,25 +208,39 @@ unit plus drop-in), useful for confirming the override took effect. ### Configuring the on-disk version The agent determines the node's current version by reading a key out of -`/etc/os-release`, defaulting to `IMAGE_VERSION` — the key ACL images -stamp. A deployment building its own images, not derived from ACL, may not -carry `IMAGE_VERSION` at all; rather than requiring every image build to -add an ACL-specific field, point the agent at the standard +`TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default), +defaulting to the key `VERSION_ID` — the standard [`os-release`](https://www.freedesktop.org/software/systemd/man/latest/os-release.html) -field `VERSION_ID` instead: +field most distributions already stamp. A deployment that instead builds +ACL images carrying an `IMAGE_VERSION` field can point the agent at that +key instead: + +```ini +[Service] +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=IMAGE_VERSION +``` + +With this set, the agent reads `IMAGE_VERSION` from +`TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. `IMAGE_VERSION=202606.29.0`) +as the node's current version, and compares it against a request's +`targetVersion` the same way it would for `VERSION_ID` — including +short-circuiting to `AlreadyAtTarget` when they already match. + +A deployment that keeps its version stamp somewhere other than +`/etc/os-release` — a different file entirely — can point the agent there +instead, as long as that file follows the `os-release` key-value schema +(`KEY=VALUE` lines, optionally quoted, blank lines and `#` comments +ignored): ```ini [Service] -Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=VERSION_ID +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH=/etc/my-app-release +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=BUILD_VERSION ``` -With this set, the agent reads `VERSION_ID` from `/etc/os-release` (e.g. -`VERSION_ID=202606.29.0`) as the node's current version, and compares it -against a request's `targetVersion` the same way it would for -`IMAGE_VERSION` — including short-circuiting to `AlreadyAtTarget` when they -already match. If the configured key is absent from `/etc/os-release` -(for example, on a dev/test host with a minimal `os-release`), the agent -falls back to `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` +If the configured key is absent from the configured file (for example, on +a dev/test host with a minimal `os-release`), the agent falls back to +`TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` (`0.0.0-unprobed-trident-acl-agent-stub` by default), a sentinel value that can never accidentally match a real requested version. From a714acc8b966d184a319d993757510f89d730347 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 21:40:02 +0000 Subject: [PATCH 16/54] docs: scope Nebraska env vars to --validate-connection nebraska TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT/APP_ID/TRACK are unused by the annotation-driven flow (requests carry their own server/appId/track); move them out of the main config table into Diagnostics where they actually apply. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/Explanation/Trident-ACL-Agent.md | 33 +++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index 794331903a..b9315b61fe 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -158,9 +158,6 @@ naming the offending variable. | `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | The file the agent reads to determine the node's currently running version. Any file works, as long as it follows the os-release format (`KEY=VALUE` lines, optionally single- or double-quoted, blank lines and `#` comments ignored) — see [below](#configuring-the-on-disk-version). | | `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `VERSION_ID` is the standard `os-release` field most images already stamp; a deployment that instead carries an ACL-specific `IMAGE_VERSION` field can point this variable at that key instead — see [below](#configuring-the-on-disk-version). | | `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `0.0.0-unprobed-trident-acl-agent-stub` | Sentinel value used as the node's "current version" when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). Deliberately shaped so it can never collide with a real release version and cause a false `AlreadyAtTarget`. | -| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Omaha server URL, never used as a fallback — `stage`/`finalize` requests must carry their own `server` field (see [above](#the-annotation-contract)). | -| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Omaha application id this node checks in as, never used as a fallback — required on the request's `appId` field instead. | -| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Omaha track this node follows, never used as a fallback — required on the request's `track` field instead. | | `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in the kubeconfig is used as-is. | | `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig used to reach the Kubernetes API server and authenticate as this node. | | `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. | @@ -170,6 +167,15 @@ naming the offending variable. | `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` is allowed to run before it's considered failed. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat. | +`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT`, `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`, +and `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` also exist, but are **not** used by +the annotation-driven flow described in this document — every `stage`/ +`finalize` request carries its own `server`/`appId`/`track` fields instead, +with no fallback to these variables (see +[above](#the-annotation-contract)). They only matter for +`trident-acl-agent --validate-connection nebraska`; see +[Diagnostics](#diagnostics). + ### Setting env vars via a systemd drop-in The agent ships as `trident-acl-agent.service`, with no `Environment=` @@ -182,14 +188,11 @@ $ sudo systemctl edit trident-acl-agent.service This opens `/etc/systemd/system/trident-acl-agent.service.d/override.conf` in an editor. For example, to point the agent at a custom annotation -namespace and Omaha server: +namespace: ```ini [Service] Environment=TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX=acl.contoso.com -Environment=TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update -Environment=TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee -Environment=TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable ``` With the prefix above, the orchestrator now reads/writes @@ -250,3 +253,19 @@ that can never accidentally match a real requested version. checks connectivity to a single dependency using the current environment and exits immediately — useful for a systemd `ExecStartPre` check or manual on-node troubleshooting without running the full orchestrator loop. + +`--validate-connection nebraska` is the one place +`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT`, `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`, +and `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` are used: it issues a real +update-check query against the configured endpoint/app id/track and reports +whether the Omaha server is reachable. They default to deliberately invalid +values (`https://nebraska.example.invalid/v1/update`, an all-zero UUID, and +`unspecified`, respectively) so this check fails loudly unless a deployment +sets them: + +```ini +[Service] +Environment=TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update +Environment=TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee +Environment=TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable +``` From eb92ad3814843e10f46d58f66a765868c2bdeef9 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 21:56:34 +0000 Subject: [PATCH 17/54] docs: drop redundant Nebraska-vars callout, fix diagnostics example Remove the duplicate note about the Nebraska env vars from the config table (already explained in Diagnostics). Fix the --validate-connection nebraska example: these vars only matter for that one-off invocation, so set them inline on the command rather than suggesting a persistent systemd [Service] drop-in. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/Explanation/Trident-ACL-Agent.md | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index b9315b61fe..b790ee9160 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -167,15 +167,6 @@ naming the offending variable. | `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` is allowed to run before it's considered failed. | | `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat. | -`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT`, `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`, -and `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` also exist, but are **not** used by -the annotation-driven flow described in this document — every `stage`/ -`finalize` request carries its own `server`/`appId`/`track` fields instead, -with no fallback to these variables (see -[above](#the-annotation-contract)). They only matter for -`trident-acl-agent --validate-connection nebraska`; see -[Diagnostics](#diagnostics). - ### Setting env vars via a systemd drop-in The agent ships as `trident-acl-agent.service`, with no `Environment=` @@ -261,11 +252,14 @@ update-check query against the configured endpoint/app id/track and reports whether the Omaha server is reachable. They default to deliberately invalid values (`https://nebraska.example.invalid/v1/update`, an all-zero UUID, and `unspecified`, respectively) so this check fails loudly unless a deployment -sets them: +sets them. Since these variables otherwise play no role in the +annotation-driven flow, there's no reason to add them to the service's +persistent environment (e.g. via a drop-in) — set them just for this +one-off invocation instead: -```ini -[Service] -Environment=TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update -Environment=TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee -Environment=TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable +```console +$ sudo TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update \ + TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee \ + TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable \ + trident-acl-agent --validate-connection nebraska ``` From 3310e3dd24023d389636b0cfb9034cca43834fcd Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 22:11:31 +0000 Subject: [PATCH 18/54] spec: build trident-acl-agent package for all builds Drop the %if %{defined rpm_ver} guards around the acl-agent subpackage, its cargo build, and its install step. trident-acl-agent should ship from both the Trident repo build and the azurelinux distro build, not just the repo one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packaging/rpm/trident.spec | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packaging/rpm/trident.spec b/packaging/rpm/trident.spec index 62a1d35ec7..ff4bc6fdc7 100644 --- a/packaging/rpm/trident.spec +++ b/packaging/rpm/trident.spec @@ -226,7 +226,6 @@ be removed once the fix is merged in AZL 4.0. # ------------------------------------------------------------------------------ -%if %{defined rpm_ver} %package acl-agent Summary: Trident ACL Agent Requires: %{name} = %{version}-%{release} @@ -247,7 +246,6 @@ The Trident ACL Agent triggers updates of ACL images. %postun acl-agent %systemd_postun_with_restart %{name}-acl-agent.service -%endif # ------------------------------------------------------------------------------ @@ -279,11 +277,7 @@ export TRIDENT_VERSION="%{version}-%{release}" # Use %{trident_version} for Trident repo build export TRIDENT_VERSION="%{trident_version}" %endif -%if %{defined rpm_ver} cargo build --release -p trident -p trident-acl-agent -%else -cargo build --release -p trident -%endif mkdir selinux cp -p packaging/selinux-policy-trident/trident.fc selinux/ @@ -313,10 +307,8 @@ cargo test --all --no-fail-fast -- --skip test_run_systemd_check --skip test_pre %install install -D -m 755 target/release/%{name} %{buildroot}/%{_bindir}/%{name} -%if %{defined rpm_ver} install -D -m 755 target/release/%{name}-acl-agent %{buildroot}/%{_bindir}/%{name}-acl-agent install -D -m 644 packaging/systemd/%{name}-acl-agent.service %{buildroot}%{_unitdir}/%{name}-acl-agent.service -%endif # Copy Trident SELinux policy module to /usr/share/selinux/packages install -D -m 0644 %{name}.pp.bz2 %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype}/%{name}.pp.bz2 From 65af8630d04e27248aa76e20a186cd5489f9dbb1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 20 Aug 2026 22:19:36 +0000 Subject: [PATCH 19/54] docs: simplify on-disk-version examples to one combined sample Drop the key-only example (a simpler subset of the path+key one) and keep just the combined TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH + TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY drop-in. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/Explanation/Trident-ACL-Agent.md | 31 ++++++++++----------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index b790ee9160..45ee8be373 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -205,26 +205,11 @@ The agent determines the node's current version by reading a key out of `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default), defaulting to the key `VERSION_ID` — the standard [`os-release`](https://www.freedesktop.org/software/systemd/man/latest/os-release.html) -field most distributions already stamp. A deployment that instead builds -ACL images carrying an `IMAGE_VERSION` field can point the agent at that -key instead: - -```ini -[Service] -Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=IMAGE_VERSION -``` - -With this set, the agent reads `IMAGE_VERSION` from -`TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. `IMAGE_VERSION=202606.29.0`) -as the node's current version, and compares it against a request's -`targetVersion` the same way it would for `VERSION_ID` — including -short-circuiting to `AlreadyAtTarget` when they already match. - -A deployment that keeps its version stamp somewhere other than -`/etc/os-release` — a different file entirely — can point the agent there -instead, as long as that file follows the `os-release` key-value schema -(`KEY=VALUE` lines, optionally quoted, blank lines and `#` comments -ignored): +field most distributions already stamp. A deployment that keeps its +version stamp under a different key, a different file entirely, or both, +can point the agent there instead, as long as that file follows the +`os-release` key-value schema (`KEY=VALUE` lines, optionally quoted, blank +lines and `#` comments ignored): ```ini [Service] @@ -232,6 +217,12 @@ Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH=/etc/my-app-release Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=BUILD_VERSION ``` +With this set, the agent reads `BUILD_VERSION` from `/etc/my-app-release` +(e.g. `BUILD_VERSION=202606.29.0`) as the node's current version, and +compares it against a request's `targetVersion` the same way it would for +`VERSION_ID`/`/etc/os-release` — including short-circuiting to +`AlreadyAtTarget` when they already match. + If the configured key is absent from the configured file (for example, on a dev/test host with a minimal `os-release`), the agent falls back to `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` From 5afa6937a372eb346a8ec0636bd5c922768c5809 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 14:59:29 +0000 Subject: [PATCH 20/54] docs: describe status annotation fields and response codes Add an example status annotation body plus a table explaining what each status code means and when it appears, so an orchestrator author can tell what a given response actually implies (retry vs. escalate vs. no-op) instead of just seeing the bare code name. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/Explanation/Trident-ACL-Agent.md | 54 ++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index 45ee8be373..b42075c82a 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -120,11 +120,55 @@ sequenceDiagram API-->>Orchestrator: terminal commit code ``` -A status annotation's `code` is one of `InProgress`, `Success`, -`AlreadyAtTarget`, `NotStaged`, `OperationFailed`, `TargetBootFailed`, -`AgentInternalError`, or `InvalidRequest` — see the request/status schema -types in `crates/trident-acl-agent/src/annotations.rs` for the full -contract, including the formal JSON Schema both sides validate against. +A status annotation (`/update-status` or +`/update-commit-status`) looks like: + +```json +{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "c9d6f0a2-3b41-4e8d-9f27-1a5b6c7d8e90", + "operation": "stage", + "code": "Success", + "message": "staged update to 202606.29.0", + "fromVersion": "202606.15.0", + "toVersion": "202606.29.0", + "startedUtc": "2026-06-29T12:00:00Z", + "lastUpdatedUtc": "2026-06-29T12:03:41Z", + "finishedUtc": "2026-06-29T12:03:41Z" +} +``` + +- `operation` is `stage`, `finalize`, `rollback`, or `commit` (`commit` only + ever appears on `/update-commit-status`, never on + `/update-status`). +- `code` is the outcome — see the table below. +- `message` is a short, human-readable explanation of `code`, useful for + logs/alerts; treat its exact wording as informational, not something to + match on (it may include error detail that varies run to run). +- `fromVersion`/`toVersion` are the versions the operation moved between + (`toVersion` is absent for `rollback`, whose target is implicit). +- `startedUtc`/`lastUpdatedUtc`/`finishedUtc` bound the operation: + `lastUpdatedUtc` refreshes on a heartbeat cadence while `code` is + `InProgress` (see [below](#pre-post-reboot-state-and-the-watchdog)); + `finishedUtc` is absent until `code` reaches a terminal value. + +`code` is one of: + +| `code` | Terminal? | Meaning | +|---|---|---| +| `InProgress` | No | The operation is running. `lastUpdatedUtc` refreshes on a heartbeat cadence; a terminal code always follows. | +| `Success` | Yes | The operation completed as requested. For `commit`, this means the reboot landed on the target partition and it was promoted. | +| `AlreadyAtTarget` | Yes | `stage`/`finalize` was requested for the version the node is already running (per `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY`); treated as a no-op success. | +| `NotStaged` | Yes | `finalize` was requested for a `nodeUpdateId` with no prior successful `stage`. Issue a `stage` first. | +| `OperationFailed` | Yes | The operation failed for a reason other than a boot/rollback outcome (e.g. the Omaha server has no update for the requested version, or the underlying `tridentd` call returned an error). See `message` for detail. | +| `TargetBootFailed` | Yes | The post-reboot `commit` found the node had rolled back to its previous partition instead of booting the target — Trident's own health checks rejected the new boot. The node is back on `fromVersion`; the orchestrator should treat this as a failed update, not retry the same `nodeUpdateId` blindly. | +| `AgentInternalError` | Yes | A failure in the agent itself rather than in Trident or the requested operation (e.g. it triggered a reboot but the reboot call failed, or it lost track of an in-flight commit). Distinct from `OperationFailed` so an orchestrator can decide to treat these differently (e.g. retry vs. escalate). | +| `InvalidRequest` | Yes | The request annotation itself was rejected before any action was taken — malformed JSON, a schema/version mismatch, a missing required field (`server`/`appId`/`track`/`targetVersion`), a `finalize` whose `targetVersion` doesn't match what was staged, or a second `finalize`/`rollback` submitted while one is already pending its post-reboot `commit`. No Trident operation runs. | + +See the request/status schema types in +`crates/trident-acl-agent/src/annotations.rs` for the full contract, +including the formal JSON Schema both sides validate against. ## Pre/post-reboot state and the watchdog From b4b4285f6d4001d477c73cc4efc48aef23c86004 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 15:07:50 +0000 Subject: [PATCH 21/54] docs: clarify progress/result annotation phrasing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/Explanation/Trident-ACL-Agent.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index b42075c82a..117fbe1795 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -17,7 +17,7 @@ An orchestrator (a Kubernetes controller with RBAC permission to PATCH the target Node object) triggers an update by writing a JSON payload to a request annotation on the Node. The agent watches that annotation, drives the requested operation against `tridentd`, and writes its progress and -result back to two status annotations on the same Node. +result via annotations on the same Node. Three annotation keys make up the contract, all sharing one configurable prefix (`acl.microsoft.com` by default — see [Configuration](#configuration) From e5ee169caa3b72f40eb581edaaa5265a75652c66 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 16:58:38 +0000 Subject: [PATCH 22/54] docs: add Deployment section for installing trident-acl-agent Describe installing the trident-acl-agent RPM subpackage, what it lays down (binary + unit, package install does not enable/start it), and that all configuration is done via standard systemd service env-var constructs (Environment=/EnvironmentFile=, most commonly a drop-in). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/Explanation/Trident-ACL-Agent.md | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index 117fbe1795..2d64aa8182 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -11,6 +11,36 @@ contract described below, provided it is willing to speak the [Omaha](https://github.com/omaha-consortium/omaha) protocol for image distribution and honors the agent's per-node protocol. +## Deployment + +`trident-acl-agent` ships as its own `trident-acl-agent` RPM subpackage +(built alongside, and `Requires:` the same version of, the main `trident` +package). Installing it: + +```console +$ tdnf install trident-acl-agent +``` + +lays down exactly two files: the `/usr/bin/trident-acl-agent` binary and +its `trident-acl-agent.service` unit +(`packaging/systemd/trident-acl-agent.service`) under the systemd unit +directory. Installing the package does not by itself enable or start the +service — a deployment decides when that happens, e.g. by running +`systemctl enable --now trident-acl-agent.service` on the node, or by +baking that enablement into the image build (as this repo's own +`updateimg-acl-agent.yaml` test image does via Image Customizer's +`services: enable` list). + +The shipped unit carries no `Environment=` lines beyond `ExecStart`, so +every deployment-specific choice — which annotation prefix to watch, where +to read the current version from, which Kubernetes API server to talk to, +and so on — is supplied the same way any other systemd service is +configured: standard `Environment=`/`EnvironmentFile=` constructs, most +commonly a drop-in applied on top of the packaged unit. See +[Configuration](#configuration) below for the full list of variables and +[Setting env vars via a systemd drop-in](#setting-env-vars-via-a-systemd-drop-in) +for how to apply them without editing the packaged unit. + ## The annotation contract An orchestrator (a Kubernetes controller with RBAC permission to PATCH the From a6d210edcbb3a907ccb575b664cf05e38dcf4cba Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 17:12:06 +0000 Subject: [PATCH 23/54] trident-acl-agent: point comments at the ADO design doc, not a local copy Replace every accepted-design-v3.md reference (a local copy, not checked in) with a link to the accepted design doc in ADO: https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md Also wrap two pre-existing bare-URL references to the same document (in orchestrator.rs/trident.rs module docs) in <> to fix rustdoc::bare_urls warnings surfaced while making this change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/trident-acl-agent/src/annotations.rs | 16 ++++++++-------- crates/trident-acl-agent/src/config.rs | 4 ++-- crates/trident-acl-agent/src/k8s.rs | 2 +- crates/trident-acl-agent/src/orchestrator.rs | 15 ++++++++------- crates/trident-acl-agent/src/state.rs | 2 +- crates/trident-acl-agent/src/trident.rs | 8 ++++---- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index a37134ed2b..203516c841 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -4,7 +4,7 @@ //! `#[cfg(test)]` design-doc conformance tests below) implements the //! `/update-request`, `/update-status`, and //! `/update-commit-status` node annotation protocol described -//! by the current accepted design (`accepted-design-v3.md`), where +//! by the current accepted design (), where //! `` defaults to `acl.microsoft.com` (see //! [`AnnotationKeys`]/[`crate::config::DEFAULT_ANNOTATION_PREFIX`]) and is //! overridable via the `TRIDENT_ACL_AGENT_ANNOTATION_PREFIX` environment @@ -181,7 +181,7 @@ pub struct UpdateStatus { impl UpdateRequest { /// Enforces the same constraints as the request annotation's formal - /// JSON Schema in `accepted-design-v3.md`: schemaVersion match, + /// JSON Schema in : schemaVersion match, /// targetVersion required for stage/finalize but disallowed for /// rollback, and server/appId/track required for stage/finalize. See /// this file's module doc. @@ -216,7 +216,7 @@ impl UpdateRequest { impl UpdateStatus { // This constructor mirrors UpdateStatus's wire schema field-for-field - // (see accepted-design-v3.md's two-status-key JSON protocol); splitting + // (see https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md's two-status-key JSON protocol); splitting // it into a builder would add ceremony across ~25 call sites in // orchestrator.rs without making any of them clearer. #[allow(clippy::too_many_arguments)] @@ -730,7 +730,7 @@ mod tests { /// (adapted to `finalize` to pair with the status/commit examples /// below, which also share this `finalize`; server/appId/track values /// are the doc's own example values for those fields, required on - /// stage/finalize per `accepted-design-v3.md`). + /// stage/finalize per ). const DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE: &str = r#"{ "schemaVersion": "1.0", "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", @@ -773,7 +773,7 @@ mod tests { }"#; /// The formal JSON Schema for the request annotation, from - /// `accepted-design-v3.md` section 2.1 "Formal JSON Schema". Keep + /// section 2.1 "Formal JSON Schema". Keep /// byte-for-byte in sync with that document. const DESIGN_DOC_REQUEST_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -805,7 +805,7 @@ mod tests { }"#; /// The formal JSON Schema for the status annotations, from - /// `accepted-design-v3.md` section 2.1 "Formal JSON Schema". Keep + /// section 2.1 "Formal JSON Schema". Keep /// byte-for-byte in sync with that document. const DESIGN_DOC_STATUS_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -843,7 +843,7 @@ mod tests { // additionalProperties, required, properties.{type,const,enum,format, // pattern}, and a single-level allOf/if/then/else). Panics loudly on any // schema keyword/pattern/type/format it doesn't recognize, so if - // accepted-design-v3.md's schemas grow new constraints, this validator's + // https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md's schemas grow new constraints, this validator's // blind spots don't silently mask them - the test fails instead, // prompting an update here. @@ -1102,7 +1102,7 @@ mod tests { #[test] fn validate_allows_rollback_without_nebraska_fields() { // Rollback reports no Nebraska event, so it carries no update - // source (accepted-design-v3.md 2.1): server/appId/track are not + // source (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md 2.1): server/appId/track are not // required, and validate() must not reject their absence. let request = UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs index 3b1878b35a..e40fdd454f 100644 --- a/crates/trident-acl-agent/src/config.rs +++ b/crates/trident-acl-agent/src/config.rs @@ -199,7 +199,7 @@ pub enum GoalSource { /// accordingly, writing progress back to /// `/update-status` and /// `/update-commit-status` (see - /// accepted-design-v3.md). `` defaults to + /// ). `` defaults to /// [`DEFAULT_ANNOTATION_PREFIX`] (`acl.microsoft.com`), overridable via /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. This is the only /// supported mode. @@ -231,7 +231,7 @@ pub struct OrchestrationConfig { pub finalize_timeout: Duration, /// Refresh cadence for in-flight InProgress heartbeats. Default is well /// below the ~10 minute watchdog staleness target proposed in - /// accepted-design-v3.md. + /// . pub heartbeat_interval: Duration, } diff --git a/crates/trident-acl-agent/src/k8s.rs b/crates/trident-acl-agent/src/k8s.rs index 4e64eb51bb..2cca4c3bfc 100644 --- a/crates/trident-acl-agent/src/k8s.rs +++ b/crates/trident-acl-agent/src/k8s.rs @@ -1,7 +1,7 @@ //! Thin Kubernetes client wrapper for Harpoon's node self-patching protocol. //! //! Implements the Node get/watch/patch access described in the current -//! accepted design (`accepted-design-v3.md`). +//! accepted design (). //! //! The design calls for get/patch access to exactly one Node object (§2.2–§2.6). //! Node changes are observed via the Kubernetes watch API (`kube::runtime::watcher`) diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs index c78c587fd8..f9f725523f 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -2,8 +2,8 @@ //! annotation, drives Trident (stage/finalize/rollback/commit) over gRPC, //! and writes the status annotation back, including post-reboot. //! -//! Implements the node-side control flow from `docs/update-trigger-design.md`: -//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md +//! Implements the node-side control flow from +//! //! (sections 2.1 "Trigger mechanism", 2.3 "Stage/finalize/rollback split //! and post-reboot commit", and 2.5 "Rollback"). See that document for the //! full state-machine rationale; keep it in sync with this file if the @@ -254,7 +254,7 @@ where // Reject on operationId, not nodeUpdateId: the actual conflict // this guard exists to prevent is "a second finalize/rollback // starts while one is still waiting for its post-reboot - // commit" (accepted-design-v3.md's in-flight conflict rule). + // commit" (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md's in-flight conflict rule). // Keying on nodeUpdateId alone let a retried/re-issued request // that reused the same nodeUpdateId but a new operationId slip // through this guard entirely and re-enter handle_finalize/ @@ -300,7 +300,7 @@ where /// across one update's lifecycle would split that state across two /// servers. /// - /// Per `accepted-design-v3.md` 2.1, `stage`/`finalize` requests must + /// Per 2.1, `stage`/`finalize` requests must /// carry `server` and there is deliberately no static-config fallback /// here: a fallback would let a node update from a source AKS-RP did /// not choose. `UpdateRequest::validate()` already rejects a @@ -807,7 +807,7 @@ where ) -> UpdateStatus { // state.json did not survive the reboot (or was never written, e.g. // the agent crashed before persisting pendingCommit). Per - // accepted-design-v3.md §2.3's degraded path, reconstruct the answer by + // https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md §2.3's degraded path, reconstruct the answer by // calling commit() unconditionally rather than guessing from labels // or the target version alone - tridentd's commit() is self-checking // and its own (ServicingKind/RebootStatus/Result) response already @@ -1336,7 +1336,7 @@ fn indicates_target_boot_failed(error: &TridentClientError) -> bool { /// needing a full `Orchestrator` instance. See `stage_result_to_status` for /// rationale. /// Pre-flight checks for the state.json-missing degraded reconstruction -/// path (accepted-design-v3.md §2.3). Returns `Some(status)` when reconstruction +/// path ( §2.3). Returns `Some(status)` when reconstruction /// cannot proceed (tridentd already known-unreachable, or the outstanding /// request isn't a finalize/rollback), or `None` when the caller should go /// on to call tridentd's commit() to determine the real outcome. @@ -1380,7 +1380,8 @@ fn reconstruct_precheck_status( } /// Maps tridentd's commit() result to the terminal status for the -/// state.json-missing degraded reconstruction path (accepted-design-v3.md +/// state.json-missing degraded reconstruction path +/// ( /// §2.3). Always reports under the original operationId, mirroring the /// normal post-reboot commit path in `commit_result_to_status`. fn reconstruct_commit_result_to_status( diff --git a/crates/trident-acl-agent/src/state.rs b/crates/trident-acl-agent/src/state.rs index 025f9f1607..c9f0c49113 100644 --- a/crates/trident-acl-agent/src/state.rs +++ b/crates/trident-acl-agent/src/state.rs @@ -2,7 +2,7 @@ //! completed-operation cache and the pending post-reboot commit record. //! //! Implements the `state.json` mechanism from the current accepted design -//! (`accepted-design-v3.md`, section 2.3), which bridges the pre-reboot +//! (, section 2.3), which bridges the pre-reboot //! finalize/rollback half and the post-reboot commit half of an operation //! across the reboot. diff --git a/crates/trident-acl-agent/src/trident.rs b/crates/trident-acl-agent/src/trident.rs index 982489a830..628007795c 100644 --- a/crates/trident-acl-agent/src/trident.rs +++ b/crates/trident-acl-agent/src/trident.rs @@ -1,7 +1,7 @@ //! gRPC helpers for talking to `tridentd`. //! -//! Implements the Trident-invocation half of `docs/update-trigger-design.md`: -//! https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md +//! Implements the Trident-invocation half of +//! //! (the "Trident invocation" column of section 2.1's operations table, //! and the stage/finalize/rollback-finalize CallerHandlesReboot split in //! section 2.3). @@ -221,7 +221,7 @@ impl TridentClient { reboot: Some(RebootManagement { // The agent, not tridentd, must own every reboot // decision: AKS-RP is the sole authority over - // reboot/rollback (accepted-design-v3.md §2.5). If commit() + // reboot/rollback (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md §2.5). If commit() // ever reports NeedsReboot (e.g. a health-check failure, // were health checks ever re-enabled), the agent needs // to see that as a RebootRequired response it controls @@ -283,7 +283,7 @@ impl TridentClient { reboot: Some(RebootManagement { // Same rationale as commit()/update_finalize(): AKS-RP, // via the agent, is the sole authority over reboot - // timing (accepted-design-v3.md §2.5). + // timing (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md §2.5). handling: RebootHandling::CallerHandlesReboot.into(), }), })) From 86d038c33498ec1ffbe3b313824289cdb79b7758 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 17:27:51 +0000 Subject: [PATCH 24/54] trident-acl-agent: remove README.md, superseded by Trident-ACL-Agent.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/README.md | 52 ------------------------------ 1 file changed, 52 deletions(-) delete mode 100644 crates/trident-acl-agent/README.md diff --git a/crates/trident-acl-agent/README.md b/crates/trident-acl-agent/README.md deleted file mode 100644 index 8303e00286..0000000000 --- a/crates/trident-acl-agent/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# trident-acl-agent - -The on-node half of Trident's annotation-driven Kubernetes A/B update -trigger. It watches its Node's `acl.microsoft.com/update-request` -annotation (the prefix is configurable, see below) and drives Trident's -stage/finalize/rollback/commit operations against `tridentd` accordingly, -reporting progress and status back to Kubernetes and to Nebraska (the -Omaha-protocol update server). - -See [`docs/Explanation/Trident-ACL-Agent.md`](../../docs/Explanation/Trident-ACL-Agent.md) -for a full description of the annotation contract and the reconcile flow. - -## Configuration - -There is no config file. Every setting is an environment variable prefixed -`TRIDENT_ACL_AGENT_`, systemd-style: set it directly in the unit's own -`Environment=` lines, via a drop-in override (`systemctl edit -trident-acl-agent.service`, which creates -`/etc/systemd/system/trident-acl-agent.service.d/override.conf`), or by any -other means that ultimately sets the process's environment before it -starts. - -A variable that is unset, or set to the empty string, falls back to that -setting's default below. A variable that is set to a malformed value (a bad -URL, a bad duration, an unrecognized `goal_source`) causes the agent to -fail to start with an error naming the offending variable. - -| Variable | Default | Description | -|---|---|---| -| `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT` | `https://nebraska.example.invalid/v1/update` (deliberately unreachable) | The Nebraska/Omaha server URL to poll for updates and report progress/completion events to. `stage`/`finalize` requests must instead carry their own `server` field on the `acl.microsoft.com/update-request` annotation - there is deliberately no fallback to this variable, since a fallback would let a node update from a source the annotation's author did not choose; a request missing it is rejected with `InvalidRequest`. | -| `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID` | An all-zero UUID (deliberately invalid) | The Nebraska application ID this node checks in as. Required on the `acl.microsoft.com/update-request` annotation's `appId` field instead, same no-fallback rule as the endpoint. | -| `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` | `unspecified` (deliberately invalid) | The Nebraska track (channel/group) this node follows. Required on the `acl.microsoft.com/update-request` annotation's `track` field instead, same no-fallback rule as the endpoint. | -| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.microsoft.com` | Prefix for the `update-request`/`update-status`/`update-commit-status` Node annotations this agent watches and writes. | -| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG`'s own kubeconfig is used as-is (e.g. the real cluster FQDN a node's own `/var/lib/kubelet/kubeconfig` already points at). Only needed when the kubeconfig's own server is wrong for this deployment. | -| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig file used to reach the Kubernetes API server and authenticate as this node. | -| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. Kubernetes Node names must be valid RFC 1123 DNS labels (lowercase), matching how kubelet itself registers the Node - so the default only needs overriding when the agent's environment can't discover the correct hostname on its own. | -| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | Path to the key-value file (must follow the `os-release` schema) this agent reads to determine the node's currently-running version. | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key read from `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` to determine the node's currently-running version. | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `CURRENT_VERSION_STUB` | Fallback value reported as the current version when the configured key is missing from the configured file. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE` | `annotations` | Selects the agent's operating mode. `annotations` is the only supported mode; other values are internal/undocumented. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file, which bridges the pre-reboot `finalize`/`rollback` half of an update and its post-reboot `commit` half across the reboot. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` operation (parsed as a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) is allowed to run before it's considered failed. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` operation is allowed to run before it's considered failed. Parsed the same way as the stage timeout. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat the agent writes while a stage/finalize/rollback operation is running, so AKS-RP and the watchdog can tell a working agent from a stuck one. Parsed the same way as the timeouts. | - -## Diagnostics - -`trident-acl-agent --validate-connection ` -checks connectivity to a single dependency using the current environment -and exits immediately - useful for a systemd `ExecStartPre` check or manual -on-node troubleshooting without running the full orchestrator loop. From f66f2eda3843463e19725bc552999333dff2a4a8 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 19:04:40 +0000 Subject: [PATCH 25/54] trident-acl-agent: address open Copilot review comments on PR 730 - annotations.rs: UpdateRequest::validate() now rejects a stage/finalize server URL whose scheme is not http/https (previously only checked presence). http is intentionally still allowed (not https-only) since the storm E2E harness points server at a plain-http local Nebraska stub. - orchestrator.rs: handle_stage's initial check_for_update() call now reports the node's real current version instead of a hardcoded 0.0.0 (which Nebraska would otherwise offer an update for on every poll, forever), and reuses the module's NEBRASKA_MACHINE_ID_SOURCE constant instead of an inline IdSource::MachineIdHashed. - lib.rs: run_omaha_only's one-shot Nebraska check has the same fix, reporting the real current version instead of 0.0.0. - Moved current_active_version() (and its os-release-reading helpers) out of annotations.rs into a new version module, since it's unrelated to the annotation wire format and is now used by both the orchestrator and the omaha-only path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/src/annotations.rs | 289 ++----------------- crates/trident-acl-agent/src/lib.rs | 10 +- crates/trident-acl-agent/src/orchestrator.rs | 11 +- crates/trident-acl-agent/src/version.rs | 273 ++++++++++++++++++ 4 files changed, 321 insertions(+), 262 deletions(-) create mode 100644 crates/trident-acl-agent/src/version.rs diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index 203516c841..fe498d2452 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -62,40 +62,6 @@ impl Default for AnnotationKeys { pub const SCHEMA_VERSION: &str = "1.0"; const MAX_MESSAGE_BYTES: usize = 2048; const TRUNCATION_MARKER: &str = "... (truncated)"; -// current_active_version() reads the `VERSION_ID` key (overridable via -// TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY, e.g. to `IMAGE_VERSION` for an ACL -// image that stamps its own per-build version there) out of os-release, but -// falls back to this stub if that key isn't present (e.g. a minimal -// dev/test os-release). The stub value below is an explicit sentinel that -// cannot collide with a real release version string, so it can never -// accidentally match a real requested target version and cause -// handle_stage/handle_finalize to incorrectly short-circuit to -// AlreadyAtTarget. Do not remove this comment when bumping the stub value; -// keep it (and its non-colliding shape). The stub itself is overridable via -// TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for dev/test hosts that want a -// specific sentinel. -pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; -/// Default path `current_active_version` reads. Overridable via -/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` so a deployment can point the -/// agent at any file that follows the os-release format (`KEY=VALUE` lines, -/// optionally quoted, blank lines and `#` comments ignored - see -/// ) -/// instead of the real `/etc/os-release`, e.g. a vendor-specific file that -/// carries the running image's version under a key `/etc/os-release` -/// doesn't have room for. -pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; -/// Default os-release key `current_active_version` looks up for the running -/// image's version: `VERSION_ID`, a standard key every os-release carries -/// (see -/// ). -/// Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` - e.g. to -/// `IMAGE_VERSION` for an ACL image that stamps its own per-build version -/// under that key instead - or point -/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` at a different file entirely. -pub const DEFAULT_CURRENT_VERSION_KEY: &str = "VERSION_ID"; -const ENV_CURRENT_VERSION_PATH: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH"; -const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; -const ENV_CURRENT_VERSION_STUB: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB"; #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] @@ -197,6 +163,13 @@ impl UpdateRequest { if self.server.is_none() { return Err("server is required for stage/finalize".to_string()); } + if self + .server + .as_ref() + .is_some_and(|u| !matches!(u.scheme(), "http" | "https")) + { + return Err("server must be an http(s) URL".to_string()); + } if self.app_id.as_deref().unwrap_or("").is_empty() { return Err("appId is required for stage/finalize".to_string()); } @@ -298,58 +271,6 @@ fn truncate_message(message: String) -> String { truncated } -/// Reads `name`, treating both "unset" and "set to the empty string" as -/// absent, matching `config::env_raw`'s convention: a drop-in override that -/// clears a variable to `""` should fall back to the default, not try to use -/// an empty value. -fn env_override(name: &str) -> Option { - std::env::var(name).ok().filter(|v| !v.is_empty()) -} - -pub fn current_active_version() -> String { - let path = env_override(ENV_CURRENT_VERSION_PATH) - .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); - let key = env_override(ENV_CURRENT_VERSION_KEY) - .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_KEY.to_string()); - read_os_release_value(&path, &key).unwrap_or_else(|| { - let stub = env_override(ENV_CURRENT_VERSION_STUB) - .unwrap_or_else(|| CURRENT_VERSION_STUB.to_string()); - log::warn!("{key} not found in {path}; falling back to stub current version {stub}"); - stub - }) -} - -/// Reads `path` (an os-release-formatted file: `KEY=VALUE` lines, blank -/// lines and `#` comments ignored, values optionally single- or -/// double-quoted - see -/// ) -/// and returns the trimmed, unquoted value for `key`, or `None` if the file -/// can't be read, `key` isn't present, or its value is empty - all of which -/// `current_active_version` treats identically: fall back to the stub. -/// Split out from `current_active_version` so tests can point it at a temp -/// file instead of the real os-release. -fn read_os_release_value(path: &str, key: &str) -> Option { - let contents = std::fs::read_to_string(path).ok()?; - for line in contents.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((line_key, raw_value)) = line.split_once('=') else { - continue; - }; - if line_key.trim() != key { - continue; - } - let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); - if value.is_empty() { - return None; - } - return Some(value.to_string()); - } - None -} - impl From for Operation { fn from(value: RequestedOperation) -> Self { match value { @@ -1075,6 +996,31 @@ mod tests { } } + #[test] + fn validate_accepts_plain_http_server() { + // Deliberately not https-only: test/dev harnesses (e.g. the storm + // E2E suite) point `server` at a local, unencrypted Nebraska stub. + for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { + let mut request = valid_nebraska_request(operation); + request.server = Some(Url::parse("http://127.0.0.1:8080/v1/update").unwrap()); + request + .validate() + .unwrap_or_else(|err| panic!("{operation:?} with http:// server: {err}")); + } + } + + #[test] + fn validate_rejects_stage_and_finalize_non_http_scheme_server() { + for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { + let mut request = valid_nebraska_request(operation); + request.server = Some(Url::parse("ftp://nebraska.example/v1/update").unwrap()); + let err = request + .validate() + .expect_err("a non-http(s) server scheme must be rejected"); + assert!(err.contains("server"), "{err}"); + } + } + #[test] fn validate_rejects_stage_and_finalize_missing_app_id() { for operation in [RequestedOperation::Stage, RequestedOperation::Finalize] { @@ -1385,175 +1331,4 @@ mod tests { let json = serde_json::to_value(&request).unwrap(); assert!(json.get("track").is_none()); } - - #[test] - fn read_os_release_value_returns_none_for_missing_file() { - assert_eq!( - read_os_release_value( - "/nonexistent/path/does-not-exist-os-release", - DEFAULT_CURRENT_VERSION_KEY - ), - None - ); - } - - #[test] - fn read_os_release_value_finds_requested_key() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-{}", Uuid::new_v4())); - std::fs::write( - &path, - "NAME=\"Azure Linux\"\nIMAGE_VERSION=202608.6.0\nVERSION_ID=3.0\n", - ) - .unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result.as_deref(), Some("202608.6.0")); - } - - #[test] - fn read_os_release_value_trims_quotes_and_whitespace() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-quoted-{}", Uuid::new_v4())); - std::fs::write(&path, " IMAGE_VERSION = \"202608.6.0\" \n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result.as_deref(), Some("202608.6.0")); - } - - #[test] - fn read_os_release_value_returns_none_for_missing_key() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-missing-key-{}", Uuid::new_v4())); - std::fs::write(&path, "NAME=\"Azure Linux\"\nVERSION_ID=3.0\n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result, None); - } - - #[test] - fn read_os_release_value_returns_none_for_empty_value() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-empty-value-{}", Uuid::new_v4())); - std::fs::write(&path, "IMAGE_VERSION=\n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result, None); - } - - #[test] - fn read_os_release_value_skips_comments_and_blank_lines() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-comments-{}", Uuid::new_v4())); - std::fs::write( - &path, - "# a comment\n\n# IMAGE_VERSION=should-be-ignored\nIMAGE_VERSION=202608.6.0\n", - ) - .unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result.as_deref(), Some("202608.6.0")); - } - - /// Clears all three env vars `current_active_version` reads. Environment - /// mutation is process-global and `std::env::remove_var`/`set_var` are - /// `unsafe` (not thread-safe against concurrent reads elsewhere in the - /// process), so the defaults/overrides/read-path cases below are - /// intentionally folded into one sequential `#[test]` rather than - /// several separate ones that `cargo test` could run in parallel - /// against the same variables. - fn clear_current_version_env() { - // SAFETY: single-threaded within this test function; no other test - // in this crate reads or writes these three variables. - unsafe { - std::env::remove_var(ENV_CURRENT_VERSION_PATH); - std::env::remove_var(ENV_CURRENT_VERSION_KEY); - std::env::remove_var(ENV_CURRENT_VERSION_STUB); - } - } - - #[test] - fn current_active_version_path_key_and_stub_are_overridable_via_env() { - clear_current_version_env(); - assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); - assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); - - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, "/custom/os-release"); - } - assert_eq!( - env_override(ENV_CURRENT_VERSION_PATH).as_deref(), - Some("/custom/os-release") - ); - - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_KEY, "CUSTOM_VERSION_KEY"); - } - assert_eq!( - env_override(ENV_CURRENT_VERSION_KEY).as_deref(), - Some("CUSTOM_VERSION_KEY") - ); - - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub"); - } - assert_eq!( - env_override(ENV_CURRENT_VERSION_STUB).as_deref(), - Some("custom-stub") - ); - - // An empty override is treated the same as unset. - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, ""); - std::env::set_var(ENV_CURRENT_VERSION_KEY, ""); - } - assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); - assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); - - clear_current_version_env(); - - // current_active_version() itself honors TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH, - // pointing it at an arbitrary os-release-formatted file instead of the - // real /etc/os-release. - let dir = std::env::temp_dir(); - let found_path = dir.join(format!( - "os-release-test-current-version-{}", - Uuid::new_v4() - )); - std::fs::write( - &found_path, - "NAME=\"Contoso Linux\"\nVERSION_ID=202608.6.0\n", - ) - .unwrap(); - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, found_path.to_str().unwrap()); - std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); - } - assert_eq!(current_active_version(), "202608.6.0"); - std::fs::remove_file(&found_path).ok(); - clear_current_version_env(); - - // When the configured key isn't present at the configured path, it - // still falls back to the (possibly also-overridden) stub, exactly - // as it does for the real /etc/os-release. - let missing_path = dir.join(format!( - "os-release-test-current-version-missing-{}", - Uuid::new_v4() - )); - std::fs::write(&missing_path, "NAME=\"Contoso Linux\"\n").unwrap(); - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, missing_path.to_str().unwrap()); - std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); - std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub-for-missing-key"); - } - assert_eq!(current_active_version(), "custom-stub-for-missing-key"); - std::fs::remove_file(&missing_path).ok(); - clear_current_version_env(); - } } diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 31761886da..89a4f0e968 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -22,6 +22,7 @@ pub mod error; pub mod id; pub mod k8s; pub mod nebraska; +pub mod version; /// The version this agent reports to Nebraska as the updater's own version, for /// [`nebraska::Client::new`]. @@ -87,9 +88,16 @@ pub async fn run_omaha_only(config: &config::AgentConfig) -> Result<(), anyhow:: let app_id = config.nebraska.app_id.clone(); let track = config.nebraska.track.clone(); let machine_id = build_machine_id(IdSource::MachineIdHashed)?; + let current_version_raw = version::current_active_version(); + let current_version = Version::parse(¤t_version_raw).unwrap_or_else(|err| { + log::warn!( + "current version {current_version_raw:?} is not valid semver ({err}); reporting 0.0.0 to Nebraska" + ); + Version::new(0, 0, 0) + }); let outcome = tokio::task::spawn_blocking(move || { let client = Client::new(endpoint, app_id, track, machine_id); - client.check_for_update(&Version::new(0, 0, 0)) + client.check_for_update(¤t_version) }) .await .context("Nebraska query task panicked")? diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs index f9f725523f..f2aaca683c 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -24,14 +24,15 @@ use osutils::dependencies::Dependency; use crate::{ annotations::{ - current_active_version, AnnotationKeys, Operation, RequestedOperation, StatusCode, - UpdateRequest, UpdateStatus, SCHEMA_VERSION, + AnnotationKeys, Operation, RequestedOperation, StatusCode, UpdateRequest, UpdateStatus, + SCHEMA_VERSION, }, config::AgentConfig, k8s::{K8sClientError, NodeClient}, nebraska::{CheckOutcome, Client as NebraskaClient, ProgressEvent}, state::{PendingCommit, StateStore}, trident::{CompletedResponse, TridentClient, TridentClientError}, + version::current_active_version, IdSource, }; @@ -381,10 +382,12 @@ where request.node_update_id ) })?; - let machine_id = crate::build_machine_id(IdSource::MachineIdHashed)?; + let machine_id = crate::build_machine_id(NEBRASKA_MACHINE_ID_SOURCE)?; + let current_version = parse_nebraska_version(&from_version, "stage current version") + .unwrap_or_else(|| Version::new(0, 0, 0)); let outcome = tokio::task::spawn_blocking(move || { let client = NebraskaClient::new(endpoint, app_id, track, machine_id); - client.check_for_update(&Version::new(0, 0, 0)) + client.check_for_update(¤t_version) }) .await .context("Nebraska query task panicked")? diff --git a/crates/trident-acl-agent/src/version.rs b/crates/trident-acl-agent/src/version.rs new file mode 100644 index 0000000000..484e811657 --- /dev/null +++ b/crates/trident-acl-agent/src/version.rs @@ -0,0 +1,273 @@ +//! Determines the node's currently-running version, for comparison against +//! a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`) +//! and for reporting the instance's current version to Nebraska. +//! +//! Split out of [`crate::annotations`] because it has nothing to do with the +//! annotation wire format - it's plain env-var-configurable file probing, +//! used by both the annotation-driven orchestrator and the `omaha-only` +//! one-shot mode. + +// current_active_version() reads the `VERSION_ID` key (overridable via +// TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY, e.g. to `IMAGE_VERSION` for an ACL +// image that stamps its own per-build version there) out of os-release, but +// falls back to this stub if that key isn't present (e.g. a minimal +// dev/test os-release). The stub value below is an explicit sentinel that +// cannot collide with a real release version string, so it can never +// accidentally match a real requested target version and cause +// handle_stage/handle_finalize to incorrectly short-circuit to +// AlreadyAtTarget. Do not remove this comment when bumping the stub value; +// keep it (and its non-colliding shape). The stub itself is overridable via +// TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for dev/test hosts that want a +// specific sentinel. +pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; +/// Default path `current_active_version` reads. Overridable via +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` so a deployment can point the +/// agent at any file that follows the os-release format (`KEY=VALUE` lines, +/// optionally quoted, blank lines and `#` comments ignored - see +/// ) +/// instead of the real `/etc/os-release`, e.g. a vendor-specific file that +/// carries the running image's version under a key `/etc/os-release` +/// doesn't have room for. +pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; +/// Default os-release key `current_active_version` looks up for the running +/// image's version: `VERSION_ID`, a standard key every os-release carries +/// (see +/// ). +/// Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` - e.g. to +/// `IMAGE_VERSION` for an ACL image that stamps its own per-build version +/// under that key instead - or point +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` at a different file entirely. +pub const DEFAULT_CURRENT_VERSION_KEY: &str = "VERSION_ID"; +const ENV_CURRENT_VERSION_PATH: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH"; +const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; +const ENV_CURRENT_VERSION_STUB: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB"; + +/// Reads `name`, treating both "unset" and "set to the empty string" as +/// absent, matching `config::env_raw`'s convention: a drop-in override that +/// clears a variable to `""` should fall back to the default, not try to use +/// an empty value. +fn env_override(name: &str) -> Option { + std::env::var(name).ok().filter(|v| !v.is_empty()) +} + +pub fn current_active_version() -> String { + let path = env_override(ENV_CURRENT_VERSION_PATH) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); + let key = env_override(ENV_CURRENT_VERSION_KEY) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_KEY.to_string()); + read_os_release_value(&path, &key).unwrap_or_else(|| { + let stub = env_override(ENV_CURRENT_VERSION_STUB) + .unwrap_or_else(|| CURRENT_VERSION_STUB.to_string()); + log::warn!("{key} not found in {path}; falling back to stub current version {stub}"); + stub + }) +} + +/// Reads `path` (an os-release-formatted file: `KEY=VALUE` lines, blank +/// lines and `#` comments ignored, values optionally single- or +/// double-quoted - see +/// ) +/// and returns the trimmed, unquoted value for `key`, or `None` if the file +/// can't be read, `key` isn't present, or its value is empty - all of which +/// `current_active_version` treats identically: fall back to the stub. +/// Split out from `current_active_version` so tests can point it at a temp +/// file instead of the real os-release. +fn read_os_release_value(path: &str, key: &str) -> Option { + let contents = std::fs::read_to_string(path).ok()?; + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((line_key, raw_value)) = line.split_once('=') else { + continue; + }; + if line_key.trim() != key { + continue; + } + let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); + if value.is_empty() { + return None; + } + return Some(value.to_string()); + } + None +} + +#[cfg(test)] +mod tests { + use uuid::Uuid; + + use super::*; + + #[test] + fn read_os_release_value_returns_none_for_missing_file() { + assert_eq!( + read_os_release_value( + "/nonexistent/path/does-not-exist-os-release", + DEFAULT_CURRENT_VERSION_KEY + ), + None + ); + } + + #[test] + fn read_os_release_value_finds_requested_key() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-{}", Uuid::new_v4())); + std::fs::write( + &path, + "NAME=\"Azure Linux\"\nIMAGE_VERSION=202608.6.0\nVERSION_ID=3.0\n", + ) + .unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + #[test] + fn read_os_release_value_trims_quotes_and_whitespace() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-quoted-{}", Uuid::new_v4())); + std::fs::write(&path, " IMAGE_VERSION = \"202608.6.0\" \n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + #[test] + fn read_os_release_value_returns_none_for_missing_key() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-missing-key-{}", Uuid::new_v4())); + std::fs::write(&path, "NAME=\"Azure Linux\"\nVERSION_ID=3.0\n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result, None); + } + + #[test] + fn read_os_release_value_returns_none_for_empty_value() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-empty-value-{}", Uuid::new_v4())); + std::fs::write(&path, "IMAGE_VERSION=\n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result, None); + } + + #[test] + fn read_os_release_value_skips_comments_and_blank_lines() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-comments-{}", Uuid::new_v4())); + std::fs::write( + &path, + "# a comment\n\n# IMAGE_VERSION=should-be-ignored\nIMAGE_VERSION=202608.6.0\n", + ) + .unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + /// Clears all three env vars `current_active_version` reads. Environment + /// mutation is process-global and `std::env::remove_var`/`set_var` are + /// `unsafe` (not thread-safe against concurrent reads elsewhere in the + /// process), so the defaults/overrides/read-path cases below are + /// intentionally folded into one sequential `#[test]` rather than + /// several separate ones that `cargo test` could run in parallel + /// against the same variables. + fn clear_current_version_env() { + // SAFETY: single-threaded within this test function; no other test + // in this crate reads or writes these three variables. + unsafe { + std::env::remove_var(ENV_CURRENT_VERSION_PATH); + std::env::remove_var(ENV_CURRENT_VERSION_KEY); + std::env::remove_var(ENV_CURRENT_VERSION_STUB); + } + } + + #[test] + fn current_active_version_path_key_and_stub_are_overridable_via_env() { + clear_current_version_env(); + assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); + assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, "/custom/os-release"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_PATH).as_deref(), + Some("/custom/os-release") + ); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_KEY, "CUSTOM_VERSION_KEY"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_KEY).as_deref(), + Some("CUSTOM_VERSION_KEY") + ); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_STUB).as_deref(), + Some("custom-stub") + ); + + // An empty override is treated the same as unset. + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, ""); + std::env::set_var(ENV_CURRENT_VERSION_KEY, ""); + } + assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); + assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + + clear_current_version_env(); + + // current_active_version() itself honors TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH, + // pointing it at an arbitrary os-release-formatted file instead of the + // real /etc/os-release. + let dir = std::env::temp_dir(); + let found_path = dir.join(format!( + "os-release-test-current-version-{}", + Uuid::new_v4() + )); + std::fs::write( + &found_path, + "NAME=\"Contoso Linux\"\nVERSION_ID=202608.6.0\n", + ) + .unwrap(); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, found_path.to_str().unwrap()); + std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); + } + assert_eq!(current_active_version(), "202608.6.0"); + std::fs::remove_file(&found_path).ok(); + clear_current_version_env(); + + // When the configured key isn't present at the configured path, it + // still falls back to the (possibly also-overridden) stub, exactly + // as it does for the real /etc/os-release. + let missing_path = dir.join(format!( + "os-release-test-current-version-missing-{}", + Uuid::new_v4() + )); + std::fs::write(&missing_path, "NAME=\"Contoso Linux\"\n").unwrap(); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, missing_path.to_str().unwrap()); + std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); + std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub-for-missing-key"); + } + assert_eq!(current_active_version(), "custom-stub-for-missing-key"); + std::fs::remove_file(&missing_path).ok(); + clear_current_version_env(); + } +} From 0a29a081265f9148c1cbe21703de28cc8890c757 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 19:29:04 +0000 Subject: [PATCH 26/54] trident-acl-agent: fix suppressed Copilot review comments + drop obsolete Harpoon naming Suppressed-comment fixes: - main.rs: is_network_target() no longer allocates a String per enabled() call; uses strip_prefix instead of format!+starts_with. - main.rs: stale comment referencing removed omaha::send now points at nebraska::transport. - k8s.rs: watch_node()'s watch timeoutSeconds no longer collapses to the 2s default poll_interval (causing constant reconnects); floored at a new WATCH_TIMEOUT_SECS constant while still honoring a larger configured poll_interval. - trident.rs: consume_servicing_stream() passes format args directly to the log macros instead of eagerly building a String regardless of whether the level is enabled. - config.rs: DEFAULT_NEBRASKA_ENDPOINT now ends in a trailing slash, matching the nebraska client's own documented /v1/update/ contract. - lib.rs: run_omaha_only's missing-endpoint error no longer references a nonexistent CLI URL argument. - annotations.rs: validate() now rejects a non-UUID operationId, per the formal schema's format: uuid requirement; schema_validate_property (test helper) now also enforces maxLength. - state.rs: save() now fsyncs the temp file before rename and best-effort fsyncs the parent directory after, for crash durability; load()'s error context now includes the actual configured path instead of a hardcoded "state.json". - nebraska/wire.rs: removed a duplicated (copy-pasted) doc comment on request_for. - Cargo.toml: dropped the unused sha2 dependency from trident-acl-agent's own manifest (still used by other crates via the workspace default) and the unused toml workspace dependency (unused anywhere in the repo). Confirmed already fixed / not applicable and left alone: - nebraska/transport.rs already has a per-request timeout. - packaging/systemd/trident-acl-agent.service already uses an absolute ExecStart path. Harpoon cleanup: - Renamed HarpoonError to AgentError (error.rs, lib.rs, id.rs). - Updated remaining "Harpoon" doc comments/strings in main.rs, k8s.rs, config.rs, and lib.rs's crate-level doc to refer to trident-acl-agent. --- Cargo.lock | 1 - Cargo.toml | 1 - crates/trident-acl-agent/Cargo.toml | 1 - crates/trident-acl-agent/src/annotations.rs | 42 ++++++++++++++++++- crates/trident-acl-agent/src/config.rs | 11 +++-- crates/trident-acl-agent/src/error.rs | 6 +-- crates/trident-acl-agent/src/id.rs | 10 ++--- crates/trident-acl-agent/src/k8s.rs | 23 +++++++++- crates/trident-acl-agent/src/lib.rs | 33 +++++++-------- crates/trident-acl-agent/src/main.rs | 16 +++---- crates/trident-acl-agent/src/nebraska/wire.rs | 2 - crates/trident-acl-agent/src/state.rs | 32 ++++++++++++-- crates/trident-acl-agent/src/trident.rs | 14 ++++--- 13 files changed, 137 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6beb8abf65..bd9bca155f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4030,7 +4030,6 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_yaml", - "sha2", "sysdefs", "systemd-journal-logger", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 15d0c73504..dde3227664 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,7 +93,6 @@ tar = "0.4.46" tempfile = "3.14.0" tera = "1.20.0" textwrap = "0.16.2" -toml = "0.8.23" thiserror = "1.0.69" tokio = { version = "1.48.0", features = ["full"] } tokio-stream = { version = "0.1.17", features = ["net"] } diff --git a/crates/trident-acl-agent/Cargo.toml b/crates/trident-acl-agent/Cargo.toml index dc075e580c..36ecde4e4e 100644 --- a/crates/trident-acl-agent/Cargo.toml +++ b/crates/trident-acl-agent/Cargo.toml @@ -21,7 +21,6 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_yaml = { workspace = true } serde_path_to_error = { workspace = true } -sha2 = { workspace = true } systemd-journal-logger = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations.rs index fe498d2452..9dc95fb71f 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations.rs @@ -155,6 +155,12 @@ impl UpdateRequest { if self.schema_version != SCHEMA_VERSION { return Err(format!("unsupported schemaVersion {}", self.schema_version)); } + if Uuid::parse_str(&self.operation_id).is_err() { + return Err(format!( + "operationId must be a UUID, got {:?}", + self.operation_id + )); + } match self.operation { RequestedOperation::Stage | RequestedOperation::Finalize => { if self.target_version.as_deref().unwrap_or("").is_empty() { @@ -935,6 +941,19 @@ mod tests { )); } } + if let Some(max_length) = prop_schema.get("maxLength").and_then(Value::as_u64) { + let s = value + .as_str() + .ok_or_else(|| format!("property {name:?}: expected string to check maxLength"))?; + // Matches truncate_message's byte-based budget (MAX_MESSAGE_BYTES) + // rather than a char count, since that's the unit actually + // enforced in production. + if s.len() as u64 > max_length { + return Err(format!( + "property {name:?}: {s:?} exceeds maxLength {max_length}" + )); + } + } Ok(()) } @@ -966,7 +985,7 @@ mod tests { UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), node_update_id: Uuid::new_v4(), - operation_id: "op-1".to_string(), + operation_id: Uuid::new_v4().to_string(), operation, target_version: Some("202606.29.0".to_string()), server: Some(Url::parse("https://nebraska.example/v1/update").unwrap()), @@ -1053,7 +1072,7 @@ mod tests { let request = UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), node_update_id: Uuid::new_v4(), - operation_id: "op-1".to_string(), + operation_id: Uuid::new_v4().to_string(), operation: RequestedOperation::Rollback, target_version: None, server: None, @@ -1065,6 +1084,25 @@ mod tests { .expect("rollback without server/appId/track must validate"); } + #[test] + fn validate_rejects_non_uuid_operation_id() { + // The formal schema requires `operationId` to be UUID-shaped + // (`format: uuid`); a non-UUID value must be rejected regardless of + // operation kind. + for operation in [ + RequestedOperation::Stage, + RequestedOperation::Finalize, + RequestedOperation::Rollback, + ] { + let mut request = valid_nebraska_request(operation); + request.operation_id = "op-1".to_string(); + let err = request + .validate() + .expect_err("a non-UUID operationId must be rejected"); + assert!(err.contains("operationId"), "{err}"); + } + } + // --- example payload parsing tests ------------------------------------- #[test] diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs index e40fdd454f..de19f49ec7 100644 --- a/crates/trident-acl-agent/src/config.rs +++ b/crates/trident-acl-agent/src/config.rs @@ -1,4 +1,4 @@ -//! Env-var-based config loading for Harpoon. +//! Env-var-based config loading for trident-acl-agent. //! //! There is no config file. Every setting is an environment variable //! prefixed `TRIDENT_ACL_AGENT_` (one constant per setting, e.g. @@ -49,7 +49,7 @@ const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); // mode does not use this default at all: stage/finalize requests must // carry their own `server` field, with no fallback to this config (see // Orchestrator::resolve_nebraska_endpoint). -pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update"; +pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update/"; const DEFAULT_STAGE_TIMEOUT: Duration = Duration::from_secs(20 * 60); const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); @@ -374,7 +374,10 @@ mod tests { env::set_var(ENV_NEBRASKA_APP_ID, "custom-app"); env::set_var(ENV_NEBRASKA_TRACK, "custom-track"); env::set_var(ENV_KUBERNETES_API_SERVER, "https://cluster.example.invalid"); - env::set_var(ENV_KUBERNETES_KUBECONFIG, "/etc/harpoon/kubeconfig"); + env::set_var( + ENV_KUBERNETES_KUBECONFIG, + "/etc/trident-acl-agent/kubeconfig", + ); env::set_var(ENV_KUBERNETES_NODE_NAME, "node-42"); env::set_var(ENV_TRIDENT_SOCKET, "unix:///custom/trident.sock"); env::set_var(ENV_ORCHESTRATION_GOAL_SOURCE, "omaha-only"); @@ -403,7 +406,7 @@ mod tests { ); assert_eq!( config.kubernetes.kubeconfig.as_str(), - "/etc/harpoon/kubeconfig" + "/etc/trident-acl-agent/kubeconfig" ); assert_eq!(config.kubernetes.node_name, "node-42"); assert_eq!(config.trident.socket, "unix:///custom/trident.sock"); diff --git a/crates/trident-acl-agent/src/error.rs b/crates/trident-acl-agent/src/error.rs index fa87674879..3eb9ba3c65 100644 --- a/crates/trident-acl-agent/src/error.rs +++ b/crates/trident-acl-agent/src/error.rs @@ -2,8 +2,8 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "kebab-case")] -pub enum HarpoonError { - #[error("Failed to initialize the Harpoon client: {0}")] +pub enum AgentError { + #[error("Failed to initialize the trident-acl-agent client: {0}")] InitializationError(String), #[error("The version provided '{version}' is not valid semver: {inner}")] @@ -21,7 +21,7 @@ pub enum HarpoonError { /// Wraps a [`nebraska::NebraskaError`](crate::nebraska::NebraskaError). /// Stored as a string rather than `#[from]` because `NebraskaError` /// doesn't derive `Serialize`/`Deserialize`/`PartialEq`, which - /// `HarpoonError` requires for annotation-status round-tripping. + /// `AgentError` requires for annotation-status round-tripping. #[error("Nebraska request failed: {0}")] Nebraska(String), } diff --git a/crates/trident-acl-agent/src/id.rs b/crates/trident-acl-agent/src/id.rs index 651356d801..3a9027f84d 100644 --- a/crates/trident-acl-agent/src/id.rs +++ b/crates/trident-acl-agent/src/id.rs @@ -2,7 +2,7 @@ use std::fmt::Display; use osutils::{hostname, machine_id::MachineId}; -use crate::error::HarpoonError; +use crate::error::AgentError; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum IdSource { @@ -22,17 +22,17 @@ impl Display for IdSource { } impl IdSource { - pub(super) fn produce_id(&self) -> Result { + pub(super) fn produce_id(&self) -> Result { Ok(match self { IdSource::MachineIdHashed => MachineId::read() - .map_err(|err| HarpoonError::MachineIdRead(err.to_string()))? + .map_err(|err| AgentError::MachineIdRead(err.to_string()))? .hashed_uuid() .to_string(), IdSource::MachineIdRaw => MachineId::read() - .map_err(|err| HarpoonError::MachineIdRead(err.to_string()))? + .map_err(|err| AgentError::MachineIdRead(err.to_string()))? .as_string(), IdSource::Hostname => { - hostname::read().map_err(|err| HarpoonError::HostnameRead(err.to_string()))? + hostname::read().map_err(|err| AgentError::HostnameRead(err.to_string()))? } }) } diff --git a/crates/trident-acl-agent/src/k8s.rs b/crates/trident-acl-agent/src/k8s.rs index 2cca4c3bfc..2b06617421 100644 --- a/crates/trident-acl-agent/src/k8s.rs +++ b/crates/trident-acl-agent/src/k8s.rs @@ -1,4 +1,5 @@ -//! Thin Kubernetes client wrapper for Harpoon's node self-patching protocol. +//! Thin Kubernetes client wrapper for trident-acl-agent's node self-patching +//! protocol. //! //! Implements the Node get/watch/patch access described in the current //! accepted design (). @@ -27,6 +28,12 @@ use serde_json::json; use crate::config::KubernetesConfig; +/// Floor for the Kubernetes watch request's `timeoutSeconds`, decoupled from +/// `poll_interval` (see [`NodeClient::watch_node`]). Sits comfortably under +/// typical apiserver request-timeout defaults (~300s) while avoiding +/// reconnect churn on an otherwise-healthy watch. +const WATCH_TIMEOUT_SECS: u32 = 290; + #[derive(Debug, thiserror::Error)] pub enum K8sClientError { #[error("failed to build Kubernetes client config: {0}")] @@ -109,9 +116,21 @@ impl NodeClient { } pub fn watch_node(&self, name: String) -> BoxStream<'static, Result> { + // The watch request's timeoutSeconds bounds how long the API server + // holds the connection open before closing it, at which point + // `kube::runtime::watcher` reconnects. `poll_interval` defaults to a + // couple of seconds (fine for a fallback-polling cadence), so using + // it directly here would force a reconnect every couple of seconds + // even on a perfectly healthy watch. Floor the request timeout at + // WATCH_TIMEOUT_SECS instead, while still honoring a larger + // configured `poll_interval` if one is ever set. + let timeout_secs = self + .poll_interval + .as_secs() + .max(u64::from(WATCH_TIMEOUT_SECS)) as u32; let watcher_config = watcher::Config::default() .fields(&format!("metadata.name={name}")) - .timeout(self.poll_interval.as_secs().max(1) as u32); + .timeout(timeout_secs); watcher(self.api.clone(), watcher_config) .default_backoff() diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 89a4f0e968..bc92d1aede 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -1,11 +1,12 @@ -//! # Harpoon +//! # trident-acl-agent //! -//! Harpoon is Trident's ACL update sidecar. Historically it was a one-shot -//! Omaha client that called Trident's combined `Update()` RPC once and exited. -//! This crate now defaults to the AKS annotation protocol described in the -//! local design doc (`aks-rp ↔ trident-acl-agent`, especially §3–§6 and -//! §12–§13), while preserving the original `omaha-only` mode as an explicit -//! opt-out (see `config::GoalSource`). +//! trident-acl-agent is Trident's ACL update sidecar. Historically it was a +//! one-shot Omaha client that called Trident's combined `Update()` RPC once +//! and exited. This crate now defaults to the Kubernetes annotation protocol +//! described in the accepted design +//! (), +//! while preserving the original `omaha-only` mode as an explicit opt-out +//! (see `config::GoalSource`). //! //! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) //! goes through the [`nebraska`] client module, a self-contained, reusable @@ -48,7 +49,7 @@ pub mod trident; #[cfg(test)] pub mod mock_tridentd; -use error::HarpoonError; +use error::AgentError; use nebraska::{CheckOutcome, Client, MachineId, NebraskaError}; use trident::TridentClient; @@ -63,9 +64,9 @@ pub const DEFAULT_NEBRASKA_APP_ID: &str = "00000000-0000-0000-0000-000000000000" pub const DEFAULT_NEBRASKA_TRACK: &str = "unspecified"; /// Builds a validated [`MachineId`] from an [`IdSource`], translating the -/// crate's own machine-id/hostname read errors into a single [`HarpoonError`]. -fn build_machine_id(source: IdSource) -> Result { - MachineId::new(source.produce_id()?).map_err(|err| HarpoonError::Nebraska(err.to_string())) +/// crate's own machine-id/hostname read errors into a single [`AgentError`]. +fn build_machine_id(source: IdSource) -> Result { + MachineId::new(source.produce_id()?).map_err(|err| AgentError::Nebraska(err.to_string())) } /// Historical one-shot flow: query the Nebraska/Omaha server at @@ -74,9 +75,7 @@ fn build_machine_id(source: IdSource) -> Result { /// involvement. pub async fn run_omaha_only(config: &config::AgentConfig) -> Result<(), anyhow::Error> { let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { - anyhow::anyhow!( - "no Nebraska endpoint configured: pass on the CLI or set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT" - ) + anyhow::anyhow!("no Nebraska endpoint configured: set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT") })?; // Client::check_for_update() is a blocking call (reqwest::blocking under @@ -137,7 +136,7 @@ pub fn check_nebraska_reachable( app_id: &str, track: &str, machine_id_source: IdSource, -) -> Result<(), HarpoonError> { +) -> Result<(), AgentError> { let machine_id = build_machine_id(machine_id_source)?; let client = Client::new(url.clone(), app_id, track, machine_id); match client.check_for_update(&Version::new(0, 0, 0)) { @@ -146,7 +145,7 @@ pub fn check_nebraska_reachable( // still proves the server is reachable and speaking Omaha; only a // transport/parse-level failure means it is not. Err(NebraskaError::ServerError(_)) => Ok(()), - Err(err) => Err(HarpoonError::Nebraska(err.to_string())), + Err(err) => Err(AgentError::Nebraska(err.to_string())), } } @@ -199,6 +198,6 @@ mod tests { IdSource::MachineIdHashed, ) .unwrap_err(); - assert!(matches!(err, HarpoonError::Nebraska(_))); + assert!(matches!(err, AgentError::Nebraska(_))); } } diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index a81f6bc087..3f6d837ba8 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -61,14 +61,16 @@ impl Log for FilteredLogger { } fn is_network_target(target: &str) -> bool { - NETWORK_LOG_TARGETS - .iter() - .any(|prefix| target == *prefix || target.starts_with(&format!("{prefix}::"))) + NETWORK_LOG_TARGETS.iter().any(|prefix| { + target + .strip_prefix(prefix) + .is_some_and(|rest| rest.is_empty() || rest.starts_with("::")) + }) } -/// Harpoon can either run the annotation-driven orchestrator (the default) -/// or fall back to its original one-shot Omaha flow. Mode selection is -/// environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`): +/// trident-acl-agent can either run the annotation-driven orchestrator (the +/// default) or fall back to its original one-shot Omaha flow. Mode selection +/// is environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`): /// shipping defaults enable the AKS annotation protocol, while a VM /// extension, systemd drop-in, or AgentBaker-set environment can opt a node /// out to `omaha-only` if needed. @@ -164,7 +166,7 @@ async fn validate_connection( })?; let app_id = config.nebraska.app_id.clone(); // check_nebraska_reachable() is a blocking call (reqwest::blocking - // under the hood, see omaha::send) - calling it directly from this + // under the hood, see nebraska::transport) - calling it directly from this // async fn can panic ("Cannot drop a runtime in a context where // blocking is not allowed") because reqwest::blocking spins up // its own inner Tokio runtime per call, which isn't safe to tear diff --git a/crates/trident-acl-agent/src/nebraska/wire.rs b/crates/trident-acl-agent/src/nebraska/wire.rs index d2f2d61016..e909aeb9f4 100644 --- a/crates/trident-acl-agent/src/nebraska/wire.rs +++ b/crates/trident-acl-agent/src/nebraska/wire.rs @@ -235,8 +235,6 @@ pub(super) struct EventElement { previous_version: Option, } -/// Builds a request from an app, setting the `` from the app version -/// so the two agree. /// Builds a request from an app, setting the `` from the app version /// so the two agree. `client_version` identifies the updater itself, and is /// omitted from the request entirely when the caller did not name one. diff --git a/crates/trident-acl-agent/src/state.rs b/crates/trident-acl-agent/src/state.rs index c9f0c49113..28f52e6ac9 100644 --- a/crates/trident-acl-agent/src/state.rs +++ b/crates/trident-acl-agent/src/state.rs @@ -9,6 +9,7 @@ use std::{ collections::BTreeMap, fs, + io::Write, path::{Path, PathBuf}, }; @@ -65,7 +66,8 @@ impl StateStore { pub fn load(&self) -> Result { match fs::read_to_string(&self.path) { - Ok(raw) => Ok(serde_json::from_str(&raw).context("failed to parse state.json")?), + Ok(raw) => Ok(serde_json::from_str(&raw) + .with_context(|| format!("failed to parse {}", self.path.display()))?), Err(err) if err.kind() == std::io::ErrorKind::NotFound => { Ok(PersistentState::default()) } @@ -94,15 +96,37 @@ impl StateStore { .unwrap_or("state.json"), std::process::id() )); - fs::write(&temp_path, serde_json::to_string_pretty(state)?) - .with_context(|| format!("failed to write {}", temp_path.display()))?; + + // Write via a File handle and fsync it before the rename: fs::write + // alone only guarantees the data reaches the OS page cache, not + // disk, so a crash between the write and a later flush could still + // leave state.json empty/corrupt after the rename below. + { + let mut file = fs::File::create(&temp_path) + .with_context(|| format!("failed to create {}", temp_path.display()))?; + file.write_all(serde_json::to_string_pretty(state)?.as_bytes()) + .with_context(|| format!("failed to write {}", temp_path.display()))?; + file.sync_all() + .with_context(|| format!("failed to fsync {}", temp_path.display()))?; + } + fs::rename(&temp_path, &self.path).with_context(|| { format!( "failed to atomically replace {} with {}", self.path.display(), temp_path.display() ) - }) + })?; + + // Best-effort: POSIX doesn't guarantee a rename is durable until the + // containing directory's metadata is also synced, so fsync it too. + // Not fatal if this fails (e.g. unsupported on some filesystems) - + // the rename itself has already succeeded. + if let Ok(dir) = fs::File::open(parent) { + let _ = dir.sync_all(); + } + + Ok(()) } pub fn remember_completed(&self, status: UpdateStatus) -> Result<(), anyhow::Error> { diff --git a/crates/trident-acl-agent/src/trident.rs b/crates/trident-acl-agent/src/trident.rs index 628007795c..5bfddea39d 100644 --- a/crates/trident-acl-agent/src/trident.rs +++ b/crates/trident-acl-agent/src/trident.rs @@ -356,13 +356,15 @@ async fn consume_servicing_stream( log::info!("[Trident:{operation}] started"); } Some(ResponseBody::Log(log_record)) => { - let msg = format!("[Trident:{operation}] {}", log_record.message); + let message = &log_record.message; match log_record.level() { - LogLevel::Unspecified | LogLevel::Trace => log::trace!("{msg}"), - LogLevel::Debug => log::debug!("{msg}"), - LogLevel::Info => log::info!("{msg}"), - LogLevel::Warn => log::warn!("{msg}"), - LogLevel::Error => log::error!("{msg}"), + LogLevel::Unspecified | LogLevel::Trace => { + log::trace!("[Trident:{operation}] {message}") + } + LogLevel::Debug => log::debug!("[Trident:{operation}] {message}"), + LogLevel::Info => log::info!("[Trident:{operation}] {message}"), + LogLevel::Warn => log::warn!("[Trident:{operation}] {message}"), + LogLevel::Error => log::error!("[Trident:{operation}] {message}"), } } Some(ResponseBody::Completed(completed)) => { From 4845800248fdd3e2799749cdebbeea94ef11e2d8 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 20:35:11 +0000 Subject: [PATCH 27/54] trident-acl-agent: fix remaining Copilot review findings (PR 730) - config.rs: drop trailing slash from DEFAULT_NEBRASKA_ENDPOINT so it matches the .../v1/update form used in docs/examples elsewhere - docs/Explanation/Trident-ACL-Agent.md: correct deployment section - the RPM installs more than just the binary and unit; it also ships LICENSE/NOTICE via %license Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/src/config.rs | 2 +- docs/Explanation/Trident-ACL-Agent.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/config.rs index de19f49ec7..7828022774 100644 --- a/crates/trident-acl-agent/src/config.rs +++ b/crates/trident-acl-agent/src/config.rs @@ -49,7 +49,7 @@ const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); // mode does not use this default at all: stage/finalize requests must // carry their own `server` field, with no fallback to this config (see // Orchestrator::resolve_nebraska_endpoint). -pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update/"; +pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update"; const DEFAULT_STAGE_TIMEOUT: Duration = Duration::from_secs(20 * 60); const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index 2d64aa8182..4f071e3792 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -21,10 +21,11 @@ package). Installing it: $ tdnf install trident-acl-agent ``` -lays down exactly two files: the `/usr/bin/trident-acl-agent` binary and -its `trident-acl-agent.service` unit +lays down the `/usr/bin/trident-acl-agent` binary and its +`trident-acl-agent.service` unit (`packaging/systemd/trident-acl-agent.service`) under the systemd unit -directory. Installing the package does not by itself enable or start the +directory, along with the package's `%license`-installed `LICENSE`/`NOTICE` +files. Installing the package does not by itself enable or start the service — a deployment decides when that happens, e.g. by running `systemctl enable --now trident-acl-agent.service` on the node, or by baking that enablement into the image build (as this repo's own From 3dbf63623e8c1c1667900dee29d6159cc76167b1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 21:08:16 +0000 Subject: [PATCH 28/54] trident-acl-agent: rework current-version fallback into explicit always/error/custom modes Renames the current-version fallback from a single descriptive stub string to a small, explicit vocabulary controlled by TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK (default "always"): - "always": report 0.0.0 as the current version (never collides with a real target, so stage/finalize always proceed) - useful for dev/test hosts. - "error": current_active_version() now returns a Result, and this mode fails the in-flight stage/finalize/rollback instead of silently proceeding with a placeholder version - lets a production deployment catch a misconfigured VERSION_ID/IMAGE_VERSION key immediately. - anything else: used verbatim as the fallback version, same as the old STUB override, with no format validation. CURRENT_VERSION_STUB / ENV_CURRENT_VERSION_STUB are renamed to DEFAULT_CURRENT_VERSION_FALLBACK / ENV_CURRENT_VERSION_FALLBACK. current_active_version()'s 3 orchestrator.rs call sites and lib.rs's omaha-only call site now propagate its Result via ?. Updated Trident-ACL-Agent.md and version.rs's tests accordingly. Verified: cargo test -p trident-acl-agent (152/152), cargo clippy, and cargo fmt all clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/src/lib.rs | 2 +- crates/trident-acl-agent/src/orchestrator.rs | 6 +- crates/trident-acl-agent/src/version.rs | 604 ++++++++++--------- docs/Explanation/Trident-ACL-Agent.md | 12 +- 4 files changed, 342 insertions(+), 282 deletions(-) diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index bc92d1aede..8a71a8208c 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -87,7 +87,7 @@ pub async fn run_omaha_only(config: &config::AgentConfig) -> Result<(), anyhow:: let app_id = config.nebraska.app_id.clone(); let track = config.nebraska.track.clone(); let machine_id = build_machine_id(IdSource::MachineIdHashed)?; - let current_version_raw = version::current_active_version(); + let current_version_raw = version::current_active_version()?; let current_version = Version::parse(¤t_version_raw).unwrap_or_else(|err| { log::warn!( "current version {current_version_raw:?} is not valid semver ({err}); reporting 0.0.0 to Nebraska" diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/orchestrator.rs index f2aaca683c..0074b4a43e 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/orchestrator.rs @@ -329,7 +329,7 @@ where async fn handle_stage(&self, request: UpdateRequest) -> Result<(), anyhow::Error> { let started = Utc::now(); - let from_version = Some(current_active_version()); + let from_version = Some(current_active_version()?); let to_version = request.target_version.clone(); if from_version == to_version { let status = UpdateStatus::new( @@ -465,7 +465,7 @@ where async fn handle_finalize(&self, request: UpdateRequest) -> Result { let started = Utc::now(); - let from_version = Some(current_active_version()); + let from_version = Some(current_active_version()?); let to_version = request.target_version.clone(); if from_version == to_version { let status = UpdateStatus::new( @@ -620,7 +620,7 @@ where async fn handle_rollback(&self, request: UpdateRequest) -> Result { let started = Utc::now(); - let from_version = Some(current_active_version()); + let from_version = Some(current_active_version()?); let mut client = TridentClient::connect(&self.config.trident.socket).await?; diff --git a/crates/trident-acl-agent/src/version.rs b/crates/trident-acl-agent/src/version.rs index 484e811657..95321dee37 100644 --- a/crates/trident-acl-agent/src/version.rs +++ b/crates/trident-acl-agent/src/version.rs @@ -1,273 +1,331 @@ -//! Determines the node's currently-running version, for comparison against -//! a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`) -//! and for reporting the instance's current version to Nebraska. -//! -//! Split out of [`crate::annotations`] because it has nothing to do with the -//! annotation wire format - it's plain env-var-configurable file probing, -//! used by both the annotation-driven orchestrator and the `omaha-only` -//! one-shot mode. - -// current_active_version() reads the `VERSION_ID` key (overridable via -// TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY, e.g. to `IMAGE_VERSION` for an ACL -// image that stamps its own per-build version there) out of os-release, but -// falls back to this stub if that key isn't present (e.g. a minimal -// dev/test os-release). The stub value below is an explicit sentinel that -// cannot collide with a real release version string, so it can never -// accidentally match a real requested target version and cause -// handle_stage/handle_finalize to incorrectly short-circuit to -// AlreadyAtTarget. Do not remove this comment when bumping the stub value; -// keep it (and its non-colliding shape). The stub itself is overridable via -// TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB, for dev/test hosts that want a -// specific sentinel. -pub const CURRENT_VERSION_STUB: &str = "0.0.0-unprobed-trident-acl-agent-stub"; -/// Default path `current_active_version` reads. Overridable via -/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` so a deployment can point the -/// agent at any file that follows the os-release format (`KEY=VALUE` lines, -/// optionally quoted, blank lines and `#` comments ignored - see -/// ) -/// instead of the real `/etc/os-release`, e.g. a vendor-specific file that -/// carries the running image's version under a key `/etc/os-release` -/// doesn't have room for. -pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; -/// Default os-release key `current_active_version` looks up for the running -/// image's version: `VERSION_ID`, a standard key every os-release carries -/// (see -/// ). -/// Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` - e.g. to -/// `IMAGE_VERSION` for an ACL image that stamps its own per-build version -/// under that key instead - or point -/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` at a different file entirely. -pub const DEFAULT_CURRENT_VERSION_KEY: &str = "VERSION_ID"; -const ENV_CURRENT_VERSION_PATH: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH"; -const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; -const ENV_CURRENT_VERSION_STUB: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB"; - -/// Reads `name`, treating both "unset" and "set to the empty string" as -/// absent, matching `config::env_raw`'s convention: a drop-in override that -/// clears a variable to `""` should fall back to the default, not try to use -/// an empty value. -fn env_override(name: &str) -> Option { - std::env::var(name).ok().filter(|v| !v.is_empty()) -} - -pub fn current_active_version() -> String { - let path = env_override(ENV_CURRENT_VERSION_PATH) - .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); - let key = env_override(ENV_CURRENT_VERSION_KEY) - .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_KEY.to_string()); - read_os_release_value(&path, &key).unwrap_or_else(|| { - let stub = env_override(ENV_CURRENT_VERSION_STUB) - .unwrap_or_else(|| CURRENT_VERSION_STUB.to_string()); - log::warn!("{key} not found in {path}; falling back to stub current version {stub}"); - stub - }) -} - -/// Reads `path` (an os-release-formatted file: `KEY=VALUE` lines, blank -/// lines and `#` comments ignored, values optionally single- or -/// double-quoted - see -/// ) -/// and returns the trimmed, unquoted value for `key`, or `None` if the file -/// can't be read, `key` isn't present, or its value is empty - all of which -/// `current_active_version` treats identically: fall back to the stub. -/// Split out from `current_active_version` so tests can point it at a temp -/// file instead of the real os-release. -fn read_os_release_value(path: &str, key: &str) -> Option { - let contents = std::fs::read_to_string(path).ok()?; - for line in contents.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((line_key, raw_value)) = line.split_once('=') else { - continue; - }; - if line_key.trim() != key { - continue; - } - let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); - if value.is_empty() { - return None; - } - return Some(value.to_string()); - } - None -} - -#[cfg(test)] -mod tests { - use uuid::Uuid; - - use super::*; - - #[test] - fn read_os_release_value_returns_none_for_missing_file() { - assert_eq!( - read_os_release_value( - "/nonexistent/path/does-not-exist-os-release", - DEFAULT_CURRENT_VERSION_KEY - ), - None - ); - } - - #[test] - fn read_os_release_value_finds_requested_key() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-{}", Uuid::new_v4())); - std::fs::write( - &path, - "NAME=\"Azure Linux\"\nIMAGE_VERSION=202608.6.0\nVERSION_ID=3.0\n", - ) - .unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result.as_deref(), Some("202608.6.0")); - } - - #[test] - fn read_os_release_value_trims_quotes_and_whitespace() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-quoted-{}", Uuid::new_v4())); - std::fs::write(&path, " IMAGE_VERSION = \"202608.6.0\" \n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result.as_deref(), Some("202608.6.0")); - } - - #[test] - fn read_os_release_value_returns_none_for_missing_key() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-missing-key-{}", Uuid::new_v4())); - std::fs::write(&path, "NAME=\"Azure Linux\"\nVERSION_ID=3.0\n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result, None); - } - - #[test] - fn read_os_release_value_returns_none_for_empty_value() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-empty-value-{}", Uuid::new_v4())); - std::fs::write(&path, "IMAGE_VERSION=\n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result, None); - } - - #[test] - fn read_os_release_value_skips_comments_and_blank_lines() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-comments-{}", Uuid::new_v4())); - std::fs::write( - &path, - "# a comment\n\n# IMAGE_VERSION=should-be-ignored\nIMAGE_VERSION=202608.6.0\n", - ) - .unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); - assert_eq!(result.as_deref(), Some("202608.6.0")); - } - - /// Clears all three env vars `current_active_version` reads. Environment - /// mutation is process-global and `std::env::remove_var`/`set_var` are - /// `unsafe` (not thread-safe against concurrent reads elsewhere in the - /// process), so the defaults/overrides/read-path cases below are - /// intentionally folded into one sequential `#[test]` rather than - /// several separate ones that `cargo test` could run in parallel - /// against the same variables. - fn clear_current_version_env() { - // SAFETY: single-threaded within this test function; no other test - // in this crate reads or writes these three variables. - unsafe { - std::env::remove_var(ENV_CURRENT_VERSION_PATH); - std::env::remove_var(ENV_CURRENT_VERSION_KEY); - std::env::remove_var(ENV_CURRENT_VERSION_STUB); - } - } - - #[test] - fn current_active_version_path_key_and_stub_are_overridable_via_env() { - clear_current_version_env(); - assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); - assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); - - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, "/custom/os-release"); - } - assert_eq!( - env_override(ENV_CURRENT_VERSION_PATH).as_deref(), - Some("/custom/os-release") - ); - - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_KEY, "CUSTOM_VERSION_KEY"); - } - assert_eq!( - env_override(ENV_CURRENT_VERSION_KEY).as_deref(), - Some("CUSTOM_VERSION_KEY") - ); - - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub"); - } - assert_eq!( - env_override(ENV_CURRENT_VERSION_STUB).as_deref(), - Some("custom-stub") - ); - - // An empty override is treated the same as unset. - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, ""); - std::env::set_var(ENV_CURRENT_VERSION_KEY, ""); - } - assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); - assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); - - clear_current_version_env(); - - // current_active_version() itself honors TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH, - // pointing it at an arbitrary os-release-formatted file instead of the - // real /etc/os-release. - let dir = std::env::temp_dir(); - let found_path = dir.join(format!( - "os-release-test-current-version-{}", - Uuid::new_v4() - )); - std::fs::write( - &found_path, - "NAME=\"Contoso Linux\"\nVERSION_ID=202608.6.0\n", - ) - .unwrap(); - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, found_path.to_str().unwrap()); - std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); - } - assert_eq!(current_active_version(), "202608.6.0"); - std::fs::remove_file(&found_path).ok(); - clear_current_version_env(); - - // When the configured key isn't present at the configured path, it - // still falls back to the (possibly also-overridden) stub, exactly - // as it does for the real /etc/os-release. - let missing_path = dir.join(format!( - "os-release-test-current-version-missing-{}", - Uuid::new_v4() - )); - std::fs::write(&missing_path, "NAME=\"Contoso Linux\"\n").unwrap(); - // SAFETY: see clear_current_version_env's doc comment. - unsafe { - std::env::set_var(ENV_CURRENT_VERSION_PATH, missing_path.to_str().unwrap()); - std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); - std::env::set_var(ENV_CURRENT_VERSION_STUB, "custom-stub-for-missing-key"); - } - assert_eq!(current_active_version(), "custom-stub-for-missing-key"); - std::fs::remove_file(&missing_path).ok(); - clear_current_version_env(); - } -} +//! Determines the node's currently-running version, for comparison against +//! a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`) +//! and for reporting the instance's current version to Nebraska. +//! +//! Split out of [`crate::annotations`] because it has nothing to do with the +//! annotation wire format - it's plain env-var-configurable file probing, +//! used by both the annotation-driven orchestrator and the `omaha-only` +//! one-shot mode. + +// current_active_version() reads the `VERSION_ID` key (overridable via +// TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY, e.g. to `IMAGE_VERSION` for an ACL +// image that stamps its own per-build version there) out of os-release, but +// falls back to TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK's behavior if +// that key isn't present (e.g. a minimal dev/test os-release). Three forms +// are recognized: +// - "always" (the default): report "0.0.0" as the current version. This +// is a sentinel that cannot collide with a real release version +// string, so it can never accidentally match a real requested target +// version and cause handle_stage/handle_finalize to incorrectly +// short-circuit to AlreadyAtTarget - useful on dev/test hosts that +// always want to treat themselves as needing whatever update is +// requested. +// - "error": current_active_version() returns an error instead of +// falling back to anything, so a misconfigured VERSION_ID/IMAGE_VERSION +// key fails the in-flight operation loudly rather than silently +// proceeding with a meaningless placeholder version - the right choice +// for a production deployment that wants to catch this class of +// misconfiguration immediately. +// - anything else: used verbatim as the fallback "current version" +// string, with no format validation - e.g. a specific sentinel a +// dev/test host wants for its own purposes. This is not checked against +// any version syntax, so it's the caller's responsibility to pick a +// value that can't collide with a real target version if that matters +// to them. +pub const DEFAULT_CURRENT_VERSION_FALLBACK: &str = "always"; +/// Default path `current_active_version` reads. Overridable via +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` so a deployment can point the +/// agent at any file that follows the os-release format (`KEY=VALUE` lines, +/// optionally quoted, blank lines and `#` comments ignored - see +/// ) +/// instead of the real `/etc/os-release`, e.g. a vendor-specific file that +/// carries the running image's version under a key `/etc/os-release` +/// doesn't have room for. +pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; +/// Default os-release key `current_active_version` looks up for the running +/// image's version: `VERSION_ID`, a standard key every os-release carries +/// (see +/// ). +/// Overridable via `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` - e.g. to +/// `IMAGE_VERSION` for an ACL image that stamps its own per-build version +/// under that key instead - or point +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` at a different file entirely. +pub const DEFAULT_CURRENT_VERSION_KEY: &str = "VERSION_ID"; +const ENV_CURRENT_VERSION_PATH: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH"; +const ENV_CURRENT_VERSION_KEY: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY"; +const ENV_CURRENT_VERSION_FALLBACK: &str = "TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK"; +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK`'s "report 0.0.0" keyword. +const FALLBACK_ALWAYS: &str = "always"; +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK`'s "fail instead" keyword. +const FALLBACK_ERROR: &str = "error"; +/// What [`current_active_version`] reports for [`FALLBACK_ALWAYS`] - see its +/// docs above for why 0.0.0 is a safe sentinel here. +const FALLBACK_ALWAYS_VERSION: &str = "0.0.0"; + +/// Reads `name`, treating both "unset" and "set to the empty string" as +/// absent, matching `config::env_raw`'s convention: a drop-in override that +/// clears a variable to `""` should fall back to the default, not try to use +/// an empty value. +fn env_override(name: &str) -> Option { + std::env::var(name).ok().filter(|v| !v.is_empty()) +} + +pub fn current_active_version() -> Result { + let path = env_override(ENV_CURRENT_VERSION_PATH) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); + let key = env_override(ENV_CURRENT_VERSION_KEY) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_KEY.to_string()); + if let Some(value) = read_os_release_value(&path, &key) { + return Ok(value); + } + let fallback = env_override(ENV_CURRENT_VERSION_FALLBACK) + .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_FALLBACK.to_string()); + match fallback.as_str() { + FALLBACK_ERROR => Err(anyhow::anyhow!( + "{key} not found in {path}, and {ENV_CURRENT_VERSION_FALLBACK} is set to \"error\"" + )), + FALLBACK_ALWAYS => { + log::warn!( + "{key} not found in {path}; falling back to \"always\" (reporting {FALLBACK_ALWAYS_VERSION} as the current version)" + ); + Ok(FALLBACK_ALWAYS_VERSION.to_string()) + } + _ => { + log::warn!( + "{key} not found in {path}; falling back to configured current version {fallback:?}" + ); + Ok(fallback) + } + } +} + +/// Reads `path` (an os-release-formatted file: `KEY=VALUE` lines, blank +/// lines and `#` comments ignored, values optionally single- or +/// double-quoted - see +/// ) +/// and returns the trimmed, unquoted value for `key`, or `None` if the file +/// can't be read, `key` isn't present, or its value is empty - all of which +/// `current_active_version` treats identically: fall back to the stub. +/// Split out from `current_active_version` so tests can point it at a temp +/// file instead of the real os-release. +fn read_os_release_value(path: &str, key: &str) -> Option { + let contents = std::fs::read_to_string(path).ok()?; + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((line_key, raw_value)) = line.split_once('=') else { + continue; + }; + if line_key.trim() != key { + continue; + } + let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); + if value.is_empty() { + return None; + } + return Some(value.to_string()); + } + None +} + +#[cfg(test)] +mod tests { + use uuid::Uuid; + + use super::*; + + #[test] + fn read_os_release_value_returns_none_for_missing_file() { + assert_eq!( + read_os_release_value( + "/nonexistent/path/does-not-exist-os-release", + DEFAULT_CURRENT_VERSION_KEY + ), + None + ); + } + + #[test] + fn read_os_release_value_finds_requested_key() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-{}", Uuid::new_v4())); + std::fs::write( + &path, + "NAME=\"Azure Linux\"\nIMAGE_VERSION=202608.6.0\nVERSION_ID=3.0\n", + ) + .unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + #[test] + fn read_os_release_value_trims_quotes_and_whitespace() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-quoted-{}", Uuid::new_v4())); + std::fs::write(&path, " IMAGE_VERSION = \"202608.6.0\" \n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + #[test] + fn read_os_release_value_returns_none_for_missing_key() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-missing-key-{}", Uuid::new_v4())); + std::fs::write(&path, "NAME=\"Azure Linux\"\nVERSION_ID=3.0\n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result, None); + } + + #[test] + fn read_os_release_value_returns_none_for_empty_value() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-empty-value-{}", Uuid::new_v4())); + std::fs::write(&path, "IMAGE_VERSION=\n").unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result, None); + } + + #[test] + fn read_os_release_value_skips_comments_and_blank_lines() { + let dir = std::env::temp_dir(); + let path = dir.join(format!("os-release-test-comments-{}", Uuid::new_v4())); + std::fs::write( + &path, + "# a comment\n\n# IMAGE_VERSION=should-be-ignored\nIMAGE_VERSION=202608.6.0\n", + ) + .unwrap(); + let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); + std::fs::remove_file(&path).ok(); + assert_eq!(result.as_deref(), Some("202608.6.0")); + } + + /// Clears all three env vars `current_active_version` reads. Environment + /// mutation is process-global and `std::env::remove_var`/`set_var` are + /// `unsafe` (not thread-safe against concurrent reads elsewhere in the + /// process), so the defaults/overrides/read-path cases below are + /// intentionally folded into one sequential `#[test]` rather than + /// several separate ones that `cargo test` could run in parallel + /// against the same variables. + fn clear_current_version_env() { + // SAFETY: single-threaded within this test function; no other test + // in this crate reads or writes these three variables. + unsafe { + std::env::remove_var(ENV_CURRENT_VERSION_PATH); + std::env::remove_var(ENV_CURRENT_VERSION_KEY); + std::env::remove_var(ENV_CURRENT_VERSION_FALLBACK); + } + } + + #[test] + fn current_active_version_path_key_and_fallback_are_overridable_via_env() { + clear_current_version_env(); + assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); + assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, "/custom/os-release"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_PATH).as_deref(), + Some("/custom/os-release") + ); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_KEY, "CUSTOM_VERSION_KEY"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_KEY).as_deref(), + Some("CUSTOM_VERSION_KEY") + ); + + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_FALLBACK, "custom-fallback"); + } + assert_eq!( + env_override(ENV_CURRENT_VERSION_FALLBACK).as_deref(), + Some("custom-fallback") + ); + + // An empty override is treated the same as unset. + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, ""); + std::env::set_var(ENV_CURRENT_VERSION_KEY, ""); + } + assert_eq!(env_override(ENV_CURRENT_VERSION_PATH), None); + assert_eq!(env_override(ENV_CURRENT_VERSION_KEY), None); + + clear_current_version_env(); + + // current_active_version() itself honors TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH, + // pointing it at an arbitrary os-release-formatted file instead of the + // real /etc/os-release. + let dir = std::env::temp_dir(); + let found_path = dir.join(format!( + "os-release-test-current-version-{}", + Uuid::new_v4() + )); + std::fs::write( + &found_path, + "NAME=\"Contoso Linux\"\nVERSION_ID=202608.6.0\n", + ) + .unwrap(); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, found_path.to_str().unwrap()); + std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); + } + assert_eq!(current_active_version().unwrap(), "202608.6.0"); + std::fs::remove_file(&found_path).ok(); + clear_current_version_env(); + + // When the configured key isn't present at the configured path, and + // no fallback override is set, it defaults to "always" - reporting + // FALLBACK_ALWAYS_VERSION ("0.0.0") as the current version. + let missing_path = dir.join(format!( + "os-release-test-current-version-missing-{}", + Uuid::new_v4() + )); + std::fs::write(&missing_path, "NAME=\"Contoso Linux\"\n").unwrap(); + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_PATH, missing_path.to_str().unwrap()); + std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); + } + assert_eq!(current_active_version().unwrap(), FALLBACK_ALWAYS_VERSION); + + // TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK="error" turns a missing + // key into a hard error instead of a placeholder version. + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var(ENV_CURRENT_VERSION_FALLBACK, FALLBACK_ERROR); + } + assert!(current_active_version().is_err()); + + // Any other TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK value is used + // verbatim, with no format validation. + // SAFETY: see clear_current_version_env's doc comment. + unsafe { + std::env::set_var( + ENV_CURRENT_VERSION_FALLBACK, + "custom-fallback-for-missing-key", + ); + } + assert_eq!( + current_active_version().unwrap(), + "custom-fallback-for-missing-key" + ); + + std::fs::remove_file(&missing_path).ok(); + clear_current_version_env(); + } +} diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index 4f071e3792..671362fd5a 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -232,7 +232,7 @@ naming the offending variable. | `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.microsoft.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.microsoft.com` in `acl.microsoft.com/update-request`). Any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | | `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | The file the agent reads to determine the node's currently running version. Any file works, as long as it follows the os-release format (`KEY=VALUE` lines, optionally single- or double-quoted, blank lines and `#` comments ignored) — see [below](#configuring-the-on-disk-version). | | `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `VERSION_ID` is the standard `os-release` field most images already stamp; a deployment that instead carries an ACL-specific `IMAGE_VERSION` field can point this variable at that key instead — see [below](#configuring-the-on-disk-version). | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` | `0.0.0-unprobed-trident-acl-agent-stub` | Sentinel value used as the node's "current version" when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). Deliberately shaped so it can never collide with a real release version and cause a false `AlreadyAtTarget`. | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` | `always` | Controls what happens when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). `always` reports `0.0.0` as the node's current version — a sentinel that can never collide with a real release version and cause a false `AlreadyAtTarget`. `error` fails the operation instead of using a placeholder version. Any other value is used verbatim as the current version, with no format validation. | | `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in the kubeconfig is used as-is. | | `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig used to reach the Kubernetes API server and authenticate as this node. | | `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. | @@ -299,10 +299,12 @@ compares it against a request's `targetVersion` the same way it would for `AlreadyAtTarget` when they already match. If the configured key is absent from the configured file (for example, on -a dev/test host with a minimal `os-release`), the agent falls back to -`TRIDENT_ACL_AGENT_CURRENT_VERSION_STUB` -(`0.0.0-unprobed-trident-acl-agent-stub` by default), a sentinel value -that can never accidentally match a real requested version. +a dev/test host with a minimal `os-release`), the agent consults +`TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` (`always` by default): `always` +reports `0.0.0` as the current version — a sentinel that can never +accidentally match a real requested version; `error` fails the operation +instead of guessing; any other value is used verbatim as the current +version, unvalidated. ## Diagnostics From aa3c0a93331a8d1201af6e38cf5425334ee76d5b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 21:22:35 +0000 Subject: [PATCH 29/54] trident-acl-agent: fix misleading watch_poll_interval doc in k8s.rs The module and watch_node comments claimed watch_poll_interval bounds the watcher's reconnect *backoff* ceiling. It doesn't: watch_node only uses poll_interval to floor the watch request's timeoutSeconds (the cadence of routine reconnects on an otherwise-healthy watch). Reconnect/backoff timing after a dropped or failed watch is governed entirely by kube::runtime::watcher's default_backoff(), independent of poll_interval. Corrected both comments to describe actual behavior. Verified: cargo build/test -p trident-acl-agent (152/152) and cargo fmt clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 98412869-e5c8-4f3f-a97c-fe870db70701 --- crates/trident-acl-agent/src/k8s.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/trident-acl-agent/src/k8s.rs b/crates/trident-acl-agent/src/k8s.rs index 2b06617421..8d6340eb00 100644 --- a/crates/trident-acl-agent/src/k8s.rs +++ b/crates/trident-acl-agent/src/k8s.rs @@ -7,10 +7,14 @@ //! The design calls for get/patch access to exactly one Node object (§2.2–§2.6). //! Node changes are observed via the Kubernetes watch API (`kube::runtime::watcher`) //! rather than polling, so annotation updates are delivered promptly and without -//! placing repeated load on the API server. `watch_poll_interval` still bounds -//! how quickly the watcher notices a dropped/re-established connection (used -//! as the watcher's backoff ceiling) and how often the fake test API server -//! needs to support being polled if it does not support real watches. +//! placing repeated load on the API server. `watch_poll_interval` only +//! bounds the watch request's `timeoutSeconds` (see +//! [`NodeClient::watch_node`]) - it floors how long a healthy watch +//! connection is held open before a routine reconnect, and how often the +//! fake test API server needs to support being polled if it does not +//! support real watches. It does not influence reconnect/backoff timing +//! after a dropped or failed watch; that is governed entirely by +//! `kube::runtime::watcher`'s built-in `default_backoff()`. use std::{collections::BTreeMap, path::Path}; @@ -123,7 +127,10 @@ impl NodeClient { // it directly here would force a reconnect every couple of seconds // even on a perfectly healthy watch. Floor the request timeout at // WATCH_TIMEOUT_SECS instead, while still honoring a larger - // configured `poll_interval` if one is ever set. + // configured `poll_interval` if one is ever set. Note this only + // affects the cadence of routine reconnects on a healthy watch - + // `default_backoff()` below (not `poll_interval`) governs + // retry/backoff timing after a dropped or failed watch. let timeout_secs = self .poll_interval .as_secs() From 9e571a09c3a616153bacafa5fbc64bf169235dd6 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 22:12:48 +0000 Subject: [PATCH 30/54] trident-acl-agent: restructure src into core/annotations/omahaonly modules Pure reorganization, no logic changes. Groups shared infrastructure under core/ (config, error, id, version, trident client, nebraska client), the Kubernetes-annotation-driven default flow under annotations/ (protocol, k8s, state, orchestrator), and the legacy one-shot flow under omahaonly/. - core/trident/client.rs (was trident.rs) and core/trident/mock.rs (was mock_tridentd.rs) avoid path stutter under core::trident::. - annotations/protocol.rs (was annotations.rs) frees up the annotations module name for the directory; annotations/mod.rs glob re-exports it so crate::annotations::* call sites are unchanged. - core/mod.rs keeps config/error/id/nebraska/trident/version nested (not flattened) since config.rs in particular has a large public surface. 152 unit tests + 1 doctest pass; clippy and fmt clean. --- .../src/{ => annotations}/k8s.rs | 2 +- .../trident-acl-agent/src/annotations/mod.rs | 15 + .../src/{ => annotations}/orchestrator.rs | 16 +- .../protocol.rs} | 6 +- .../src/{ => annotations}/state.rs | 0 .../src/{ => core}/config.rs | 0 .../trident-acl-agent/src/{ => core}/error.rs | 54 +- crates/trident-acl-agent/src/{ => core}/id.rs | 4 +- crates/trident-acl-agent/src/core/mod.rs | 12 + .../src/{ => core}/nebraska/README.md | 0 .../src/{ => core}/nebraska/client.rs | 0 .../src/{ => core}/nebraska/error.rs | 0 .../src/{ => core}/nebraska/event.rs | 0 .../src/{ => core}/nebraska/id.rs | 0 .../src/{ => core}/nebraska/mod.rs | 2 +- .../src/{ => core}/nebraska/status.rs | 0 .../src/{ => core}/nebraska/transport.rs | 0 .../src/{ => core}/nebraska/wire.rs | 0 .../{trident.rs => core/trident/client.rs} | 0 .../trident/mock.rs} | 2 +- .../trident-acl-agent/src/core/trident/mod.rs | 8 + .../src/{ => core}/version.rs | 2 +- crates/trident-acl-agent/src/lib.rs | 340 ++++----- crates/trident-acl-agent/src/main.rs | 11 +- crates/trident-acl-agent/src/omahaonly/mod.rs | 68 ++ docs/Explanation/Trident-ACL-Agent.md | 666 +++++++++--------- 26 files changed, 624 insertions(+), 584 deletions(-) rename crates/trident-acl-agent/src/{ => annotations}/k8s.rs (99%) create mode 100644 crates/trident-acl-agent/src/annotations/mod.rs rename crates/trident-acl-agent/src/{ => annotations}/orchestrator.rs (99%) rename crates/trident-acl-agent/src/{annotations.rs => annotations/protocol.rs} (99%) rename crates/trident-acl-agent/src/{ => annotations}/state.rs (100%) rename crates/trident-acl-agent/src/{ => core}/config.rs (100%) rename crates/trident-acl-agent/src/{ => core}/error.rs (89%) rename crates/trident-acl-agent/src/{ => core}/id.rs (91%) create mode 100644 crates/trident-acl-agent/src/core/mod.rs rename crates/trident-acl-agent/src/{ => core}/nebraska/README.md (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/client.rs (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/error.rs (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/event.rs (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/id.rs (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/mod.rs (97%) rename crates/trident-acl-agent/src/{ => core}/nebraska/status.rs (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/transport.rs (100%) rename crates/trident-acl-agent/src/{ => core}/nebraska/wire.rs (100%) rename crates/trident-acl-agent/src/{trident.rs => core/trident/client.rs} (100%) rename crates/trident-acl-agent/src/{mock_tridentd.rs => core/trident/mock.rs} (99%) create mode 100644 crates/trident-acl-agent/src/core/trident/mod.rs rename crates/trident-acl-agent/src/{ => core}/version.rs (99%) create mode 100644 crates/trident-acl-agent/src/omahaonly/mod.rs diff --git a/crates/trident-acl-agent/src/k8s.rs b/crates/trident-acl-agent/src/annotations/k8s.rs similarity index 99% rename from crates/trident-acl-agent/src/k8s.rs rename to crates/trident-acl-agent/src/annotations/k8s.rs index 8d6340eb00..3324405875 100644 --- a/crates/trident-acl-agent/src/k8s.rs +++ b/crates/trident-acl-agent/src/annotations/k8s.rs @@ -30,7 +30,7 @@ use kube::{ }; use serde_json::json; -use crate::config::KubernetesConfig; +use crate::core::config::KubernetesConfig; /// Floor for the Kubernetes watch request's `timeoutSeconds`, decoupled from /// `poll_interval` (see [`NodeClient::watch_node`]). Sits comfortably under diff --git a/crates/trident-acl-agent/src/annotations/mod.rs b/crates/trident-acl-agent/src/annotations/mod.rs new file mode 100644 index 0000000000..666e15af6e --- /dev/null +++ b/crates/trident-acl-agent/src/annotations/mod.rs @@ -0,0 +1,15 @@ +//! The Kubernetes annotation-driven update protocol - the crate's default +//! mode (see [`crate::core::config::GoalSource`]). +//! +//! [`protocol`] defines the annotation schema (request/status types, keys, +//! schema version); [`k8s`] is the Kubernetes Node get/watch/patch client; +//! [`state`] persists in-flight/completed operations across the reboot that +//! finalize triggers; [`orchestrator`] is the reconcile loop tying them all +//! together. + +mod protocol; +pub use protocol::*; + +pub mod k8s; +pub mod orchestrator; +pub mod state; diff --git a/crates/trident-acl-agent/src/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs similarity index 99% rename from crates/trident-acl-agent/src/orchestrator.rs rename to crates/trident-acl-agent/src/annotations/orchestrator.rs index 0074b4a43e..e5c528b439 100644 --- a/crates/trident-acl-agent/src/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -24,15 +24,17 @@ use osutils::dependencies::Dependency; use crate::{ annotations::{ + k8s::{K8sClientError, NodeClient}, + state::{PendingCommit, StateStore}, AnnotationKeys, Operation, RequestedOperation, StatusCode, UpdateRequest, UpdateStatus, SCHEMA_VERSION, }, - config::AgentConfig, - k8s::{K8sClientError, NodeClient}, - nebraska::{CheckOutcome, Client as NebraskaClient, ProgressEvent}, - state::{PendingCommit, StateStore}, - trident::{CompletedResponse, TridentClient, TridentClientError}, - version::current_active_version, + core::{ + config::AgentConfig, + nebraska::{CheckOutcome, Client as NebraskaClient, ProgressEvent}, + trident::{CompletedResponse, TridentClient, TridentClientError}, + version::current_active_version, + }, IdSource, }; @@ -1569,7 +1571,7 @@ mod tests { use super::*; use crate::{ annotations::{RequestedOperation, SCHEMA_VERSION}, - mock_tridentd::{connect_mock_client, MockTridentdConfig, Outcome}, + core::trident::mock::{connect_mock_client, MockTridentdConfig, Outcome}, }; fn request(operation: RequestedOperation) -> UpdateRequest { diff --git a/crates/trident-acl-agent/src/annotations.rs b/crates/trident-acl-agent/src/annotations/protocol.rs similarity index 99% rename from crates/trident-acl-agent/src/annotations.rs rename to crates/trident-acl-agent/src/annotations/protocol.rs index 9dc95fb71f..3f9ffb87a5 100644 --- a/crates/trident-acl-agent/src/annotations.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -6,7 +6,7 @@ //! `/update-commit-status` node annotation protocol described //! by the current accepted design (), where //! `` defaults to `acl.microsoft.com` (see -//! [`AnnotationKeys`]/[`crate::config::DEFAULT_ANNOTATION_PREFIX`]) and is +//! [`AnnotationKeys`]/[`crate::core::config::DEFAULT_ANNOTATION_PREFIX`]) and is //! overridable via the `TRIDENT_ACL_AGENT_ANNOTATION_PREFIX` environment //! variable. Keep `UpdateRequest`/`UpdateStatus`/`StatusCode` and //! `validate()` in sync with that document's formal JSON Schema (its @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use uuid::Uuid; -use crate::config::DEFAULT_ANNOTATION_PREFIX; +use crate::core::config::DEFAULT_ANNOTATION_PREFIX; /// Suffix (appended to the configured annotation prefix) for the request /// annotation, e.g. `acl.microsoft.com/update-request`. @@ -33,7 +33,7 @@ pub const UPDATE_STATUS_SUFFIX: &str = "update-status"; pub const UPDATE_COMMIT_STATUS_SUFFIX: &str = "update-commit-status"; /// The full annotation keys for one deployment's configured annotation -/// prefix. Built once from [`crate::config::KubernetesConfig::annotation_prefix`] +/// prefix. Built once from [`crate::core::config::KubernetesConfig::annotation_prefix`] /// and threaded through instead of hardcoding a fixed /// `acl.microsoft.com` prefix. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/trident-acl-agent/src/state.rs b/crates/trident-acl-agent/src/annotations/state.rs similarity index 100% rename from crates/trident-acl-agent/src/state.rs rename to crates/trident-acl-agent/src/annotations/state.rs diff --git a/crates/trident-acl-agent/src/config.rs b/crates/trident-acl-agent/src/core/config.rs similarity index 100% rename from crates/trident-acl-agent/src/config.rs rename to crates/trident-acl-agent/src/core/config.rs diff --git a/crates/trident-acl-agent/src/error.rs b/crates/trident-acl-agent/src/core/error.rs similarity index 89% rename from crates/trident-acl-agent/src/error.rs rename to crates/trident-acl-agent/src/core/error.rs index 3eb9ba3c65..e79f54f842 100644 --- a/crates/trident-acl-agent/src/error.rs +++ b/crates/trident-acl-agent/src/core/error.rs @@ -1,27 +1,27 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "kebab-case")] -pub enum AgentError { - #[error("Failed to initialize the trident-acl-agent client: {0}")] - InitializationError(String), - - #[error("The version provided '{version}' is not valid semver: {inner}")] - InvalidVersion { version: String, inner: String }, - - #[error("Failed to read machine-id: {0}")] - MachineIdRead(String), - - #[error("Failed to read hostname: {0}")] - HostnameRead(String), - - #[error("Internal error: {0}")] - Internal(String), - - /// Wraps a [`nebraska::NebraskaError`](crate::nebraska::NebraskaError). - /// Stored as a string rather than `#[from]` because `NebraskaError` - /// doesn't derive `Serialize`/`Deserialize`/`PartialEq`, which - /// `AgentError` requires for annotation-status round-tripping. - #[error("Nebraska request failed: {0}")] - Nebraska(String), -} +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum AgentError { + #[error("Failed to initialize the trident-acl-agent client: {0}")] + InitializationError(String), + + #[error("The version provided '{version}' is not valid semver: {inner}")] + InvalidVersion { version: String, inner: String }, + + #[error("Failed to read machine-id: {0}")] + MachineIdRead(String), + + #[error("Failed to read hostname: {0}")] + HostnameRead(String), + + #[error("Internal error: {0}")] + Internal(String), + + /// Wraps a [`nebraska::NebraskaError`](crate::core::nebraska::NebraskaError). + /// Stored as a string rather than `#[from]` because `NebraskaError` + /// doesn't derive `Serialize`/`Deserialize`/`PartialEq`, which + /// `AgentError` requires for annotation-status round-tripping. + #[error("Nebraska request failed: {0}")] + Nebraska(String), +} diff --git a/crates/trident-acl-agent/src/id.rs b/crates/trident-acl-agent/src/core/id.rs similarity index 91% rename from crates/trident-acl-agent/src/id.rs rename to crates/trident-acl-agent/src/core/id.rs index 3a9027f84d..f9c971fc92 100644 --- a/crates/trident-acl-agent/src/id.rs +++ b/crates/trident-acl-agent/src/core/id.rs @@ -2,7 +2,7 @@ use std::fmt::Display; use osutils::{hostname, machine_id::MachineId}; -use crate::error::AgentError; +use crate::core::error::AgentError; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum IdSource { @@ -22,7 +22,7 @@ impl Display for IdSource { } impl IdSource { - pub(super) fn produce_id(&self) -> Result { + pub(crate) fn produce_id(&self) -> Result { Ok(match self { IdSource::MachineIdHashed => MachineId::read() .map_err(|err| AgentError::MachineIdRead(err.to_string()))? diff --git a/crates/trident-acl-agent/src/core/mod.rs b/crates/trident-acl-agent/src/core/mod.rs new file mode 100644 index 0000000000..21c4bd0402 --- /dev/null +++ b/crates/trident-acl-agent/src/core/mod.rs @@ -0,0 +1,12 @@ +//! Shared building blocks used by both the annotation-driven protocol +//! ([`crate::annotations`]) and the legacy one-shot Omaha flow +//! ([`crate::omahaonly`]): configuration, error types, machine-id resolution, +//! the current-version fallback, the `tridentd` gRPC client, and the generic +//! Nebraska/Omaha protocol client. + +pub mod config; +pub mod error; +pub mod id; +pub mod nebraska; +pub mod trident; +pub mod version; diff --git a/crates/trident-acl-agent/src/nebraska/README.md b/crates/trident-acl-agent/src/core/nebraska/README.md similarity index 100% rename from crates/trident-acl-agent/src/nebraska/README.md rename to crates/trident-acl-agent/src/core/nebraska/README.md diff --git a/crates/trident-acl-agent/src/nebraska/client.rs b/crates/trident-acl-agent/src/core/nebraska/client.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/client.rs rename to crates/trident-acl-agent/src/core/nebraska/client.rs diff --git a/crates/trident-acl-agent/src/nebraska/error.rs b/crates/trident-acl-agent/src/core/nebraska/error.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/error.rs rename to crates/trident-acl-agent/src/core/nebraska/error.rs diff --git a/crates/trident-acl-agent/src/nebraska/event.rs b/crates/trident-acl-agent/src/core/nebraska/event.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/event.rs rename to crates/trident-acl-agent/src/core/nebraska/event.rs diff --git a/crates/trident-acl-agent/src/nebraska/id.rs b/crates/trident-acl-agent/src/core/nebraska/id.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/id.rs rename to crates/trident-acl-agent/src/core/nebraska/id.rs diff --git a/crates/trident-acl-agent/src/nebraska/mod.rs b/crates/trident-acl-agent/src/core/nebraska/mod.rs similarity index 97% rename from crates/trident-acl-agent/src/nebraska/mod.rs rename to crates/trident-acl-agent/src/core/nebraska/mod.rs index f0586a22a1..d2ed9f0323 100644 --- a/crates/trident-acl-agent/src/nebraska/mod.rs +++ b/crates/trident-acl-agent/src/core/nebraska/mod.rs @@ -48,7 +48,7 @@ //! ```no_run //! use semver::Version; //! use url::Url; -//! use trident_acl_agent::nebraska::{Client, CheckOutcome, MachineId, ProgressEvent}; +//! use trident_acl_agent::core::nebraska::{Client, CheckOutcome, MachineId, ProgressEvent}; //! //! # fn example() -> Result<(), Box> { //! let client = Client::new( diff --git a/crates/trident-acl-agent/src/nebraska/status.rs b/crates/trident-acl-agent/src/core/nebraska/status.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/status.rs rename to crates/trident-acl-agent/src/core/nebraska/status.rs diff --git a/crates/trident-acl-agent/src/nebraska/transport.rs b/crates/trident-acl-agent/src/core/nebraska/transport.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/transport.rs rename to crates/trident-acl-agent/src/core/nebraska/transport.rs diff --git a/crates/trident-acl-agent/src/nebraska/wire.rs b/crates/trident-acl-agent/src/core/nebraska/wire.rs similarity index 100% rename from crates/trident-acl-agent/src/nebraska/wire.rs rename to crates/trident-acl-agent/src/core/nebraska/wire.rs diff --git a/crates/trident-acl-agent/src/trident.rs b/crates/trident-acl-agent/src/core/trident/client.rs similarity index 100% rename from crates/trident-acl-agent/src/trident.rs rename to crates/trident-acl-agent/src/core/trident/client.rs diff --git a/crates/trident-acl-agent/src/mock_tridentd.rs b/crates/trident-acl-agent/src/core/trident/mock.rs similarity index 99% rename from crates/trident-acl-agent/src/mock_tridentd.rs rename to crates/trident-acl-agent/src/core/trident/mock.rs index d0e60670f0..5052328418 100644 --- a/crates/trident-acl-agent/src/mock_tridentd.rs +++ b/crates/trident-acl-agent/src/core/trident/mock.rs @@ -26,7 +26,7 @@ use trident_proto::v1::{ StatusCode as ProtoStatusCode, TridentError, UpdateRequest, }; -use crate::trident::TridentClient; +use crate::core::trident::TridentClient; /// Canned outcome a `MockTridentd` should return for a given RPC call. #[derive(Clone, Debug)] diff --git a/crates/trident-acl-agent/src/core/trident/mod.rs b/crates/trident-acl-agent/src/core/trident/mod.rs new file mode 100644 index 0000000000..15edaef560 --- /dev/null +++ b/crates/trident-acl-agent/src/core/trident/mod.rs @@ -0,0 +1,8 @@ +//! gRPC client for talking to `tridentd`, plus its in-process mock used only +//! by unit tests. + +mod client; +pub use client::*; + +#[cfg(test)] +pub mod mock; diff --git a/crates/trident-acl-agent/src/version.rs b/crates/trident-acl-agent/src/core/version.rs similarity index 99% rename from crates/trident-acl-agent/src/version.rs rename to crates/trident-acl-agent/src/core/version.rs index 95321dee37..5e30777128 100644 --- a/crates/trident-acl-agent/src/version.rs +++ b/crates/trident-acl-agent/src/core/version.rs @@ -2,7 +2,7 @@ //! a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`) //! and for reporting the instance's current version to Nebraska. //! -//! Split out of [`crate::annotations`] because it has nothing to do with the +//! Lives in `core`, not `annotations`, because it has nothing to do with the //! annotation wire format - it's plain env-var-configurable file probing, //! used by both the annotation-driven orchestrator and the `omaha-only` //! one-shot mode. diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 8a71a8208c..1757f8a616 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -1,203 +1,137 @@ -//! # trident-acl-agent -//! -//! trident-acl-agent is Trident's ACL update sidecar. Historically it was a -//! one-shot Omaha client that called Trident's combined `Update()` RPC once -//! and exited. This crate now defaults to the Kubernetes annotation protocol -//! described in the accepted design -//! (), -//! while preserving the original `omaha-only` mode as an explicit opt-out -//! (see `config::GoalSource`). -//! -//! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) -//! goes through the [`nebraska`] client module, a self-contained, reusable -//! implementation of the Nebraska/Omaha update protocol. It is usable both by -//! this crate's agent binary and by a future Trident ACL Agent that -//! orchestrates updates differently. - -use anyhow::Context; -use semver::Version; - -pub mod annotations; -pub mod config; -pub mod error; -pub mod id; -pub mod k8s; -pub mod nebraska; -pub mod version; - -/// The version this agent reports to Nebraska as the updater's own version, for -/// [`nebraska::Client::new`]. -/// -/// Prefers the build-time `TRIDENT_VERSION` (the version the shipped product is -/// stamped with) over this crate's package version, which is not released -/// independently and is a placeholder. Nebraska itself ignores the value, so -/// this is for whoever reads the raw requests. It lives here, not in -/// [`nebraska`], because that module is a generic Omaha client: which product is -/// doing the updating is the caller's business. -pub const AGENT_VERSION: &str = match option_env!("TRIDENT_VERSION") { - Some(version) => version, - None => env!("CARGO_PKG_VERSION"), -}; - -pub mod orchestrator; -pub mod state; -pub mod trident; - -/// Only built for `cargo test` (relies on trident-proto's `server` feature, -/// which is only enabled via trident-acl-agent's dev-dependencies - see -/// mock_tridentd.rs's module docs). -#[cfg(test)] -pub mod mock_tridentd; - -use error::AgentError; -use nebraska::{CheckOutcome, Client, MachineId, NebraskaError}; -use trident::TridentClient; - -pub use id::IdSource; - -// Deliberately invalid sentinels, mirroring DEFAULT_NEBRASKA_ENDPOINT's -// `.invalid` domain trick: a deployment that forgets to configure (or -// override via the update-request annotation's `appId`/`track` fields) a -// real app_id/track fails loudly against Nebraska instead of silently -// querying a real-looking but wrong app/group. -pub const DEFAULT_NEBRASKA_APP_ID: &str = "00000000-0000-0000-0000-000000000000"; -pub const DEFAULT_NEBRASKA_TRACK: &str = "unspecified"; - -/// Builds a validated [`MachineId`] from an [`IdSource`], translating the -/// crate's own machine-id/hostname read errors into a single [`AgentError`]. -fn build_machine_id(source: IdSource) -> Result { - MachineId::new(source.produce_id()?).map_err(|err| AgentError::Nebraska(err.to_string())) -} - -/// Historical one-shot flow: query the Nebraska/Omaha server at -/// `config.nebraska.endpoint` once, and if an update is offered, call -/// tridentd's combined `Update()` RPC once and exit. No Kubernetes/annotation -/// involvement. -pub async fn run_omaha_only(config: &config::AgentConfig) -> Result<(), anyhow::Error> { - let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { - anyhow::anyhow!("no Nebraska endpoint configured: set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT") - })?; - - // Client::check_for_update() is a blocking call (reqwest::blocking under - // the hood, see nebraska::transport) - calling it directly from this - // async fn can panic ("Cannot drop a runtime in a context where blocking - // is not allowed") because reqwest::blocking spins up its own inner - // Tokio runtime per call, which isn't safe to tear down from inside an - // already-running async task. Run it on a dedicated blocking thread. - let app_id = config.nebraska.app_id.clone(); - let track = config.nebraska.track.clone(); - let machine_id = build_machine_id(IdSource::MachineIdHashed)?; - let current_version_raw = version::current_active_version()?; - let current_version = Version::parse(¤t_version_raw).unwrap_or_else(|err| { - log::warn!( - "current version {current_version_raw:?} is not valid semver ({err}); reporting 0.0.0 to Nebraska" - ); - Version::new(0, 0, 0) - }); - let outcome = tokio::task::spawn_blocking(move || { - let client = Client::new(endpoint, app_id, track, machine_id); - client.check_for_update(¤t_version) - }) - .await - .context("Nebraska query task panicked")? - .map_err(|err| anyhow::anyhow!("Nebraska query failed: {err}"))?; - - match outcome { - CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { - log::debug!("No update available from Nebraska"); - Ok(()) - } - CheckOutcome::UpdateAvailable(offer) => { - log::info!("Triggering one-shot Omaha update to {}", offer.version); - let mut client = TridentClient::connect(&config.trident.socket).await?; - let combined_timeout = - config.orchestration.stage_timeout + config.orchestration.finalize_timeout; - // Integrity of the downloaded image is verified by Trident itself - // via the image's own COSI metadata, so the Nebraska-reported hash - // (offer.primary.hash) is not passed here. - client - .update(&offer.primary.url, None, combined_timeout) - .await?; - Ok(()) - } - } -} - -/// Checks that the Omaha/Nebraska server at `url` is reachable and speaking -/// the Omaha protocol, without treating any app-level result (including a -/// non-OK app/update-check status) as a failure. Unlike -/// [`Client::check_for_update`], this only fails on network/transport -/// problems or a response that isn't well-formed Omaha XML -- it's meant for -/// a pure "can we talk to this server at all" check (e.g. -/// `--validate-connection nebraska`), not for deciding whether an update is -/// available. -pub fn check_nebraska_reachable( - url: &url::Url, - app_id: &str, - track: &str, - machine_id_source: IdSource, -) -> Result<(), AgentError> { - let machine_id = build_machine_id(machine_id_source)?; - let client = Client::new(url.clone(), app_id, track, machine_id); - match client.check_for_update(&Version::new(0, 0, 0)) { - Ok(_) => Ok(()), - // A well-formed response reporting a non-OK app/update-check status - // still proves the server is reachable and speaking Omaha; only a - // transport/parse-level failure means it is not. - Err(NebraskaError::ServerError(_)) => Ok(()), - Err(err) => Err(AgentError::Nebraska(err.to_string())), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_check_nebraska_reachable_succeeds_on_error_app_status() { - // check_nebraska_reachable() is meant to be a pure "can we reach this - // server and does it speak Omaha" check, unlike check_for_update() - // which also validates app-level semantics. A well-formed response - // with a non-OK app status should still count as "reachable" here, - // even though check_for_update() would reject the same response as a - // NebraskaError::ServerError. - let mut server = mockito::Server::new(); - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .with_body(indoc::indoc! {r#" - - - - - - - "#}) - .expect(1) - .create(); - - check_nebraska_reachable( - &url::Url::parse(&server.url()).unwrap(), - "test", - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - - omaha_mock.assert(); - } - - #[test] - fn test_check_nebraska_reachable_fails_on_transport_error() { - let err = check_nebraska_reachable( - // Port 0 never accepts a connection. - &url::Url::parse("http://127.0.0.1:0/").unwrap(), - "test", - "track", - IdSource::MachineIdHashed, - ) - .unwrap_err(); - assert!(matches!(err, AgentError::Nebraska(_))); - } -} +//! # trident-acl-agent +//! +//! trident-acl-agent is Trident's ACL update sidecar. Historically it was a +//! one-shot Omaha client that called Trident's combined `Update()` RPC once +//! and exited. This crate now defaults to the Kubernetes annotation protocol +//! described in the accepted design +//! (), +//! while preserving the original `omaha-only` mode as an explicit opt-out +//! (see `core::config::GoalSource`). +//! +//! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) +//! goes through the [`core::nebraska`] client module, a self-contained, +//! reusable implementation of the Nebraska/Omaha update protocol. It is usable +//! both by this crate's agent binary and by a future Trident ACL Agent that +//! orchestrates updates differently. +//! +//! - [`core`]: building blocks shared by both modes (config, errors, +//! machine-id, current-version, the `tridentd` client, the Nebraska +//! client). +//! - [`annotations`]: the default Kubernetes annotation-driven protocol. +//! - [`omahaonly`]: the legacy one-shot Omaha flow. + +pub mod annotations; +pub mod core; +pub mod omahaonly; + +/// The version this agent reports to Nebraska as the updater's own version, for +/// [`core::nebraska::Client::new`]. +/// +/// Prefers the build-time `TRIDENT_VERSION` (the version the shipped product is +/// stamped with) over this crate's package version, which is not released +/// independently and is a placeholder. Nebraska itself ignores the value, so +/// this is for whoever reads the raw requests. It lives here, not in +/// [`core::nebraska`], because that module is a generic Omaha client: which +/// product is doing the updating is the caller's business. +pub const AGENT_VERSION: &str = match option_env!("TRIDENT_VERSION") { + Some(version) => version, + None => env!("CARGO_PKG_VERSION"), +}; + +use crate::core::error::AgentError; +use crate::core::nebraska::{Client, MachineId, NebraskaError}; + +pub use crate::core::id::IdSource; + +// Deliberately invalid sentinels, mirroring DEFAULT_NEBRASKA_ENDPOINT's +// `.invalid` domain trick: a deployment that forgets to configure (or +// override via the update-request annotation's `appId`/`track` fields) a +// real app_id/track fails loudly against Nebraska instead of silently +// querying a real-looking but wrong app/group. +pub const DEFAULT_NEBRASKA_APP_ID: &str = "00000000-0000-0000-0000-000000000000"; +pub const DEFAULT_NEBRASKA_TRACK: &str = "unspecified"; + +/// Builds a validated [`MachineId`] from an [`IdSource`], translating the +/// crate's own machine-id/hostname read errors into a single [`AgentError`]. +fn build_machine_id(source: IdSource) -> Result { + MachineId::new(source.produce_id()?).map_err(|err| AgentError::Nebraska(err.to_string())) +} + +/// Checks that the Omaha/Nebraska server at `url` is reachable and speaking +/// the Omaha protocol, without treating any app-level result (including a +/// non-OK app/update-check status) as a failure. Unlike +/// [`Client::check_for_update`], this only fails on network/transport +/// problems or a response that isn't well-formed Omaha XML -- it's meant for +/// a pure "can we talk to this server at all" check (e.g. +/// `--validate-connection nebraska`), not for deciding whether an update is +/// available. +pub fn check_nebraska_reachable( + url: &url::Url, + app_id: &str, + track: &str, + machine_id_source: IdSource, +) -> Result<(), AgentError> { + let machine_id = build_machine_id(machine_id_source)?; + let client = Client::new(url.clone(), app_id, track, machine_id); + match client.check_for_update(&semver::Version::new(0, 0, 0)) { + Ok(_) => Ok(()), + // A well-formed response reporting a non-OK app/update-check status + // still proves the server is reachable and speaking Omaha; only a + // transport/parse-level failure means it is not. + Err(NebraskaError::ServerError(_)) => Ok(()), + Err(err) => Err(AgentError::Nebraska(err.to_string())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_check_nebraska_reachable_succeeds_on_error_app_status() { + // check_nebraska_reachable() is meant to be a pure "can we reach this + // server and does it speak Omaha" check, unlike check_for_update() + // which also validates app-level semantics. A well-formed response + // with a non-OK app status should still count as "reachable" here, + // even though check_for_update() would reject the same response as a + // NebraskaError::ServerError. + let mut server = mockito::Server::new(); + + let omaha_mock = server + .mock("POST", "/") + .with_status(200) + .with_body(indoc::indoc! {r#" + + + + + + + "#}) + .expect(1) + .create(); + + check_nebraska_reachable( + &url::Url::parse(&server.url()).unwrap(), + "test", + "track", + IdSource::MachineIdHashed, + ) + .unwrap(); + + omaha_mock.assert(); + } + + #[test] + fn test_check_nebraska_reachable_fails_on_transport_error() { + let err = check_nebraska_reachable( + // Port 0 never accepts a connection. + &url::Url::parse("http://127.0.0.1:0/").unwrap(), + "test", + "track", + IdSource::MachineIdHashed, + ) + .unwrap_err(); + assert!(matches!(err, AgentError::Nebraska(_))); + } +} diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 3f6d837ba8..9ae29e0531 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -3,12 +3,13 @@ use clap::Parser; use log::{LevelFilter, Log, Metadata, Record}; use trident_acl_agent::{ + annotations::{k8s::NodeClient, orchestrator::Orchestrator}, check_nebraska_reachable, - config::{AgentConfig, GoalSource}, - k8s::NodeClient, - orchestrator::Orchestrator, - run_omaha_only, - trident::TridentClient, + core::{ + config::{AgentConfig, GoalSource}, + trident::TridentClient, + }, + omahaonly::run_omaha_only, IdSource, }; diff --git a/crates/trident-acl-agent/src/omahaonly/mod.rs b/crates/trident-acl-agent/src/omahaonly/mod.rs new file mode 100644 index 0000000000..fea7e1d3bc --- /dev/null +++ b/crates/trident-acl-agent/src/omahaonly/mod.rs @@ -0,0 +1,68 @@ +//! The historical one-shot Omaha flow, preserved as an explicit opt-out from +//! the default annotation-driven protocol (see +//! [`crate::core::config::GoalSource`]). + +use anyhow::Context; +use semver::Version; + +use crate::core::{ + config::AgentConfig, + nebraska::{CheckOutcome, Client}, + trident::TridentClient, + version, +}; +use crate::IdSource; + +/// Historical one-shot flow: query the Nebraska/Omaha server at +/// `config.nebraska.endpoint` once, and if an update is offered, call +/// tridentd's combined `Update()` RPC once and exit. No Kubernetes/annotation +/// involvement. +pub async fn run_omaha_only(config: &AgentConfig) -> Result<(), anyhow::Error> { + let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { + anyhow::anyhow!("no Nebraska endpoint configured: set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT") + })?; + + // Client::check_for_update() is a blocking call (reqwest::blocking under + // the hood, see nebraska::transport) - calling it directly from this + // async fn can panic ("Cannot drop a runtime in a context where blocking + // is not allowed") because reqwest::blocking spins up its own inner + // Tokio runtime per call, which isn't safe to tear down from inside an + // already-running async task. Run it on a dedicated blocking thread. + let app_id = config.nebraska.app_id.clone(); + let track = config.nebraska.track.clone(); + let machine_id = crate::build_machine_id(IdSource::MachineIdHashed)?; + let current_version_raw = version::current_active_version()?; + let current_version = Version::parse(¤t_version_raw).unwrap_or_else(|err| { + log::warn!( + "current version {current_version_raw:?} is not valid semver ({err}); reporting 0.0.0 to Nebraska" + ); + Version::new(0, 0, 0) + }); + let outcome = tokio::task::spawn_blocking(move || { + let client = Client::new(endpoint, app_id, track, machine_id); + client.check_for_update(¤t_version) + }) + .await + .context("Nebraska query task panicked")? + .map_err(|err| anyhow::anyhow!("Nebraska query failed: {err}"))?; + + match outcome { + CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { + log::debug!("No update available from Nebraska"); + Ok(()) + } + CheckOutcome::UpdateAvailable(offer) => { + log::info!("Triggering one-shot Omaha update to {}", offer.version); + let mut client = TridentClient::connect(&config.trident.socket).await?; + let combined_timeout = + config.orchestration.stage_timeout + config.orchestration.finalize_timeout; + // Integrity of the downloaded image is verified by Trident itself + // via the image's own COSI metadata, so the Nebraska-reported hash + // (offer.primary.hash) is not passed here. + client + .update(&offer.primary.url, None, combined_timeout) + .await?; + Ok(()) + } + } +} diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index 671362fd5a..c5ee4c13cd 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -1,333 +1,333 @@ -# Trident ACL Agent - -`trident-acl-agent` is an on-node daemon that drives Trident -[A/B updates](./AB-Update.md) from a Kubernetes control plane, using node -annotations instead of a direct API call as the trigger. It is the on-node -half of Azure Container Linux (ACL)'s update mechanism. Any Kubernetes -control-plane component (a custom controller, an operator, or a script -driven by `kubectl patch`) can -orchestrate updates across a fleet of nodes by writing to the annotation -contract described below, provided it is willing to speak the -[Omaha](https://github.com/omaha-consortium/omaha) protocol for image -distribution and honors the agent's per-node protocol. - -## Deployment - -`trident-acl-agent` ships as its own `trident-acl-agent` RPM subpackage -(built alongside, and `Requires:` the same version of, the main `trident` -package). Installing it: - -```console -$ tdnf install trident-acl-agent -``` - -lays down the `/usr/bin/trident-acl-agent` binary and its -`trident-acl-agent.service` unit -(`packaging/systemd/trident-acl-agent.service`) under the systemd unit -directory, along with the package's `%license`-installed `LICENSE`/`NOTICE` -files. Installing the package does not by itself enable or start the -service — a deployment decides when that happens, e.g. by running -`systemctl enable --now trident-acl-agent.service` on the node, or by -baking that enablement into the image build (as this repo's own -`updateimg-acl-agent.yaml` test image does via Image Customizer's -`services: enable` list). - -The shipped unit carries no `Environment=` lines beyond `ExecStart`, so -every deployment-specific choice — which annotation prefix to watch, where -to read the current version from, which Kubernetes API server to talk to, -and so on — is supplied the same way any other systemd service is -configured: standard `Environment=`/`EnvironmentFile=` constructs, most -commonly a drop-in applied on top of the packaged unit. See -[Configuration](#configuration) below for the full list of variables and -[Setting env vars via a systemd drop-in](#setting-env-vars-via-a-systemd-drop-in) -for how to apply them without editing the packaged unit. - -## The annotation contract - -An orchestrator (a Kubernetes controller with RBAC permission to PATCH the -target Node object) triggers an update by writing a JSON payload to a -request annotation on the Node. The agent watches that annotation, drives -the requested operation against `tridentd`, and writes its progress and -result via annotations on the same Node. - -Three annotation keys make up the contract, all sharing one configurable -prefix (`acl.microsoft.com` by default — see [Configuration](#configuration) -below): - -| Annotation | Written by | Purpose | -|---|---|---| -| `/update-request` | Orchestrator | Requests `stage`, `finalize`, or `rollback` for this node. | -| `/update-status` | Agent | Reports the status of the requested operation. | -| `/update-commit-status` | Agent | Reports the status of the implicit post-reboot `commit` that follows a `finalize` or `rollback`. | - -A request annotation looks like: - -```json -{ - "schemaVersion": "1.0", - "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", - "operationId": "c9d6f0a2-3b41-4e8d-9f27-1a5b6c7d8e90", - "operation": "stage", - "targetVersion": "202606.29.0", - "server": "https://nebraska.example.com/v1/update", - "appId": "11111111-2222-3333-4444-555555555555", - "track": "pin-202606.29.0" -} -``` - -- `nodeUpdateId` identifies one node's update sequence and is held constant - across `stage` → `finalize` → `commit`. -- `operationId` identifies this specific step; the agent uses it to decide - whether to start new work, resume in-flight work, or re-emit a cached - terminal status as a no-op on a duplicate PATCH. -- `targetVersion` is the image release version to update to. Required for - `stage`/`finalize`; omitted for `rollback`, whose target (the previous - partition) is implicit. -- `server`, `appId`, and `track` name the Omaha instance that serves the - target image and receives progress events for it. They are required on - `stage`/`finalize` requests, with **no static fallback** — a request - missing them is rejected with `InvalidRequest` rather than falling back to - a built-in endpoint, so a node can never update from a source the - orchestrator did not explicitly choose. - -`operation` maps to Trident invocations as follows: - -| `operation` | Trident invocation | Effect | -|---|---|---| -| `stage` | `trident update --allowed-operations=stage` | Queries the `server`/`appId`/`track` Omaha endpoint for `targetVersion`, then streams the resulting image to the inactive partition. No reboot. | -| `finalize` | `trident update --allowed-operations=finalize` (gRPC `UpdateFinalize`, caller-handled reboot) | Arms boot for the staged target, writes a terminal `finalize` status, then triggers the reboot. | -| `rollback` | `trident rollback --ab` (gRPC `RollbackStage`/`RollbackFinalize`, caller-handled reboot) | Swaps back to the previous partition, mirroring `finalize` on the return path. Only the last update can be undone this way. | - -A fourth phase, `commit`, runs implicitly after the post-`finalize`/ -`rollback` reboot: the agent runs `trident commit` on the new partition and -writes a `commit` status without needing a separate annotation request. The -orchestrator watches `/update-commit-status` as the terminal signal -that the reboot half of the update succeeded. - -`stage` end-to-end: - -```mermaid -sequenceDiagram - actor Orchestrator - participant API as K8s API Server - participant Agent as Trident ACL Agent
(on the node) - participant Nebraska as Omaha server - participant Trident - - Orchestrator->>API: 1. PATCH request: stage (opId A) - Note over Agent: agent picks up the request
on its next poll - Agent->>API: 2. read request, PATCH status: stage InProgress - Agent->>Nebraska: 3. query targetVersion (server/appId/track) - Nebraska-->>Agent: image location - Agent->>Trident: 4. Stage (image to inactive partition) - Trident-->>Agent: staged | error - Agent->>API: 5. PATCH status: stage Success | - API-->>Orchestrator: terminal stage code -``` - -`finalize` / `rollback`, spanning the reboot: - -```mermaid -sequenceDiagram - actor Orchestrator - participant API as K8s API Server - participant Agent as Trident ACL Agent
(on the node) - participant Trident - - Note over Orchestrator,Trident: pre-reboot half - Orchestrator->>API: 1. PATCH request: finalize (opId A) - Agent->>API: 2. read request, PATCH status: finalize InProgress - Agent->>Trident: 3. UpdateFinalize (caller-handled reboot) - Trident-->>Agent: boot armed, reboot required - Agent->>Agent: 4. persist pendingCommit + boot marker to state.json - Agent->>API: 5. PATCH status: finalize Success - API-->>Orchestrator: finalize Success (reboot pending) - Note over Agent,Trident: 6. agent triggers reboot, boots new partition - Note over Orchestrator,Trident: post-reboot half - Agent->>Agent: 7. read state.json, confirm a boot happened since the marker - Agent->>Trident: 8. Commit (validate volume, promote boot order) - Trident-->>Agent: committed | reverted to previous - Agent->>API: 9. PATCH status: commit Success | TargetBootFailed - API-->>Orchestrator: terminal commit code -``` - -A status annotation (`/update-status` or -`/update-commit-status`) looks like: - -```json -{ - "schemaVersion": "1.0", - "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", - "operationId": "c9d6f0a2-3b41-4e8d-9f27-1a5b6c7d8e90", - "operation": "stage", - "code": "Success", - "message": "staged update to 202606.29.0", - "fromVersion": "202606.15.0", - "toVersion": "202606.29.0", - "startedUtc": "2026-06-29T12:00:00Z", - "lastUpdatedUtc": "2026-06-29T12:03:41Z", - "finishedUtc": "2026-06-29T12:03:41Z" -} -``` - -- `operation` is `stage`, `finalize`, `rollback`, or `commit` (`commit` only - ever appears on `/update-commit-status`, never on - `/update-status`). -- `code` is the outcome — see the table below. -- `message` is a short, human-readable explanation of `code`, useful for - logs/alerts; treat its exact wording as informational, not something to - match on (it may include error detail that varies run to run). -- `fromVersion`/`toVersion` are the versions the operation moved between - (`toVersion` is absent for `rollback`, whose target is implicit). -- `startedUtc`/`lastUpdatedUtc`/`finishedUtc` bound the operation: - `lastUpdatedUtc` refreshes on a heartbeat cadence while `code` is - `InProgress` (see [below](#pre-post-reboot-state-and-the-watchdog)); - `finishedUtc` is absent until `code` reaches a terminal value. - -`code` is one of: - -| `code` | Terminal? | Meaning | -|---|---|---| -| `InProgress` | No | The operation is running. `lastUpdatedUtc` refreshes on a heartbeat cadence; a terminal code always follows. | -| `Success` | Yes | The operation completed as requested. For `commit`, this means the reboot landed on the target partition and it was promoted. | -| `AlreadyAtTarget` | Yes | `stage`/`finalize` was requested for the version the node is already running (per `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY`); treated as a no-op success. | -| `NotStaged` | Yes | `finalize` was requested for a `nodeUpdateId` with no prior successful `stage`. Issue a `stage` first. | -| `OperationFailed` | Yes | The operation failed for a reason other than a boot/rollback outcome (e.g. the Omaha server has no update for the requested version, or the underlying `tridentd` call returned an error). See `message` for detail. | -| `TargetBootFailed` | Yes | The post-reboot `commit` found the node had rolled back to its previous partition instead of booting the target — Trident's own health checks rejected the new boot. The node is back on `fromVersion`; the orchestrator should treat this as a failed update, not retry the same `nodeUpdateId` blindly. | -| `AgentInternalError` | Yes | A failure in the agent itself rather than in Trident or the requested operation (e.g. it triggered a reboot but the reboot call failed, or it lost track of an in-flight commit). Distinct from `OperationFailed` so an orchestrator can decide to treat these differently (e.g. retry vs. escalate). | -| `InvalidRequest` | Yes | The request annotation itself was rejected before any action was taken — malformed JSON, a schema/version mismatch, a missing required field (`server`/`appId`/`track`/`targetVersion`), a `finalize` whose `targetVersion` doesn't match what was staged, or a second `finalize`/`rollback` submitted while one is already pending its post-reboot `commit`. No Trident operation runs. | - -See the request/status schema types in -`crates/trident-acl-agent/src/annotations.rs` for the full contract, -including the formal JSON Schema both sides validate against. - -## Pre/post-reboot state and the watchdog - -Because `finalize`/`rollback` spans a reboot, the agent persists a small -state file (`TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH`) recording that a -commit is pending and a marker for "a boot happened after this point". On -restart, the agent checks this state to resume the post-reboot `commit` -step rather than re-running `finalize` from scratch. - -While an operation is in flight, the agent refreshes the `InProgress` -status's `lastUpdatedUtc` on a heartbeat cadence -(`TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL`), so an external -watchdog can distinguish a working agent from a stuck one and reprovision a -node that never reports a terminal `commit` status within its SLA. - -## Configuration - -There is no config file. Every setting is an environment variable prefixed -`TRIDENT_ACL_AGENT_`, systemd-style: set it in the unit's own -`Environment=` lines, via a drop-in override, or by any other means that -sets the process's environment before it starts. - -A variable that is unset, or set to the empty string, falls back to its -default. A variable set to a malformed value (a bad URL, a bad duration, an -unrecognized `goal_source`) causes the agent to fail to start with an error -naming the offending variable. - -| Variable | Default | Description | -|---|---|---| -| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.microsoft.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.microsoft.com` in `acl.microsoft.com/update-request`). Any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | The file the agent reads to determine the node's currently running version. Any file works, as long as it follows the os-release format (`KEY=VALUE` lines, optionally single- or double-quoted, blank lines and `#` comments ignored) — see [below](#configuring-the-on-disk-version). | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `VERSION_ID` is the standard `os-release` field most images already stamp; a deployment that instead carries an ACL-specific `IMAGE_VERSION` field can point this variable at that key instead — see [below](#configuring-the-on-disk-version). | -| `TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` | `always` | Controls what happens when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). `always` reports `0.0.0` as the node's current version — a sentinel that can never collide with a real release version and cause a false `AlreadyAtTarget`. `error` fails the operation instead of using a placeholder version. Any other value is used verbatim as the current version, with no format validation. | -| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in the kubeconfig is used as-is. | -| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig used to reach the Kubernetes API server and authenticate as this node. | -| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. | -| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file bridging the pre-reboot and post-reboot halves of `finalize`/`rollback` across the reboot. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` is allowed to run (a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) before it's considered failed. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` is allowed to run before it's considered failed. | -| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat. | - -### Setting env vars via a systemd drop-in - -The agent ships as `trident-acl-agent.service`, with no `Environment=` -lines of its own beyond `ExecStart`. Any setting is overridden with a -drop-in file, without editing the packaged unit: - -```console -$ sudo systemctl edit trident-acl-agent.service -``` - -This opens `/etc/systemd/system/trident-acl-agent.service.d/override.conf` -in an editor. For example, to point the agent at a custom annotation -namespace: - -```ini -[Service] -Environment=TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX=acl.contoso.com -``` - -With the prefix above, the orchestrator now reads/writes -`acl.contoso.com/update-request`, `acl.contoso.com/update-status`, and -`acl.contoso.com/update-commit-status` instead of the `acl.microsoft.com/*` -defaults. Reload and restart to apply: - -```console -$ sudo systemctl daemon-reload -$ sudo systemctl restart trident-acl-agent.service -``` - -`systemctl cat trident-acl-agent.service` shows the merged unit (packaged -unit plus drop-in), useful for confirming the override took effect. - -### Configuring the on-disk version - -The agent determines the node's current version by reading a key out of -`TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default), -defaulting to the key `VERSION_ID` — the standard -[`os-release`](https://www.freedesktop.org/software/systemd/man/latest/os-release.html) -field most distributions already stamp. A deployment that keeps its -version stamp under a different key, a different file entirely, or both, -can point the agent there instead, as long as that file follows the -`os-release` key-value schema (`KEY=VALUE` lines, optionally quoted, blank -lines and `#` comments ignored): - -```ini -[Service] -Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH=/etc/my-app-release -Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=BUILD_VERSION -``` - -With this set, the agent reads `BUILD_VERSION` from `/etc/my-app-release` -(e.g. `BUILD_VERSION=202606.29.0`) as the node's current version, and -compares it against a request's `targetVersion` the same way it would for -`VERSION_ID`/`/etc/os-release` — including short-circuiting to -`AlreadyAtTarget` when they already match. - -If the configured key is absent from the configured file (for example, on -a dev/test host with a minimal `os-release`), the agent consults -`TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` (`always` by default): `always` -reports `0.0.0` as the current version — a sentinel that can never -accidentally match a real requested version; `error` fails the operation -instead of guessing; any other value is used verbatim as the current -version, unvalidated. - -## Diagnostics - -`trident-acl-agent --validate-connection ` -checks connectivity to a single dependency using the current environment -and exits immediately — useful for a systemd `ExecStartPre` check or manual -on-node troubleshooting without running the full orchestrator loop. - -`--validate-connection nebraska` is the one place -`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT`, `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`, -and `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` are used: it issues a real -update-check query against the configured endpoint/app id/track and reports -whether the Omaha server is reachable. They default to deliberately invalid -values (`https://nebraska.example.invalid/v1/update`, an all-zero UUID, and -`unspecified`, respectively) so this check fails loudly unless a deployment -sets them. Since these variables otherwise play no role in the -annotation-driven flow, there's no reason to add them to the service's -persistent environment (e.g. via a drop-in) — set them just for this -one-off invocation instead: - -```console -$ sudo TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update \ - TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee \ - TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable \ - trident-acl-agent --validate-connection nebraska -``` +# Trident ACL Agent + +`trident-acl-agent` is an on-node daemon that drives Trident +[A/B updates](./AB-Update.md) from a Kubernetes control plane, using node +annotations instead of a direct API call as the trigger. It is the on-node +half of Azure Container Linux (ACL)'s update mechanism. Any Kubernetes +control-plane component (a custom controller, an operator, or a script +driven by `kubectl patch`) can +orchestrate updates across a fleet of nodes by writing to the annotation +contract described below, provided it is willing to speak the +[Omaha](https://github.com/omaha-consortium/omaha) protocol for image +distribution and honors the agent's per-node protocol. + +## Deployment + +`trident-acl-agent` ships as its own `trident-acl-agent` RPM subpackage +(built alongside, and `Requires:` the same version of, the main `trident` +package). Installing it: + +```console +$ tdnf install trident-acl-agent +``` + +lays down the `/usr/bin/trident-acl-agent` binary and its +`trident-acl-agent.service` unit +(`packaging/systemd/trident-acl-agent.service`) under the systemd unit +directory, along with the package's `%license`-installed `LICENSE`/`NOTICE` +files. Installing the package does not by itself enable or start the +service — a deployment decides when that happens, e.g. by running +`systemctl enable --now trident-acl-agent.service` on the node, or by +baking that enablement into the image build (as this repo's own +`updateimg-acl-agent.yaml` test image does via Image Customizer's +`services: enable` list). + +The shipped unit carries no `Environment=` lines beyond `ExecStart`, so +every deployment-specific choice — which annotation prefix to watch, where +to read the current version from, which Kubernetes API server to talk to, +and so on — is supplied the same way any other systemd service is +configured: standard `Environment=`/`EnvironmentFile=` constructs, most +commonly a drop-in applied on top of the packaged unit. See +[Configuration](#configuration) below for the full list of variables and +[Setting env vars via a systemd drop-in](#setting-env-vars-via-a-systemd-drop-in) +for how to apply them without editing the packaged unit. + +## The annotation contract + +An orchestrator (a Kubernetes controller with RBAC permission to PATCH the +target Node object) triggers an update by writing a JSON payload to a +request annotation on the Node. The agent watches that annotation, drives +the requested operation against `tridentd`, and writes its progress and +result via annotations on the same Node. + +Three annotation keys make up the contract, all sharing one configurable +prefix (`acl.microsoft.com` by default — see [Configuration](#configuration) +below): + +| Annotation | Written by | Purpose | +|---|---|---| +| `/update-request` | Orchestrator | Requests `stage`, `finalize`, or `rollback` for this node. | +| `/update-status` | Agent | Reports the status of the requested operation. | +| `/update-commit-status` | Agent | Reports the status of the implicit post-reboot `commit` that follows a `finalize` or `rollback`. | + +A request annotation looks like: + +```json +{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "c9d6f0a2-3b41-4e8d-9f27-1a5b6c7d8e90", + "operation": "stage", + "targetVersion": "202606.29.0", + "server": "https://nebraska.example.com/v1/update", + "appId": "11111111-2222-3333-4444-555555555555", + "track": "pin-202606.29.0" +} +``` + +- `nodeUpdateId` identifies one node's update sequence and is held constant + across `stage` → `finalize` → `commit`. +- `operationId` identifies this specific step; the agent uses it to decide + whether to start new work, resume in-flight work, or re-emit a cached + terminal status as a no-op on a duplicate PATCH. +- `targetVersion` is the image release version to update to. Required for + `stage`/`finalize`; omitted for `rollback`, whose target (the previous + partition) is implicit. +- `server`, `appId`, and `track` name the Omaha instance that serves the + target image and receives progress events for it. They are required on + `stage`/`finalize` requests, with **no static fallback** — a request + missing them is rejected with `InvalidRequest` rather than falling back to + a built-in endpoint, so a node can never update from a source the + orchestrator did not explicitly choose. + +`operation` maps to Trident invocations as follows: + +| `operation` | Trident invocation | Effect | +|---|---|---| +| `stage` | `trident update --allowed-operations=stage` | Queries the `server`/`appId`/`track` Omaha endpoint for `targetVersion`, then streams the resulting image to the inactive partition. No reboot. | +| `finalize` | `trident update --allowed-operations=finalize` (gRPC `UpdateFinalize`, caller-handled reboot) | Arms boot for the staged target, writes a terminal `finalize` status, then triggers the reboot. | +| `rollback` | `trident rollback --ab` (gRPC `RollbackStage`/`RollbackFinalize`, caller-handled reboot) | Swaps back to the previous partition, mirroring `finalize` on the return path. Only the last update can be undone this way. | + +A fourth phase, `commit`, runs implicitly after the post-`finalize`/ +`rollback` reboot: the agent runs `trident commit` on the new partition and +writes a `commit` status without needing a separate annotation request. The +orchestrator watches `/update-commit-status` as the terminal signal +that the reboot half of the update succeeded. + +`stage` end-to-end: + +```mermaid +sequenceDiagram + actor Orchestrator + participant API as K8s API Server + participant Agent as Trident ACL Agent
(on the node) + participant Nebraska as Omaha server + participant Trident + + Orchestrator->>API: 1. PATCH request: stage (opId A) + Note over Agent: agent picks up the request
on its next poll + Agent->>API: 2. read request, PATCH status: stage InProgress + Agent->>Nebraska: 3. query targetVersion (server/appId/track) + Nebraska-->>Agent: image location + Agent->>Trident: 4. Stage (image to inactive partition) + Trident-->>Agent: staged | error + Agent->>API: 5. PATCH status: stage Success | + API-->>Orchestrator: terminal stage code +``` + +`finalize` / `rollback`, spanning the reboot: + +```mermaid +sequenceDiagram + actor Orchestrator + participant API as K8s API Server + participant Agent as Trident ACL Agent
(on the node) + participant Trident + + Note over Orchestrator,Trident: pre-reboot half + Orchestrator->>API: 1. PATCH request: finalize (opId A) + Agent->>API: 2. read request, PATCH status: finalize InProgress + Agent->>Trident: 3. UpdateFinalize (caller-handled reboot) + Trident-->>Agent: boot armed, reboot required + Agent->>Agent: 4. persist pendingCommit + boot marker to state.json + Agent->>API: 5. PATCH status: finalize Success + API-->>Orchestrator: finalize Success (reboot pending) + Note over Agent,Trident: 6. agent triggers reboot, boots new partition + Note over Orchestrator,Trident: post-reboot half + Agent->>Agent: 7. read state.json, confirm a boot happened since the marker + Agent->>Trident: 8. Commit (validate volume, promote boot order) + Trident-->>Agent: committed | reverted to previous + Agent->>API: 9. PATCH status: commit Success | TargetBootFailed + API-->>Orchestrator: terminal commit code +``` + +A status annotation (`/update-status` or +`/update-commit-status`) looks like: + +```json +{ + "schemaVersion": "1.0", + "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", + "operationId": "c9d6f0a2-3b41-4e8d-9f27-1a5b6c7d8e90", + "operation": "stage", + "code": "Success", + "message": "staged update to 202606.29.0", + "fromVersion": "202606.15.0", + "toVersion": "202606.29.0", + "startedUtc": "2026-06-29T12:00:00Z", + "lastUpdatedUtc": "2026-06-29T12:03:41Z", + "finishedUtc": "2026-06-29T12:03:41Z" +} +``` + +- `operation` is `stage`, `finalize`, `rollback`, or `commit` (`commit` only + ever appears on `/update-commit-status`, never on + `/update-status`). +- `code` is the outcome — see the table below. +- `message` is a short, human-readable explanation of `code`, useful for + logs/alerts; treat its exact wording as informational, not something to + match on (it may include error detail that varies run to run). +- `fromVersion`/`toVersion` are the versions the operation moved between + (`toVersion` is absent for `rollback`, whose target is implicit). +- `startedUtc`/`lastUpdatedUtc`/`finishedUtc` bound the operation: + `lastUpdatedUtc` refreshes on a heartbeat cadence while `code` is + `InProgress` (see [below](#pre-post-reboot-state-and-the-watchdog)); + `finishedUtc` is absent until `code` reaches a terminal value. + +`code` is one of: + +| `code` | Terminal? | Meaning | +|---|---|---| +| `InProgress` | No | The operation is running. `lastUpdatedUtc` refreshes on a heartbeat cadence; a terminal code always follows. | +| `Success` | Yes | The operation completed as requested. For `commit`, this means the reboot landed on the target partition and it was promoted. | +| `AlreadyAtTarget` | Yes | `stage`/`finalize` was requested for the version the node is already running (per `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY`); treated as a no-op success. | +| `NotStaged` | Yes | `finalize` was requested for a `nodeUpdateId` with no prior successful `stage`. Issue a `stage` first. | +| `OperationFailed` | Yes | The operation failed for a reason other than a boot/rollback outcome (e.g. the Omaha server has no update for the requested version, or the underlying `tridentd` call returned an error). See `message` for detail. | +| `TargetBootFailed` | Yes | The post-reboot `commit` found the node had rolled back to its previous partition instead of booting the target — Trident's own health checks rejected the new boot. The node is back on `fromVersion`; the orchestrator should treat this as a failed update, not retry the same `nodeUpdateId` blindly. | +| `AgentInternalError` | Yes | A failure in the agent itself rather than in Trident or the requested operation (e.g. it triggered a reboot but the reboot call failed, or it lost track of an in-flight commit). Distinct from `OperationFailed` so an orchestrator can decide to treat these differently (e.g. retry vs. escalate). | +| `InvalidRequest` | Yes | The request annotation itself was rejected before any action was taken — malformed JSON, a schema/version mismatch, a missing required field (`server`/`appId`/`track`/`targetVersion`), a `finalize` whose `targetVersion` doesn't match what was staged, or a second `finalize`/`rollback` submitted while one is already pending its post-reboot `commit`. No Trident operation runs. | + +See the request/status schema types in +`crates/trident-acl-agent/src/annotations/protocol.rs` for the full contract, +including the formal JSON Schema both sides validate against. + +## Pre/post-reboot state and the watchdog + +Because `finalize`/`rollback` spans a reboot, the agent persists a small +state file (`TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH`) recording that a +commit is pending and a marker for "a boot happened after this point". On +restart, the agent checks this state to resume the post-reboot `commit` +step rather than re-running `finalize` from scratch. + +While an operation is in flight, the agent refreshes the `InProgress` +status's `lastUpdatedUtc` on a heartbeat cadence +(`TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL`), so an external +watchdog can distinguish a working agent from a stuck one and reprovision a +node that never reports a terminal `commit` status within its SLA. + +## Configuration + +There is no config file. Every setting is an environment variable prefixed +`TRIDENT_ACL_AGENT_`, systemd-style: set it in the unit's own +`Environment=` lines, via a drop-in override, or by any other means that +sets the process's environment before it starts. + +A variable that is unset, or set to the empty string, falls back to its +default. A variable set to a malformed value (a bad URL, a bad duration, an +unrecognized `goal_source`) causes the agent to fail to start with an error +naming the offending variable. + +| Variable | Default | Description | +|---|---|---| +| `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` | `acl.microsoft.com` | The annotation-key prefix for the request/status/commit-status annotations (e.g. the `acl.microsoft.com` in `acl.microsoft.com/update-request`). Any orchestrator can pick its own namespace here so its annotations don't collide with another controller's. | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` | `/etc/os-release` | The file the agent reads to determine the node's currently running version. Any file works, as long as it follows the os-release format (`KEY=VALUE` lines, optionally single- or double-quoted, blank lines and `#` comments ignored) — see [below](#configuring-the-on-disk-version). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` | `VERSION_ID` | The key the agent looks up in `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default) to determine the node's currently running version, used to compare against a request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`). `VERSION_ID` is the standard `os-release` field most images already stamp; a deployment that instead carries an ACL-specific `IMAGE_VERSION` field can point this variable at that key instead — see [below](#configuring-the-on-disk-version). | +| `TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` | `always` | Controls what happens when `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` isn't present at `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (e.g. a dev/test host, or an image that hasn't started stamping that key yet). `always` reports `0.0.0` as the node's current version — a sentinel that can never collide with a real release version and cause a false `AlreadyAtTarget`. `error` fails the operation instead of using a placeholder version. Any other value is used verbatim as the current version, with no format validation. | +| `TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER` | unset | Explicit override for the Kubernetes API server URL. When unset, the server embedded in the kubeconfig is used as-is. | +| `TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG` | `/var/lib/kubelet/kubeconfig` | Path to the kubeconfig used to reach the Kubernetes API server and authenticate as this node. | +| `TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME` | The node's own hostname, lowercased | The Node object this agent watches/patches. | +| `TRIDENT_ACL_AGENT_TRIDENT_SOCKET` | `unix:///run/trident/trident.sock` | The gRPC Unix socket URI used to reach `tridentd`. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH` | `/var/lib/trident-acl-agent/state.json` | Path to the agent's persistent state file bridging the pre-reboot and post-reboot halves of `finalize`/`rollback` across the reboot. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT` | `20m` | How long a `stage` is allowed to run (a [`humantime`](https://docs.rs/humantime) duration, e.g. `20m`, `1h`) before it's considered failed. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT` | `10m` | How long a `finalize` is allowed to run before it's considered failed. | +| `TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL` | `60s` | Refresh cadence for the `InProgress` status heartbeat. | + +### Setting env vars via a systemd drop-in + +The agent ships as `trident-acl-agent.service`, with no `Environment=` +lines of its own beyond `ExecStart`. Any setting is overridden with a +drop-in file, without editing the packaged unit: + +```console +$ sudo systemctl edit trident-acl-agent.service +``` + +This opens `/etc/systemd/system/trident-acl-agent.service.d/override.conf` +in an editor. For example, to point the agent at a custom annotation +namespace: + +```ini +[Service] +Environment=TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX=acl.contoso.com +``` + +With the prefix above, the orchestrator now reads/writes +`acl.contoso.com/update-request`, `acl.contoso.com/update-status`, and +`acl.contoso.com/update-commit-status` instead of the `acl.microsoft.com/*` +defaults. Reload and restart to apply: + +```console +$ sudo systemctl daemon-reload +$ sudo systemctl restart trident-acl-agent.service +``` + +`systemctl cat trident-acl-agent.service` shows the merged unit (packaged +unit plus drop-in), useful for confirming the override took effect. + +### Configuring the on-disk version + +The agent determines the node's current version by reading a key out of +`TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (`/etc/os-release` by default), +defaulting to the key `VERSION_ID` — the standard +[`os-release`](https://www.freedesktop.org/software/systemd/man/latest/os-release.html) +field most distributions already stamp. A deployment that keeps its +version stamp under a different key, a different file entirely, or both, +can point the agent there instead, as long as that file follows the +`os-release` key-value schema (`KEY=VALUE` lines, optionally quoted, blank +lines and `#` comments ignored): + +```ini +[Service] +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH=/etc/my-app-release +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=BUILD_VERSION +``` + +With this set, the agent reads `BUILD_VERSION` from `/etc/my-app-release` +(e.g. `BUILD_VERSION=202606.29.0`) as the node's current version, and +compares it against a request's `targetVersion` the same way it would for +`VERSION_ID`/`/etc/os-release` — including short-circuiting to +`AlreadyAtTarget` when they already match. + +If the configured key is absent from the configured file (for example, on +a dev/test host with a minimal `os-release`), the agent consults +`TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` (`always` by default): `always` +reports `0.0.0` as the current version — a sentinel that can never +accidentally match a real requested version; `error` fails the operation +instead of guessing; any other value is used verbatim as the current +version, unvalidated. + +## Diagnostics + +`trident-acl-agent --validate-connection ` +checks connectivity to a single dependency using the current environment +and exits immediately — useful for a systemd `ExecStartPre` check or manual +on-node troubleshooting without running the full orchestrator loop. + +`--validate-connection nebraska` is the one place +`TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT`, `TRIDENT_ACL_AGENT_NEBRASKA_APP_ID`, +and `TRIDENT_ACL_AGENT_NEBRASKA_TRACK` are used: it issues a real +update-check query against the configured endpoint/app id/track and reports +whether the Omaha server is reachable. They default to deliberately invalid +values (`https://nebraska.example.invalid/v1/update`, an all-zero UUID, and +`unspecified`, respectively) so this check fails loudly unless a deployment +sets them. Since these variables otherwise play no role in the +annotation-driven flow, there's no reason to add them to the service's +persistent environment (e.g. via a drop-in) — set them just for this +one-off invocation instead: + +```console +$ sudo TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT=https://updates.contoso.com/v1/update \ + TRIDENT_ACL_AGENT_NEBRASKA_APP_ID=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee \ + TRIDENT_ACL_AGENT_NEBRASKA_TRACK=stable \ + trident-acl-agent --validate-connection nebraska +``` From 3052c5d681127ecc3b5c21e5e8943cdbdff10149 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 22:42:46 +0000 Subject: [PATCH 31/54] trident-acl-agent: fix remaining Copilot review findings (PR 730) - annotations/protocol.rs: fix stale env var name in module doc (TRIDENT_ACL_AGENT_ANNOTATION_PREFIX -> the real TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX, per core/config.rs). - annotations/state.rs: create the state.json temp file with owner-only permissions (0600) instead of relying on the process umask. Persisted state embeds the full UpdateRequest, including a potentially secret-bearing Omaha server URL. 152 unit tests + 1 doctest pass; clippy and fmt clean. --- .../trident-acl-agent/src/annotations/protocol.rs | 4 ++-- crates/trident-acl-agent/src/annotations/state.rs | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/protocol.rs b/crates/trident-acl-agent/src/annotations/protocol.rs index 3f9ffb87a5..a6b75be690 100644 --- a/crates/trident-acl-agent/src/annotations/protocol.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -7,8 +7,8 @@ //! by the current accepted design (), where //! `` defaults to `acl.microsoft.com` (see //! [`AnnotationKeys`]/[`crate::core::config::DEFAULT_ANNOTATION_PREFIX`]) and is -//! overridable via the `TRIDENT_ACL_AGENT_ANNOTATION_PREFIX` environment -//! variable. Keep `UpdateRequest`/`UpdateStatus`/`StatusCode` and +//! overridable via the `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` +//! environment variable. Keep `UpdateRequest`/`UpdateStatus`/`StatusCode` and //! `validate()` in sync with that document's formal JSON Schema (its //! section "Formal JSON Schema") - the //! `design_doc_*`/`agent_built_*_conform_to_formal_schema` tests in this diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index 28f52e6ac9..7c1856b7e4 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -10,6 +10,7 @@ use std::{ collections::BTreeMap, fs, io::Write, + os::unix::fs::OpenOptionsExt, path::{Path, PathBuf}, }; @@ -101,8 +102,18 @@ impl StateStore { // alone only guarantees the data reaches the OS page cache, not // disk, so a crash between the write and a later flush could still // leave state.json empty/corrupt after the rename below. + // + // The persisted state embeds the full UpdateRequest, which can + // include a secret-bearing Omaha `server` URL, so create the file + // with owner-only permissions (0600) rather than relying on the + // process umask. { - let mut file = fs::File::create(&temp_path) + let mut file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(&temp_path) .with_context(|| format!("failed to create {}", temp_path.display()))?; file.write_all(serde_json::to_string_pretty(state)?.as_bytes()) .with_context(|| format!("failed to write {}", temp_path.display()))?; From e9fd7159e078d977069c9a596288183df2550f44 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 21 Aug 2026 23:25:45 +0000 Subject: [PATCH 32/54] trident-acl-agent: redact Nebraska endpoint in connectivity-check logs --validate-connection nebraska logged the raw configured endpoint URL in both an error-context string and an info log. A Nebraska endpoint can carry an Omaha secret in its path/query (per core/nebraska::redacted's own doc comment), so this could leak a secret into journald/log aggregation. Widen the existing (previously private) core::nebraska::redacted() helper to pub, export it from core::nebraska, and use it for both main.rs log sites instead of the raw endpoint. 152 unit tests + 1 doctest pass; clippy and fmt clean. --- .../trident-acl-agent/src/core/nebraska/client.rs | 2 +- crates/trident-acl-agent/src/core/nebraska/mod.rs | 2 +- crates/trident-acl-agent/src/main.rs | 13 +++++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/trident-acl-agent/src/core/nebraska/client.rs b/crates/trident-acl-agent/src/core/nebraska/client.rs index 94c0c685be..5b30fabd09 100644 --- a/crates/trident-acl-agent/src/core/nebraska/client.rs +++ b/crates/trident-acl-agent/src/core/nebraska/client.rs @@ -237,7 +237,7 @@ fn summarize(response: &wire::Response) -> String { /// and host are safe to emit; the path, query, fragment, and any userinfo are /// dropped rather than selectively scrubbed, so a secret cannot leak from a /// component this code did not anticipate. -fn redacted(endpoint: &Url) -> String { +pub fn redacted(endpoint: &Url) -> String { let Some(host) = endpoint.host_str() else { return "".to_string(); }; diff --git a/crates/trident-acl-agent/src/core/nebraska/mod.rs b/crates/trident-acl-agent/src/core/nebraska/mod.rs index d2ed9f0323..484efc3328 100644 --- a/crates/trident-acl-agent/src/core/nebraska/mod.rs +++ b/crates/trident-acl-agent/src/core/nebraska/mod.rs @@ -83,7 +83,7 @@ mod status; mod transport; mod wire; -pub use client::{CheckOutcome, Client, PackageFile, PackageHash, UpdateOffer}; +pub use client::{redacted, CheckOutcome, Client, PackageFile, PackageHash, UpdateOffer}; pub use error::NebraskaError; pub use event::ProgressEvent; pub use id::MachineId; diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 9ae29e0531..471625430f 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -7,6 +7,7 @@ use trident_acl_agent::{ check_nebraska_reachable, core::{ config::{AgentConfig, GoalSource}, + nebraska, trident::TridentClient, }, omahaonly::run_omaha_only, @@ -192,8 +193,16 @@ async fn validate_connection( }) .await .context("Nebraska connectivity check task panicked")? - .with_context(|| format!("failed to reach Nebraska server at {endpoint}"))?; - log::info!("nebraska: reached server at {endpoint}"); + .with_context(|| { + format!( + "failed to reach Nebraska server at {}", + nebraska::redacted(&endpoint) + ) + })?; + log::info!( + "nebraska: reached server at {}", + nebraska::redacted(&endpoint) + ); } } Ok(()) From 468b22e3c307b72333096f70c98095592bbc82af Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 19:09:24 +0000 Subject: [PATCH 33/54] trident-acl-agent: address frhuelsz PR 730 review comments - Move FilteredLogger into osutils::logging as a reusable component, parameterized over verbosity/network_verbosity/network targets, and use it from trident-acl-agent (addresses the suggestion to reuse logging infra instead of duplicating it). - Split the clap Args/ConnectionTarget definitions out of main.rs into a new cli.rs module. - Move validate_connection() out of main.rs into a new connection_check.rs module. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/osutils/src/logging.rs | 172 +++++++++ crates/trident-acl-agent/src/cli.rs | 50 +++ .../trident-acl-agent/src/connection_check.rs | 100 ++++++ crates/trident-acl-agent/src/main.rs | 339 +++++------------- 4 files changed, 403 insertions(+), 258 deletions(-) create mode 100644 crates/osutils/src/logging.rs create mode 100644 crates/trident-acl-agent/src/cli.rs create mode 100644 crates/trident-acl-agent/src/connection_check.rs diff --git a/crates/osutils/src/logging.rs b/crates/osutils/src/logging.rs new file mode 100644 index 0000000000..103ceb0b4b --- /dev/null +++ b/crates/osutils/src/logging.rs @@ -0,0 +1,172 @@ +use log::{LevelFilter, Log, Metadata, Record}; + +/// A `log::Log` wrapper that applies a separate level filter to a configurable +/// list of noisy "network" targets (e.g. HTTP/gRPC/watch client crates) while +/// leaving every other target at a main verbosity level. +/// +/// This is useful for binaries that talk to chatty client stacks (hyper, h2, +/// tonic, kube, reqwest, ...) whose per-frame/per-request logging would +/// otherwise drown out the binary's own orchestration logs at the same +/// verbosity. +pub struct FilteredLogger { + inner: L, + verbosity: LevelFilter, + network_verbosity: LevelFilter, + network_targets: &'static [&'static str], +} + +impl FilteredLogger { + /// Builds a new [`FilteredLogger`] wrapping `inner`. Targets in + /// `network_targets` (matched by prefix, e.g. `"hyper"` matches + /// `hyper::client`) are filtered at `network_verbosity`; every other + /// target is filtered at `verbosity`. + pub fn new( + inner: L, + verbosity: LevelFilter, + network_verbosity: LevelFilter, + network_targets: &'static [&'static str], + ) -> Self { + Self { + inner, + verbosity, + network_verbosity, + network_targets, + } + } + + /// The maximum of `verbosity` and `network_verbosity`, suitable for + /// passing to [`log::set_max_level`] so the log facade doesn't drop + /// records before they reach this filter. + pub fn max_level(&self) -> LevelFilter { + self.verbosity.max(self.network_verbosity) + } + + fn is_network_target(&self, target: &str) -> bool { + self.network_targets.iter().any(|prefix| { + target + .strip_prefix(prefix) + .is_some_and(|rest| rest.is_empty() || rest.starts_with("::")) + }) + } +} + +impl Log for FilteredLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + let level = if self.is_network_target(metadata.target()) { + self.network_verbosity + } else { + self.verbosity + }; + metadata.level() <= level + } + + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + self.inner.log(record); + } + } + + fn flush(&self) { + self.inner.flush(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::{Arc, Mutex}; + + use log::{Level, Metadata, Record}; + + #[derive(Clone)] + struct TestLogger { + logged: Arc>>, + } + + impl TestLogger { + fn new() -> Self { + Self { + logged: Arc::new(Mutex::new(Vec::new())), + } + } + } + + impl Log for TestLogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + self.logged + .lock() + .unwrap() + .push(format!("{} {}", record.target(), record.args())); + } + + fn flush(&self) {} + } + + const NETWORK_TARGETS: &[&str] = &["hyper", "kube"]; + + #[test] + fn test_network_target_uses_network_verbosity() { + let inner = TestLogger::new(); + let logged = inner.logged.clone(); + let logger = FilteredLogger::new( + inner, + LevelFilter::Debug, + LevelFilter::Warn, + NETWORK_TARGETS, + ); + + assert!(logger.enabled( + &Metadata::builder() + .level(Level::Warn) + .target("hyper::client") + .build() + )); + assert!(!logger.enabled( + &Metadata::builder() + .level(Level::Debug) + .target("hyper::client") + .build() + )); + drop(logged); + } + + #[test] + fn test_non_network_target_uses_verbosity() { + let inner = TestLogger::new(); + let logger = FilteredLogger::new( + inner, + LevelFilter::Debug, + LevelFilter::Warn, + NETWORK_TARGETS, + ); + + assert!(logger.enabled( + &Metadata::builder() + .level(Level::Debug) + .target("trident_acl_agent") + .build() + )); + assert!(!logger.enabled( + &Metadata::builder() + .level(Level::Trace) + .target("trident_acl_agent") + .build() + )); + } + + #[test] + fn test_max_level_is_max_of_both() { + let logger = FilteredLogger::new( + TestLogger::new(), + LevelFilter::Warn, + LevelFilter::Debug, + NETWORK_TARGETS, + ); + assert_eq!(logger.max_level(), LevelFilter::Debug); + } +} diff --git a/crates/trident-acl-agent/src/cli.rs b/crates/trident-acl-agent/src/cli.rs new file mode 100644 index 0000000000..cd8ac2271c --- /dev/null +++ b/crates/trident-acl-agent/src/cli.rs @@ -0,0 +1,50 @@ +use clap::Parser; +use log::LevelFilter; + +/// trident-acl-agent can either run the annotation-driven orchestrator (the +/// default) or fall back to its original one-shot Omaha flow. Mode selection +/// is environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`): +/// shipping defaults enable the AKS annotation protocol, while a VM +/// extension, systemd drop-in, or AgentBaker-set environment can opt a node +/// out to `omaha-only` if needed. +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + /// Logging verbosity [OFF, ERROR, WARN, INFO, DEBUG, TRACE] + #[arg(global = true, short, long, default_value_t = LevelFilter::Debug)] + pub verbosity: LevelFilter, + + /// Logging verbosity for the underlying HTTP/gRPC/watch client stack + /// (hyper, h2, tower, tonic, reqwest, rustls, kube). Kept separate from + /// `--verbosity` because it can be extremely noisy (per-frame HTTP2 + /// detail, watch reconnect churn) [OFF, ERROR, WARN, INFO, DEBUG, TRACE]. + #[arg(global = true, long, default_value_t = LevelFilter::Warn)] + pub network_verbosity: LevelFilter, + + /// Validate connectivity to a single dependency and exit immediately, + /// instead of running the agent. Useful for troubleshooting one + /// connection in isolation (e.g. a systemd ExecStartPre check, or manual + /// diagnostics on-node) without running the full orchestrator loop. + /// Exits with status 0 if the connection could be established, non-zero + /// (with an error message) otherwise. + #[arg(long, value_enum)] + pub validate_connection: Option, +} + +/// A single dependency `--validate-connection` can check. +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +pub enum ConnectionTarget { + /// Validates reachability of the Kubernetes API server by fetching this + /// node's own Node object (the same access the agent's reconcile loop + /// already requires). + Kubernetes, + /// Validates reachability of tridentd by connecting to its gRPC Unix + /// socket. Connecting is sufficient - no RPC call is needed, since the + /// connection itself fails immediately if nothing is listening. + Tridentd, + /// Validates reachability of the Nebraska/Omaha server by issuing a real + /// update-check query. Any well-formed Omaha response (including "no + /// update available") counts as success - only a network/transport + /// failure is treated as unreachable. + Nebraska, +} diff --git a/crates/trident-acl-agent/src/connection_check.rs b/crates/trident-acl-agent/src/connection_check.rs new file mode 100644 index 0000000000..03aaba1460 --- /dev/null +++ b/crates/trident-acl-agent/src/connection_check.rs @@ -0,0 +1,100 @@ +use anyhow::Context; + +use trident_acl_agent::{ + annotations::k8s::NodeClient, + check_nebraska_reachable, + core::{config::AgentConfig, nebraska, trident::TridentClient}, + IdSource, +}; + +use crate::cli::ConnectionTarget; + +/// Checks connectivity to exactly one of `target`'s dependencies and returns +/// `Ok(())` on success. The caller (`main`) surfaces any `Err` the normal way +/// (`anyhow`'s `Termination` impl prints the error and exits non-zero), so +/// this function only needs to produce a descriptive error on failure - no +/// explicit `process::exit` is required. +pub async fn validate_connection( + target: ConnectionTarget, + config: &AgentConfig, +) -> Result<(), anyhow::Error> { + match target { + ConnectionTarget::Kubernetes => { + let client = NodeClient::new(&config.kubernetes) + .await + .context("failed to build Kubernetes client")?; + // Report the actually-resolved server (kubeconfig's own server, + // unless overridden by kubernetes.api_server), not a value + // guessed from config - the two only match when an override is + // set. + let cluster_url = client.cluster_url(); + client + .get_node(&config.kubernetes.node_name) + .await + .with_context(|| { + format!( + "failed to reach Kubernetes API server at {} (get Node {:?})", + cluster_url, config.kubernetes.node_name + ) + })?; + log::info!( + "kubernetes: reached API server at {} and fetched Node {:?}", + cluster_url, + config.kubernetes.node_name + ); + } + ConnectionTarget::Tridentd => { + TridentClient::connect(&config.trident.socket) + .await + .with_context(|| { + format!("failed to reach tridentd at {}", config.trident.socket) + })?; + log::info!("tridentd: connected to {}", config.trident.socket); + } + ConnectionTarget::Nebraska => { + let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { + anyhow::anyhow!( + "nebraska.endpoint is not configured (set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT)" + ) + })?; + let app_id = config.nebraska.app_id.clone(); + // check_nebraska_reachable() is a blocking call (reqwest::blocking + // under the hood, see nebraska::transport) - calling it directly from this + // async fn can panic ("Cannot drop a runtime in a context where + // blocking is not allowed") because reqwest::blocking spins up + // its own inner Tokio runtime per call, which isn't safe to tear + // down from inside an already-running async task. Run it on a + // dedicated blocking thread instead. + // + // Deliberately uses check_nebraska_reachable() rather than + // query_for_update(): the latter also validates app-level + // semantics (app ID match, non-error app/update-check status), + // which would make this a "can we get a valid update check" test + // rather than the pure reachability check documented on + // ConnectionTarget::Nebraska above. + let endpoint_for_task = endpoint.clone(); + let track = config.nebraska.track.clone(); + tokio::task::spawn_blocking(move || { + check_nebraska_reachable( + &endpoint_for_task, + &app_id, + &track, + IdSource::MachineIdHashed, + ) + }) + .await + .context("Nebraska connectivity check task panicked")? + .with_context(|| { + format!( + "failed to reach Nebraska server at {}", + nebraska::redacted(&endpoint) + ) + })?; + log::info!( + "nebraska: reached server at {}", + nebraska::redacted(&endpoint) + ); + } + } + Ok(()) +} diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 471625430f..d266482855 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -1,258 +1,81 @@ -use anyhow::Context; -use clap::Parser; -use log::{LevelFilter, Log, Metadata, Record}; - -use trident_acl_agent::{ - annotations::{k8s::NodeClient, orchestrator::Orchestrator}, - check_nebraska_reachable, - core::{ - config::{AgentConfig, GoalSource}, - nebraska, - trident::TridentClient, - }, - omahaonly::run_omaha_only, - IdSource, -}; - -/// Module/target prefixes for the underlying HTTP/gRPC/watch client stack. -/// These crates emit very verbose `log`-facade tracing (connection setup, -/// per-frame HTTP2 detail, watch reconnect churn) that is rarely useful at -/// the same verbosity as the agent's own orchestration logic, so it's -/// filtered independently via `--network-verbosity`. -const NETWORK_LOG_TARGETS: &[&str] = &[ - "hyper", - "h2", - "tower", - "tonic", - "reqwest", - "rustls", - "kube", - "kube_client", - "kube_runtime", -]; - -/// A `log::Log` wrapper that applies a separate level filter to the noisy -/// HTTP/gRPC/watch client crates (see [`NETWORK_LOG_TARGETS`]) while leaving -/// every other target (the agent's own code) at the main `--verbosity` -/// level. -struct FilteredLogger { - inner: L, - verbosity: LevelFilter, - network_verbosity: LevelFilter, -} - -impl Log for FilteredLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - let level = if is_network_target(metadata.target()) { - self.network_verbosity - } else { - self.verbosity - }; - metadata.level() <= level - } - - fn log(&self, record: &Record) { - if self.enabled(record.metadata()) { - self.inner.log(record); - } - } - - fn flush(&self) { - self.inner.flush(); - } -} - -fn is_network_target(target: &str) -> bool { - NETWORK_LOG_TARGETS.iter().any(|prefix| { - target - .strip_prefix(prefix) - .is_some_and(|rest| rest.is_empty() || rest.starts_with("::")) - }) -} - -/// trident-acl-agent can either run the annotation-driven orchestrator (the -/// default) or fall back to its original one-shot Omaha flow. Mode selection -/// is environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`): -/// shipping defaults enable the AKS annotation protocol, while a VM -/// extension, systemd drop-in, or AgentBaker-set environment can opt a node -/// out to `omaha-only` if needed. -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - /// Logging verbosity [OFF, ERROR, WARN, INFO, DEBUG, TRACE] - #[arg(global = true, short, long, default_value_t = LevelFilter::Debug)] - verbosity: LevelFilter, - - /// Logging verbosity for the underlying HTTP/gRPC/watch client stack - /// (hyper, h2, tower, tonic, reqwest, rustls, kube). Kept separate from - /// `--verbosity` because it can be extremely noisy (per-frame HTTP2 - /// detail, watch reconnect churn) [OFF, ERROR, WARN, INFO, DEBUG, TRACE]. - #[arg(global = true, long, default_value_t = LevelFilter::Warn)] - network_verbosity: LevelFilter, - - /// Validate connectivity to a single dependency and exit immediately, - /// instead of running the agent. Useful for troubleshooting one - /// connection in isolation (e.g. a systemd ExecStartPre check, or manual - /// diagnostics on-node) without running the full orchestrator loop. - /// Exits with status 0 if the connection could be established, non-zero - /// (with an error message) otherwise. - #[arg(long, value_enum)] - validate_connection: Option, -} - -/// A single dependency `--validate-connection` can check. -#[derive(Clone, Copy, Debug, clap::ValueEnum)] -enum ConnectionTarget { - /// Validates reachability of the Kubernetes API server by fetching this - /// node's own Node object (the same access the agent's reconcile loop - /// already requires). - Kubernetes, - /// Validates reachability of tridentd by connecting to its gRPC Unix - /// socket. Connecting is sufficient - no RPC call is needed, since the - /// connection itself fails immediately if nothing is listening. - Tridentd, - /// Validates reachability of the Nebraska/Omaha server by issuing a real - /// update-check query. Any well-formed Omaha response (including "no - /// update available") counts as success - only a network/transport - /// failure is treated as unreachable. - Nebraska, -} - -/// Checks connectivity to exactly one of `target`'s dependencies and returns -/// `Ok(())` on success. The caller (`main`) surfaces any `Err` the normal way -/// (`anyhow`'s `Termination` impl prints the error and exits non-zero), so -/// this function only needs to produce a descriptive error on failure - no -/// explicit `process::exit` is required. -async fn validate_connection( - target: ConnectionTarget, - config: &AgentConfig, -) -> Result<(), anyhow::Error> { - match target { - ConnectionTarget::Kubernetes => { - let client = NodeClient::new(&config.kubernetes) - .await - .context("failed to build Kubernetes client")?; - // Report the actually-resolved server (kubeconfig's own server, - // unless overridden by kubernetes.api_server), not a value - // guessed from config - the two only match when an override is - // set. - let cluster_url = client.cluster_url(); - client - .get_node(&config.kubernetes.node_name) - .await - .with_context(|| { - format!( - "failed to reach Kubernetes API server at {} (get Node {:?})", - cluster_url, config.kubernetes.node_name - ) - })?; - log::info!( - "kubernetes: reached API server at {} and fetched Node {:?}", - cluster_url, - config.kubernetes.node_name - ); - } - ConnectionTarget::Tridentd => { - TridentClient::connect(&config.trident.socket) - .await - .with_context(|| { - format!("failed to reach tridentd at {}", config.trident.socket) - })?; - log::info!("tridentd: connected to {}", config.trident.socket); - } - ConnectionTarget::Nebraska => { - let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { - anyhow::anyhow!( - "nebraska.endpoint is not configured (set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT)" - ) - })?; - let app_id = config.nebraska.app_id.clone(); - // check_nebraska_reachable() is a blocking call (reqwest::blocking - // under the hood, see nebraska::transport) - calling it directly from this - // async fn can panic ("Cannot drop a runtime in a context where - // blocking is not allowed") because reqwest::blocking spins up - // its own inner Tokio runtime per call, which isn't safe to tear - // down from inside an already-running async task. Run it on a - // dedicated blocking thread instead. - // - // Deliberately uses check_nebraska_reachable() rather than - // query_for_update(): the latter also validates app-level - // semantics (app ID match, non-error app/update-check status), - // which would make this a "can we get a valid update check" test - // rather than the pure reachability check documented on - // ConnectionTarget::Nebraska above. - let endpoint_for_task = endpoint.clone(); - let track = config.nebraska.track.clone(); - tokio::task::spawn_blocking(move || { - check_nebraska_reachable( - &endpoint_for_task, - &app_id, - &track, - IdSource::MachineIdHashed, - ) - }) - .await - .context("Nebraska connectivity check task panicked")? - .with_context(|| { - format!( - "failed to reach Nebraska server at {}", - nebraska::redacted(&endpoint) - ) - })?; - log::info!( - "nebraska: reached server at {}", - nebraska::redacted(&endpoint) - ); - } - } - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<(), anyhow::Error> { - let args = Args::parse(); - - let max_level = args.verbosity.max(args.network_verbosity); - if let Some(Ok(journal_logger)) = - systemd_journal_logger::connected_to_journal().then(systemd_journal_logger::JournalLog::new) - { - log::set_boxed_logger(Box::new(FilteredLogger { - inner: journal_logger, - verbosity: args.verbosity, - network_verbosity: args.network_verbosity, - })) - .expect("Failed to install systemd journal logger"); - log::set_max_level(max_level); - } else { - let inner = env_logger::builder() - .format_timestamp(None) - .filter_level(max_level) - .build(); - log::set_boxed_logger(Box::new(FilteredLogger { - inner, - verbosity: args.verbosity, - network_verbosity: args.network_verbosity, - })) - .expect("Failed to install env logger"); - log::set_max_level(max_level); - } - - let config = AgentConfig::from_env()?; - - if let Some(target) = args.validate_connection { - return validate_connection(target, &config).await; - } - - match config.orchestration.goal_source { - // Historical one-shot flow: query Nebraska once, apply an update if - // offered, and exit. No Kubernetes/annotation involvement. Not a - // documented/supported deployment option (see config::GoalSource). - GoalSource::OmahaOnly => run_omaha_only(&config).await, - // The only supported mode: the annotation-driven reconcile loop - // (watches /update-request, drives stage/finalize/rollback/ - // commit against tridentd, writes /update-status; prefix - // defaults to acl.microsoft.com, overridable via - // TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX). - GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, - } -} +use clap::Parser; +use osutils::logging::FilteredLogger; + +use trident_acl_agent::{ + annotations::orchestrator::Orchestrator, + core::config::{AgentConfig, GoalSource}, + omahaonly::run_omaha_only, +}; + +mod cli; +mod connection_check; + +use cli::Args; +use connection_check::validate_connection; + +/// Module/target prefixes for the underlying HTTP/gRPC/watch client stack. +/// These crates emit very verbose `log`-facade tracing (connection setup, +/// per-frame HTTP2 detail, watch reconnect churn) that is rarely useful at +/// the same verbosity as the agent's own orchestration logic, so it's +/// filtered independently via `--network-verbosity`. +const NETWORK_LOG_TARGETS: &[&str] = &[ + "hyper", + "h2", + "tower", + "tonic", + "reqwest", + "rustls", + "kube", + "kube_client", + "kube_runtime", +]; + +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + let args = Args::parse(); + + if let Some(Ok(journal_logger)) = + systemd_journal_logger::connected_to_journal().then(systemd_journal_logger::JournalLog::new) + { + let logger = FilteredLogger::new( + journal_logger, + args.verbosity, + args.network_verbosity, + NETWORK_LOG_TARGETS, + ); + log::set_max_level(logger.max_level()); + log::set_boxed_logger(Box::new(logger)).expect("Failed to install systemd journal logger"); + } else { + let inner = env_logger::builder() + .format_timestamp(None) + .filter_level(args.verbosity.max(args.network_verbosity)) + .build(); + let logger = FilteredLogger::new( + inner, + args.verbosity, + args.network_verbosity, + NETWORK_LOG_TARGETS, + ); + log::set_max_level(logger.max_level()); + log::set_boxed_logger(Box::new(logger)).expect("Failed to install env logger"); + } + + let config = AgentConfig::from_env()?; + + if let Some(target) = args.validate_connection { + return validate_connection(target, &config).await; + } + + match config.orchestration.goal_source { + // Historical one-shot flow: query Nebraska once, apply an update if + // offered, and exit. No Kubernetes/annotation involvement. Not a + // documented/supported deployment option (see config::GoalSource). + GoalSource::OmahaOnly => run_omaha_only(&config).await, + // The only supported mode: the annotation-driven reconcile loop + // (watches /update-request, drives stage/finalize/rollback/ + // commit against tridentd, writes /update-status; prefix + // defaults to acl.microsoft.com, overridable via + // TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX). + GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, + } +} From 72113544e55005c537857696cf155a5f577584fd Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 19:40:12 +0000 Subject: [PATCH 34/54] trident-acl-agent: address PR 730 nit feedback Address frhuelsz review nits on PR 730. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/osutils/src/machine_id.rs | 19 +- crates/osutils/src/osrelease.rs | 28 +- .../trident-acl-agent/src/annotations/k8s.rs | 36 ++- .../src/annotations/orchestrator.rs | 248 +++++++-------- .../src/annotations/protocol.rs | 67 ++-- .../src/annotations/state.rs | 56 ++-- .../trident-acl-agent/src/connection_check.rs | 201 ++++++------ crates/trident-acl-agent/src/core/config.rs | 45 +-- crates/trident-acl-agent/src/core/error.rs | 8 +- .../src/core/trident/client.rs | 57 ++-- .../src/core/trident/mock.rs | 30 +- crates/trident-acl-agent/src/core/version.rs | 128 ++++---- crates/trident-acl-agent/src/lib.rs | 287 +++++++++--------- crates/trident-acl-agent/src/main.rs | 168 +++++----- crates/trident-acl-agent/src/omahaonly/mod.rs | 34 ++- 15 files changed, 752 insertions(+), 660 deletions(-) diff --git a/crates/osutils/src/machine_id.rs b/crates/osutils/src/machine_id.rs index 60b77b2f11..8d3b32bf8a 100644 --- a/crates/osutils/src/machine_id.rs +++ b/crates/osutils/src/machine_id.rs @@ -1,10 +1,11 @@ -use std::path::Path; +use std::{fs, path::Path}; -use anyhow::{Context, Error}; +use anyhow::{ensure, Context, Error}; use sha2::{Digest, Sha384}; use uuid::Uuid; const MACHINE_ID_FILE: &str = "/etc/machine-id"; +const BOOT_ID_FILE: &str = "/proc/sys/kernel/random/boot_id"; #[derive(Debug, Clone, Copy)] pub struct MachineId(u128); @@ -15,7 +16,7 @@ impl MachineId { } fn read_inner(path: impl AsRef) -> Result { - let id = std::fs::read_to_string(MACHINE_ID_FILE).with_context(|| { + let id = fs::read_to_string(path.as_ref()).with_context(|| { format!( "Failed to read machine ID from '{}'", path.as_ref().display() @@ -34,6 +35,14 @@ impl MachineId { )?)) } + pub fn boot_id() -> Result { + let raw = fs::read_to_string(BOOT_ID_FILE) + .with_context(|| format!("Failed to read boot ID from '{BOOT_ID_FILE}'"))?; + let boot_id = raw.trim(); + ensure!(!boot_id.is_empty(), "Boot ID in '{BOOT_ID_FILE}' was empty"); + Ok(boot_id.to_string()) + } + pub fn as_bytes(&self) -> [u8; 16] { self.0.to_be_bytes() } @@ -61,3 +70,7 @@ impl MachineId { Uuid::from_bytes(self.hashed()) } } + +pub fn boot_id() -> Result { + MachineId::boot_id() +} diff --git a/crates/osutils/src/osrelease.rs b/crates/osutils/src/osrelease.rs index e51926e745..d034909bc2 100644 --- a/crates/osutils/src/osrelease.rs +++ b/crates/osutils/src/osrelease.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{fs, path::Path}; use anyhow::{ensure, Context, Error}; use const_format::formatcp; @@ -31,6 +31,32 @@ pub fn is_azl3() -> Result { Ok(OsRelease::read()?.get_distro().is_azl3()) } +/// Reads a single key from an arbitrary os-release-formatted file. Returns +/// `None` when the file is unreadable, the key is absent, or its value is +/// empty. +pub fn read_key(path: impl AsRef, key: &str) -> Option { + let path = path.as_ref(); + let contents = fs::read_to_string(path).ok()?; + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((line_key, raw_value)) = line.split_once('=') else { + continue; + }; + if line_key.trim() != key { + continue; + } + let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); + if value.is_empty() { + return None; + } + return Some(value.to_string()); + } + None +} + /// Represents the contents of the /etc/os-release file. /// /// See diff --git a/crates/trident-acl-agent/src/annotations/k8s.rs b/crates/trident-acl-agent/src/annotations/k8s.rs index 3324405875..1dbf51b9e8 100644 --- a/crates/trident-acl-agent/src/annotations/k8s.rs +++ b/crates/trident-acl-agent/src/annotations/k8s.rs @@ -16,19 +16,24 @@ //! after a dropped or failed watch; that is governed entirely by //! `kube::runtime::watcher`'s built-in `default_backoff()`. -use std::{collections::BTreeMap, path::Path}; +use std::{collections::BTreeMap, path::Path, time::Duration}; -use anyhow::Context; +use anyhow::{Context, Error}; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::Node; use kube::{ api::{Patch, PatchParams}, config::{KubeConfigOptions, Kubeconfig}, error::ErrorResponse, - runtime::{watcher, WatchStreamExt}, - Api, Client, Config, + runtime::{ + watcher::{self, Error as WatchError}, + WatchStreamExt, + }, + Api, Client, Config, Error as KubeError, }; +use reqwest::StatusCode; use serde_json::json; +use thiserror::Error; use crate::core::config::KubernetesConfig; @@ -38,22 +43,22 @@ use crate::core::config::KubernetesConfig; /// reconnect churn on an otherwise-healthy watch. const WATCH_TIMEOUT_SECS: u32 = 290; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Error)] pub enum K8sClientError { #[error("failed to build Kubernetes client config: {0}")] - Config(#[from] anyhow::Error), + Config(#[from] Error), #[error("node object no longer exists")] NodeGone, #[error("failed Kubernetes API call: {0}")] - Api(#[source] kube::Error), + Api(#[source] KubeError), #[error("Kubernetes watch stream failed: {0}")] - Watch(#[from] kube::runtime::watcher::Error), + Watch(#[from] WatchError), } #[derive(Clone)] pub struct NodeClient { api: Api, - poll_interval: std::time::Duration, + poll_interval: Duration, cluster_url: String, } @@ -61,7 +66,7 @@ impl NodeClient { pub async fn new(config: &KubernetesConfig) -> Result { let client_config = load_client_config(config).await?; let cluster_url = client_config.cluster_url.to_string(); - let client = Client::try_from(client_config).map_err(anyhow::Error::new)?; + let client = Client::try_from(client_config).map_err(Error::new)?; Ok(Self { api: Api::all(client), poll_interval: config.watch_poll_interval, @@ -139,7 +144,7 @@ impl NodeClient { .fields(&format!("metadata.name={name}")) .timeout(timeout_secs); - watcher(self.api.clone(), watcher_config) + watcher::watcher(self.api.clone(), watcher_config) .default_backoff() .touched_objects() .map_err(K8sClientError::from) @@ -147,15 +152,16 @@ impl NodeClient { } } -fn map_kube_error(err: kube::Error) -> K8sClientError { - if matches!(&err, kube::Error::Api(ErrorResponse { code: 404, .. })) { +fn map_kube_error(err: KubeError) -> K8sClientError { + if matches!(&err, KubeError::Api(ErrorResponse { code, .. }) if *code == StatusCode::NOT_FOUND.as_u16()) + { K8sClientError::NodeGone } else { K8sClientError::Api(err) } } -async fn load_client_config(config: &KubernetesConfig) -> Result { +async fn load_client_config(config: &KubernetesConfig) -> Result { let path = Path::new(&config.kubeconfig); let kubeconfig = Kubeconfig::read_from(path) .with_context(|| format!("failed to read kubeconfig {}", path.display()))?; @@ -169,8 +175,6 @@ async fn load_client_config(config: &KubernetesConfig) -> Result Result<(), anyhow::Error>; + fn reboot(&self) -> Result<(), Error>; } impl RebootHandle for SystemRebooter { - fn reboot(&self) -> Result<(), anyhow::Error> { + fn reboot(&self) -> Result<(), Error> { // Route through the repo's centralized dependency runner so a // missing systemctl binary or non-zero exit produces the same // uniform, actionable error type used everywhere else in the @@ -99,7 +101,7 @@ impl RebootHandle for SystemRebooter { .cmd() .arg("reboot") .run_and_check() - .map_err(|err| anyhow::anyhow!("failed to issue systemctl reboot: {err}")) + .context("failed to issue systemctl reboot") } } @@ -112,7 +114,7 @@ pub struct Orchestrator { } impl Orchestrator { - pub async fn from_config(config: AgentConfig) -> Result { + pub async fn from_config(config: AgentConfig) -> Result { let k8s = NodeClient::new(&config.kubernetes).await?; let annotation_keys = AnnotationKeys::new(&config.kubernetes.annotation_prefix); Ok(Self { @@ -129,7 +131,7 @@ impl Orchestrator where R: RebootHandle, { - pub async fn run(&self) -> Result<(), anyhow::Error> { + pub async fn run(&self) -> Result<(), Error> { if let Err(err) = self.recover_from_trident_state().await { if self.log_and_swallow_node_gone(&err, "recovering persisted state") { return Ok(()); @@ -155,7 +157,7 @@ where Ok(()) } - async fn recover_from_trident_state(&self) -> Result<(), anyhow::Error> { + async fn recover_from_trident_state(&self) -> Result<(), Error> { let node = self.k8s.get_node(&self.config.kubernetes.node_name).await?; let snapshot = Snapshot::from_node(&node, &self.annotation_keys); let persisted = self.state.load()?; @@ -200,13 +202,11 @@ where Ok(()) } - async fn reconcile_node(&self, node: &Node) -> Result { + async fn reconcile_node(&self, node: &Node) -> Result { let snapshot = Snapshot::from_node(node, &self.annotation_keys); - log::debug!( + debug!( "received node update: request={:?} operation_status={:?} commit_status={:?}", - snapshot.request, - snapshot.operation_status, - snapshot.commit_status + snapshot.request, snapshot.operation_status, snapshot.commit_status ); let persisted = self.state.load()?; @@ -329,7 +329,7 @@ where request.track.clone() } - async fn handle_stage(&self, request: UpdateRequest) -> Result<(), anyhow::Error> { + async fn handle_stage(&self, request: UpdateRequest) -> Result<(), Error> { let started = Utc::now(); let from_version = Some(current_active_version()?); let to_version = request.target_version.clone(); @@ -367,33 +367,36 @@ where // an agent-internal error rather than silently defaulting - there // is deliberately no static-config fallback for the annotation flow. let endpoint = self.resolve_nebraska_endpoint(&request).ok_or_else(|| { - anyhow::anyhow!( + anyhow!( "stage request has no request.server despite passing validation (nodeUpdateId {})", request.node_update_id ) })?; let app_id = self.resolve_nebraska_app_id(&request).ok_or_else(|| { - anyhow::anyhow!( + anyhow!( "stage request has no request.appId despite passing validation (nodeUpdateId {})", request.node_update_id ) })?; let track = self.resolve_nebraska_track(&request).ok_or_else(|| { - anyhow::anyhow!( + anyhow!( "stage request has no request.track despite passing validation (nodeUpdateId {})", request.node_update_id ) })?; let machine_id = crate::build_machine_id(NEBRASKA_MACHINE_ID_SOURCE)?; let current_version = parse_nebraska_version(&from_version, "stage current version") - .unwrap_or_else(|| Version::new(0, 0, 0)); - let outcome = tokio::task::spawn_blocking(move || { + .unwrap_or_else(|| { + Version::parse(FALLBACK_ALWAYS_VERSION) + .expect("invariant: FALLBACK_ALWAYS_VERSION is valid semver") + }); + let outcome = task::spawn_blocking(move || { let client = NebraskaClient::new(endpoint, app_id, track, machine_id); client.check_for_update(¤t_version) }) .await .context("Nebraska query task panicked")? - .map_err(|err| anyhow::anyhow!("Nebraska query failed: {err}"))?; + .context("Nebraska query failed")?; let offered = match outcome { CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { let status = UpdateStatus::new( @@ -465,7 +468,7 @@ where self.record_and_publish(status).await } - async fn handle_finalize(&self, request: UpdateRequest) -> Result { + async fn handle_finalize(&self, request: UpdateRequest) -> Result { let started = Utc::now(); let from_version = Some(current_active_version()?); let to_version = request.target_version.clone(); @@ -577,7 +580,7 @@ where started, ); if let Err(err) = self.state.remember_completed(terminal.clone()) { - log::warn!("failed to record finalize completion in state.json: {err}"); + warn!("failed to record finalize completion in state.json: {err}"); } self.best_effort_publish_terminal(&terminal).await; match self.rebooter.reboot() { @@ -620,7 +623,7 @@ where } } - async fn handle_rollback(&self, request: UpdateRequest) -> Result { + async fn handle_rollback(&self, request: UpdateRequest) -> Result { let started = Utc::now(); let from_version = Some(current_active_version()?); @@ -706,7 +709,7 @@ where let terminal = rollback_finalize_success_status(&request, from_version.clone(), started); if let Err(err) = self.state.remember_completed(terminal.clone()) { - log::warn!("failed to record rollback completion in state.json: {err}"); + warn!("failed to record rollback completion in state.json: {err}"); } self.best_effort_publish_terminal(&terminal).await; match self.rebooter.reboot() { @@ -739,10 +742,10 @@ where } } - async fn resume_pending_commit(&self, pending: PendingCommit) -> Result<(), anyhow::Error> { + async fn resume_pending_commit(&self, pending: PendingCommit) -> Result<(), Error> { let current_boot = current_boot_marker()?; if current_boot == pending.boot_marker { - log::info!( + info!( "pending commit {} is still waiting for the reboot to happen", pending.operation_id ); @@ -791,19 +794,11 @@ where .await; } } - let status = self.map_commit_result(&pending, result); + let status = commit_result_to_status(&pending, result); self.state.clear_pending_commit()?; self.record_and_publish(status).await } - fn map_commit_result( - &self, - pending: &PendingCommit, - result: Result, - ) -> UpdateStatus { - commit_result_to_status(pending, result) - } - async fn reconstruct_without_state( &self, request: &UpdateRequest, @@ -865,14 +860,14 @@ where reconstruct_commit_result_to_status(request, from_version, started, result) } - async fn record_and_publish(&self, status: UpdateStatus) -> Result<(), anyhow::Error> { + async fn record_and_publish(&self, status: UpdateStatus) -> Result<(), Error> { let status = status.refreshed_for_write(); self.state.remember_completed(status.clone())?; self.best_effort_publish_terminal(&status).await; Ok(()) } - async fn publish_status(&self, status: &UpdateStatus) -> Result<(), anyhow::Error> { + async fn publish_status(&self, status: &UpdateStatus) -> Result<(), Error> { let status = status.refreshed_for_write(); let mut annotations = BTreeMap::new(); let annotation_key = match status.operation { @@ -883,7 +878,7 @@ where annotation_key.to_string(), Some(serde_json::to_string(&status)?), ); - log::info!( + info!( "sending {annotation_key} annotation to node {}: {status:?}", self.config.kubernetes.node_name ); @@ -902,29 +897,28 @@ where match self.publish_status(status).await { Ok(()) => return, Err(err) if self.is_node_gone_error(&err) => { - log::info!( + info!( "stopping terminal status publish because node {} no longer exists", self.config.kubernetes.node_name ); return; } - Err(_) => tokio::time::sleep(FINAL_STATUS_PATCH_BACKOFF).await, + Err(_) => time::sleep(FINAL_STATUS_PATCH_BACKOFF).await, } } } - fn is_node_gone_error(&self, err: &anyhow::Error) -> bool { + fn is_node_gone_error(&self, err: &Error) -> bool { matches!( err.downcast_ref::(), Some(K8sClientError::NodeGone) ) } - fn log_and_swallow_node_gone(&self, err: &anyhow::Error, context: &str) -> bool { + fn log_and_swallow_node_gone(&self, err: &Error, context: &str) -> bool { if self.is_node_gone_error(err) { - log::info!( - "stopping trident-acl-agent while {}: node {} no longer exists", - context, + info!( + "stopping trident-acl-agent while {context}: node {} no longer exists", self.config.kubernetes.node_name ); true @@ -937,23 +931,23 @@ where where F: Future, { - tokio::pin!(future); - let mut interval = tokio::time::interval(self.config.orchestration.heartbeat_interval); + pin!(future); + let mut interval = time::interval(self.config.orchestration.heartbeat_interval); interval.tick().await; let mut stop_heartbeats = false; loop { - tokio::select! { + select! { result = &mut future => return result, _ = interval.tick(), if !stop_heartbeats => { if let Err(err) = self.publish_status(&status).await { if self.is_node_gone_error(&err) { - log::info!( + info!( "stopping heartbeats because node {} no longer exists", self.config.kubernetes.node_name ); stop_heartbeats = true; } else { - log::warn!("failed to refresh in-progress status heartbeat: {err}"); + warn!("failed to refresh in-progress status heartbeat: {err}"); } } } @@ -986,7 +980,7 @@ where // deliberately no static-config fallback (see // resolve_nebraska_endpoint's docs). Guard anyway since this is // best-effort telemetry, not something worth panicking over. - log::warn!( + warn!( "skipping Nebraska '{}' report: request has no server (nodeUpdateId {})", report.label(), request.node_update_id @@ -994,7 +988,7 @@ where return; }; let Some(app_id) = self.resolve_nebraska_app_id(request) else { - log::warn!( + warn!( "skipping Nebraska '{}' report: request has no appId (nodeUpdateId {})", report.label(), request.node_update_id @@ -1002,7 +996,7 @@ where return; }; let Some(track) = self.resolve_nebraska_track(request) else { - log::warn!( + warn!( "skipping Nebraska '{}' report: request has no track (nodeUpdateId {})", report.label(), request.node_update_id @@ -1012,7 +1006,7 @@ where let machine_id = match crate::build_machine_id(NEBRASKA_MACHINE_ID_SOURCE) { Ok(id) => id, Err(err) => { - log::warn!( + warn!( "skipping Nebraska '{}' report: failed to build machine id: {err}", report.label() ); @@ -1020,7 +1014,7 @@ where } }; let label = report.label(); - let result = tokio::task::spawn_blocking(move || { + let result = task::spawn_blocking(move || { let client = NebraskaClient::new(endpoint, app_id, track, machine_id); match report { NebraskaReport::Progress { version, event } => { @@ -1036,13 +1030,21 @@ where }) .await; match result { - Ok(Ok(())) => log::debug!("reported Nebraska '{label}' event"), - Ok(Err(err)) => log::warn!("Nebraska '{label}' event report failed: {err}"), - Err(err) => log::warn!("Nebraska '{label}' event report task panicked: {err}"), + Ok(Ok(())) => debug!("reported Nebraska '{label}' event"), + Ok(Err(err)) => warn!("Nebraska '{label}' event report failed: {err}"), + Err(err) => warn!("Nebraska '{label}' event report task panicked: {err}"), } } } +/// Uses the kernel boot ID as the reboot marker because it changes on every +/// successful reboot but remains stable for the lifetime of the current boot. +/// That makes it a simple, durable fence for deciding whether a pending +/// finalize/rollback has crossed the reboot boundary yet. +fn current_boot_marker() -> Result { + machine_id::boot_id() +} + /// Parses `version` (e.g. an `UpdateStatus::from_version`/`to_version` /// field) as a semver [`Version`] for use in a Nebraska event report, /// logging and returning `None` rather than failing if it's absent or not @@ -1050,22 +1052,12 @@ where /// `Orchestrator::report_nebraska_event`), so a malformed/missing version /// string must only skip the report, never the Trident operation it /// describes. -fn current_boot_marker() -> Result { - let raw = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") - .context("failed to read /proc/sys/kernel/random/boot_id")?; - let marker = raw.trim().to_string(); - if marker.is_empty() { - anyhow::bail!("/proc/sys/kernel/random/boot_id was empty"); - } - Ok(marker) -} - fn parse_nebraska_version(version: &Option, context: &str) -> Option { let raw = version.as_deref()?; match Version::parse(raw) { Ok(v) => Some(v), Err(err) => { - log::warn!( + warn!( "skipping Nebraska event report for {context}: {raw:?} is not valid semver: {err}" ); None @@ -1109,7 +1101,7 @@ impl Snapshot { node_update_id: candidate.node_update_id, operation_id: candidate.operation_id, operation: candidate.operation.into(), - reason, + reason: reason.to_string(), }), ), }, @@ -1118,7 +1110,7 @@ impl Snapshot { // even parse out of the annotation - log loudly instead so // this doesn't fail silently, but there's no request to // surface an InvalidRequest status against. - log::warn!( + warn!( "ignoring malformed {} annotation (JSON parse failed): {err}", keys.request ); @@ -1336,10 +1328,6 @@ fn indicates_target_boot_failed(error: &TridentClientError) -> bool { .unwrap_or(false) } -/// Pure function extracted from `Orchestrator::map_commit_result` so tests -/// can exercise it directly (with a mock-tridentd-driven `Result`) without -/// needing a full `Orchestrator` instance. See `stage_result_to_status` for -/// rationale. /// Pre-flight checks for the state.json-missing degraded reconstruction /// path ( §2.3). Returns `Some(status)` when reconstruction /// cannot proceed (tridentd already known-unreachable, or the outstanding @@ -1445,6 +1433,10 @@ fn reconstruct_commit_result_to_status( } } +/// Pure function extracted from `Orchestrator::map_commit_result` so tests +/// can exercise it directly (with a mock-tridentd-driven `Result`) without +/// needing a full `Orchestrator` instance. See `stage_result_to_status` for +/// rationale. fn commit_result_to_status( pending: &PendingCommit, result: Result, @@ -1563,12 +1555,14 @@ fn rollback_finalize_failure_status( #[cfg(test)] mod tests { + use super::*; + use std::sync::{Arc, Mutex}; use chrono::Utc; use uuid::Uuid; - use super::*; + const MOCK_RPC_TIMEOUT: Duration = Duration::from_secs(5); use crate::{ annotations::{RequestedOperation, SCHEMA_VERSION}, core::trident::mock::{connect_mock_client, MockTridentdConfig, Outcome}, @@ -1611,9 +1605,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client - .rollback_stage(std::time::Duration::from_secs(5)) - .await; + let result = client.rollback_stage(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Rollback); let status = rollback_stage_failure_status( @@ -1647,7 +1639,7 @@ mod tests { })); let mut client = connect_mock_client(config).await; let response = client - .rollback_stage(std::time::Duration::from_secs(5)) + .rollback_stage(MOCK_RPC_TIMEOUT) .await .expect("mocked rollback_stage should succeed"); assert_eq!( @@ -1667,7 +1659,7 @@ mod tests { })); let mut client = connect_mock_client(config).await; let response = client - .rollback_stage(std::time::Duration::from_secs(5)) + .rollback_stage(MOCK_RPC_TIMEOUT) .await .expect("mocked rollback_stage should succeed"); assert_eq!(response.servicing_kind, Some(ServicingKind::NoneRequired)); @@ -1683,10 +1675,8 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client - .rollback_finalize(std::time::Duration::from_secs(5)) - .await; - assert!(result.is_ok()); + let result = client.rollback_finalize(MOCK_RPC_TIMEOUT).await; + result.expect("mocked rollback_finalize should succeed"); let request = request(RequestedOperation::Rollback); let status = @@ -1707,9 +1697,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client - .rollback_finalize(std::time::Duration::from_secs(5)) - .await; + let result = client.rollback_finalize(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Rollback); let status = rollback_finalize_failure_status( @@ -1734,9 +1722,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client - .rollback_finalize(std::time::Duration::from_secs(5)) - .await; + let result = client.rollback_finalize(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Rollback); let status = rollback_finalize_failure_status( @@ -1765,7 +1751,7 @@ mod tests { .update_stage( &"http://example.test/image".parse().unwrap(), None, - std::time::Duration::from_secs(5), + MOCK_RPC_TIMEOUT, ) .await; @@ -1798,7 +1784,7 @@ mod tests { .update_stage( &"http://example.test/image".parse().unwrap(), None, - std::time::Duration::from_secs(5), + MOCK_RPC_TIMEOUT, ) .await; @@ -1824,11 +1810,11 @@ mod tests { .update_stage( &"http://example.test/image".parse().unwrap(), None, - std::time::Duration::from_secs(5), + MOCK_RPC_TIMEOUT, ) .await; - let version = semver::Version::new(1, 0, 0); + let version = Version::new(1, 0, 0); let report = stage_nebraska_report(&version, &result); assert_eq!( report, @@ -1853,11 +1839,11 @@ mod tests { .update_stage( &"http://example.test/image".parse().unwrap(), None, - std::time::Duration::from_secs(5), + MOCK_RPC_TIMEOUT, ) .await; - let version = semver::Version::new(1, 0, 0); + let version = Version::new(1, 0, 0); let report = stage_nebraska_report(&version, &result); assert_eq!( report, @@ -1894,10 +1880,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let err = client - .update_finalize(std::time::Duration::from_secs(5)) - .await - .unwrap_err(); + let err = client.update_finalize(MOCK_RPC_TIMEOUT).await.unwrap_err(); let status = finalize_failure_status( &request(RequestedOperation::Finalize), @@ -1922,10 +1905,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let err = client - .update_finalize(std::time::Duration::from_secs(5)) - .await - .unwrap_err(); + let err = client.update_finalize(MOCK_RPC_TIMEOUT).await.unwrap_err(); let status = finalize_failure_status( &request(RequestedOperation::Finalize), @@ -1948,11 +1928,9 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client - .update_finalize(std::time::Duration::from_secs(5)) - .await; + let result = client.update_finalize(MOCK_RPC_TIMEOUT).await; - let version = semver::Version::new(1, 0, 0); + let version = Version::new(1, 0, 0); let report = finalize_nebraska_report(&version, &result); assert_eq!( report, @@ -1973,11 +1951,9 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client - .update_finalize(std::time::Duration::from_secs(5)) - .await; + let result = client.update_finalize(MOCK_RPC_TIMEOUT).await; - let version = semver::Version::new(1, 0, 0); + let version = Version::new(1, 0, 0); let report = finalize_nebraska_report(&version, &result); assert_eq!( report, @@ -2000,7 +1976,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let status = commit_result_to_status(&pending(Operation::Finalize), result); @@ -2018,7 +1994,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let status = commit_result_to_status(&pending(Operation::Finalize), result); @@ -2036,7 +2012,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let status = commit_result_to_status(&pending(Operation::Finalize), result); @@ -2054,7 +2030,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let status = commit_result_to_status(&pending(Operation::Finalize), result); @@ -2071,7 +2047,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let status = commit_result_to_status(&pending(Operation::Finalize), result); @@ -2088,10 +2064,10 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; - let previous = semver::Version::new(1, 0, 0); - let current = semver::Version::new(2, 0, 0); + let previous = Version::new(1, 0, 0); + let current = Version::new(2, 0, 0); let report = commit_nebraska_report(&previous, ¤t, &result); assert_eq!(report, NebraskaReport::Completed { previous, current }); } @@ -2110,10 +2086,10 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; - let previous = semver::Version::new(1, 0, 0); - let current = semver::Version::new(2, 0, 0); + let previous = Version::new(1, 0, 0); + let current = Version::new(2, 0, 0); let report = commit_nebraska_report(&previous, ¤t, &result); assert_eq!( report, @@ -2134,10 +2110,10 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; - let previous = semver::Version::new(1, 0, 0); - let current = semver::Version::new(2, 0, 0); + let previous = Version::new(1, 0, 0); + let current = Version::new(2, 0, 0); let report = commit_nebraska_report(&previous, ¤t, &result); assert_eq!( report, @@ -2198,7 +2174,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Finalize); let status = reconstruct_commit_result_to_status( @@ -2224,7 +2200,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Finalize); let status = reconstruct_commit_result_to_status(&request, None, Utc::now(), result); @@ -2243,7 +2219,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Rollback); let status = reconstruct_commit_result_to_status(&request, None, Utc::now(), result); @@ -2262,7 +2238,7 @@ mod tests { ..Default::default() })); let mut client = connect_mock_client(config).await; - let result = client.commit(std::time::Duration::from_secs(5)).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; let request = request(RequestedOperation::Finalize); let status = reconstruct_commit_result_to_status(&request, None, Utc::now(), result); @@ -2283,7 +2259,7 @@ mod tests { fn parse_nebraska_version_parses_valid_semver() { assert_eq!( parse_nebraska_version(&Some("1.2.3".to_string()), "test"), - Some(semver::Version::new(1, 2, 3)) + Some(Version::new(1, 2, 3)) ); } diff --git a/crates/trident-acl-agent/src/annotations/protocol.rs b/crates/trident-acl-agent/src/annotations/protocol.rs index a6b75be690..b23ece8dda 100644 --- a/crates/trident-acl-agent/src/annotations/protocol.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use uuid::Uuid; -use crate::core::config::DEFAULT_ANNOTATION_PREFIX; +use crate::core::{config::DEFAULT_ANNOTATION_PREFIX, error::AgentError}; /// Suffix (appended to the configured annotation prefix) for the request /// annotation, e.g. `acl.microsoft.com/update-request`. @@ -63,7 +63,7 @@ pub const SCHEMA_VERSION: &str = "1.0"; const MAX_MESSAGE_BYTES: usize = 2048; const TRUNCATION_MARKER: &str = "... (truncated)"; -#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] pub enum RequestedOperation { Stage, @@ -71,7 +71,7 @@ pub enum RequestedOperation { Rollback, } -#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] pub enum Operation { Stage, @@ -80,7 +80,8 @@ pub enum Operation { Commit, } -#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[serde(rename_all = "PascalCase")] pub enum StatusCode { InProgress, Success, @@ -92,7 +93,7 @@ pub enum StatusCode { InvalidRequest, } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct UpdateRequest { pub schema_version: String, @@ -126,7 +127,7 @@ pub struct UpdateRequest { pub track: Option, } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct UpdateStatus { pub schema_version: String, @@ -151,41 +152,56 @@ impl UpdateRequest { /// targetVersion required for stage/finalize but disallowed for /// rollback, and server/appId/track required for stage/finalize. See /// this file's module doc. - pub fn validate(self) -> Result { + pub fn validate(self) -> Result { if self.schema_version != SCHEMA_VERSION { - return Err(format!("unsupported schemaVersion {}", self.schema_version)); + return Err(AgentError::InvalidRequest(format!( + "unsupported schemaVersion {}", + self.schema_version + ))); } if Uuid::parse_str(&self.operation_id).is_err() { - return Err(format!( + return Err(AgentError::InvalidRequest(format!( "operationId must be a UUID, got {:?}", self.operation_id - )); + ))); } match self.operation { RequestedOperation::Stage | RequestedOperation::Finalize => { if self.target_version.as_deref().unwrap_or("").is_empty() { - return Err("targetVersion is required for stage/finalize".to_string()); + return Err(AgentError::InvalidRequest( + "targetVersion is required for stage/finalize".to_string(), + )); } if self.server.is_none() { - return Err("server is required for stage/finalize".to_string()); + return Err(AgentError::InvalidRequest( + "server is required for stage/finalize".to_string(), + )); } if self .server .as_ref() .is_some_and(|u| !matches!(u.scheme(), "http" | "https")) { - return Err("server must be an http(s) URL".to_string()); + return Err(AgentError::InvalidRequest( + "server must be an http(s) URL".to_string(), + )); } if self.app_id.as_deref().unwrap_or("").is_empty() { - return Err("appId is required for stage/finalize".to_string()); + return Err(AgentError::InvalidRequest( + "appId is required for stage/finalize".to_string(), + )); } if self.track.as_deref().unwrap_or("").is_empty() { - return Err("track is required for stage/finalize".to_string()); + return Err(AgentError::InvalidRequest( + "track is required for stage/finalize".to_string(), + )); } } RequestedOperation::Rollback => { if self.target_version.is_some() { - return Err("targetVersion must be omitted for rollback".to_string()); + return Err(AgentError::InvalidRequest( + "targetVersion must be omitted for rollback".to_string(), + )); } } } @@ -289,12 +305,11 @@ impl From for Operation { #[cfg(test)] mod tests { - use chrono::TimeZone; - use serde_json::Value; - use uuid::Uuid; - use super::*; + use chrono::TimeZone; + use serde_json::{Map, Value}; + #[test] fn annotation_keys_default_uses_acl_microsoft_com_prefix() { let keys = AnnotationKeys::default(); @@ -836,7 +851,7 @@ mod tests { Ok(()) } - fn schema_if_matches(if_schema: &Value, obj: &serde_json::Map) -> bool { + fn schema_if_matches(if_schema: &Value, obj: &Map) -> bool { let Some(if_obj) = if_schema.as_object() else { return false; }; @@ -1011,7 +1026,7 @@ mod tests { let err = request .validate() .expect_err("missing server must be rejected for stage/finalize"); - assert!(err.contains("server"), "{err}"); + assert!(err.to_string().contains("server"), "{err}"); } } @@ -1036,7 +1051,7 @@ mod tests { let err = request .validate() .expect_err("a non-http(s) server scheme must be rejected"); - assert!(err.contains("server"), "{err}"); + assert!(err.to_string().contains("server"), "{err}"); } } @@ -1048,7 +1063,7 @@ mod tests { let err = request .validate() .expect_err("missing appId must be rejected for stage/finalize"); - assert!(err.contains("appId"), "{err}"); + assert!(err.to_string().contains("appId"), "{err}"); } } @@ -1060,7 +1075,7 @@ mod tests { let err = request .validate() .expect_err("missing track must be rejected for stage/finalize"); - assert!(err.contains("track"), "{err}"); + assert!(err.to_string().contains("track"), "{err}"); } } @@ -1099,7 +1114,7 @@ mod tests { let err = request .validate() .expect_err("a non-UUID operationId must be rejected"); - assert!(err.contains("operationId"), "{err}"); + assert!(err.to_string().contains("operationId"), "{err}"); } } diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index 7c1856b7e4..2f085aaf50 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -9,17 +9,24 @@ use std::{ collections::BTreeMap, fs, - io::Write, + io::{ErrorKind, Write}, os::unix::fs::OpenOptionsExt, path::{Path, PathBuf}, + process, }; -use anyhow::Context; +use anyhow::{Context, Error}; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::annotations::{Operation, UpdateRequest, UpdateStatus}; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +const DEFAULT_STATE_FILE_NAME: &str = "state.json"; +// state.json persists the full UpdateRequest, which can include a +// secret-bearing Omaha `server` URL, so store it with owner-only permissions. +const STATE_FILE_MODE: u32 = 0o600; + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct PersistentState { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -28,7 +35,7 @@ pub struct PersistentState { pub completed: BTreeMap, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct CompletedEntry { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -37,7 +44,7 @@ pub struct CompletedEntry { pub commit: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct PendingCommit { pub request: UpdateRequest, @@ -47,7 +54,7 @@ pub struct PendingCommit { pub from_version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub to_version: Option, - pub started_utc: chrono::DateTime, + pub started_utc: DateTime, pub boot_marker: String, } @@ -65,21 +72,16 @@ impl StateStore { &self.path } - pub fn load(&self) -> Result { + pub fn load(&self) -> Result { match fs::read_to_string(&self.path) { Ok(raw) => Ok(serde_json::from_str(&raw) .with_context(|| format!("failed to parse {}", self.path.display()))?), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - Ok(PersistentState::default()) - } - Err(err) => { - Err(anyhow::Error::new(err) - .context(format!("failed to read {}", self.path.display()))) - } + Err(err) if err.kind() == ErrorKind::NotFound => Ok(PersistentState::default()), + Err(err) => Err(err).with_context(|| format!("failed to read {}", self.path.display())), } } - pub fn save(&self, state: &PersistentState) -> Result<(), anyhow::Error> { + pub fn save(&self, state: &PersistentState) -> Result<(), Error> { let parent = match self.path.parent() { Some(parent) => { fs::create_dir_all(parent) @@ -94,8 +96,8 @@ impl StateStore { self.path .file_name() .and_then(|name| name.to_str()) - .unwrap_or("state.json"), - std::process::id() + .unwrap_or(DEFAULT_STATE_FILE_NAME), + process::id() )); // Write via a File handle and fsync it before the rename: fs::write @@ -112,7 +114,7 @@ impl StateStore { .write(true) .create(true) .truncate(true) - .mode(0o600) + .mode(STATE_FILE_MODE) .open(&temp_path) .with_context(|| format!("failed to create {}", temp_path.display()))?; file.write_all(serde_json::to_string_pretty(state)?.as_bytes()) @@ -140,7 +142,7 @@ impl StateStore { Ok(()) } - pub fn remember_completed(&self, status: UpdateStatus) -> Result<(), anyhow::Error> { + pub fn remember_completed(&self, status: UpdateStatus) -> Result<(), Error> { let mut state = self.load()?; let entry = state .completed @@ -153,13 +155,13 @@ impl StateStore { self.save(&state) } - pub fn set_pending_commit(&self, pending: PendingCommit) -> Result<(), anyhow::Error> { + pub fn set_pending_commit(&self, pending: PendingCommit) -> Result<(), Error> { let mut state = self.load()?; state.pending_commit = Some(pending); self.save(&state) } - pub fn clear_pending_commit(&self) -> Result<(), anyhow::Error> { + pub fn clear_pending_commit(&self) -> Result<(), Error> { let mut state = self.load()?; state.pending_commit = None; self.save(&state) @@ -168,13 +170,13 @@ impl StateStore { #[cfg(test)] mod tests { + use super::*; + + use crate::annotations::{RequestedOperation, StatusCode, SCHEMA_VERSION}; use chrono::Utc; use url::Url; use uuid::Uuid; - use super::*; - use crate::annotations::{RequestedOperation, StatusCode, SCHEMA_VERSION}; - fn store() -> (tempfile::TempDir, StateStore) { let dir = tempfile::tempdir().expect("failed to create temp dir"); let path = dir.path().join("state.json"); @@ -235,7 +237,7 @@ mod tests { #[test] fn save_then_load_round_trips_full_state() { let (_dir, store) = store(); - let mut completed = std::collections::BTreeMap::new(); + let mut completed = BTreeMap::new(); completed.insert( "op-1".to_string(), CompletedEntry { @@ -365,14 +367,14 @@ mod tests { .save(&PersistentState::default()) .expect("initial save should succeed"); - let metadata_before = std::fs::metadata(store.path()).expect("state file should exist"); + let metadata_before = fs::metadata(store.path()).expect("state file should exist"); let state = PersistentState { pending_commit: Some(sample_pending()), completed: BTreeMap::new(), }; store.save(&state).expect("second save should succeed"); - let metadata_after = std::fs::metadata(store.path()).expect("state file should exist"); + let metadata_after = fs::metadata(store.path()).expect("state file should exist"); assert!(metadata_after.len() > 0); assert!(metadata_before.modified().is_ok()); } diff --git a/crates/trident-acl-agent/src/connection_check.rs b/crates/trident-acl-agent/src/connection_check.rs index 03aaba1460..62f04f00f5 100644 --- a/crates/trident-acl-agent/src/connection_check.rs +++ b/crates/trident-acl-agent/src/connection_check.rs @@ -1,100 +1,101 @@ -use anyhow::Context; - -use trident_acl_agent::{ - annotations::k8s::NodeClient, - check_nebraska_reachable, - core::{config::AgentConfig, nebraska, trident::TridentClient}, - IdSource, -}; - -use crate::cli::ConnectionTarget; - -/// Checks connectivity to exactly one of `target`'s dependencies and returns -/// `Ok(())` on success. The caller (`main`) surfaces any `Err` the normal way -/// (`anyhow`'s `Termination` impl prints the error and exits non-zero), so -/// this function only needs to produce a descriptive error on failure - no -/// explicit `process::exit` is required. -pub async fn validate_connection( - target: ConnectionTarget, - config: &AgentConfig, -) -> Result<(), anyhow::Error> { - match target { - ConnectionTarget::Kubernetes => { - let client = NodeClient::new(&config.kubernetes) - .await - .context("failed to build Kubernetes client")?; - // Report the actually-resolved server (kubeconfig's own server, - // unless overridden by kubernetes.api_server), not a value - // guessed from config - the two only match when an override is - // set. - let cluster_url = client.cluster_url(); - client - .get_node(&config.kubernetes.node_name) - .await - .with_context(|| { - format!( - "failed to reach Kubernetes API server at {} (get Node {:?})", - cluster_url, config.kubernetes.node_name - ) - })?; - log::info!( - "kubernetes: reached API server at {} and fetched Node {:?}", - cluster_url, - config.kubernetes.node_name - ); - } - ConnectionTarget::Tridentd => { - TridentClient::connect(&config.trident.socket) - .await - .with_context(|| { - format!("failed to reach tridentd at {}", config.trident.socket) - })?; - log::info!("tridentd: connected to {}", config.trident.socket); - } - ConnectionTarget::Nebraska => { - let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { - anyhow::anyhow!( - "nebraska.endpoint is not configured (set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT)" - ) - })?; - let app_id = config.nebraska.app_id.clone(); - // check_nebraska_reachable() is a blocking call (reqwest::blocking - // under the hood, see nebraska::transport) - calling it directly from this - // async fn can panic ("Cannot drop a runtime in a context where - // blocking is not allowed") because reqwest::blocking spins up - // its own inner Tokio runtime per call, which isn't safe to tear - // down from inside an already-running async task. Run it on a - // dedicated blocking thread instead. - // - // Deliberately uses check_nebraska_reachable() rather than - // query_for_update(): the latter also validates app-level - // semantics (app ID match, non-error app/update-check status), - // which would make this a "can we get a valid update check" test - // rather than the pure reachability check documented on - // ConnectionTarget::Nebraska above. - let endpoint_for_task = endpoint.clone(); - let track = config.nebraska.track.clone(); - tokio::task::spawn_blocking(move || { - check_nebraska_reachable( - &endpoint_for_task, - &app_id, - &track, - IdSource::MachineIdHashed, - ) - }) - .await - .context("Nebraska connectivity check task panicked")? - .with_context(|| { - format!( - "failed to reach Nebraska server at {}", - nebraska::redacted(&endpoint) - ) - })?; - log::info!( - "nebraska: reached server at {}", - nebraska::redacted(&endpoint) - ); - } - } - Ok(()) -} +use anyhow::{anyhow, Context, Error}; +use log::info; +use tokio::task; + +use trident_acl_agent::{ + annotations::k8s::NodeClient, + check_nebraska_reachable, + core::{config::AgentConfig, nebraska, trident::TridentClient}, + IdSource, +}; + +use crate::cli::ConnectionTarget; + +/// Checks connectivity to exactly one of `target`'s dependencies and returns +/// `Ok(())` on success. The caller (`main`) surfaces any `Err` the normal way +/// (`anyhow`'s `Termination` impl prints the error and exits non-zero), so +/// this function only needs to produce a descriptive error on failure - no +/// explicit `process::exit` is required. +pub async fn validate_connection( + target: ConnectionTarget, + config: &AgentConfig, +) -> Result<(), Error> { + match target { + ConnectionTarget::Kubernetes => { + let client = NodeClient::new(&config.kubernetes) + .await + .context("failed to build Kubernetes client")?; + // Report the actually-resolved server (kubeconfig's own server, + // unless overridden by kubernetes.api_server), not a value + // guessed from config - the two only match when an override is + // set. + let cluster_url = client.cluster_url(); + client + .get_node(&config.kubernetes.node_name) + .await + .with_context(|| { + format!( + "failed to reach Kubernetes API server at {cluster_url} (get Node {:?})", + config.kubernetes.node_name + ) + })?; + info!( + "kubernetes: reached API server at {cluster_url} and fetched Node {:?}", + config.kubernetes.node_name + ); + } + ConnectionTarget::Tridentd => { + TridentClient::connect(&config.trident.socket) + .await + .with_context(|| { + format!("failed to reach tridentd at {}", config.trident.socket) + })?; + info!("tridentd: connected to {}", config.trident.socket); + } + ConnectionTarget::Nebraska => { + let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { + anyhow!( + "nebraska.endpoint is not configured (set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT)" + ) + })?; + let app_id = config.nebraska.app_id.clone(); + // check_nebraska_reachable() is a blocking call (reqwest::blocking + // under the hood, see nebraska::transport) - calling it directly from this + // async fn can panic ("Cannot drop a runtime in a context where + // blocking is not allowed") because reqwest::blocking spins up + // its own inner Tokio runtime per call, which isn't safe to tear + // down from inside an already-running async task. Run it on a + // dedicated blocking thread instead. + // + // Deliberately uses check_nebraska_reachable() rather than + // query_for_update(): the latter also validates app-level + // semantics (app ID match, non-error app/update-check status), + // which would make this a "can we get a valid update check" test + // rather than the pure reachability check documented on + // ConnectionTarget::Nebraska above. + let endpoint_for_task = endpoint.clone(); + let track = config.nebraska.track.clone(); + task::spawn_blocking(move || { + check_nebraska_reachable( + &endpoint_for_task, + &app_id, + &track, + IdSource::MachineIdHashed, + ) + }) + .await + .context("Nebraska connectivity check task panicked")? + .with_context(|| { + format!( + "failed to reach Nebraska server at {}", + nebraska::redacted(&endpoint) + ) + })?; + info!( + "nebraska: reached server at {}", + nebraska::redacted(&endpoint) + ); + } + } + Ok(()) +} diff --git a/crates/trident-acl-agent/src/core/config.rs b/crates/trident-acl-agent/src/core/config.rs index 7828022774..281be4f26a 100644 --- a/crates/trident-acl-agent/src/core/config.rs +++ b/crates/trident-acl-agent/src/core/config.rs @@ -16,6 +16,9 @@ use std::{env, path::PathBuf, str::FromStr, time::Duration}; +use anyhow::{anyhow, Context, Error}; +use osutils::hostname; +use trident_proto::TRIDENT_DEFAULT_SOCKET_URI; use url::Url; use crate::{DEFAULT_NEBRASKA_APP_ID, DEFAULT_NEBRASKA_TRACK}; @@ -50,6 +53,7 @@ const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); // carry their own `server` field, with no fallback to this config (see // Orchestrator::resolve_nebraska_endpoint). pub const DEFAULT_NEBRASKA_ENDPOINT: &str = "https://nebraska.example.invalid/v1/update"; +const DEFAULT_NODE_NAME: &str = "localhost"; const DEFAULT_STAGE_TIMEOUT: Duration = Duration::from_secs(20 * 60); const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); @@ -73,11 +77,11 @@ impl AgentConfig { /// is never an error - it just falls back to that setting's default - /// but a present-and-malformed value (bad URL, bad duration, unknown /// `goal_source`, etc.) is. - pub fn from_env() -> Result { + pub fn from_env() -> Result { Ok(Self { nebraska: NebraskaConfig { endpoint: env_url(ENV_NEBRASKA_ENDPOINT)? - .or_else(|| Some(Url::parse(DEFAULT_NEBRASKA_ENDPOINT).expect("static url"))), + .or_else(|| Some(default_nebraska_endpoint())), app_id: env_string(ENV_NEBRASKA_APP_ID) .unwrap_or_else(|| DEFAULT_NEBRASKA_APP_ID.to_string()), track: env_string(ENV_NEBRASKA_TRACK) @@ -94,7 +98,7 @@ impl AgentConfig { }, trident: TridentConfig { socket: env_string(ENV_TRIDENT_SOCKET) - .unwrap_or_else(|| trident_proto::TRIDENT_DEFAULT_SOCKET_URI.to_string()), + .unwrap_or_else(|| TRIDENT_DEFAULT_SOCKET_URI.to_string()), }, orchestration: OrchestrationConfig { goal_source: env_parse(ENV_ORCHESTRATION_GOAL_SOURCE)?.unwrap_or_default(), @@ -128,7 +132,7 @@ pub struct NebraskaConfig { impl Default for NebraskaConfig { fn default() -> Self { Self { - endpoint: Some(Url::parse(DEFAULT_NEBRASKA_ENDPOINT).expect("static url")), + endpoint: Some(default_nebraska_endpoint()), app_id: DEFAULT_NEBRASKA_APP_ID.to_string(), track: DEFAULT_NEBRASKA_TRACK.to_string(), } @@ -177,7 +181,7 @@ pub struct TridentConfig { impl Default for TridentConfig { fn default() -> Self { Self { - socket: trident_proto::TRIDENT_DEFAULT_SOCKET_URI.to_string(), + socket: TRIDENT_DEFAULT_SOCKET_URI.to_string(), } } } @@ -208,13 +212,13 @@ pub enum GoalSource { } impl FromStr for GoalSource { - type Err = anyhow::Error; + type Err = Error; fn from_str(s: &str) -> Result { match s { "omaha-only" => Ok(GoalSource::OmahaOnly), "annotations" => Ok(GoalSource::Annotations), - other => Err(anyhow::anyhow!( + other => Err(anyhow!( "unknown goal_source {other:?} (expected \"annotations\" or \"omaha-only\")" )), } @@ -258,25 +262,29 @@ fn env_string(name: &str) -> Option { env_raw(name) } -fn env_url(name: &str) -> Result, anyhow::Error> { +fn default_nebraska_endpoint() -> Url { + Url::parse(DEFAULT_NEBRASKA_ENDPOINT) + .expect("invariant: DEFAULT_NEBRASKA_ENDPOINT is a compile-time-valid URL") +} + +fn env_url(name: &str) -> Result, Error> { env_raw(name) - .map(|v| Url::parse(&v).map_err(|err| anyhow::anyhow!("invalid URL for {name}: {err}"))) + .map(|v| Url::parse(&v).with_context(|| format!("invalid URL for {name}"))) .transpose() } -fn env_duration(name: &str, default: Duration) -> Result { +fn env_duration(name: &str, default: Duration) -> Result { env_raw(name) .map(|v| { - humantime::parse_duration(&v) - .map_err(|err| anyhow::anyhow!("invalid duration for {name}: {err}")) + humantime::parse_duration(&v).with_context(|| format!("invalid duration for {name}")) }) .transpose() .map(|parsed| parsed.unwrap_or(default)) } -fn env_parse(name: &str) -> Result, anyhow::Error> +fn env_parse(name: &str) -> Result, Error> where - T: FromStr, + T: FromStr, { env_raw(name).map(|v| v.parse::()).transpose() } @@ -287,8 +295,8 @@ fn default_node_name() -> String { // registers the Node object. Match that behavior here so a mixed-case // hostname doesn't produce a node_name that can never match the actual // Node the agent is supposed to reconcile against. - osutils::hostname::read() - .unwrap_or_else(|_| "localhost".to_string()) + hostname::read() + .unwrap_or_else(|_| DEFAULT_NODE_NAME.to_string()) .to_lowercase() } @@ -342,10 +350,7 @@ mod tests { config.kubernetes.kubeconfig, DEFAULT_KUBELET_KUBECONFIG.to_string() ); - assert_eq!( - config.trident.socket, - trident_proto::TRIDENT_DEFAULT_SOCKET_URI - ); + assert_eq!(config.trident.socket, TRIDENT_DEFAULT_SOCKET_URI); assert_eq!(config.orchestration.goal_source, GoalSource::Annotations); assert_eq!( config.orchestration.state_path, diff --git a/crates/trident-acl-agent/src/core/error.rs b/crates/trident-acl-agent/src/core/error.rs index e79f54f842..f55ecdcb47 100644 --- a/crates/trident-acl-agent/src/core/error.rs +++ b/crates/trident-acl-agent/src/core/error.rs @@ -1,7 +1,8 @@ use serde::{Deserialize, Serialize}; +use thiserror::Error; -#[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "kebab-case")] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Error)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] pub enum AgentError { #[error("Failed to initialize the trident-acl-agent client: {0}")] InitializationError(String), @@ -18,6 +19,9 @@ pub enum AgentError { #[error("Internal error: {0}")] Internal(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + /// Wraps a [`nebraska::NebraskaError`](crate::core::nebraska::NebraskaError). /// Stored as a string rather than `#[from]` because `NebraskaError` /// doesn't derive `Serialize`/`Deserialize`/`PartialEq`, which diff --git a/crates/trident-acl-agent/src/core/trident/client.rs b/crates/trident-acl-agent/src/core/trident/client.rs index 5bfddea39d..57f56e702c 100644 --- a/crates/trident-acl-agent/src/core/trident/client.rs +++ b/crates/trident-acl-agent/src/core/trident/client.rs @@ -15,11 +15,18 @@ //! progress for anything commit() reports nothing to do for. See //! orchestrator.rs's recover_from_trident_state for the full rationale. -use std::time::Duration; +use std::{future::Future, time::Duration}; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use futures::StreamExt; -use tonic::{transport::Endpoint, Request, Streaming}; +use log::{debug, error, info, trace, warn}; +use serde::Serialize; +use thiserror::Error; +use tokio::time; +use tonic::{ + transport::{Channel, Endpoint, Error as TransportError}, + Request, Streaming, +}; use trident_proto::v1::{ commit_service_client::CommitServiceClient, rollback_service_client::RollbackServiceClient, servicing_response::Response as ResponseBody, update_service_client::UpdateServiceClient, @@ -44,13 +51,13 @@ pub struct RemoteError { pub error_message: String, } -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Error)] pub enum TridentClientError { #[error("failed to connect to trident socket {socket}: {source}")] Connect { socket: String, #[source] - source: tonic::transport::Error, + source: TransportError, }, #[error("failed to start trident request {operation}: {source}")] Request { @@ -64,7 +71,7 @@ pub enum TridentClientError { Stream { operation: &'static str, #[source] - source: anyhow::Error, + source: Error, }, #[error("trident reported {operation} failure: {details:?}")] Remote { @@ -87,10 +94,16 @@ impl TridentClientError { } } +const PLACEHOLDER_IMAGE_HASH: &str = "ignored"; +// Trident reports a structured error object for real remote failures; if it +// omits one entirely, keep a sentinel subkind so callers can still surface +// that contract explicitly. +const UNKNOWN_REMOTE_ERROR_SUBKIND: &str = "unknown"; + pub struct TridentClient { - update_client: UpdateServiceClient, - commit_client: CommitServiceClient, - rollback_client: RollbackServiceClient, + update_client: UpdateServiceClient, + commit_client: CommitServiceClient, + rollback_client: RollbackServiceClient, } impl TridentClient { @@ -118,7 +131,7 @@ impl TridentClient { /// over an in-memory duplex stream) and exercise the exact same /// request/response/error-mapping code as production, without a real /// unix socket or subprocess. - pub fn from_channel(channel: tonic::transport::Channel) -> Self { + pub fn from_channel(channel: Channel) -> Self { Self { update_client: UpdateServiceClient::new(channel.clone()), commit_client: CommitServiceClient::new(channel.clone()), @@ -303,13 +316,13 @@ impl TridentClient { } } -#[derive(serde::Serialize)] +#[derive(Serialize)] struct ImageSpec<'a> { url: &'a str, sha384: &'a str, } -#[derive(serde::Serialize)] +#[derive(Serialize)] struct HostConfigurationYaml<'a> { image: ImageSpec<'a>, } @@ -322,7 +335,7 @@ pub fn host_configuration_from_image(url: &Url, hash: Option<&str>) -> HostConfi let spec = HostConfigurationYaml { image: ImageSpec { url: url.as_str(), - sha384: hash.unwrap_or("ignored"), + sha384: hash.unwrap_or(PLACEHOLDER_IMAGE_HASH), }, }; HostConfiguration { @@ -334,9 +347,9 @@ pub fn host_configuration_from_image(url: &Url, hash: Option<&str>) -> HostConfi async fn run_with_timeout( operation: &'static str, timeout: Duration, - future: impl std::future::Future>, + future: impl Future>, ) -> Result { - tokio::time::timeout(timeout, future) + time::timeout(timeout, future) .await .map_err(|_| TridentClientError::Timeout { operation, timeout })? } @@ -353,18 +366,18 @@ async fn consume_servicing_stream( match response.response { Some(ResponseBody::Started(_)) => { - log::info!("[Trident:{operation}] started"); + info!("[Trident:{operation}] started"); } Some(ResponseBody::Log(log_record)) => { let message = &log_record.message; match log_record.level() { LogLevel::Unspecified | LogLevel::Trace => { - log::trace!("[Trident:{operation}] {message}") + trace!("[Trident:{operation}] {message}") } - LogLevel::Debug => log::debug!("[Trident:{operation}] {message}"), - LogLevel::Info => log::info!("[Trident:{operation}] {message}"), - LogLevel::Warn => log::warn!("[Trident:{operation}] {message}"), - LogLevel::Error => log::error!("[Trident:{operation}] {message}"), + LogLevel::Debug => debug!("[Trident:{operation}] {message}"), + LogLevel::Info => info!("[Trident:{operation}] {message}"), + LogLevel::Warn => warn!("[Trident:{operation}] {message}"), + LogLevel::Error => error!("[Trident:{operation}] {message}"), } } Some(ResponseBody::Completed(completed)) => { @@ -387,7 +400,7 @@ async fn consume_servicing_stream( }) .unwrap_or(RemoteError { kind: None, - subkind: "unknown".to_string(), + subkind: UNKNOWN_REMOTE_ERROR_SUBKIND.to_string(), message: format!("Trident {operation} failed without structured error"), error_message: String::new(), }); diff --git a/crates/trident-acl-agent/src/core/trident/mock.rs b/crates/trident-acl-agent/src/core/trident/mock.rs index 5052328418..82cfc362bb 100644 --- a/crates/trident-acl-agent/src/core/trident/mock.rs +++ b/crates/trident-acl-agent/src/core/trident/mock.rs @@ -11,11 +11,18 @@ //! `tokio::io::duplex` transport via `Endpoint::connect_with_connector` + //! `TridentClient::from_channel` - see `connect_mock_client` below. -use std::sync::{Arc, Mutex}; +use std::{ + io::Error as IoError, + sync::{Arc, Mutex}, +}; use hyper_util::rt::TokioIo; +use tokio::{io, sync::mpsc}; use tokio_stream::wrappers::ReceiverStream; -use tonic::{transport::Endpoint, Request, Response, Status}; +use tonic::{ + transport::{Endpoint, Server, Uri}, + Request, Response, Status, +}; use trident_proto::v1::{ commit_service_server::{CommitService, CommitServiceServer}, rollback_service_server::{RollbackService, RollbackServiceServer}, @@ -28,6 +35,9 @@ use trident_proto::v1::{ use crate::core::trident::TridentClient; +const MOCK_RESPONSE_CHANNEL_CAPACITY: usize = 4; +const MOCK_DUPLEX_BUFFER_BYTES: usize = 64 * 1024; + /// Canned outcome a `MockTridentd` should return for a given RPC call. #[derive(Clone, Debug)] pub enum Outcome { @@ -105,7 +115,7 @@ struct MockTridentd { async fn respond_with( outcome: Outcome, ) -> Result>>, Status> { - let (tx, rx) = tokio::sync::mpsc::channel(4); + let (tx, rx) = mpsc::channel(MOCK_RESPONSE_CHANNEL_CAPACITY); tx.send(Ok(outcome.into_servicing_response())) .await .expect("mock tridentd channel send should not fail"); @@ -222,15 +232,15 @@ impl RollbackService for MockTridentd { /// so the caller can reconfigure outcomes between calls if a test needs to /// simulate stage-then-finalize-then-commit in one session. pub async fn connect_mock_client(config: Arc>) -> TridentClient { - let (client_io, server_io) = tokio::io::duplex(64 * 1024); + let (client_io, server_io) = io::duplex(MOCK_DUPLEX_BUFFER_BYTES); let mock = MockTridentd { config }; tokio::spawn(async move { - tonic::transport::Server::builder() + Server::builder() .add_service(UpdateServiceServer::new(mock.clone())) .add_service(CommitServiceServer::new(mock.clone())) .add_service(RollbackServiceServer::new(mock)) - .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server_io))) + .serve_with_incoming(tokio_stream::once(Ok::<_, IoError>(server_io))) .await .expect("mock tridentd server should not fail"); }); @@ -238,12 +248,12 @@ pub async fn connect_mock_client(config: Arc>) -> Trid let mut client_io = Some(client_io); let channel = Endpoint::try_from("http://[::]:50051") .expect("static endpoint URI should always parse") - .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { + .connect_with_connector(tower::service_fn(move |_: Uri| { let client_io = client_io.take(); async move { - client_io.map(TokioIo::new).ok_or_else(|| { - std::io::Error::other("mock client connector called more than once") - }) + client_io + .map(TokioIo::new) + .ok_or_else(|| IoError::other("mock client connector called more than once")) } })) .await diff --git a/crates/trident-acl-agent/src/core/version.rs b/crates/trident-acl-agent/src/core/version.rs index 5e30777128..1953aec927 100644 --- a/crates/trident-acl-agent/src/core/version.rs +++ b/crates/trident-acl-agent/src/core/version.rs @@ -7,6 +7,12 @@ //! used by both the annotation-driven orchestrator and the `omaha-only` //! one-shot mode. +use std::{env, path::Path}; + +use anyhow::{anyhow, Error}; +use log::warn; +use osutils::osrelease; + // current_active_version() reads the `VERSION_ID` key (overridable via // TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY, e.g. to `IMAGE_VERSION` for an ACL // image that stamps its own per-build version there) out of os-release, but @@ -41,7 +47,7 @@ pub const DEFAULT_CURRENT_VERSION_FALLBACK: &str = "always"; /// instead of the real `/etc/os-release`, e.g. a vendor-specific file that /// carries the running image's version under a key `/etc/os-release` /// doesn't have room for. -pub const DEFAULT_CURRENT_VERSION_PATH: &str = osutils::osrelease::OS_RELEASE_PATH; +pub const DEFAULT_CURRENT_VERSION_PATH: &str = osrelease::OS_RELEASE_PATH; /// Default os-release key `current_active_version` looks up for the running /// image's version: `VERSION_ID`, a standard key every os-release carries /// (see @@ -60,17 +66,17 @@ const FALLBACK_ALWAYS: &str = "always"; const FALLBACK_ERROR: &str = "error"; /// What [`current_active_version`] reports for [`FALLBACK_ALWAYS`] - see its /// docs above for why 0.0.0 is a safe sentinel here. -const FALLBACK_ALWAYS_VERSION: &str = "0.0.0"; +pub(crate) const FALLBACK_ALWAYS_VERSION: &str = "0.0.0"; /// Reads `name`, treating both "unset" and "set to the empty string" as /// absent, matching `config::env_raw`'s convention: a drop-in override that /// clears a variable to `""` should fall back to the default, not try to use /// an empty value. fn env_override(name: &str) -> Option { - std::env::var(name).ok().filter(|v| !v.is_empty()) + env::var(name).ok().filter(|v| !v.is_empty()) } -pub fn current_active_version() -> Result { +pub fn current_active_version() -> Result { let path = env_override(ENV_CURRENT_VERSION_PATH) .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); let key = env_override(ENV_CURRENT_VERSION_KEY) @@ -81,17 +87,17 @@ pub fn current_active_version() -> Result { let fallback = env_override(ENV_CURRENT_VERSION_FALLBACK) .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_FALLBACK.to_string()); match fallback.as_str() { - FALLBACK_ERROR => Err(anyhow::anyhow!( + FALLBACK_ERROR => Err(anyhow!( "{key} not found in {path}, and {ENV_CURRENT_VERSION_FALLBACK} is set to \"error\"" )), FALLBACK_ALWAYS => { - log::warn!( + warn!( "{key} not found in {path}; falling back to \"always\" (reporting {FALLBACK_ALWAYS_VERSION} as the current version)" ); Ok(FALLBACK_ALWAYS_VERSION.to_string()) } _ => { - log::warn!( + warn!( "{key} not found in {path}; falling back to configured current version {fallback:?}" ); Ok(fallback) @@ -108,34 +114,18 @@ pub fn current_active_version() -> Result { /// `current_active_version` treats identically: fall back to the stub. /// Split out from `current_active_version` so tests can point it at a temp /// file instead of the real os-release. -fn read_os_release_value(path: &str, key: &str) -> Option { - let contents = std::fs::read_to_string(path).ok()?; - for line in contents.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((line_key, raw_value)) = line.split_once('=') else { - continue; - }; - if line_key.trim() != key { - continue; - } - let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); - if value.is_empty() { - return None; - } - return Some(value.to_string()); - } - None +fn read_os_release_value(path: impl AsRef, key: &str) -> Option { + let path = path.as_ref(); + osrelease::read_key(path, key) } #[cfg(test)] mod tests { - use uuid::Uuid; - use super::*; + use indoc::indoc; + use tempfile::tempdir; + #[test] fn read_os_release_value_returns_none_for_missing_file() { assert_eq!( @@ -149,59 +139,76 @@ mod tests { #[test] fn read_os_release_value_finds_requested_key() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-{}", Uuid::new_v4())); + let dir = tempdir().unwrap(); + let path = dir.path().join("os-release"); std::fs::write( &path, - "NAME=\"Azure Linux\"\nIMAGE_VERSION=202608.6.0\nVERSION_ID=3.0\n", + indoc! {r#" + NAME="Azure Linux" + IMAGE_VERSION=202608.6.0 + VERSION_ID=3.0 + "#}, ) .unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); + let result = read_os_release_value(&path, "IMAGE_VERSION"); assert_eq!(result.as_deref(), Some("202608.6.0")); } #[test] fn read_os_release_value_trims_quotes_and_whitespace() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-quoted-{}", Uuid::new_v4())); - std::fs::write(&path, " IMAGE_VERSION = \"202608.6.0\" \n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); + let dir = tempdir().unwrap(); + let path = dir.path().join("os-release"); + std::fs::write( + &path, + indoc! {r#" + IMAGE_VERSION = "202608.6.0" + "#}, + ) + .unwrap(); + let result = read_os_release_value(&path, "IMAGE_VERSION"); assert_eq!(result.as_deref(), Some("202608.6.0")); } #[test] fn read_os_release_value_returns_none_for_missing_key() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-missing-key-{}", Uuid::new_v4())); - std::fs::write(&path, "NAME=\"Azure Linux\"\nVERSION_ID=3.0\n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); + let dir = tempdir().unwrap(); + let path = dir.path().join("os-release"); + std::fs::write( + &path, + indoc! {r#" + NAME="Azure Linux" + VERSION_ID=3.0 + "#}, + ) + .unwrap(); + let result = read_os_release_value(&path, "IMAGE_VERSION"); assert_eq!(result, None); } #[test] fn read_os_release_value_returns_none_for_empty_value() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-empty-value-{}", Uuid::new_v4())); + let dir = tempdir().unwrap(); + let path = dir.path().join("os-release"); std::fs::write(&path, "IMAGE_VERSION=\n").unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); + let result = read_os_release_value(&path, "IMAGE_VERSION"); assert_eq!(result, None); } #[test] fn read_os_release_value_skips_comments_and_blank_lines() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("os-release-test-comments-{}", Uuid::new_v4())); + let dir = tempdir().unwrap(); + let path = dir.path().join("os-release"); std::fs::write( &path, - "# a comment\n\n# IMAGE_VERSION=should-be-ignored\nIMAGE_VERSION=202608.6.0\n", + indoc! {r#" + # a comment + + # IMAGE_VERSION=should-be-ignored + IMAGE_VERSION=202608.6.0 + "#}, ) .unwrap(); - let result = read_os_release_value(path.to_str().unwrap(), "IMAGE_VERSION"); - std::fs::remove_file(&path).ok(); + let result = read_os_release_value(&path, "IMAGE_VERSION"); assert_eq!(result.as_deref(), Some("202608.6.0")); } @@ -269,11 +276,8 @@ mod tests { // current_active_version() itself honors TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH, // pointing it at an arbitrary os-release-formatted file instead of the // real /etc/os-release. - let dir = std::env::temp_dir(); - let found_path = dir.join(format!( - "os-release-test-current-version-{}", - Uuid::new_v4() - )); + let found_dir = tempdir().unwrap(); + let found_path = found_dir.path().join("os-release"); std::fs::write( &found_path, "NAME=\"Contoso Linux\"\nVERSION_ID=202608.6.0\n", @@ -285,16 +289,13 @@ mod tests { std::env::set_var(ENV_CURRENT_VERSION_KEY, "VERSION_ID"); } assert_eq!(current_active_version().unwrap(), "202608.6.0"); - std::fs::remove_file(&found_path).ok(); clear_current_version_env(); // When the configured key isn't present at the configured path, and // no fallback override is set, it defaults to "always" - reporting // FALLBACK_ALWAYS_VERSION ("0.0.0") as the current version. - let missing_path = dir.join(format!( - "os-release-test-current-version-missing-{}", - Uuid::new_v4() - )); + let missing_dir = tempdir().unwrap(); + let missing_path = missing_dir.path().join("os-release"); std::fs::write(&missing_path, "NAME=\"Contoso Linux\"\n").unwrap(); // SAFETY: see clear_current_version_env's doc comment. unsafe { @@ -325,7 +326,6 @@ mod tests { "custom-fallback-for-missing-key" ); - std::fs::remove_file(&missing_path).ok(); clear_current_version_env(); } } diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 1757f8a616..dc8de47b27 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -1,137 +1,150 @@ -//! # trident-acl-agent -//! -//! trident-acl-agent is Trident's ACL update sidecar. Historically it was a -//! one-shot Omaha client that called Trident's combined `Update()` RPC once -//! and exited. This crate now defaults to the Kubernetes annotation protocol -//! described in the accepted design -//! (), -//! while preserving the original `omaha-only` mode as an explicit opt-out -//! (see `core::config::GoalSource`). -//! -//! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) -//! goes through the [`core::nebraska`] client module, a self-contained, -//! reusable implementation of the Nebraska/Omaha update protocol. It is usable -//! both by this crate's agent binary and by a future Trident ACL Agent that -//! orchestrates updates differently. -//! -//! - [`core`]: building blocks shared by both modes (config, errors, -//! machine-id, current-version, the `tridentd` client, the Nebraska -//! client). -//! - [`annotations`]: the default Kubernetes annotation-driven protocol. -//! - [`omahaonly`]: the legacy one-shot Omaha flow. - -pub mod annotations; -pub mod core; -pub mod omahaonly; - -/// The version this agent reports to Nebraska as the updater's own version, for -/// [`core::nebraska::Client::new`]. -/// -/// Prefers the build-time `TRIDENT_VERSION` (the version the shipped product is -/// stamped with) over this crate's package version, which is not released -/// independently and is a placeholder. Nebraska itself ignores the value, so -/// this is for whoever reads the raw requests. It lives here, not in -/// [`core::nebraska`], because that module is a generic Omaha client: which -/// product is doing the updating is the caller's business. -pub const AGENT_VERSION: &str = match option_env!("TRIDENT_VERSION") { - Some(version) => version, - None => env!("CARGO_PKG_VERSION"), -}; - -use crate::core::error::AgentError; -use crate::core::nebraska::{Client, MachineId, NebraskaError}; - -pub use crate::core::id::IdSource; - -// Deliberately invalid sentinels, mirroring DEFAULT_NEBRASKA_ENDPOINT's -// `.invalid` domain trick: a deployment that forgets to configure (or -// override via the update-request annotation's `appId`/`track` fields) a -// real app_id/track fails loudly against Nebraska instead of silently -// querying a real-looking but wrong app/group. -pub const DEFAULT_NEBRASKA_APP_ID: &str = "00000000-0000-0000-0000-000000000000"; -pub const DEFAULT_NEBRASKA_TRACK: &str = "unspecified"; - -/// Builds a validated [`MachineId`] from an [`IdSource`], translating the -/// crate's own machine-id/hostname read errors into a single [`AgentError`]. -fn build_machine_id(source: IdSource) -> Result { - MachineId::new(source.produce_id()?).map_err(|err| AgentError::Nebraska(err.to_string())) -} - -/// Checks that the Omaha/Nebraska server at `url` is reachable and speaking -/// the Omaha protocol, without treating any app-level result (including a -/// non-OK app/update-check status) as a failure. Unlike -/// [`Client::check_for_update`], this only fails on network/transport -/// problems or a response that isn't well-formed Omaha XML -- it's meant for -/// a pure "can we talk to this server at all" check (e.g. -/// `--validate-connection nebraska`), not for deciding whether an update is -/// available. -pub fn check_nebraska_reachable( - url: &url::Url, - app_id: &str, - track: &str, - machine_id_source: IdSource, -) -> Result<(), AgentError> { - let machine_id = build_machine_id(machine_id_source)?; - let client = Client::new(url.clone(), app_id, track, machine_id); - match client.check_for_update(&semver::Version::new(0, 0, 0)) { - Ok(_) => Ok(()), - // A well-formed response reporting a non-OK app/update-check status - // still proves the server is reachable and speaking Omaha; only a - // transport/parse-level failure means it is not. - Err(NebraskaError::ServerError(_)) => Ok(()), - Err(err) => Err(AgentError::Nebraska(err.to_string())), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_check_nebraska_reachable_succeeds_on_error_app_status() { - // check_nebraska_reachable() is meant to be a pure "can we reach this - // server and does it speak Omaha" check, unlike check_for_update() - // which also validates app-level semantics. A well-formed response - // with a non-OK app status should still count as "reachable" here, - // even though check_for_update() would reject the same response as a - // NebraskaError::ServerError. - let mut server = mockito::Server::new(); - - let omaha_mock = server - .mock("POST", "/") - .with_status(200) - .with_body(indoc::indoc! {r#" - - - - - - - "#}) - .expect(1) - .create(); - - check_nebraska_reachable( - &url::Url::parse(&server.url()).unwrap(), - "test", - "track", - IdSource::MachineIdHashed, - ) - .unwrap(); - - omaha_mock.assert(); - } - - #[test] - fn test_check_nebraska_reachable_fails_on_transport_error() { - let err = check_nebraska_reachable( - // Port 0 never accepts a connection. - &url::Url::parse("http://127.0.0.1:0/").unwrap(), - "test", - "track", - IdSource::MachineIdHashed, - ) - .unwrap_err(); - assert!(matches!(err, AgentError::Nebraska(_))); - } -} +//! # trident-acl-agent +//! +//! trident-acl-agent is Trident's ACL update sidecar. Historically it was a +//! one-shot Omaha client that called Trident's combined `Update()` RPC once +//! and exited. This crate now defaults to the Kubernetes annotation protocol +//! described in the accepted design +//! (), +//! while preserving the original `omaha-only` mode as an explicit opt-out +//! (see `core::config::GoalSource`). +//! +//! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) +//! goes through the [`core::nebraska`] client module, a self-contained, +//! reusable implementation of the Nebraska/Omaha update protocol. It is usable +//! both by this crate's agent binary and by a future Trident ACL Agent that +//! orchestrates updates differently. +//! +//! - [`core`]: building blocks shared by both modes (config, errors, +//! machine-id, current-version, the `tridentd` client, the Nebraska +//! client). +//! - [`annotations`]: the default Kubernetes annotation-driven protocol. +//! - [`omahaonly`]: the legacy one-shot Omaha flow. + +use semver::Version; +use url::Url; + +use crate::core::{ + error::AgentError, + nebraska::{Client, MachineId, NebraskaError}, + version::FALLBACK_ALWAYS_VERSION, +}; + +pub use crate::core::id::IdSource; + +pub mod annotations; +pub mod core; +pub mod omahaonly; + +/// The version this agent reports to Nebraska as the updater's own version, for +/// [`core::nebraska::Client::new`]. +/// +/// Prefers the build-time `TRIDENT_VERSION` (the version the shipped product is +/// stamped with) over this crate's package version, which is not released +/// independently and is a placeholder. Nebraska itself ignores the value, so +/// this is for whoever reads the raw requests. It lives here, not in +/// [`core::nebraska`], because that module is a generic Omaha client: which +/// product is doing the updating is the caller's business. +pub const AGENT_VERSION: &str = match option_env!("TRIDENT_VERSION") { + Some(version) => version, + None => env!("CARGO_PKG_VERSION"), +}; + +// Deliberately invalid sentinels, mirroring DEFAULT_NEBRASKA_ENDPOINT's +// `.invalid` domain trick: a deployment that forgets to configure (or +// override via the update-request annotation's `appId`/`track` fields) a +// real app_id/track fails loudly against Nebraska instead of silently +// querying a real-looking but wrong app/group. +pub const DEFAULT_NEBRASKA_APP_ID: &str = "00000000-0000-0000-0000-000000000000"; +pub const DEFAULT_NEBRASKA_TRACK: &str = "unspecified"; + +/// Builds a validated [`MachineId`] from an [`IdSource`], translating the +/// crate's own machine-id/hostname read errors into a single [`AgentError`]. +fn build_machine_id(source: IdSource) -> Result { + MachineId::new(source.produce_id()?).map_err(|err| AgentError::Nebraska(err.to_string())) +} + +/// Checks that the Omaha/Nebraska server at `url` is reachable and speaking +/// the Omaha protocol, without treating any app-level result (including a +/// non-OK app/update-check status) as a failure. Unlike +/// [`Client::check_for_update`], this only fails on network/transport +/// problems or a response that isn't well-formed Omaha XML -- it's meant for +/// a pure "can we talk to this server at all" check (e.g. +/// `--validate-connection nebraska`), not for deciding whether an update is +/// available. +pub fn check_nebraska_reachable( + url: &Url, + app_id: &str, + track: &str, + machine_id_source: IdSource, +) -> Result<(), AgentError> { + let machine_id = build_machine_id(machine_id_source)?; + let client = Client::new(url.clone(), app_id, track, machine_id); + match client.check_for_update( + &Version::parse(FALLBACK_ALWAYS_VERSION) + .expect("invariant: FALLBACK_ALWAYS_VERSION is valid semver"), + ) { + Ok(_) => Ok(()), + // A well-formed response reporting a non-OK app/update-check status + // still proves the server is reachable and speaking Omaha; only a + // transport/parse-level failure means it is not. + Err(NebraskaError::ServerError(_)) => Ok(()), + Err(err) => Err(AgentError::Nebraska(err.to_string())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use indoc::indoc; + use mockito::Server; + use url::Url; + + #[test] + fn test_check_nebraska_reachable_succeeds_on_error_app_status() { + // check_nebraska_reachable() is meant to be a pure "can we reach this + // server and does it speak Omaha" check, unlike check_for_update() + // which also validates app-level semantics. A well-formed response + // with a non-OK app status should still count as "reachable" here, + // even though check_for_update() would reject the same response as a + // NebraskaError::ServerError. + let mut server = Server::new(); + + let omaha_mock = server + .mock("POST", "/") + .with_status(200) + .with_body(indoc! {r#" + + + + + + + "#}) + .expect(1) + .create(); + + check_nebraska_reachable( + &Url::parse(&server.url()).unwrap(), + "test", + "track", + IdSource::MachineIdHashed, + ) + .unwrap(); + + omaha_mock.assert(); + } + + #[test] + fn test_check_nebraska_reachable_fails_on_transport_error() { + let err = check_nebraska_reachable( + // Port 0 never accepts a connection. + &Url::parse("http://127.0.0.1:0/").unwrap(), + "test", + "track", + IdSource::MachineIdHashed, + ) + .unwrap_err(); + assert!(matches!(err, AgentError::Nebraska(_))); + } +} diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index d266482855..0095d13dc1 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -1,81 +1,87 @@ -use clap::Parser; -use osutils::logging::FilteredLogger; - -use trident_acl_agent::{ - annotations::orchestrator::Orchestrator, - core::config::{AgentConfig, GoalSource}, - omahaonly::run_omaha_only, -}; - -mod cli; -mod connection_check; - -use cli::Args; -use connection_check::validate_connection; - -/// Module/target prefixes for the underlying HTTP/gRPC/watch client stack. -/// These crates emit very verbose `log`-facade tracing (connection setup, -/// per-frame HTTP2 detail, watch reconnect churn) that is rarely useful at -/// the same verbosity as the agent's own orchestration logic, so it's -/// filtered independently via `--network-verbosity`. -const NETWORK_LOG_TARGETS: &[&str] = &[ - "hyper", - "h2", - "tower", - "tonic", - "reqwest", - "rustls", - "kube", - "kube_client", - "kube_runtime", -]; - -#[tokio::main] -async fn main() -> Result<(), anyhow::Error> { - let args = Args::parse(); - - if let Some(Ok(journal_logger)) = - systemd_journal_logger::connected_to_journal().then(systemd_journal_logger::JournalLog::new) - { - let logger = FilteredLogger::new( - journal_logger, - args.verbosity, - args.network_verbosity, - NETWORK_LOG_TARGETS, - ); - log::set_max_level(logger.max_level()); - log::set_boxed_logger(Box::new(logger)).expect("Failed to install systemd journal logger"); - } else { - let inner = env_logger::builder() - .format_timestamp(None) - .filter_level(args.verbosity.max(args.network_verbosity)) - .build(); - let logger = FilteredLogger::new( - inner, - args.verbosity, - args.network_verbosity, - NETWORK_LOG_TARGETS, - ); - log::set_max_level(logger.max_level()); - log::set_boxed_logger(Box::new(logger)).expect("Failed to install env logger"); - } - - let config = AgentConfig::from_env()?; - - if let Some(target) = args.validate_connection { - return validate_connection(target, &config).await; - } - - match config.orchestration.goal_source { - // Historical one-shot flow: query Nebraska once, apply an update if - // offered, and exit. No Kubernetes/annotation involvement. Not a - // documented/supported deployment option (see config::GoalSource). - GoalSource::OmahaOnly => run_omaha_only(&config).await, - // The only supported mode: the annotation-driven reconcile loop - // (watches /update-request, drives stage/finalize/rollback/ - // commit against tridentd, writes /update-status; prefix - // defaults to acl.microsoft.com, overridable via - // TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX). - GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, - } -} +use anyhow::{Context, Error}; +use clap::Parser; +use osutils::logging::FilteredLogger; +use systemd_journal_logger::{self, JournalLog}; + +use trident_acl_agent::{ + annotations::orchestrator::Orchestrator, + core::config::{AgentConfig, GoalSource}, + omahaonly::run_omaha_only, +}; + +mod cli; +mod connection_check; + +use cli::Args; +use connection_check::validate_connection; + +/// Module/target prefixes for the underlying HTTP/gRPC/watch client stack. +/// These crates emit very verbose `log`-facade tracing (connection setup, +/// per-frame HTTP2 detail, watch reconnect churn) that is rarely useful at +/// the same verbosity as the agent's own orchestration logic, so it's +/// filtered independently via `--network-verbosity`. +const NETWORK_LOG_TARGETS: &[&str] = &[ + "hyper", + "h2", + "tower", + "tonic", + "reqwest", + "rustls", + "kube", + "kube_client", + "kube_runtime", +]; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let args = Args::parse(); + + if let Some(Ok(journal_logger)) = + systemd_journal_logger::connected_to_journal().then(JournalLog::new) + { + let logger = FilteredLogger::new( + journal_logger, + args.verbosity, + args.network_verbosity, + NETWORK_LOG_TARGETS, + ); + log::set_max_level(logger.max_level()); + log::set_boxed_logger(Box::new(logger)) + .map_err(Error::new) + .context("failed to install systemd journal logger")?; + } else { + let inner = env_logger::builder() + .format_timestamp(None) + .filter_level(args.verbosity.max(args.network_verbosity)) + .build(); + let logger = FilteredLogger::new( + inner, + args.verbosity, + args.network_verbosity, + NETWORK_LOG_TARGETS, + ); + log::set_max_level(logger.max_level()); + log::set_boxed_logger(Box::new(logger)) + .map_err(Error::new) + .context("failed to install env logger")?; + } + + let config = AgentConfig::from_env()?; + + if let Some(target) = args.validate_connection { + return validate_connection(target, &config).await; + } + + match config.orchestration.goal_source { + // Historical one-shot flow: query Nebraska once, apply an update if + // offered, and exit. No Kubernetes/annotation involvement. Not a + // documented/supported deployment option (see config::GoalSource). + GoalSource::OmahaOnly => run_omaha_only(&config).await, + // The only supported mode: the annotation-driven reconcile loop + // (watches /update-request, drives stage/finalize/rollback/ + // commit against tridentd, writes /update-status; prefix + // defaults to acl.microsoft.com, overridable via + // TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX). + GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, + } +} diff --git a/crates/trident-acl-agent/src/omahaonly/mod.rs b/crates/trident-acl-agent/src/omahaonly/mod.rs index fea7e1d3bc..22750e77ab 100644 --- a/crates/trident-acl-agent/src/omahaonly/mod.rs +++ b/crates/trident-acl-agent/src/omahaonly/mod.rs @@ -2,24 +2,28 @@ //! the default annotation-driven protocol (see //! [`crate::core::config::GoalSource`]). -use anyhow::Context; +use anyhow::{anyhow, Context, Error}; +use log::{debug, info, warn}; use semver::Version; +use tokio::task; -use crate::core::{ - config::AgentConfig, - nebraska::{CheckOutcome, Client}, - trident::TridentClient, - version, +use crate::{ + core::{ + config::AgentConfig, + nebraska::{CheckOutcome, Client}, + trident::TridentClient, + version::{self, FALLBACK_ALWAYS_VERSION}, + }, + IdSource, }; -use crate::IdSource; /// Historical one-shot flow: query the Nebraska/Omaha server at /// `config.nebraska.endpoint` once, and if an update is offered, call /// tridentd's combined `Update()` RPC once and exit. No Kubernetes/annotation /// involvement. -pub async fn run_omaha_only(config: &AgentConfig) -> Result<(), anyhow::Error> { +pub async fn run_omaha_only(config: &AgentConfig) -> Result<(), Error> { let endpoint = config.nebraska.endpoint.clone().ok_or_else(|| { - anyhow::anyhow!("no Nebraska endpoint configured: set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT") + anyhow!("no Nebraska endpoint configured: set TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT") })?; // Client::check_for_update() is a blocking call (reqwest::blocking under @@ -33,26 +37,26 @@ pub async fn run_omaha_only(config: &AgentConfig) -> Result<(), anyhow::Error> { let machine_id = crate::build_machine_id(IdSource::MachineIdHashed)?; let current_version_raw = version::current_active_version()?; let current_version = Version::parse(¤t_version_raw).unwrap_or_else(|err| { - log::warn!( + warn!( "current version {current_version_raw:?} is not valid semver ({err}); reporting 0.0.0 to Nebraska" ); - Version::new(0, 0, 0) + Version::parse(FALLBACK_ALWAYS_VERSION).expect("invariant: FALLBACK_ALWAYS_VERSION is valid semver") }); - let outcome = tokio::task::spawn_blocking(move || { + let outcome = task::spawn_blocking(move || { let client = Client::new(endpoint, app_id, track, machine_id); client.check_for_update(¤t_version) }) .await .context("Nebraska query task panicked")? - .map_err(|err| anyhow::anyhow!("Nebraska query failed: {err}"))?; + .context("Nebraska query failed")?; match outcome { CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { - log::debug!("No update available from Nebraska"); + debug!("No update available from Nebraska"); Ok(()) } CheckOutcome::UpdateAvailable(offer) => { - log::info!("Triggering one-shot Omaha update to {}", offer.version); + info!("Triggering one-shot Omaha update to {}", offer.version); let mut client = TridentClient::connect(&config.trident.socket).await?; let combined_timeout = config.orchestration.stage_timeout + config.orchestration.finalize_timeout; From 37e113e8256a7a03a83e0385a203e7d486bee30d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 21:29:10 +0000 Subject: [PATCH 35/54] trident-acl-agent: derive DEFAULT_STATE_PATH from a single STATE_FILE_NAME const Use const_format::formatcp! to build DEFAULT_STATE_PATH from a single STATE_FILE_NAME constant, instead of hardcoding the "state.json" literal separately in config.rs and annotations/state.rs. --- Cargo.lock | 1 + crates/trident-acl-agent/Cargo.toml | 1 + crates/trident-acl-agent/src/annotations/state.rs | 8 +++++--- crates/trident-acl-agent/src/core/config.rs | 5 ++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd9bca155f..a1097a0994 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4012,6 +4012,7 @@ dependencies = [ "anyhow", "chrono", "clap", + "const_format", "env_logger 0.11.5", "futures", "humantime", diff --git a/crates/trident-acl-agent/Cargo.toml b/crates/trident-acl-agent/Cargo.toml index 36ecde4e4e..c3b435b001 100644 --- a/crates/trident-acl-agent/Cargo.toml +++ b/crates/trident-acl-agent/Cargo.toml @@ -8,6 +8,7 @@ publish = false anyhow = { workspace = true, features = ["backtrace"] } clap = { workspace = true, features = ["derive"] } chrono = { workspace = true } +const_format = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } humantime = { workspace = true } diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index 2f085aaf50..5bdd2caa73 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -19,9 +19,11 @@ use anyhow::{Context, Error}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::annotations::{Operation, UpdateRequest, UpdateStatus}; +use crate::{ + annotations::{Operation, UpdateRequest, UpdateStatus}, + core::config::STATE_FILE_NAME, +}; -const DEFAULT_STATE_FILE_NAME: &str = "state.json"; // state.json persists the full UpdateRequest, which can include a // secret-bearing Omaha `server` URL, so store it with owner-only permissions. const STATE_FILE_MODE: u32 = 0o600; @@ -96,7 +98,7 @@ impl StateStore { self.path .file_name() .and_then(|name| name.to_str()) - .unwrap_or(DEFAULT_STATE_FILE_NAME), + .unwrap_or(STATE_FILE_NAME), process::id() )); diff --git a/crates/trident-acl-agent/src/core/config.rs b/crates/trident-acl-agent/src/core/config.rs index 281be4f26a..bc144a8ad7 100644 --- a/crates/trident-acl-agent/src/core/config.rs +++ b/crates/trident-acl-agent/src/core/config.rs @@ -17,6 +17,7 @@ use std::{env, path::PathBuf, str::FromStr, time::Duration}; use anyhow::{anyhow, Context, Error}; +use const_format::formatcp; use osutils::hostname; use trident_proto::TRIDENT_DEFAULT_SOCKET_URI; use url::Url; @@ -57,7 +58,9 @@ const DEFAULT_NODE_NAME: &str = "localhost"; const DEFAULT_STAGE_TIMEOUT: Duration = Duration::from_secs(20 * 60); const DEFAULT_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10 * 60); const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60); -pub const DEFAULT_STATE_PATH: &str = "/var/lib/trident-acl-agent/state.json"; +/// File name for the persisted agent state (see `annotations::state`). +pub const STATE_FILE_NAME: &str = "state.json"; +pub const DEFAULT_STATE_PATH: &str = formatcp!("/var/lib/trident-acl-agent/{STATE_FILE_NAME}"); pub const DEFAULT_KUBELET_KUBECONFIG: &str = "/var/lib/kubelet/kubeconfig"; /// Default annotation-key prefix. /// Override with `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. From 5d6b9bab542952ffa7656954a36c127ac438fae8 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 21:51:42 +0000 Subject: [PATCH 36/54] trident-acl-agent: address remaining PR 730 nit-review gaps - annotations/orchestrator.rs: unqualify DateTime (chrono already imported); replace hardcoded reboot-check subkind strings with new ServicingError/HealthChecksError subkind consts. - trident_api::error: add AB_UPDATE_REBOOT_CHECK_SUBKIND, MANUAL_ROLLBACK_REBOOT_CHECK_SUBKIND and AB_UPDATE_HEALTH_CHECK_COMMIT_CHECK_SUBKIND consts next to their ServicingError/HealthChecksError variants, with a test asserting they stay in sync with serde_variant::to_variant_name. - trident-acl-agent: add trident_api as a dependency to reference the new consts instead of duplicating the kebab-case literals. - main.rs: import connected_to_journal directly instead of calling it through the systemd_journal_logger:: path. - annotations/k8s.rs: use the already-aliased KubeError in tests instead of the fully-qualified kube::Error::Api. --- Cargo.lock | 1 + crates/trident-acl-agent/Cargo.toml | 1 + .../trident-acl-agent/src/annotations/k8s.rs | 4 +- .../src/annotations/orchestrator.rs | 19 ++++---- crates/trident-acl-agent/src/main.rs | 6 +-- crates/trident_api/src/error.rs | 46 +++++++++++++++++++ 6 files changed, 62 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1097a0994..f6405ec1e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4040,6 +4040,7 @@ dependencies = [ "tonic", "tower", "trident-proto", + "trident_api", "url", "uuid", ] diff --git a/crates/trident-acl-agent/Cargo.toml b/crates/trident-acl-agent/Cargo.toml index c3b435b001..d89079ecfd 100644 --- a/crates/trident-acl-agent/Cargo.toml +++ b/crates/trident-acl-agent/Cargo.toml @@ -31,6 +31,7 @@ url = { workspace = true, features = ["serde"] } uuid = { workspace = true, features = ["v4", "serde"] } sysdefs = { path = "../sysdefs" } +trident_api = { path = "../trident_api" } osutils = { path = "../osutils" } trident-proto = { path = "../trident-proto" } diff --git a/crates/trident-acl-agent/src/annotations/k8s.rs b/crates/trident-acl-agent/src/annotations/k8s.rs index 1dbf51b9e8..0f10f1e00a 100644 --- a/crates/trident-acl-agent/src/annotations/k8s.rs +++ b/crates/trident-acl-agent/src/annotations/k8s.rs @@ -179,7 +179,7 @@ mod tests { #[test] fn maps_404_to_node_gone() { - let err = kube::Error::Api(ErrorResponse { + let err = KubeError::Api(ErrorResponse { status: "Failure".to_string(), message: "nodes \"n\" not found".to_string(), reason: "NotFound".to_string(), @@ -191,7 +191,7 @@ mod tests { #[test] fn leaves_other_api_errors_as_api() { - let err = kube::Error::Api(ErrorResponse { + let err = KubeError::Api(ErrorResponse { status: "Failure".to_string(), message: "forbidden".to_string(), reason: "Forbidden".to_string(), diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index a5a0f889a8..4ae24d8630 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -22,6 +22,7 @@ use url::Url; use uuid::Uuid; use osutils::{dependencies::Dependency, machine_id}; +use trident_api::error::{HealthChecksError, ServicingError}; use trident_proto::v1::{RebootStatus, ServicingKind}; use crate::{ @@ -1225,7 +1226,7 @@ fn stage_result_to_status( request: &UpdateRequest, from_version: Option, to_version: Option, - started: chrono::DateTime, + started: DateTime, result: Result, ) -> UpdateStatus { match result { @@ -1260,7 +1261,7 @@ fn finalize_success_status( request: &UpdateRequest, from_version: Option, to_version: Option, - started: chrono::DateTime, + started: DateTime, ) -> UpdateStatus { UpdateStatus::new( request, @@ -1281,7 +1282,7 @@ fn finalize_failure_status( request: &UpdateRequest, from_version: Option, to_version: Option, - started: chrono::DateTime, + started: DateTime, err: &TridentClientError, ) -> UpdateStatus { UpdateStatus::new( @@ -1321,9 +1322,9 @@ fn indicates_target_boot_failed(error: &TridentClientError) -> bool { // one - both must be checked here, or a real rollback // boot-fallback silently reports as generic OperationFailed // instead of TargetBootFailed. - remote.subkind == "ab-update-reboot-check" - || remote.subkind == "ab-update-health-check-commit-check" - || remote.subkind == "manual-rollback-reboot-check" + remote.subkind == ServicingError::AB_UPDATE_REBOOT_CHECK_SUBKIND + || remote.subkind == HealthChecksError::AB_UPDATE_HEALTH_CHECK_COMMIT_CHECK_SUBKIND + || remote.subkind == ServicingError::MANUAL_ROLLBACK_REBOOT_CHECK_SUBKIND }) .unwrap_or(false) } @@ -1496,7 +1497,7 @@ fn commit_result_to_status( fn rollback_stage_failure_status( request: &UpdateRequest, from_version: Option, - started: chrono::DateTime, + started: DateTime, err: &TridentClientError, ) -> UpdateStatus { UpdateStatus::new( @@ -1517,7 +1518,7 @@ fn rollback_stage_failure_status( fn rollback_finalize_success_status( request: &UpdateRequest, from_version: Option, - started: chrono::DateTime, + started: DateTime, ) -> UpdateStatus { UpdateStatus::new( request, @@ -1537,7 +1538,7 @@ fn rollback_finalize_success_status( fn rollback_finalize_failure_status( request: &UpdateRequest, from_version: Option, - started: chrono::DateTime, + started: DateTime, err: &TridentClientError, ) -> UpdateStatus { UpdateStatus::new( diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 0095d13dc1..741b9ab8d0 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Error}; use clap::Parser; use osutils::logging::FilteredLogger; -use systemd_journal_logger::{self, JournalLog}; +use systemd_journal_logger::{connected_to_journal, JournalLog}; use trident_acl_agent::{ annotations::orchestrator::Orchestrator, @@ -36,9 +36,7 @@ const NETWORK_LOG_TARGETS: &[&str] = &[ async fn main() -> Result<(), Error> { let args = Args::parse(); - if let Some(Ok(journal_logger)) = - systemd_journal_logger::connected_to_journal().then(JournalLog::new) - { + if let Some(Ok(journal_logger)) = connected_to_journal().then(JournalLog::new) { let logger = FilteredLogger::new( journal_logger, args.verbosity, diff --git a/crates/trident_api/src/error.rs b/crates/trident_api/src/error.rs index 70c88e1a16..064be997dc 100644 --- a/crates/trident_api/src/error.rs +++ b/crates/trident_api/src/error.rs @@ -749,6 +749,16 @@ pub enum DatastoreError { WriteToDatastore, } +impl ServicingError { + /// Kebab-case serde subkind for [`ServicingError::AbUpdateRebootCheck`], + /// matching this enum's `#[serde(rename_all = "kebab-case")]` attribute. + /// Verified against `serde_variant::to_variant_name` in this module's tests. + pub const AB_UPDATE_REBOOT_CHECK_SUBKIND: &str = "ab-update-reboot-check"; + + /// Kebab-case serde subkind for [`ServicingError::ManualRollbackRebootCheck`]. + pub const MANUAL_ROLLBACK_REBOOT_CHECK_SUBKIND: &str = "manual-rollback-reboot-check"; +} + /// Identifies errors that occur when interacting with failed health checks. #[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "kebab-case")] @@ -766,6 +776,15 @@ pub enum HealthChecksError { }, } +impl HealthChecksError { + /// Kebab-case serde subkind for + /// [`HealthChecksError::AbUpdateHealthCheckCommitCheck`], matching this + /// enum's `#[serde(rename_all = "kebab-case")]` attribute. Verified + /// against `serde_variant::to_variant_name` in this module's tests. + pub const AB_UPDATE_HEALTH_CHECK_COMMIT_CHECK_SUBKIND: &str = + "ab-update-health-check-commit-check"; +} + /// Identifies errors that occur when clean install or update fail due to the current configuration /// of the host. #[derive(Debug, Eq, thiserror::Error, Serialize, Deserialize, PartialEq)] @@ -1212,4 +1231,31 @@ mod tests { }; assert_eq!(format!("{error:?}"), expected); } + + #[test] + fn test_subkind_consts_match_serde_variant_names() { + assert_eq!( + serde_variant::to_variant_name(&ServicingError::AbUpdateRebootCheck { + root_device_path: String::new(), + expected_device_path: String::new(), + }) + .unwrap(), + ServicingError::AB_UPDATE_REBOOT_CHECK_SUBKIND, + ); + assert_eq!( + serde_variant::to_variant_name(&ServicingError::ManualRollbackRebootCheck { + root_device_path: String::new(), + expected_device_path: String::new(), + }) + .unwrap(), + ServicingError::MANUAL_ROLLBACK_REBOOT_CHECK_SUBKIND, + ); + assert_eq!( + serde_variant::to_variant_name(&HealthChecksError::AbUpdateHealthCheckCommitCheck { + expected_device_path: String::new(), + }) + .unwrap(), + HealthChecksError::AB_UPDATE_HEALTH_CHECK_COMMIT_CHECK_SUBKIND, + ); + } } From e1099da8de34a16a6bd981525a87d4e0dd0fef98 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 22:03:01 +0000 Subject: [PATCH 37/54] Remove private msazure.visualstudio.com links from doc comments References to the accepted design doc used a private msazure.visualstudio.com link, which is not appropriate for public GitHub code. Reword the surrounding prose to refer to "the accepted design" implicitly, without exposing the private link. No behavior change. --- .../trident-acl-agent/src/annotations/k8s.rs | 2 +- .../src/annotations/orchestrator.rs | 15 +++++------ .../src/annotations/protocol.rs | 26 +++++++++---------- .../src/annotations/state.rs | 2 +- crates/trident-acl-agent/src/core/config.rs | 8 +++--- .../src/core/trident/client.rs | 7 +++-- crates/trident-acl-agent/src/lib.rs | 5 ++-- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/k8s.rs b/crates/trident-acl-agent/src/annotations/k8s.rs index 0f10f1e00a..eb65bfd634 100644 --- a/crates/trident-acl-agent/src/annotations/k8s.rs +++ b/crates/trident-acl-agent/src/annotations/k8s.rs @@ -2,7 +2,7 @@ //! protocol. //! //! Implements the Node get/watch/patch access described in the current -//! accepted design (). +//! accepted design. //! //! The design calls for get/patch access to exactly one Node object (§2.2–§2.6). //! Node changes are observed via the Kubernetes watch API (`kube::runtime::watcher`) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 4ae24d8630..285de8c5bc 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -2,8 +2,7 @@ //! annotation, drives Trident (stage/finalize/rollback/commit) over gRPC, //! and writes the status annotation back, including post-reboot. //! -//! Implements the node-side control flow from -//! +//! Implements the node-side control flow from the accepted design //! (sections 2.1 "Trigger mechanism", 2.3 "Stage/finalize/rollback split //! and post-reboot commit", and 2.5 "Rollback"). See that document for the //! full state-machine rationale; keep it in sync with this file if the @@ -258,7 +257,7 @@ where // Reject on operationId, not nodeUpdateId: the actual conflict // this guard exists to prevent is "a second finalize/rollback // starts while one is still waiting for its post-reboot - // commit" (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md's in-flight conflict rule). + // commit" (the accepted design's in-flight conflict rule). // Keying on nodeUpdateId alone let a retried/re-issued request // that reused the same nodeUpdateId but a new operationId slip // through this guard entirely and re-enter handle_finalize/ @@ -304,7 +303,7 @@ where /// across one update's lifecycle would split that state across two /// servers. /// - /// Per 2.1, `stage`/`finalize` requests must + /// Per the accepted design (§2.1), `stage`/`finalize` requests must /// carry `server` and there is deliberately no static-config fallback /// here: a fallback would let a node update from a source AKS-RP did /// not choose. `UpdateRequest::validate()` already rejects a @@ -808,7 +807,7 @@ where ) -> UpdateStatus { // state.json did not survive the reboot (or was never written, e.g. // the agent crashed before persisting pendingCommit). Per - // https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md §2.3's degraded path, reconstruct the answer by + // the accepted design's §2.3 degraded path, reconstruct the answer by // calling commit() unconditionally rather than guessing from labels // or the target version alone - tridentd's commit() is self-checking // and its own (ServicingKind/RebootStatus/Result) response already @@ -1330,7 +1329,7 @@ fn indicates_target_boot_failed(error: &TridentClientError) -> bool { } /// Pre-flight checks for the state.json-missing degraded reconstruction -/// path ( §2.3). Returns `Some(status)` when reconstruction +/// path (the accepted design's §2.3). Returns `Some(status)` when reconstruction /// cannot proceed (tridentd already known-unreachable, or the outstanding /// request isn't a finalize/rollback), or `None` when the caller should go /// on to call tridentd's commit() to determine the real outcome. @@ -1375,8 +1374,8 @@ fn reconstruct_precheck_status( /// Maps tridentd's commit() result to the terminal status for the /// state.json-missing degraded reconstruction path -/// ( -/// §2.3). Always reports under the original operationId, mirroring the +/// (the accepted design's §2.3). Always reports under the original +/// operationId, mirroring the /// normal post-reboot commit path in `commit_result_to_status`. fn reconstruct_commit_result_to_status( request: &UpdateRequest, diff --git a/crates/trident-acl-agent/src/annotations/protocol.rs b/crates/trident-acl-agent/src/annotations/protocol.rs index b23ece8dda..dc3faef05e 100644 --- a/crates/trident-acl-agent/src/annotations/protocol.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -4,7 +4,7 @@ //! `#[cfg(test)]` design-doc conformance tests below) implements the //! `/update-request`, `/update-status`, and //! `/update-commit-status` node annotation protocol described -//! by the current accepted design (), where +//! by the current accepted design, where //! `` defaults to `acl.microsoft.com` (see //! [`AnnotationKeys`]/[`crate::core::config::DEFAULT_ANNOTATION_PREFIX`]) and is //! overridable via the `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` @@ -148,7 +148,7 @@ pub struct UpdateStatus { impl UpdateRequest { /// Enforces the same constraints as the request annotation's formal - /// JSON Schema in : schemaVersion match, + /// JSON Schema in the accepted design: schemaVersion match, /// targetVersion required for stage/finalize but disallowed for /// rollback, and server/appId/track required for stage/finalize. See /// this file's module doc. @@ -211,7 +211,7 @@ impl UpdateRequest { impl UpdateStatus { // This constructor mirrors UpdateStatus's wire schema field-for-field - // (see https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md's two-status-key JSON protocol); splitting + // (see the accepted design's two-status-key JSON protocol); splitting // it into a builder would add ceremony across ~25 call sites in // orchestrator.rs without making any of them clearer. #[allow(clippy::too_many_arguments)] @@ -656,9 +656,9 @@ mod tests { // --- docs/update-trigger-design.md conformance -------------------------- // // Pins our annotation (de)serialization/validation code against two - // things lifted verbatim from docs/update-trigger-design.md - // (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md), - // section 2.1 "Trigger mechanism", so a doc/code drift shows up as a + // things lifted verbatim from the accepted design's + // docs/update-trigger-design.md, section 2.1 "Trigger mechanism", so a + // doc/code drift shows up as a // test failure instead of being discovered against a real AKS-RP: // 1. The three example JSON payloads (request, finalize status, and // the derived commit status) parse with our real UpdateRequest / @@ -672,7 +672,7 @@ mod tests { /// (adapted to `finalize` to pair with the status/commit examples /// below, which also share this `finalize`; server/appId/track values /// are the doc's own example values for those fields, required on - /// stage/finalize per ). + /// stage/finalize per the accepted design). const DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE: &str = r#"{ "schemaVersion": "1.0", "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", @@ -714,8 +714,8 @@ mod tests { "finishedUtc": "2026-06-04T12:01:32Z" }"#; - /// The formal JSON Schema for the request annotation, from - /// section 2.1 "Formal JSON Schema". Keep + /// The formal JSON Schema for the request annotation, from the + /// accepted design's section 2.1 "Formal JSON Schema". Keep /// byte-for-byte in sync with that document. const DESIGN_DOC_REQUEST_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -746,8 +746,8 @@ mod tests { ] }"#; - /// The formal JSON Schema for the status annotations, from - /// section 2.1 "Formal JSON Schema". Keep + /// The formal JSON Schema for the status annotations, from the + /// accepted design's section 2.1 "Formal JSON Schema". Keep /// byte-for-byte in sync with that document. const DESIGN_DOC_STATUS_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -785,7 +785,7 @@ mod tests { // additionalProperties, required, properties.{type,const,enum,format, // pattern}, and a single-level allOf/if/then/else). Panics loudly on any // schema keyword/pattern/type/format it doesn't recognize, so if - // https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md's schemas grow new constraints, this validator's + // the accepted design's schemas grow new constraints, this validator's // blind spots don't silently mask them - the test fails instead, // prompting an update here. @@ -1082,7 +1082,7 @@ mod tests { #[test] fn validate_allows_rollback_without_nebraska_fields() { // Rollback reports no Nebraska event, so it carries no update - // source (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md 2.1): server/appId/track are not + // source (the accepted design, §2.1): server/appId/track are not // required, and validate() must not reject their absence. let request = UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index 5bdd2caa73..2311f577ef 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -2,7 +2,7 @@ //! completed-operation cache and the pending post-reboot commit record. //! //! Implements the `state.json` mechanism from the current accepted design -//! (, section 2.3), which bridges the pre-reboot +//! (section 2.3), which bridges the pre-reboot //! finalize/rollback half and the post-reboot commit half of an operation //! across the reboot. diff --git a/crates/trident-acl-agent/src/core/config.rs b/crates/trident-acl-agent/src/core/config.rs index bc144a8ad7..011a6a95a2 100644 --- a/crates/trident-acl-agent/src/core/config.rs +++ b/crates/trident-acl-agent/src/core/config.rs @@ -205,8 +205,8 @@ pub enum GoalSource { /// stage/finalize/rollback/commit operations against tridentd /// accordingly, writing progress back to /// `/update-status` and - /// `/update-commit-status` (see - /// ). `` defaults to + /// `/update-commit-status` (see the accepted design). + /// `` defaults to /// [`DEFAULT_ANNOTATION_PREFIX`] (`acl.microsoft.com`), overridable via /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. This is the only /// supported mode. @@ -237,8 +237,8 @@ pub struct OrchestrationConfig { /// Placeholder default pending real data from storm aclagent scenario runs. pub finalize_timeout: Duration, /// Refresh cadence for in-flight InProgress heartbeats. Default is well - /// below the ~10 minute watchdog staleness target proposed in - /// . + /// below the ~10 minute watchdog staleness target proposed in the + /// accepted design. pub heartbeat_interval: Duration, } diff --git a/crates/trident-acl-agent/src/core/trident/client.rs b/crates/trident-acl-agent/src/core/trident/client.rs index 57f56e702c..c9ea535103 100644 --- a/crates/trident-acl-agent/src/core/trident/client.rs +++ b/crates/trident-acl-agent/src/core/trident/client.rs @@ -1,7 +1,6 @@ //! gRPC helpers for talking to `tridentd`. //! -//! Implements the Trident-invocation half of -//! +//! Implements the Trident-invocation half of the accepted design //! (the "Trident invocation" column of section 2.1's operations table, //! and the stage/finalize/rollback-finalize CallerHandlesReboot split in //! section 2.3). @@ -234,7 +233,7 @@ impl TridentClient { reboot: Some(RebootManagement { // The agent, not tridentd, must own every reboot // decision: AKS-RP is the sole authority over - // reboot/rollback (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md §2.5). If commit() + // reboot/rollback (the accepted design's §2.5). If commit() // ever reports NeedsReboot (e.g. a health-check failure, // were health checks ever re-enabled), the agent needs // to see that as a RebootRequired response it controls @@ -296,7 +295,7 @@ impl TridentClient { reboot: Some(RebootManagement { // Same rationale as commit()/update_finalize(): AKS-RP, // via the agent, is the sole authority over reboot - // timing (https://msazure.visualstudio.com/One/_git/Compute-ACL-Update-Service?version=GCeb7e534b2415ad52b37ef22fd49685e81e56c8aa&path=/docs/update-trigger-design.md §2.5). + // timing (the accepted design's §2.5). handling: RebootHandling::CallerHandlesReboot.into(), }), })) diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index dc8de47b27..84cbb250b2 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -3,9 +3,8 @@ //! trident-acl-agent is Trident's ACL update sidecar. Historically it was a //! one-shot Omaha client that called Trident's combined `Update()` RPC once //! and exited. This crate now defaults to the Kubernetes annotation protocol -//! described in the accepted design -//! (), -//! while preserving the original `omaha-only` mode as an explicit opt-out +//! (the currently accepted design for triggering updates), while +//! preserving the original `omaha-only` mode as an explicit opt-out //! (see `core::config::GoalSource`). //! //! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) From fabf34f94f3949cf4594fca6e7c9f466dd226de1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 22:10:27 +0000 Subject: [PATCH 38/54] trident-acl-agent: drop "accepted design" phrasing and doc section refs Doc comments no longer name or cite section numbers of the (unlinked) accepted design doc; reworded prose to stand on its own. Also updated lib.rs module doc to describe the current annotation-protocol behavior directly rather than as a change from history. No behavior change. --- .../trident-acl-agent/src/annotations/k8s.rs | 5 ++-- .../src/annotations/orchestrator.rs | 24 ++++++++--------- .../src/annotations/protocol.rs | 26 ++++++++----------- .../src/annotations/state.rs | 3 +-- crates/trident-acl-agent/src/core/config.rs | 7 +++-- .../src/core/trident/client.rs | 19 +++++++------- crates/trident-acl-agent/src/lib.rs | 11 ++++---- 7 files changed, 42 insertions(+), 53 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/k8s.rs b/crates/trident-acl-agent/src/annotations/k8s.rs index eb65bfd634..a5f6b5474e 100644 --- a/crates/trident-acl-agent/src/annotations/k8s.rs +++ b/crates/trident-acl-agent/src/annotations/k8s.rs @@ -1,10 +1,9 @@ //! Thin Kubernetes client wrapper for trident-acl-agent's node self-patching //! protocol. //! -//! Implements the Node get/watch/patch access described in the current -//! accepted design. +//! Implements the Node get/watch/patch access. //! -//! The design calls for get/patch access to exactly one Node object (§2.2–§2.6). +//! The design calls for get/patch access to exactly one Node object. //! Node changes are observed via the Kubernetes watch API (`kube::runtime::watcher`) //! rather than polling, so annotation updates are delivered promptly and without //! placing repeated load on the API server. `watch_poll_interval` only diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 285de8c5bc..2b98039e10 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -2,11 +2,10 @@ //! annotation, drives Trident (stage/finalize/rollback/commit) over gRPC, //! and writes the status annotation back, including post-reboot. //! -//! Implements the node-side control flow from the accepted design -//! (sections 2.1 "Trigger mechanism", 2.3 "Stage/finalize/rollback split -//! and post-reboot commit", and 2.5 "Rollback"). See that document for the -//! full state-machine rationale; keep it in sync with this file if the -//! design changes. +//! Implements the node-side control flow (covering the trigger +//! mechanism, stage/finalize/rollback split with post-reboot commit, and +//! rollback). See the design doc for the full state-machine rationale; +//! keep it in sync with this file if the design changes. use std::{collections::BTreeMap, future::Future, time::Duration}; @@ -257,7 +256,7 @@ where // Reject on operationId, not nodeUpdateId: the actual conflict // this guard exists to prevent is "a second finalize/rollback // starts while one is still waiting for its post-reboot - // commit" (the accepted design's in-flight conflict rule). + // commit" (the in-flight conflict rule). // Keying on nodeUpdateId alone let a retried/re-issued request // that reused the same nodeUpdateId but a new operationId slip // through this guard entirely and re-enter handle_finalize/ @@ -303,7 +302,7 @@ where /// across one update's lifecycle would split that state across two /// servers. /// - /// Per the accepted design (§2.1), `stage`/`finalize` requests must + /// `stage`/`finalize` requests must /// carry `server` and there is deliberately no static-config fallback /// here: a fallback would let a node update from a source AKS-RP did /// not choose. `UpdateRequest::validate()` already rejects a @@ -806,8 +805,8 @@ where connect_error: Option, ) -> UpdateStatus { // state.json did not survive the reboot (or was never written, e.g. - // the agent crashed before persisting pendingCommit). Per - // the accepted design's §2.3 degraded path, reconstruct the answer by + // the agent crashed before persisting pendingCommit). Reconstruct + // the answer by // calling commit() unconditionally rather than guessing from labels // or the target version alone - tridentd's commit() is self-checking // and its own (ServicingKind/RebootStatus/Result) response already @@ -1329,7 +1328,7 @@ fn indicates_target_boot_failed(error: &TridentClientError) -> bool { } /// Pre-flight checks for the state.json-missing degraded reconstruction -/// path (the accepted design's §2.3). Returns `Some(status)` when reconstruction +/// path. Returns `Some(status)` when reconstruction /// cannot proceed (tridentd already known-unreachable, or the outstanding /// request isn't a finalize/rollback), or `None` when the caller should go /// on to call tridentd's commit() to determine the real outcome. @@ -1373,9 +1372,8 @@ fn reconstruct_precheck_status( } /// Maps tridentd's commit() result to the terminal status for the -/// state.json-missing degraded reconstruction path -/// (the accepted design's §2.3). Always reports under the original -/// operationId, mirroring the +/// state.json-missing degraded reconstruction path. Always reports +/// under the original operationId, mirroring the /// normal post-reboot commit path in `commit_result_to_status`. fn reconstruct_commit_result_to_status( request: &UpdateRequest, diff --git a/crates/trident-acl-agent/src/annotations/protocol.rs b/crates/trident-acl-agent/src/annotations/protocol.rs index dc3faef05e..ace31824d8 100644 --- a/crates/trident-acl-agent/src/annotations/protocol.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -3,8 +3,7 @@ //! This module (schema types, `UpdateRequest::validate()`, and the //! `#[cfg(test)]` design-doc conformance tests below) implements the //! `/update-request`, `/update-status`, and -//! `/update-commit-status` node annotation protocol described -//! by the current accepted design, where +//! `/update-commit-status` node annotation protocol, where //! `` defaults to `acl.microsoft.com` (see //! [`AnnotationKeys`]/[`crate::core::config::DEFAULT_ANNOTATION_PREFIX`]) and is //! overridable via the `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX` @@ -148,7 +147,7 @@ pub struct UpdateStatus { impl UpdateRequest { /// Enforces the same constraints as the request annotation's formal - /// JSON Schema in the accepted design: schemaVersion match, + /// JSON Schema: schemaVersion match, /// targetVersion required for stage/finalize but disallowed for /// rollback, and server/appId/track required for stage/finalize. See /// this file's module doc. @@ -211,7 +210,7 @@ impl UpdateRequest { impl UpdateStatus { // This constructor mirrors UpdateStatus's wire schema field-for-field - // (see the accepted design's two-status-key JSON protocol); splitting + // (the two-status-key JSON protocol); splitting // it into a builder would add ceremony across ~25 call sites in // orchestrator.rs without making any of them clearer. #[allow(clippy::too_many_arguments)] @@ -656,8 +655,7 @@ mod tests { // --- docs/update-trigger-design.md conformance -------------------------- // // Pins our annotation (de)serialization/validation code against two - // things lifted verbatim from the accepted design's - // docs/update-trigger-design.md, section 2.1 "Trigger mechanism", so a + // things lifted verbatim from docs/update-trigger-design.md, so a // doc/code drift shows up as a // test failure instead of being discovered against a real AKS-RP: // 1. The three example JSON payloads (request, finalize status, and @@ -672,7 +670,7 @@ mod tests { /// (adapted to `finalize` to pair with the status/commit examples /// below, which also share this `finalize`; server/appId/track values /// are the doc's own example values for those fields, required on - /// stage/finalize per the accepted design). + /// stage/finalize). const DESIGN_DOC_FINALIZE_REQUEST_EXAMPLE: &str = r#"{ "schemaVersion": "1.0", "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", @@ -714,9 +712,8 @@ mod tests { "finishedUtc": "2026-06-04T12:01:32Z" }"#; - /// The formal JSON Schema for the request annotation, from the - /// accepted design's section 2.1 "Formal JSON Schema". Keep - /// byte-for-byte in sync with that document. + /// The formal JSON Schema for the request annotation. Keep + /// byte-for-byte in sync with the design doc. const DESIGN_DOC_REQUEST_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://acl.azure.com/schemas/update-request/1.0.json", @@ -746,9 +743,8 @@ mod tests { ] }"#; - /// The formal JSON Schema for the status annotations, from the - /// accepted design's section 2.1 "Formal JSON Schema". Keep - /// byte-for-byte in sync with that document. + /// The formal JSON Schema for the status annotations. Keep + /// byte-for-byte in sync with the design doc. const DESIGN_DOC_STATUS_SCHEMA: &str = r#"{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://acl.azure.com/schemas/update-status/1.0.json", @@ -785,7 +781,7 @@ mod tests { // additionalProperties, required, properties.{type,const,enum,format, // pattern}, and a single-level allOf/if/then/else). Panics loudly on any // schema keyword/pattern/type/format it doesn't recognize, so if - // the accepted design's schemas grow new constraints, this validator's + // the schemas grow new constraints, this validator's // blind spots don't silently mask them - the test fails instead, // prompting an update here. @@ -1082,7 +1078,7 @@ mod tests { #[test] fn validate_allows_rollback_without_nebraska_fields() { // Rollback reports no Nebraska event, so it carries no update - // source (the accepted design, §2.1): server/appId/track are not + // source: server/appId/track are not // required, and validate() must not reject their absence. let request = UpdateRequest { schema_version: SCHEMA_VERSION.to_string(), diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index 2311f577ef..e5b6e1ac0f 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -1,8 +1,7 @@ //! Persistent agent state (`/var/lib/trident-acl-agent/state.json`): //! completed-operation cache and the pending post-reboot commit record. //! -//! Implements the `state.json` mechanism from the current accepted design -//! (section 2.3), which bridges the pre-reboot +//! Implements the `state.json` mechanism, which bridges the pre-reboot //! finalize/rollback half and the post-reboot commit half of an operation //! across the reboot. diff --git a/crates/trident-acl-agent/src/core/config.rs b/crates/trident-acl-agent/src/core/config.rs index 011a6a95a2..06963d013b 100644 --- a/crates/trident-acl-agent/src/core/config.rs +++ b/crates/trident-acl-agent/src/core/config.rs @@ -205,8 +205,8 @@ pub enum GoalSource { /// stage/finalize/rollback/commit operations against tridentd /// accordingly, writing progress back to /// `/update-status` and - /// `/update-commit-status` (see the accepted design). - /// `` defaults to + /// `/update-commit-status`. `` + /// defaults to /// [`DEFAULT_ANNOTATION_PREFIX`] (`acl.microsoft.com`), overridable via /// `TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX`. This is the only /// supported mode. @@ -237,8 +237,7 @@ pub struct OrchestrationConfig { /// Placeholder default pending real data from storm aclagent scenario runs. pub finalize_timeout: Duration, /// Refresh cadence for in-flight InProgress heartbeats. Default is well - /// below the ~10 minute watchdog staleness target proposed in the - /// accepted design. + /// below the ~10 minute watchdog staleness target. pub heartbeat_interval: Duration, } diff --git a/crates/trident-acl-agent/src/core/trident/client.rs b/crates/trident-acl-agent/src/core/trident/client.rs index c9ea535103..37432c793f 100644 --- a/crates/trident-acl-agent/src/core/trident/client.rs +++ b/crates/trident-acl-agent/src/core/trident/client.rs @@ -1,12 +1,11 @@ //! gRPC helpers for talking to `tridentd`. //! -//! Implements the Trident-invocation half of the accepted design -//! (the "Trident invocation" column of section 2.1's operations table, -//! and the stage/finalize/rollback-finalize CallerHandlesReboot split in -//! section 2.3). +//! Implements the Trident-invocation half of the update-trigger flow +//! (covering operation invocation and the +//! stage/finalize/rollback-finalize CallerHandlesReboot split). //! //! The annotation protocol drives stage/finalize/commit directly against -//! tridentd's stable v1 API (§4–§5). Startup recovery no longer pre-queries +//! tridentd's stable v1 API. Startup recovery no longer pre-queries //! the preview `StatusService::GetServicingState`: commit() is self-checking //! (tridentd only commits from a valid servicing_state and otherwise returns //! ServicingKind::NoneRequired as a harmless no-op), so the orchestrator @@ -233,7 +232,7 @@ impl TridentClient { reboot: Some(RebootManagement { // The agent, not tridentd, must own every reboot // decision: AKS-RP is the sole authority over - // reboot/rollback (the accepted design's §2.5). If commit() + // reboot/rollback. If commit() // ever reports NeedsReboot (e.g. a health-check failure, // were health checks ever re-enabled), the agent needs // to see that as a RebootRequired response it controls @@ -257,9 +256,9 @@ impl TridentClient { .await } - /// Stages an A/B rollback. Only `AbRollbackRequested` is used - per the - /// accepted design, trident-acl-agent only ever drives AB-kind manual - /// rollback; runtime-kind and "any" rollback are out of scope for the + /// Stages an A/B rollback. Only `AbRollbackRequested` is used - + /// trident-acl-agent only ever drives AB-kind manual rollback; + /// runtime-kind and "any" rollback are out of scope for the /// annotation-driven protocol. pub async fn rollback_stage( &mut self, @@ -295,7 +294,7 @@ impl TridentClient { reboot: Some(RebootManagement { // Same rationale as commit()/update_finalize(): AKS-RP, // via the agent, is the sole authority over reboot - // timing (the accepted design's §2.5). + // timing. handling: RebootHandling::CallerHandlesReboot.into(), }), })) diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 84cbb250b2..a339490c67 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -1,11 +1,10 @@ //! # trident-acl-agent //! -//! trident-acl-agent is Trident's ACL update sidecar. Historically it was a -//! one-shot Omaha client that called Trident's combined `Update()` RPC once -//! and exited. This crate now defaults to the Kubernetes annotation protocol -//! (the currently accepted design for triggering updates), while -//! preserving the original `omaha-only` mode as an explicit opt-out -//! (see `core::config::GoalSource`). +//! trident-acl-agent is Trident's ACL update sidecar. By default it drives +//! Trident (stage/finalize/rollback/commit) through a Kubernetes node +//! annotation protocol; a one-shot `omaha-only` mode that calls Trident's +//! combined `Update()` RPC once and exits is also available as an explicit +//! opt-out (see `core::config::GoalSource`). //! //! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) //! goes through the [`core::nebraska`] client module, a self-contained, From 65c0c83132146cddd77e6033573d17fcc43a294b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 22:15:42 +0000 Subject: [PATCH 39/54] trident-acl-agent: drop docs/update-trigger-design.md path references This filename referred to the same private accepted-design doc whose links were already removed; it does not correspond to a file in this repo. Reworded to "the design doc" without a path. No behavior change. --- crates/trident-acl-agent/src/annotations/protocol.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/protocol.rs b/crates/trident-acl-agent/src/annotations/protocol.rs index ace31824d8..5498b53159 100644 --- a/crates/trident-acl-agent/src/annotations/protocol.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -652,10 +652,10 @@ mod tests { assert_eq!(json["fromVersion"], "1.0.0"); } - // --- docs/update-trigger-design.md conformance -------------------------- + // --- design-doc conformance -------------------------- // // Pins our annotation (de)serialization/validation code against two - // things lifted verbatim from docs/update-trigger-design.md, so a + // things lifted verbatim from the design doc, so a // doc/code drift shows up as a // test failure instead of being discovered against a real AKS-RP: // 1. The three example JSON payloads (request, finalize status, and @@ -666,7 +666,7 @@ mod tests { // // Keep these constants byte-for-byte in sync with the design doc. - /// docs/update-trigger-design.md 2.1, "Request annotation" example + /// The design doc's "Request annotation" example /// (adapted to `finalize` to pair with the status/commit examples /// below, which also share this `finalize`; server/appId/track values /// are the doc's own example values for those fields, required on @@ -682,7 +682,7 @@ mod tests { "track": "pin-202606.29.0" }"#; - /// docs/update-trigger-design.md 2.1, "Status annotation" example. + /// The design doc's "Status annotation" example. const DESIGN_DOC_FINALIZE_STATUS_EXAMPLE: &str = r#"{ "schemaVersion": "1.0", "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", @@ -697,7 +697,7 @@ mod tests { "finishedUtc": "2026-06-04T12:00:32Z" }"#; - /// docs/update-trigger-design.md 2.1, the derived post-reboot commit status example. + /// The design doc's derived post-reboot commit status example. const DESIGN_DOC_COMMIT_STATUS_EXAMPLE: &str = r#"{ "schemaVersion": "1.0", "nodeUpdateId": "550e8400-e29b-41d4-a716-446655440000", From 4d1bd7ab0149ba95ac41a531164e01703f1dbc6c Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 22:17:12 +0000 Subject: [PATCH 40/54] trident-acl-agent: remove unnecessary design-doc conformance heading comment --- crates/trident-acl-agent/src/annotations/protocol.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/protocol.rs b/crates/trident-acl-agent/src/annotations/protocol.rs index 5498b53159..515440cc4f 100644 --- a/crates/trident-acl-agent/src/annotations/protocol.rs +++ b/crates/trident-acl-agent/src/annotations/protocol.rs @@ -652,8 +652,6 @@ mod tests { assert_eq!(json["fromVersion"], "1.0.0"); } - // --- design-doc conformance -------------------------- - // // Pins our annotation (de)serialization/validation code against two // things lifted verbatim from the design doc, so a // doc/code drift shows up as a From ee6e18e5912e74eed2da317aa9ba580710ba8c9e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 22:32:25 +0000 Subject: [PATCH 41/54] trident-acl-agent: remove duplicate top-level module docs annotations/mod.rs, core/mod.rs, and omahaonly/mod.rs each restated a description of their own module already summarized in lib.rs ' s top-level module list. Removed the duplicate module-level doc comments and folded annotations ' submodule breakdown into its lib.rs bullet instead of keeping it in two places. Also re-verified all other outstanding frhuelsz PR 730 review threads: the main.rs logging-filter/cli.rs/connection_check.rs restructuring comments are already satisfied by the current code (FilteredLogger already lives in osutils::logging and is reused here; Args/ ConnectionTarget already live in cli.rs; validate_connection already lives in connection_check.rs), and the suggestion to fold build_machine_id() into its caller no longer applies now that it has multiple callers (orchestrator.rs, omahaonly). No behavior change. --- .../trident-acl-agent/src/annotations/mod.rs | 21 ++++++------------- crates/trident-acl-agent/src/core/mod.rs | 18 ++++++---------- crates/trident-acl-agent/src/lib.rs | 7 ++++++- crates/trident-acl-agent/src/omahaonly/mod.rs | 4 ---- 4 files changed, 18 insertions(+), 32 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/mod.rs b/crates/trident-acl-agent/src/annotations/mod.rs index 666e15af6e..964c95470a 100644 --- a/crates/trident-acl-agent/src/annotations/mod.rs +++ b/crates/trident-acl-agent/src/annotations/mod.rs @@ -1,15 +1,6 @@ -//! The Kubernetes annotation-driven update protocol - the crate's default -//! mode (see [`crate::core::config::GoalSource`]). -//! -//! [`protocol`] defines the annotation schema (request/status types, keys, -//! schema version); [`k8s`] is the Kubernetes Node get/watch/patch client; -//! [`state`] persists in-flight/completed operations across the reboot that -//! finalize triggers; [`orchestrator`] is the reconcile loop tying them all -//! together. - -mod protocol; -pub use protocol::*; - -pub mod k8s; -pub mod orchestrator; -pub mod state; +mod protocol; +pub use protocol::*; + +pub mod k8s; +pub mod orchestrator; +pub mod state; diff --git a/crates/trident-acl-agent/src/core/mod.rs b/crates/trident-acl-agent/src/core/mod.rs index 21c4bd0402..4fd36d5691 100644 --- a/crates/trident-acl-agent/src/core/mod.rs +++ b/crates/trident-acl-agent/src/core/mod.rs @@ -1,12 +1,6 @@ -//! Shared building blocks used by both the annotation-driven protocol -//! ([`crate::annotations`]) and the legacy one-shot Omaha flow -//! ([`crate::omahaonly`]): configuration, error types, machine-id resolution, -//! the current-version fallback, the `tridentd` gRPC client, and the generic -//! Nebraska/Omaha protocol client. - -pub mod config; -pub mod error; -pub mod id; -pub mod nebraska; -pub mod trident; -pub mod version; +pub mod config; +pub mod error; +pub mod id; +pub mod nebraska; +pub mod trident; +pub mod version; diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index a339490c67..6fd8a88b0e 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -15,7 +15,12 @@ //! - [`core`]: building blocks shared by both modes (config, errors, //! machine-id, current-version, the `tridentd` client, the Nebraska //! client). -//! - [`annotations`]: the default Kubernetes annotation-driven protocol. +//! - [`annotations`]: the default Kubernetes annotation-driven protocol - +//! [`annotations::protocol`] defines the annotation schema, +//! [`annotations::k8s`] is the Kubernetes Node get/watch/patch client, +//! [`annotations::state`] persists in-flight/completed operations across +//! the reboot that finalize triggers, and [`annotations::orchestrator`] +//! is the reconcile loop tying them all together. //! - [`omahaonly`]: the legacy one-shot Omaha flow. use semver::Version; diff --git a/crates/trident-acl-agent/src/omahaonly/mod.rs b/crates/trident-acl-agent/src/omahaonly/mod.rs index 22750e77ab..f85f4c5975 100644 --- a/crates/trident-acl-agent/src/omahaonly/mod.rs +++ b/crates/trident-acl-agent/src/omahaonly/mod.rs @@ -1,7 +1,3 @@ -//! The historical one-shot Omaha flow, preserved as an explicit opt-out from -//! the default annotation-driven protocol (see -//! [`crate::core::config::GoalSource`]). - use anyhow::{anyhow, Context, Error}; use log::{debug, info, warn}; use semver::Version; From 91cb35f110ea984240efa34f28311a43b35aaff0 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 25 Aug 2026 23:41:11 +0000 Subject: [PATCH 42/54] trident-acl-agent: PathBuf kubeconfig, hostname crate, envy-based config, rename GoalSource to Mode --- Cargo.lock | 11 + Cargo.toml | 1 + crates/trident-acl-agent/Cargo.toml | 2 + .../trident-acl-agent/src/annotations/k8s.rs | 4 +- crates/trident-acl-agent/src/cli.rs | 2 +- crates/trident-acl-agent/src/core/config.rs | 451 +++++++++++------- crates/trident-acl-agent/src/lib.rs | 2 +- crates/trident-acl-agent/src/main.rs | 10 +- docs/Explanation/Trident-ACL-Agent.md | 2 +- 9 files changed, 291 insertions(+), 194 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6405ec1e8..e04caea25d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -863,6 +863,15 @@ dependencies = [ "log", ] +[[package]] +name = "envy" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965" +dependencies = [ + "serde", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -4014,7 +4023,9 @@ dependencies = [ "clap", "const_format", "env_logger 0.11.5", + "envy", "futures", + "hostname", "humantime", "hyper-util", "indoc", diff --git a/Cargo.toml b/Cargo.toml index dde3227664..9919febebd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ documented = "0.6.0" duct = "0.13.7" enumflags2 = { version = "0.7", features = ["serde"] } env_logger = "0.11.5" +envy = "0.4.2" futures = "0.3.32" glob = "0.3.1" gpt = "4.1.0" diff --git a/crates/trident-acl-agent/Cargo.toml b/crates/trident-acl-agent/Cargo.toml index d89079ecfd..185ff263bd 100644 --- a/crates/trident-acl-agent/Cargo.toml +++ b/crates/trident-acl-agent/Cargo.toml @@ -10,7 +10,9 @@ clap = { workspace = true, features = ["derive"] } chrono = { workspace = true } const_format = { workspace = true } env_logger = { workspace = true } +envy = { workspace = true } futures = { workspace = true } +hostname = { workspace = true } humantime = { workspace = true } k8s-openapi = { workspace = true } kube = { workspace = true } diff --git a/crates/trident-acl-agent/src/annotations/k8s.rs b/crates/trident-acl-agent/src/annotations/k8s.rs index a5f6b5474e..b5603b173f 100644 --- a/crates/trident-acl-agent/src/annotations/k8s.rs +++ b/crates/trident-acl-agent/src/annotations/k8s.rs @@ -15,7 +15,7 @@ //! after a dropped or failed watch; that is governed entirely by //! `kube::runtime::watcher`'s built-in `default_backoff()`. -use std::{collections::BTreeMap, path::Path, time::Duration}; +use std::{collections::BTreeMap, time::Duration}; use anyhow::{Context, Error}; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; @@ -161,7 +161,7 @@ fn map_kube_error(err: KubeError) -> K8sClientError { } async fn load_client_config(config: &KubernetesConfig) -> Result { - let path = Path::new(&config.kubeconfig); + let path = config.kubeconfig.as_path(); let kubeconfig = Kubeconfig::read_from(path) .with_context(|| format!("failed to read kubeconfig {}", path.display()))?; let mut client_config = diff --git a/crates/trident-acl-agent/src/cli.rs b/crates/trident-acl-agent/src/cli.rs index cd8ac2271c..641f870915 100644 --- a/crates/trident-acl-agent/src/cli.rs +++ b/crates/trident-acl-agent/src/cli.rs @@ -3,7 +3,7 @@ use log::LevelFilter; /// trident-acl-agent can either run the annotation-driven orchestrator (the /// default) or fall back to its original one-shot Omaha flow. Mode selection -/// is environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE`): +/// is environment-variable only (`TRIDENT_ACL_AGENT_ORCHESTRATION_MODE`): /// shipping defaults enable the AKS annotation protocol, while a VM /// extension, systemd drop-in, or AgentBaker-set environment can opt a node /// out to `omaha-only` if needed. diff --git a/crates/trident-acl-agent/src/core/config.rs b/crates/trident-acl-agent/src/core/config.rs index 06963d013b..043fd8710d 100644 --- a/crates/trident-acl-agent/src/core/config.rs +++ b/crates/trident-acl-agent/src/core/config.rs @@ -1,48 +1,43 @@ //! Env-var-based config loading for trident-acl-agent. //! //! There is no config file. Every setting is an environment variable -//! prefixed `TRIDENT_ACL_AGENT_` (one constant per setting, e.g. -//! [`ENV_NEBRASKA_ENDPOINT`]), systemd-style: set it directly in the unit's -//! own `Environment=` lines, via a drop-in override (`systemctl edit -//! trident-acl-agent.service`, which creates +//! prefixed `TRIDENT_ACL_AGENT_
_` (e.g. +//! `TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT`), systemd-style: set it directly in +//! the unit's own `Environment=` lines, via a drop-in override (`systemctl +//! edit trident-acl-agent.service`, which creates //! `/etc/systemd/system/trident-acl-agent.service.d/override.conf`), or by //! any other means that ultimately sets the process's environment before it -//! starts. All are equivalent from the agent's point of view - it just reads -//! `std::env::var`. +//! starts. All are equivalent from the agent's point of view. +//! +//! Loading goes through [`envy`], which deserializes a prefixed subset of +//! the environment into small `Raw*` structs below - one per section, with a +//! field per setting - via [`envy::prefixed`]. Every field is optional, so a +//! merely-absent variable is never an error; it just falls back to that +//! setting's default (applied by [`AgentConfig::from_vars`]). A +//! present-and-malformed value (bad URL, bad duration, unknown +//! `mode`, etc.) is. `envy::prefixed(..).from_iter(..)` also means +//! this module's own unit tests can build config from a plain iterator of +//! `(name, value)` pairs instead of mutating real (process-global, `unsafe`) +//! environment variables. //! //! Annotation mode is the default; `omaha-only` (the historical one-shot //! behavior) remains available as an explicit opt-out via -//! `TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE=omaha-only`. +//! `TRIDENT_ACL_AGENT_ORCHESTRATION_MODE=omaha-only`. -use std::{env, path::PathBuf, str::FromStr, time::Duration}; +use std::{path::PathBuf, str::FromStr, time::Duration}; use anyhow::{anyhow, Context, Error}; use const_format::formatcp; -use osutils::hostname; +use serde::{de::Error as _, Deserialize, Deserializer}; use trident_proto::TRIDENT_DEFAULT_SOCKET_URI; use url::Url; use crate::{DEFAULT_NEBRASKA_APP_ID, DEFAULT_NEBRASKA_TRACK}; -/// The environment variables this module reads, one constant per setting. -const ENV_NEBRASKA_ENDPOINT: &str = "TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT"; -const ENV_NEBRASKA_APP_ID: &str = "TRIDENT_ACL_AGENT_NEBRASKA_APP_ID"; -const ENV_NEBRASKA_TRACK: &str = "TRIDENT_ACL_AGENT_NEBRASKA_TRACK"; -const ENV_KUBERNETES_API_SERVER: &str = "TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER"; -const ENV_KUBERNETES_KUBECONFIG: &str = "TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG"; -const ENV_KUBERNETES_NODE_NAME: &str = "TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME"; -const ENV_TRIDENT_SOCKET: &str = "TRIDENT_ACL_AGENT_TRIDENT_SOCKET"; -const ENV_ORCHESTRATION_GOAL_SOURCE: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_GOAL_SOURCE"; -const ENV_ORCHESTRATION_STATE_PATH: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH"; -const ENV_ORCHESTRATION_STAGE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT"; -const ENV_ORCHESTRATION_FINALIZE_TIMEOUT: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT"; -const ENV_ORCHESTRATION_HEARTBEAT_INTERVAL: &str = - "TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL"; -/// Overrides the annotation-key prefix (e.g. `acl.microsoft.com` in -/// `acl.microsoft.com/update-request`). Defaults to -/// [`DEFAULT_ANNOTATION_PREFIX`] so a deployment can point the agent at its -/// own annotation namespace without a code change. -const ENV_KUBERNETES_ANNOTATION_PREFIX: &str = "TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX"; +const ENV_PREFIX_NEBRASKA: &str = "TRIDENT_ACL_AGENT_NEBRASKA_"; +const ENV_PREFIX_KUBERNETES: &str = "TRIDENT_ACL_AGENT_KUBERNETES_"; +const ENV_PREFIX_TRIDENT: &str = "TRIDENT_ACL_AGENT_TRIDENT_"; +const ENV_PREFIX_ORCHESTRATION: &str = "TRIDENT_ACL_AGENT_ORCHESTRATION_"; const DEFAULT_KUBERNETES_POLL_INTERVAL: Duration = Duration::from_secs(2); // TODO: placeholder until the real production Nebraska/Omaha endpoint is @@ -76,55 +71,180 @@ pub struct AgentConfig { impl AgentConfig { /// Loads the effective config purely from `TRIDENT_ACL_AGENT_*` - /// environment variables (see the module doc). A merely-absent variable - /// is never an error - it just falls back to that setting's default - - /// but a present-and-malformed value (bad URL, bad duration, unknown - /// `goal_source`, etc.) is. + /// environment variables (see the module doc). pub fn from_env() -> Result { + Self::from_vars(std::env::vars().collect()) + } + + /// Same as [`Self::from_env`], but reads from a plain `Vec` of `(name, + /// value)` pairs instead of the real process environment. + fn from_vars(vars: Vec<(String, String)>) -> Result { + let nebraska: RawNebraskaConfig = envy::prefixed(ENV_PREFIX_NEBRASKA) + .from_iter(vars.iter().cloned()) + .context("invalid TRIDENT_ACL_AGENT_NEBRASKA_* environment variable")?; + let kubernetes: RawKubernetesConfig = envy::prefixed(ENV_PREFIX_KUBERNETES) + .from_iter(vars.iter().cloned()) + .context("invalid TRIDENT_ACL_AGENT_KUBERNETES_* environment variable")?; + let trident: RawTridentConfig = envy::prefixed(ENV_PREFIX_TRIDENT) + .from_iter(vars.iter().cloned()) + .context("invalid TRIDENT_ACL_AGENT_TRIDENT_* environment variable")?; + let orchestration: RawOrchestrationConfig = envy::prefixed(ENV_PREFIX_ORCHESTRATION) + .from_iter(vars.iter().cloned()) + .context("invalid TRIDENT_ACL_AGENT_ORCHESTRATION_* environment variable")?; + Ok(Self { nebraska: NebraskaConfig { - endpoint: env_url(ENV_NEBRASKA_ENDPOINT)? - .or_else(|| Some(default_nebraska_endpoint())), - app_id: env_string(ENV_NEBRASKA_APP_ID) + endpoint: Some(nebraska.endpoint.unwrap_or_else(default_nebraska_endpoint)), + app_id: nebraska + .app_id .unwrap_or_else(|| DEFAULT_NEBRASKA_APP_ID.to_string()), - track: env_string(ENV_NEBRASKA_TRACK) + track: nebraska + .track .unwrap_or_else(|| DEFAULT_NEBRASKA_TRACK.to_string()), }, kubernetes: KubernetesConfig { - api_server: env_url(ENV_KUBERNETES_API_SERVER)?, - kubeconfig: env_string(ENV_KUBERNETES_KUBECONFIG) - .unwrap_or_else(|| DEFAULT_KUBELET_KUBECONFIG.to_string()), - node_name: env_string(ENV_KUBERNETES_NODE_NAME).unwrap_or_else(default_node_name), + api_server: kubernetes.api_server, + kubeconfig: kubernetes + .kubeconfig + .unwrap_or_else(|| PathBuf::from(DEFAULT_KUBELET_KUBECONFIG)), + node_name: kubernetes.node_name.unwrap_or_else(default_node_name), watch_poll_interval: DEFAULT_KUBERNETES_POLL_INTERVAL, - annotation_prefix: env_string(ENV_KUBERNETES_ANNOTATION_PREFIX) + annotation_prefix: kubernetes + .annotation_prefix .unwrap_or_else(|| DEFAULT_ANNOTATION_PREFIX.to_string()), }, trident: TridentConfig { - socket: env_string(ENV_TRIDENT_SOCKET) + socket: trident + .socket .unwrap_or_else(|| TRIDENT_DEFAULT_SOCKET_URI.to_string()), }, orchestration: OrchestrationConfig { - goal_source: env_parse(ENV_ORCHESTRATION_GOAL_SOURCE)?.unwrap_or_default(), - state_path: env_string(ENV_ORCHESTRATION_STATE_PATH) - .map(PathBuf::from) + mode: orchestration.mode.unwrap_or_default(), + state_path: orchestration + .state_path .unwrap_or_else(|| PathBuf::from(DEFAULT_STATE_PATH)), - stage_timeout: env_duration( - ENV_ORCHESTRATION_STAGE_TIMEOUT, - DEFAULT_STAGE_TIMEOUT, - )?, - finalize_timeout: env_duration( - ENV_ORCHESTRATION_FINALIZE_TIMEOUT, - DEFAULT_FINALIZE_TIMEOUT, - )?, - heartbeat_interval: env_duration( - ENV_ORCHESTRATION_HEARTBEAT_INTERVAL, - DEFAULT_HEARTBEAT_INTERVAL, - )?, + stage_timeout: orchestration.stage_timeout.unwrap_or(DEFAULT_STAGE_TIMEOUT), + finalize_timeout: orchestration + .finalize_timeout + .unwrap_or(DEFAULT_FINALIZE_TIMEOUT), + heartbeat_interval: orchestration + .heartbeat_interval + .unwrap_or(DEFAULT_HEARTBEAT_INTERVAL), }, }) } } +/// Mirrors [`NebraskaConfig`], with every field optional: [`envy`] leaves a +/// field `None` when its environment variable is unset, so +/// [`AgentConfig::from_vars`] can apply this section's defaults itself. +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RawNebraskaConfig { + #[serde(deserialize_with = "empty_url_as_none")] + endpoint: Option, + #[serde(deserialize_with = "empty_string_as_none")] + app_id: Option, + #[serde(deserialize_with = "empty_string_as_none")] + track: Option, +} + +/// Mirrors [`KubernetesConfig`] (see [`RawNebraskaConfig`]). +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RawKubernetesConfig { + #[serde(deserialize_with = "empty_url_as_none")] + api_server: Option, + #[serde(deserialize_with = "empty_path_as_none")] + kubeconfig: Option, + #[serde(deserialize_with = "empty_string_as_none")] + node_name: Option, + #[serde(deserialize_with = "empty_string_as_none")] + annotation_prefix: Option, +} + +/// Mirrors [`TridentConfig`] (see [`RawNebraskaConfig`]). +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RawTridentConfig { + #[serde(deserialize_with = "empty_string_as_none")] + socket: Option, +} + +/// Mirrors [`OrchestrationConfig`] (see [`RawNebraskaConfig`]). +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RawOrchestrationConfig { + #[serde(deserialize_with = "empty_mode_as_none")] + mode: Option, + #[serde(deserialize_with = "empty_path_as_none")] + state_path: Option, + #[serde(deserialize_with = "empty_duration_as_none")] + stage_timeout: Option, + #[serde(deserialize_with = "empty_duration_as_none")] + finalize_timeout: Option, + #[serde(deserialize_with = "empty_duration_as_none")] + heartbeat_interval: Option, +} + +/// Treats "set to the empty string" the same as "unset": a drop-in override +/// that clears a variable to `""` should fall back to the default, not try +/// to parse an empty value. +fn empty_as_none(value: String) -> Option { + if value.is_empty() { + None + } else { + Some(value) + } +} + +fn empty_string_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + Ok(empty_as_none(String::deserialize(deserializer)?)) +} + +fn empty_url_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + empty_as_none(String::deserialize(deserializer)?) + .map(|value| { + Url::parse(&value) + .map_err(|err| D::Error::custom(format!("invalid URL {value:?}: {err}"))) + }) + .transpose() +} + +fn empty_path_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + Ok(empty_as_none(String::deserialize(deserializer)?).map(PathBuf::from)) +} + +fn empty_duration_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + empty_as_none(String::deserialize(deserializer)?) + .map(|value| { + humantime::parse_duration(&value) + .map_err(|err| D::Error::custom(format!("invalid duration {value:?}: {err}"))) + }) + .transpose() +} + +fn empty_mode_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + empty_as_none(String::deserialize(deserializer)?) + .map(|value| value.parse::().map_err(D::Error::custom)) + .transpose() +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct NebraskaConfig { pub endpoint: Option, @@ -152,7 +272,7 @@ pub struct KubernetesConfig { /// `https://kubernetes.default.svc` name, which a plain node-level /// kubeconfig has no reason to contain. pub api_server: Option, - pub kubeconfig: String, + pub kubeconfig: PathBuf, pub node_name: String, pub watch_poll_interval: Duration, /// Annotation-key prefix used for the request/status/commit-status @@ -168,7 +288,7 @@ impl Default for KubernetesConfig { fn default() -> Self { Self { api_server: None, - kubeconfig: DEFAULT_KUBELET_KUBECONFIG.to_string(), + kubeconfig: PathBuf::from(DEFAULT_KUBELET_KUBECONFIG), node_name: default_node_name(), watch_poll_interval: DEFAULT_KUBERNETES_POLL_INTERVAL, annotation_prefix: DEFAULT_ANNOTATION_PREFIX.to_string(), @@ -190,7 +310,7 @@ impl Default for TridentConfig { } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum GoalSource { +pub enum Mode { /// Historical one-shot behavior: query Nebraska/Omaha once, and if an /// update is offered, call tridentd's combined `update()` RPC once and /// exit. No Kubernetes involvement at all - no annotations, no watch, @@ -214,15 +334,15 @@ pub enum GoalSource { Annotations, } -impl FromStr for GoalSource { +impl FromStr for Mode { type Err = Error; fn from_str(s: &str) -> Result { match s { - "omaha-only" => Ok(GoalSource::OmahaOnly), - "annotations" => Ok(GoalSource::Annotations), + "omaha-only" => Ok(Mode::OmahaOnly), + "annotations" => Ok(Mode::Annotations), other => Err(anyhow!( - "unknown goal_source {other:?} (expected \"annotations\" or \"omaha-only\")" + "unknown mode {other:?} (expected \"annotations\" or \"omaha-only\")" )), } } @@ -230,7 +350,7 @@ impl FromStr for GoalSource { #[derive(Debug, Clone, PartialEq, Eq)] pub struct OrchestrationConfig { - pub goal_source: GoalSource, + pub mode: Mode, pub state_path: PathBuf, /// Placeholder default pending real data from storm aclagent scenario runs. pub stage_timeout: Duration, @@ -244,7 +364,7 @@ pub struct OrchestrationConfig { impl Default for OrchestrationConfig { fn default() -> Self { Self { - goal_source: GoalSource::Annotations, + mode: Mode::Annotations, state_path: PathBuf::from(DEFAULT_STATE_PATH), stage_timeout: DEFAULT_STAGE_TIMEOUT, finalize_timeout: DEFAULT_FINALIZE_TIMEOUT, @@ -253,52 +373,21 @@ impl Default for OrchestrationConfig { } } -/// Reads `name`, treating both "unset" and "set to the empty string" as -/// absent - a drop-in override that clears a variable to `""` should fall -/// back to the default, not try to parse an empty value. -fn env_raw(name: &str) -> Option { - env::var(name).ok().filter(|v| !v.is_empty()) -} - -fn env_string(name: &str) -> Option { - env_raw(name) -} - fn default_nebraska_endpoint() -> Url { Url::parse(DEFAULT_NEBRASKA_ENDPOINT) .expect("invariant: DEFAULT_NEBRASKA_ENDPOINT is a compile-time-valid URL") } -fn env_url(name: &str) -> Result, Error> { - env_raw(name) - .map(|v| Url::parse(&v).with_context(|| format!("invalid URL for {name}"))) - .transpose() -} - -fn env_duration(name: &str, default: Duration) -> Result { - env_raw(name) - .map(|v| { - humantime::parse_duration(&v).with_context(|| format!("invalid duration for {name}")) - }) - .transpose() - .map(|parsed| parsed.unwrap_or(default)) -} - -fn env_parse(name: &str) -> Result, Error> -where - T: FromStr, -{ - env_raw(name).map(|v| v.parse::()).transpose() -} - fn default_node_name() -> String { // Kubernetes Node names must be valid RFC 1123 DNS labels, which are // lowercase-only; kubelet itself lowercases the hostname when it // registers the Node object. Match that behavior here so a mixed-case // hostname doesn't produce a node_name that can never match the actual // Node the agent is supposed to reconcile against. - hostname::read() - .unwrap_or_else(|_| DEFAULT_NODE_NAME.to_string()) + hostname::get() + .ok() + .and_then(|name| name.into_string().ok()) + .unwrap_or_else(|| DEFAULT_NODE_NAME.to_string()) .to_lowercase() } @@ -306,38 +395,17 @@ fn default_node_name() -> String { mod tests { use super::*; - /// Clears every var this module reads. Environment mutation is process- - /// global and `std::env::remove_var`/`set_var` are `unsafe` (not - /// thread-safe against concurrent reads elsewhere in the process), so - /// all of the defaults/overrides/empty-value/malformed-value cases below - /// are intentionally folded into one sequential `#[test]` rather than - /// several separate ones that `cargo test` could run in parallel against - /// the same variables. - fn clear_env() { - // SAFETY: single-threaded within this test function; no other test - // in this crate reads or writes these TRIDENT_ACL_AGENT_* variables. - unsafe { - env::remove_var(ENV_NEBRASKA_ENDPOINT); - env::remove_var(ENV_NEBRASKA_APP_ID); - env::remove_var(ENV_NEBRASKA_TRACK); - env::remove_var(ENV_KUBERNETES_API_SERVER); - env::remove_var(ENV_KUBERNETES_KUBECONFIG); - env::remove_var(ENV_KUBERNETES_NODE_NAME); - env::remove_var(ENV_TRIDENT_SOCKET); - env::remove_var(ENV_ORCHESTRATION_GOAL_SOURCE); - env::remove_var(ENV_ORCHESTRATION_STATE_PATH); - env::remove_var(ENV_ORCHESTRATION_STAGE_TIMEOUT); - env::remove_var(ENV_ORCHESTRATION_FINALIZE_TIMEOUT); - env::remove_var(ENV_ORCHESTRATION_HEARTBEAT_INTERVAL); - env::remove_var(ENV_KUBERNETES_ANNOTATION_PREFIX); - } + fn vars(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() } #[test] - fn env_config_defaults_then_overrides() { - clear_env(); + fn defaults_when_all_vars_unset() { + let config = AgentConfig::from_vars(vec![]).unwrap(); - let config = AgentConfig::from_env().unwrap(); assert_eq!( config.nebraska.endpoint.unwrap().as_str(), DEFAULT_NEBRASKA_ENDPOINT @@ -350,10 +418,10 @@ mod tests { ); assert_eq!( config.kubernetes.kubeconfig, - DEFAULT_KUBELET_KUBECONFIG.to_string() + PathBuf::from(DEFAULT_KUBELET_KUBECONFIG) ); assert_eq!(config.trident.socket, TRIDENT_DEFAULT_SOCKET_URI); - assert_eq!(config.orchestration.goal_source, GoalSource::Annotations); + assert_eq!(config.orchestration.mode, Mode::Annotations); assert_eq!( config.orchestration.state_path, PathBuf::from(DEFAULT_STATE_PATH) @@ -371,35 +439,44 @@ mod tests { config.kubernetes.annotation_prefix, DEFAULT_ANNOTATION_PREFIX ); + } - // SAFETY: see clear_env's doc comment. - unsafe { - env::set_var( - ENV_NEBRASKA_ENDPOINT, + #[test] + fn overrides_apply_when_vars_set() { + let config = AgentConfig::from_vars(vars(&[ + ( + "TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT", "https://custom-nebraska.example.invalid/v1/update", - ); - env::set_var(ENV_NEBRASKA_APP_ID, "custom-app"); - env::set_var(ENV_NEBRASKA_TRACK, "custom-track"); - env::set_var(ENV_KUBERNETES_API_SERVER, "https://cluster.example.invalid"); - env::set_var( - ENV_KUBERNETES_KUBECONFIG, + ), + ("TRIDENT_ACL_AGENT_NEBRASKA_APP_ID", "custom-app"), + ("TRIDENT_ACL_AGENT_NEBRASKA_TRACK", "custom-track"), + ( + "TRIDENT_ACL_AGENT_KUBERNETES_API_SERVER", + "https://cluster.example.invalid", + ), + ( + "TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG", "/etc/trident-acl-agent/kubeconfig", - ); - env::set_var(ENV_KUBERNETES_NODE_NAME, "node-42"); - env::set_var(ENV_TRIDENT_SOCKET, "unix:///custom/trident.sock"); - env::set_var(ENV_ORCHESTRATION_GOAL_SOURCE, "omaha-only"); - env::set_var( - ENV_ORCHESTRATION_STATE_PATH, + ), + ("TRIDENT_ACL_AGENT_KUBERNETES_NODE_NAME", "node-42"), + ( + "TRIDENT_ACL_AGENT_TRIDENT_SOCKET", + "unix:///custom/trident.sock", + ), + ("TRIDENT_ACL_AGENT_ORCHESTRATION_MODE", "omaha-only"), + ( + "TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH", "/var/lib/trident-acl-agent/custom-state.json", - ); - env::set_var(ENV_ORCHESTRATION_STAGE_TIMEOUT, "21m"); - env::set_var(ENV_ORCHESTRATION_FINALIZE_TIMEOUT, "11m"); - env::set_var(ENV_ORCHESTRATION_HEARTBEAT_INTERVAL, "45s"); - env::set_var(ENV_KUBERNETES_ANNOTATION_PREFIX, "contoso.example.com"); - } - - let config = AgentConfig::from_env().unwrap(); - clear_env(); + ), + ("TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT", "21m"), + ("TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT", "11m"), + ("TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL", "45s"), + ( + "TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX", + "contoso.example.com", + ), + ])) + .unwrap(); assert_eq!( config.nebraska.endpoint.unwrap().as_str(), @@ -412,12 +489,12 @@ mod tests { "https://cluster.example.invalid/" ); assert_eq!( - config.kubernetes.kubeconfig.as_str(), - "/etc/trident-acl-agent/kubeconfig" + config.kubernetes.kubeconfig, + PathBuf::from("/etc/trident-acl-agent/kubeconfig") ); assert_eq!(config.kubernetes.node_name, "node-42"); assert_eq!(config.trident.socket, "unix:///custom/trident.sock"); - assert_eq!(config.orchestration.goal_source, GoalSource::OmahaOnly); + assert_eq!(config.orchestration.mode, Mode::OmahaOnly); assert_eq!( config.orchestration.state_path, PathBuf::from("/var/lib/trident-acl-agent/custom-state.json") @@ -435,34 +512,40 @@ mod tests { Duration::from_secs(45) ); assert_eq!(config.kubernetes.annotation_prefix, "contoso.example.com"); + } - // --- empty value falls back to default, same as unset ------------- - clear_env(); - // SAFETY: see clear_env's doc comment. - unsafe { - env::set_var(ENV_NEBRASKA_APP_ID, ""); - } - let config = AgentConfig::from_env().unwrap(); + #[test] + fn empty_value_falls_back_to_default() { + let config = + AgentConfig::from_vars(vars(&[("TRIDENT_ACL_AGENT_NEBRASKA_APP_ID", "")])).unwrap(); assert_eq!(config.nebraska.app_id, DEFAULT_NEBRASKA_APP_ID); + } - // --- a present-but-malformed URL is a parse error ------------------ - clear_env(); - // SAFETY: see clear_env's doc comment. - unsafe { - env::set_var(ENV_NEBRASKA_ENDPOINT, "not a url"); - } - let err = AgentConfig::from_env().unwrap_err(); - assert!(err.to_string().contains(ENV_NEBRASKA_ENDPOINT), "{err}"); - - // --- a present-but-unknown goal_source is a parse error ------------ - clear_env(); - // SAFETY: see clear_env's doc comment. - unsafe { - env::set_var(ENV_ORCHESTRATION_GOAL_SOURCE, "bogus"); - } - let err = AgentConfig::from_env().unwrap_err(); - assert!(err.to_string().contains("bogus"), "{err}"); + #[test] + fn malformed_url_is_a_parse_error() { + let err = AgentConfig::from_vars(vars(&[( + "TRIDENT_ACL_AGENT_NEBRASKA_ENDPOINT", + "not a url", + )])) + .unwrap_err(); + assert!(format!("{err:#}").contains("not a url"), "{err:#}"); + } - clear_env(); + #[test] + fn malformed_mode_is_a_parse_error() { + let err = + AgentConfig::from_vars(vars(&[("TRIDENT_ACL_AGENT_ORCHESTRATION_MODE", "bogus")])) + .unwrap_err(); + assert!(format!("{err:#}").contains("bogus"), "{err:#}"); + } + + #[test] + fn malformed_duration_is_a_parse_error() { + let err = AgentConfig::from_vars(vars(&[( + "TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT", + "not a duration", + )])) + .unwrap_err(); + assert!(format!("{err:#}").contains("not a duration"), "{err:#}"); } } diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 6fd8a88b0e..8738786843 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -4,7 +4,7 @@ //! Trident (stage/finalize/rollback/commit) through a Kubernetes node //! annotation protocol; a one-shot `omaha-only` mode that calls Trident's //! combined `Update()` RPC once and exits is also available as an explicit -//! opt-out (see `core::config::GoalSource`). +//! opt-out (see `core::config::Mode`). //! //! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) //! goes through the [`core::nebraska`] client module, a self-contained, diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 741b9ab8d0..2dbbbf5a9d 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -5,7 +5,7 @@ use systemd_journal_logger::{connected_to_journal, JournalLog}; use trident_acl_agent::{ annotations::orchestrator::Orchestrator, - core::config::{AgentConfig, GoalSource}, + core::config::{AgentConfig, Mode}, omahaonly::run_omaha_only, }; @@ -70,16 +70,16 @@ async fn main() -> Result<(), Error> { return validate_connection(target, &config).await; } - match config.orchestration.goal_source { + match config.orchestration.mode { // Historical one-shot flow: query Nebraska once, apply an update if // offered, and exit. No Kubernetes/annotation involvement. Not a - // documented/supported deployment option (see config::GoalSource). - GoalSource::OmahaOnly => run_omaha_only(&config).await, + // documented/supported deployment option (see config::Mode). + Mode::OmahaOnly => run_omaha_only(&config).await, // The only supported mode: the annotation-driven reconcile loop // (watches /update-request, drives stage/finalize/rollback/ // commit against tridentd, writes /update-status; prefix // defaults to acl.microsoft.com, overridable via // TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX). - GoalSource::Annotations => Orchestrator::from_config(config).await?.run().await, + Mode::Annotations => Orchestrator::from_config(config).await?.run().await, } } diff --git a/docs/Explanation/Trident-ACL-Agent.md b/docs/Explanation/Trident-ACL-Agent.md index c5ee4c13cd..d92f9d704d 100644 --- a/docs/Explanation/Trident-ACL-Agent.md +++ b/docs/Explanation/Trident-ACL-Agent.md @@ -224,7 +224,7 @@ sets the process's environment before it starts. A variable that is unset, or set to the empty string, falls back to its default. A variable set to a malformed value (a bad URL, a bad duration, an -unrecognized `goal_source`) causes the agent to fail to start with an error +unrecognized `mode`) causes the agent to fail to start with an error naming the offending variable. | Variable | Default | Description | From 91e6d1d0c2b349985f1ce2dff5963ae34aa195f3 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 26 Aug 2026 00:12:15 +0000 Subject: [PATCH 43/54] trident-acl-agent: add module-level docs, drop unused RebootHandle abstraction --- .../trident-acl-agent/src/annotations/mod.rs | 9 ++++ .../src/annotations/orchestrator.rs | 48 +++++++------------ crates/trident-acl-agent/src/core/mod.rs | 6 +++ crates/trident-acl-agent/src/lib.rs | 19 ++------ crates/trident-acl-agent/src/omahaonly/mod.rs | 4 ++ 5 files changed, 40 insertions(+), 46 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/mod.rs b/crates/trident-acl-agent/src/annotations/mod.rs index 964c95470a..ff29fed9b6 100644 --- a/crates/trident-acl-agent/src/annotations/mod.rs +++ b/crates/trident-acl-agent/src/annotations/mod.rs @@ -1,3 +1,12 @@ +//! The default Kubernetes annotation-driven update protocol. +//! +//! - [`protocol`] defines the annotation schema (`UpdateRequest`/ +//! `UpdateStatus`/`StatusCode`). +//! - [`k8s`] is the Kubernetes Node get/watch/patch client. +//! - [`state`] persists in-flight/completed operations across the reboot +//! that finalize triggers. +//! - [`orchestrator`] is the reconcile loop tying them all together. + mod protocol; pub use protocol::*; diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 2b98039e10..22c7bff030 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -83,36 +83,14 @@ impl NebraskaReport { } } -#[derive(Clone, Default)] -pub struct SystemRebooter; - -pub trait RebootHandle: Clone + Send + Sync + 'static { - fn reboot(&self) -> Result<(), Error>; -} - -impl RebootHandle for SystemRebooter { - fn reboot(&self) -> Result<(), Error> { - // Route through the repo's centralized dependency runner so a - // missing systemctl binary or non-zero exit produces the same - // uniform, actionable error type used everywhere else in the - // codebase (see crates/trident/src/reboot.rs for the same pattern). - Dependency::Systemctl - .cmd() - .arg("reboot") - .run_and_check() - .context("failed to issue systemctl reboot") - } -} - -pub struct Orchestrator { +pub struct Orchestrator { config: AgentConfig, k8s: NodeClient, - rebooter: R, state: StateStore, annotation_keys: AnnotationKeys, } -impl Orchestrator { +impl Orchestrator { pub async fn from_config(config: AgentConfig) -> Result { let k8s = NodeClient::new(&config.kubernetes).await?; let annotation_keys = AnnotationKeys::new(&config.kubernetes.annotation_prefix); @@ -120,16 +98,22 @@ impl Orchestrator { state: StateStore::new(config.orchestration.state_path.clone()), config, k8s, - rebooter: SystemRebooter, annotation_keys, }) } -} -impl Orchestrator -where - R: RebootHandle, -{ + /// Issues a real `systemctl reboot`. Routed through the repo's + /// centralized dependency runner so a missing systemctl binary or + /// non-zero exit produces the same uniform, actionable error type used + /// everywhere else in the codebase (see crates/trident/src/reboot.rs for + /// the same pattern). + fn reboot(&self) -> Result<(), Error> { + Dependency::Systemctl + .cmd() + .arg("reboot") + .run_and_check() + .context("failed to issue systemctl reboot") + } pub async fn run(&self) -> Result<(), Error> { if let Err(err) = self.recover_from_trident_state().await { if self.log_and_swallow_node_gone(&err, "recovering persisted state") { @@ -582,7 +566,7 @@ where warn!("failed to record finalize completion in state.json: {err}"); } self.best_effort_publish_terminal(&terminal).await; - match self.rebooter.reboot() { + match self.reboot() { Ok(()) => Ok(LoopControl::ExitForReboot), Err(err) => { self.state.clear_pending_commit()?; @@ -711,7 +695,7 @@ where warn!("failed to record rollback completion in state.json: {err}"); } self.best_effort_publish_terminal(&terminal).await; - match self.rebooter.reboot() { + match self.reboot() { Ok(()) => Ok(LoopControl::ExitForReboot), Err(err) => { self.state.clear_pending_commit()?; diff --git a/crates/trident-acl-agent/src/core/mod.rs b/crates/trident-acl-agent/src/core/mod.rs index 4fd36d5691..05c9e3a515 100644 --- a/crates/trident-acl-agent/src/core/mod.rs +++ b/crates/trident-acl-agent/src/core/mod.rs @@ -1,3 +1,9 @@ +//! Building blocks shared by both the annotation-driven and `omaha-only` +//! modes: env-var config loading ([`config`]), the crate's unified error +//! type ([`error`]), machine-id derivation ([`id`]), current-version +//! detection ([`version`]), the `tridentd` gRPC client ([`trident`]), and +//! the Nebraska/Omaha protocol client ([`nebraska`]). + pub mod config; pub mod error; pub mod id; diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 8738786843..669037be82 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -2,26 +2,17 @@ //! //! trident-acl-agent is Trident's ACL update sidecar. By default it drives //! Trident (stage/finalize/rollback/commit) through a Kubernetes node -//! annotation protocol; a one-shot `omaha-only` mode that calls Trident's -//! combined `Update()` RPC once and exits is also available as an explicit -//! opt-out (see `core::config::Mode`). +//! annotation protocol ([`annotations`]); a one-shot `omaha-only` mode +//! ([`omahaonly`]) that calls Trident's combined `Update()` RPC once and +//! exits is also available as an explicit opt-out (see +//! `core::config::Mode`). Building blocks shared by both modes live in +//! [`core`]. //! //! All Omaha/Nebraska protocol traffic (both `omaha-only` and annotation mode) //! goes through the [`core::nebraska`] client module, a self-contained, //! reusable implementation of the Nebraska/Omaha update protocol. It is usable //! both by this crate's agent binary and by a future Trident ACL Agent that //! orchestrates updates differently. -//! -//! - [`core`]: building blocks shared by both modes (config, errors, -//! machine-id, current-version, the `tridentd` client, the Nebraska -//! client). -//! - [`annotations`]: the default Kubernetes annotation-driven protocol - -//! [`annotations::protocol`] defines the annotation schema, -//! [`annotations::k8s`] is the Kubernetes Node get/watch/patch client, -//! [`annotations::state`] persists in-flight/completed operations across -//! the reboot that finalize triggers, and [`annotations::orchestrator`] -//! is the reconcile loop tying them all together. -//! - [`omahaonly`]: the legacy one-shot Omaha flow. use semver::Version; use url::Url; diff --git a/crates/trident-acl-agent/src/omahaonly/mod.rs b/crates/trident-acl-agent/src/omahaonly/mod.rs index f85f4c5975..4fae03a7de 100644 --- a/crates/trident-acl-agent/src/omahaonly/mod.rs +++ b/crates/trident-acl-agent/src/omahaonly/mod.rs @@ -1,3 +1,7 @@ +//! The legacy one-shot Omaha flow: query the Nebraska/Omaha server once, +//! and if an update is offered, call tridentd's combined `Update()` RPC +//! once and exit. No Kubernetes/annotation involvement. + use anyhow::{anyhow, Context, Error}; use log::{debug, info, warn}; use semver::Version; From a518f73d0118d0f99f474c2cc1e5bc8a1035534e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 26 Aug 2026 00:24:45 +0000 Subject: [PATCH 44/54] notice: update third-party attribution (envy crate) --- NOTICE | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 07901dd517..da8c2f679d 100644 --- a/NOTICE +++ b/NOTICE @@ -4089,6 +4089,33 @@ DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- MIT License (MIT) +Used by: + - envy 0.4.2 + +Copyright (c) 2016-2019 Doug Tangren + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +MIT License (MIT) + Used by: - scopeguard 1.2.0 @@ -6661,7 +6688,6 @@ Used by: - thiserror-impl 2.0.12 - unicode-ident 1.0.14 - unsafe-libyaml 0.2.11 - - zerocopy 0.7.35 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated From 8681a3ee465216405e17f1ca327d98c379aa0d3e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 26 Aug 2026 00:36:23 +0000 Subject: [PATCH 45/54] trident-acl-agent: fix stale doc comment referencing removed map_commit_result --- crates/trident-acl-agent/src/annotations/orchestrator.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 22c7bff030..24f735159a 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -1415,10 +1415,10 @@ fn reconstruct_commit_result_to_status( } } -/// Pure function extracted from `Orchestrator::map_commit_result` so tests -/// can exercise it directly (with a mock-tridentd-driven `Result`) without -/// needing a full `Orchestrator` instance. See `stage_result_to_status` for -/// rationale. +/// Maps a post-reboot commit RPC outcome to the status annotation. Pure +/// function so tests can exercise it directly (with a mock-tridentd-driven +/// `Result`) without needing a full `Orchestrator` instance. See +/// `stage_result_to_status` for rationale. fn commit_result_to_status( pending: &PendingCommit, result: Result, From 56ba5eecf2657184ab262df285c519e26fe45156 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 26 Aug 2026 00:47:24 +0000 Subject: [PATCH 46/54] trident-acl-agent, osutils: fix 3 copilot review comments - re-export core::nebraska at crate root so crate::nebraska intra-doc links and README examples resolve (was silently broken by the earlier core/ module reorg) - FilteredLogger::log() now also honors the wrapped inner loggers own enabled(), instead of only checking its own level/target filter - StateStore::save() creates its temp file with create_new (not create+truncate) so a stale temp file left by a prior crash cannot keep broader permissions than STATE_FILE_MODE; a stale file is removed and recreated rather than reused --- crates/osutils/src/logging.rs | 389 ++++++++++-------- .../src/annotations/state.rs | 61 ++- crates/trident-acl-agent/src/lib.rs | 2 +- 3 files changed, 272 insertions(+), 180 deletions(-) diff --git a/crates/osutils/src/logging.rs b/crates/osutils/src/logging.rs index 103ceb0b4b..9ac518122a 100644 --- a/crates/osutils/src/logging.rs +++ b/crates/osutils/src/logging.rs @@ -1,172 +1,217 @@ -use log::{LevelFilter, Log, Metadata, Record}; - -/// A `log::Log` wrapper that applies a separate level filter to a configurable -/// list of noisy "network" targets (e.g. HTTP/gRPC/watch client crates) while -/// leaving every other target at a main verbosity level. -/// -/// This is useful for binaries that talk to chatty client stacks (hyper, h2, -/// tonic, kube, reqwest, ...) whose per-frame/per-request logging would -/// otherwise drown out the binary's own orchestration logs at the same -/// verbosity. -pub struct FilteredLogger { - inner: L, - verbosity: LevelFilter, - network_verbosity: LevelFilter, - network_targets: &'static [&'static str], -} - -impl FilteredLogger { - /// Builds a new [`FilteredLogger`] wrapping `inner`. Targets in - /// `network_targets` (matched by prefix, e.g. `"hyper"` matches - /// `hyper::client`) are filtered at `network_verbosity`; every other - /// target is filtered at `verbosity`. - pub fn new( - inner: L, - verbosity: LevelFilter, - network_verbosity: LevelFilter, - network_targets: &'static [&'static str], - ) -> Self { - Self { - inner, - verbosity, - network_verbosity, - network_targets, - } - } - - /// The maximum of `verbosity` and `network_verbosity`, suitable for - /// passing to [`log::set_max_level`] so the log facade doesn't drop - /// records before they reach this filter. - pub fn max_level(&self) -> LevelFilter { - self.verbosity.max(self.network_verbosity) - } - - fn is_network_target(&self, target: &str) -> bool { - self.network_targets.iter().any(|prefix| { - target - .strip_prefix(prefix) - .is_some_and(|rest| rest.is_empty() || rest.starts_with("::")) - }) - } -} - -impl Log for FilteredLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - let level = if self.is_network_target(metadata.target()) { - self.network_verbosity - } else { - self.verbosity - }; - metadata.level() <= level - } - - fn log(&self, record: &Record) { - if self.enabled(record.metadata()) { - self.inner.log(record); - } - } - - fn flush(&self) { - self.inner.flush(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use std::sync::{Arc, Mutex}; - - use log::{Level, Metadata, Record}; - - #[derive(Clone)] - struct TestLogger { - logged: Arc>>, - } - - impl TestLogger { - fn new() -> Self { - Self { - logged: Arc::new(Mutex::new(Vec::new())), - } - } - } - - impl Log for TestLogger { - fn enabled(&self, _metadata: &Metadata) -> bool { - true - } - - fn log(&self, record: &Record) { - self.logged - .lock() - .unwrap() - .push(format!("{} {}", record.target(), record.args())); - } - - fn flush(&self) {} - } - - const NETWORK_TARGETS: &[&str] = &["hyper", "kube"]; - - #[test] - fn test_network_target_uses_network_verbosity() { - let inner = TestLogger::new(); - let logged = inner.logged.clone(); - let logger = FilteredLogger::new( - inner, - LevelFilter::Debug, - LevelFilter::Warn, - NETWORK_TARGETS, - ); - - assert!(logger.enabled( - &Metadata::builder() - .level(Level::Warn) - .target("hyper::client") - .build() - )); - assert!(!logger.enabled( - &Metadata::builder() - .level(Level::Debug) - .target("hyper::client") - .build() - )); - drop(logged); - } - - #[test] - fn test_non_network_target_uses_verbosity() { - let inner = TestLogger::new(); - let logger = FilteredLogger::new( - inner, - LevelFilter::Debug, - LevelFilter::Warn, - NETWORK_TARGETS, - ); - - assert!(logger.enabled( - &Metadata::builder() - .level(Level::Debug) - .target("trident_acl_agent") - .build() - )); - assert!(!logger.enabled( - &Metadata::builder() - .level(Level::Trace) - .target("trident_acl_agent") - .build() - )); - } - - #[test] - fn test_max_level_is_max_of_both() { - let logger = FilteredLogger::new( - TestLogger::new(), - LevelFilter::Warn, - LevelFilter::Debug, - NETWORK_TARGETS, - ); - assert_eq!(logger.max_level(), LevelFilter::Debug); - } -} +use log::{LevelFilter, Log, Metadata, Record}; + +/// A `log::Log` wrapper that applies a separate level filter to a configurable +/// list of noisy "network" targets (e.g. HTTP/gRPC/watch client crates) while +/// leaving every other target at a main verbosity level. +/// +/// This is useful for binaries that talk to chatty client stacks (hyper, h2, +/// tonic, kube, reqwest, ...) whose per-frame/per-request logging would +/// otherwise drown out the binary's own orchestration logs at the same +/// verbosity. +pub struct FilteredLogger { + inner: L, + verbosity: LevelFilter, + network_verbosity: LevelFilter, + network_targets: &'static [&'static str], +} + +impl FilteredLogger { + /// Builds a new [`FilteredLogger`] wrapping `inner`. Targets in + /// `network_targets` (matched by prefix, e.g. `"hyper"` matches + /// `hyper::client`) are filtered at `network_verbosity`; every other + /// target is filtered at `verbosity`. + pub fn new( + inner: L, + verbosity: LevelFilter, + network_verbosity: LevelFilter, + network_targets: &'static [&'static str], + ) -> Self { + Self { + inner, + verbosity, + network_verbosity, + network_targets, + } + } + + /// The maximum of `verbosity` and `network_verbosity`, suitable for + /// passing to [`log::set_max_level`] so the log facade doesn't drop + /// records before they reach this filter. + pub fn max_level(&self) -> LevelFilter { + self.verbosity.max(self.network_verbosity) + } + + fn is_network_target(&self, target: &str) -> bool { + self.network_targets.iter().any(|prefix| { + target + .strip_prefix(prefix) + .is_some_and(|rest| rest.is_empty() || rest.starts_with("::")) + }) + } +} + +impl Log for FilteredLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + let level = if self.is_network_target(metadata.target()) { + self.network_verbosity + } else { + self.verbosity + }; + metadata.level() <= level && self.inner.enabled(metadata) + } + + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + self.inner.log(record); + } + } + + fn flush(&self) { + self.inner.flush(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::{Arc, Mutex}; + + use log::{Level, Metadata, Record}; + + #[derive(Clone)] + struct TestLogger { + logged: Arc>>, + } + + impl TestLogger { + fn new() -> Self { + Self { + logged: Arc::new(Mutex::new(Vec::new())), + } + } + } + + impl Log for TestLogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + self.logged + .lock() + .unwrap() + .push(format!("{} {}", record.target(), record.args())); + } + + fn flush(&self) {} + } + + /// A logger whose `enabled()` always returns `false`, to verify + /// `FilteredLogger` honors the inner logger's own filter rather than + /// bypassing it once its own level/target filter passes. + struct AlwaysDisabledLogger; + + impl Log for AlwaysDisabledLogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + false + } + + fn log(&self, _record: &Record) { + panic!("log() must not be called when enabled() is false"); + } + + fn flush(&self) {} + } + + const NETWORK_TARGETS: &[&str] = &["hyper", "kube"]; + + #[test] + fn test_network_target_uses_network_verbosity() { + let inner = TestLogger::new(); + let logged = inner.logged.clone(); + let logger = FilteredLogger::new( + inner, + LevelFilter::Debug, + LevelFilter::Warn, + NETWORK_TARGETS, + ); + + assert!(logger.enabled( + &Metadata::builder() + .level(Level::Warn) + .target("hyper::client") + .build() + )); + assert!(!logger.enabled( + &Metadata::builder() + .level(Level::Debug) + .target("hyper::client") + .build() + )); + drop(logged); + } + + #[test] + fn test_non_network_target_uses_verbosity() { + let inner = TestLogger::new(); + let logger = FilteredLogger::new( + inner, + LevelFilter::Debug, + LevelFilter::Warn, + NETWORK_TARGETS, + ); + + assert!(logger.enabled( + &Metadata::builder() + .level(Level::Debug) + .target("trident_acl_agent") + .build() + )); + assert!(!logger.enabled( + &Metadata::builder() + .level(Level::Trace) + .target("trident_acl_agent") + .build() + )); + } + + #[test] + fn test_inner_enabled_is_respected() { + let logger = FilteredLogger::new( + AlwaysDisabledLogger, + LevelFilter::Debug, + LevelFilter::Debug, + NETWORK_TARGETS, + ); + + let metadata = Metadata::builder() + .level(Level::Error) + .target("trident_acl_agent") + .build(); + + // FilteredLogger's own filter passes (Error <= Debug), but the inner + // logger's enabled() returns false, so the combined result must too. + assert!(!logger.enabled(&metadata)); + + // log() must therefore be a no-op (AlwaysDisabledLogger panics if + // its log() is ever reached). + logger.log( + &Record::builder() + .metadata(metadata) + .args(format_args!("should not be logged")) + .build(), + ); + } + + #[test] + fn test_max_level_is_max_of_both() { + let logger = FilteredLogger::new( + TestLogger::new(), + LevelFilter::Warn, + LevelFilter::Debug, + NETWORK_TARGETS, + ); + assert_eq!(logger.max_level(), LevelFilter::Debug); + } +} diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index e5b6e1ac0f..f652a8571a 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -111,13 +111,31 @@ impl StateStore { // with owner-only permissions (0600) rather than relying on the // process umask. { - let mut file = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(STATE_FILE_MODE) - .open(&temp_path) - .with_context(|| format!("failed to create {}", temp_path.display()))?; + // create_new (not create+truncate) so the file is always newly + // created with STATE_FILE_MODE: opening/truncating a stale temp + // file left behind by a prior crash would silently keep that + // file's existing (possibly broader) permissions, since .mode() + // only applies on creation. A leftover temp file (same + // process::id() reused across reboots) is removed and retried + // once, since it can only be this process's own abandoned + // write, never another process's live file. + let mut open_opts = fs::OpenOptions::new(); + open_opts.write(true).create_new(true).mode(STATE_FILE_MODE); + let mut file = match open_opts.open(&temp_path) { + Ok(file) => file, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + fs::remove_file(&temp_path).with_context(|| { + format!("failed to remove stale {}", temp_path.display()) + })?; + open_opts + .open(&temp_path) + .with_context(|| format!("failed to create {}", temp_path.display()))? + } + Err(err) => { + return Err(err) + .with_context(|| format!("failed to create {}", temp_path.display())) + } + }; file.write_all(serde_json::to_string_pretty(state)?.as_bytes()) .with_context(|| format!("failed to write {}", temp_path.display()))?; file.sync_all() @@ -380,6 +398,35 @@ mod tests { assert!(metadata_before.modified().is_ok()); } + #[test] + fn save_recreates_stale_world_readable_temp_file_with_owner_only_mode() { + use std::os::unix::fs::PermissionsExt; + + let (_dir, store) = store(); + + // Simulate a temp file left behind by a prior crash, deliberately + // world-readable, at the exact path save() will compute for this + // process (same process::id()-derived name). + let temp_path = store.path().parent().unwrap().join(format!( + "{}.tmp-{}", + STATE_FILE_NAME, + process::id() + )); + fs::write(&temp_path, b"stale leftover data").expect("failed to write stale temp file"); + fs::set_permissions(&temp_path, fs::Permissions::from_mode(0o644)) + .expect("failed to widen stale temp file permissions"); + + store + .save(&PersistentState::default()) + .expect("save should recover from a stale temp file"); + + // The stale file must not still exist post-rename (it becomes + // state.json), and the final state.json must carry owner-only mode, + // proving the stale file's permissions were never inherited. + let metadata = fs::metadata(store.path()).expect("state file should exist"); + assert_eq!(metadata.permissions().mode() & 0o777, STATE_FILE_MODE); + } + #[test] fn deserialize_rejects_unknown_top_level_fields() { let err = serde_json::from_str::( diff --git a/crates/trident-acl-agent/src/lib.rs b/crates/trident-acl-agent/src/lib.rs index 669037be82..1451fd6772 100644 --- a/crates/trident-acl-agent/src/lib.rs +++ b/crates/trident-acl-agent/src/lib.rs @@ -23,7 +23,7 @@ use crate::core::{ version::FALLBACK_ALWAYS_VERSION, }; -pub use crate::core::id::IdSource; +pub use crate::core::{id::IdSource, nebraska}; pub mod annotations; pub mod core; From 1d7ca394b2f32779b17bae32498d201bf219e44b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 26 Aug 2026 21:49:55 +0000 Subject: [PATCH 47/54] trident-acl-agent: resolve logging module conflict with PR #751 PR #751 (merged into main) moved MultiLogger/LogFilter from trident into a new osutils::logging/ directory module, while this branch had independently added its own osutils::logging.rs (single file) containing FilteredLogger - same module path, different shapes, causing an E0761 module-ambiguity build error after rebasing onto post-#751 main. Resolved by treating #751's shared LogFilter as the canonical implementation (it already supports everything FilteredLogger did, via its existing with_global_filter builder, matching the exact pattern trident/src/main.rs already uses for its own noisy-target suppression) and porting trident-acl-agent's two call sites onto it, rather than keeping a parallel duplicate wrapper type: - Deleted crates/osutils/src/logging.rs (FilteredLogger). - crates/trident-acl-agent/src/main.rs: added a small build_logger() helper that chains LogFilter::with_global_filter() over NETWORK_LOG_TARGETS, replacing FilteredLogger::new(). max_level is now computed inline (verbosity.max(network_verbosity)) since LogFilter has no equivalent to FilteredLogger::max_level(). Verified: cargo build/test/clippy/fmt clean for osutils, trident-acl-agent, and trident (386+148+158 tests pass). --- crates/osutils/src/logging.rs | 217 --------------------------- crates/trident-acl-agent/src/main.rs | 41 +++-- 2 files changed, 25 insertions(+), 233 deletions(-) delete mode 100644 crates/osutils/src/logging.rs diff --git a/crates/osutils/src/logging.rs b/crates/osutils/src/logging.rs deleted file mode 100644 index 9ac518122a..0000000000 --- a/crates/osutils/src/logging.rs +++ /dev/null @@ -1,217 +0,0 @@ -use log::{LevelFilter, Log, Metadata, Record}; - -/// A `log::Log` wrapper that applies a separate level filter to a configurable -/// list of noisy "network" targets (e.g. HTTP/gRPC/watch client crates) while -/// leaving every other target at a main verbosity level. -/// -/// This is useful for binaries that talk to chatty client stacks (hyper, h2, -/// tonic, kube, reqwest, ...) whose per-frame/per-request logging would -/// otherwise drown out the binary's own orchestration logs at the same -/// verbosity. -pub struct FilteredLogger { - inner: L, - verbosity: LevelFilter, - network_verbosity: LevelFilter, - network_targets: &'static [&'static str], -} - -impl FilteredLogger { - /// Builds a new [`FilteredLogger`] wrapping `inner`. Targets in - /// `network_targets` (matched by prefix, e.g. `"hyper"` matches - /// `hyper::client`) are filtered at `network_verbosity`; every other - /// target is filtered at `verbosity`. - pub fn new( - inner: L, - verbosity: LevelFilter, - network_verbosity: LevelFilter, - network_targets: &'static [&'static str], - ) -> Self { - Self { - inner, - verbosity, - network_verbosity, - network_targets, - } - } - - /// The maximum of `verbosity` and `network_verbosity`, suitable for - /// passing to [`log::set_max_level`] so the log facade doesn't drop - /// records before they reach this filter. - pub fn max_level(&self) -> LevelFilter { - self.verbosity.max(self.network_verbosity) - } - - fn is_network_target(&self, target: &str) -> bool { - self.network_targets.iter().any(|prefix| { - target - .strip_prefix(prefix) - .is_some_and(|rest| rest.is_empty() || rest.starts_with("::")) - }) - } -} - -impl Log for FilteredLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - let level = if self.is_network_target(metadata.target()) { - self.network_verbosity - } else { - self.verbosity - }; - metadata.level() <= level && self.inner.enabled(metadata) - } - - fn log(&self, record: &Record) { - if self.enabled(record.metadata()) { - self.inner.log(record); - } - } - - fn flush(&self) { - self.inner.flush(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use std::sync::{Arc, Mutex}; - - use log::{Level, Metadata, Record}; - - #[derive(Clone)] - struct TestLogger { - logged: Arc>>, - } - - impl TestLogger { - fn new() -> Self { - Self { - logged: Arc::new(Mutex::new(Vec::new())), - } - } - } - - impl Log for TestLogger { - fn enabled(&self, _metadata: &Metadata) -> bool { - true - } - - fn log(&self, record: &Record) { - self.logged - .lock() - .unwrap() - .push(format!("{} {}", record.target(), record.args())); - } - - fn flush(&self) {} - } - - /// A logger whose `enabled()` always returns `false`, to verify - /// `FilteredLogger` honors the inner logger's own filter rather than - /// bypassing it once its own level/target filter passes. - struct AlwaysDisabledLogger; - - impl Log for AlwaysDisabledLogger { - fn enabled(&self, _metadata: &Metadata) -> bool { - false - } - - fn log(&self, _record: &Record) { - panic!("log() must not be called when enabled() is false"); - } - - fn flush(&self) {} - } - - const NETWORK_TARGETS: &[&str] = &["hyper", "kube"]; - - #[test] - fn test_network_target_uses_network_verbosity() { - let inner = TestLogger::new(); - let logged = inner.logged.clone(); - let logger = FilteredLogger::new( - inner, - LevelFilter::Debug, - LevelFilter::Warn, - NETWORK_TARGETS, - ); - - assert!(logger.enabled( - &Metadata::builder() - .level(Level::Warn) - .target("hyper::client") - .build() - )); - assert!(!logger.enabled( - &Metadata::builder() - .level(Level::Debug) - .target("hyper::client") - .build() - )); - drop(logged); - } - - #[test] - fn test_non_network_target_uses_verbosity() { - let inner = TestLogger::new(); - let logger = FilteredLogger::new( - inner, - LevelFilter::Debug, - LevelFilter::Warn, - NETWORK_TARGETS, - ); - - assert!(logger.enabled( - &Metadata::builder() - .level(Level::Debug) - .target("trident_acl_agent") - .build() - )); - assert!(!logger.enabled( - &Metadata::builder() - .level(Level::Trace) - .target("trident_acl_agent") - .build() - )); - } - - #[test] - fn test_inner_enabled_is_respected() { - let logger = FilteredLogger::new( - AlwaysDisabledLogger, - LevelFilter::Debug, - LevelFilter::Debug, - NETWORK_TARGETS, - ); - - let metadata = Metadata::builder() - .level(Level::Error) - .target("trident_acl_agent") - .build(); - - // FilteredLogger's own filter passes (Error <= Debug), but the inner - // logger's enabled() returns false, so the combined result must too. - assert!(!logger.enabled(&metadata)); - - // log() must therefore be a no-op (AlwaysDisabledLogger panics if - // its log() is ever reached). - logger.log( - &Record::builder() - .metadata(metadata) - .args(format_args!("should not be logged")) - .build(), - ); - } - - #[test] - fn test_max_level_is_max_of_both() { - let logger = FilteredLogger::new( - TestLogger::new(), - LevelFilter::Warn, - LevelFilter::Debug, - NETWORK_TARGETS, - ); - assert_eq!(logger.max_level(), LevelFilter::Debug); - } -} diff --git a/crates/trident-acl-agent/src/main.rs b/crates/trident-acl-agent/src/main.rs index 2dbbbf5a9d..f711ee478b 100644 --- a/crates/trident-acl-agent/src/main.rs +++ b/crates/trident-acl-agent/src/main.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Error}; use clap::Parser; -use osutils::logging::FilteredLogger; +use osutils::logging::filter::LogFilter; use systemd_journal_logger::{connected_to_journal, JournalLog}; use trident_acl_agent::{ @@ -32,33 +32,42 @@ const NETWORK_LOG_TARGETS: &[&str] = &[ "kube_runtime", ]; +/// Wraps `inner` in a [`LogFilter`] that caps overall verbosity at +/// `args.verbosity`, then further caps every [`NETWORK_LOG_TARGETS`] prefix +/// down to `args.network_verbosity` (matching osutils::logging's shared +/// `LogFilter`/`MultiLogger` pattern already used by `trident`'s own +/// `main.rs`, rather than a bespoke wrapper type). +fn build_logger(inner: L, args: &Args) -> LogFilter { + NETWORK_LOG_TARGETS.iter().fold( + LogFilter::new(inner).with_max_level(args.verbosity), + |logger, target| logger.with_global_filter(*target, args.network_verbosity), + ) +} + #[tokio::main] async fn main() -> Result<(), Error> { let args = Args::parse(); + // LogFilter has no single accessor for "the loosest level anything could + // be logged at" (unlike the old FilteredLogger::max_level()), so compute + // it the same way FilteredLogger did: the looser of the two configured + // verbosities, so the `log` facade doesn't drop records before this + // filter gets a chance to apply the per-target ceiling. + let max_level = args.verbosity.max(args.network_verbosity); + if let Some(Ok(journal_logger)) = connected_to_journal().then(JournalLog::new) { - let logger = FilteredLogger::new( - journal_logger, - args.verbosity, - args.network_verbosity, - NETWORK_LOG_TARGETS, - ); - log::set_max_level(logger.max_level()); + let logger = build_logger(journal_logger, &args); + log::set_max_level(max_level); log::set_boxed_logger(Box::new(logger)) .map_err(Error::new) .context("failed to install systemd journal logger")?; } else { let inner = env_logger::builder() .format_timestamp(None) - .filter_level(args.verbosity.max(args.network_verbosity)) + .filter_level(max_level) .build(); - let logger = FilteredLogger::new( - inner, - args.verbosity, - args.network_verbosity, - NETWORK_LOG_TARGETS, - ); - log::set_max_level(logger.max_level()); + let logger = build_logger(inner, &args); + log::set_max_level(max_level); log::set_boxed_logger(Box::new(logger)) .map_err(Error::new) .context("failed to install env logger")?; From 76417dded859c22a2de3e9a211f1d06abea49b71 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 18:13:04 +0000 Subject: [PATCH 48/54] trident-acl-agent: fix 8 soundness issues flagged by frhuelsz review Addresses PR 730 review comments on annotations/orchestrator.rs: - state.rs: add StateStore::update() atomic load->mutate->save combinator and remember_completed_and_clear_pending(), so a completed record and a pending-commit clear happen in one write instead of two separate saves with a crash window between them. Used by every post-reboot completion path (resume_pending_commit success, reboot-failure branches, and the connect-error degraded path). - recover_from_trident_state: reorder so a pending post-reboot commit is resumed before the first Kubernetes call, not after - commit() is a purely local tridentd call and is exactly the time-sensitive step a k8s outage must not block. Return type changed to Result so a request reconstructed via the new function below can also trigger a real reboot during startup recovery, not just at request time. - New reconstruct_without_pending_record(): when state.json has no pendingCommit at all for an in-flight finalize/rollback (missing entirely, or lost across the reboot), compare current_active_version() against the request's target before ever calling commit() - which is state-changing and can silently report Success for a no-op or destroy an armed-but-unbooted update if called as an unconditional probe. active == target hands off to the existing reconstruction path; active != target runs the request fresh instead of guessing further. Distinguishing never rebooted from rebooted rebooted" from "rebooted, firmware fell back would need Trident to expose boot history over gRPC, which it does not today - documented as a known remaining gap. - reconcile_node: a retried request with the same operationId as an outstanding pendingCommit now resumes that commit instead of falling through to handle_finalize/handle_rollback, which would re-drive UpdateFinalize/RollbackFinalize against a boot that may already be armed or in flight. - resume_pending_commit: re-issue the reboot when the boot marker still matches (agent restarted before the original reboot ever took effect), instead of only logging and waiting forever. - handle_finalize: require reboot_status == RebootRequired before arming pendingCommit and rebooting, mirroring handle_rollback's existing servicing_kind check - previously any Ok(_) from update_finalize was treated as boot armed, so a no-op finalize could reboot a node and report a false-positive Success. - handle_stage: on Nebraska CheckOutcome::UpdateInProgress (a stage interrupted mid-download by a crash/reboot), send a compensating Failed event to clear the wedge before reporting failure, instead of silently leaving the instance permanently stuck. - commit_result_to_status / reconstruct_commit_result_to_status: treat servicing_kind == NoneRequired as nothing committed rather than Success. - map_trident_failure renamed to map_trident_commit_failure and restricted to the post-reboot commit-status builders; finalize_failure_status, rollback_stage_failure_status, and rollback_finalize_failure_status now always report OperationFailed, since TargetBootFailed is contractually reserved for the commit status. - orchestrator.rs:116 nit: added the missing blank line between reboot() and run(). Verified: cargo build/test/clippy/fmt clean across the workspace (901 tests pass, including 2 new tests covering the NoneRequired servicing_kind fix and 2 existing tests updated to assert the corrected OperationFailed-not-TargetBootFailed pre-reboot behavior). --- .../src/annotations/orchestrator.rs | 404 +++++++++++++++--- .../src/annotations/state.rs | 86 +++- 2 files changed, 425 insertions(+), 65 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 24f735159a..8bcffb79b6 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -114,12 +114,17 @@ impl Orchestrator { .run_and_check() .context("failed to issue systemctl reboot") } + pub async fn run(&self) -> Result<(), Error> { - if let Err(err) = self.recover_from_trident_state().await { - if self.log_and_swallow_node_gone(&err, "recovering persisted state") { - return Ok(()); + match self.recover_from_trident_state().await { + Ok(LoopControl::Continue) => {} + Ok(LoopControl::ExitForReboot) => return Ok(()), + Err(err) => { + if self.log_and_swallow_node_gone(&err, "recovering persisted state") { + return Ok(()); + } + return Err(err); } - return Err(err); } let mut stream = self .k8s @@ -140,15 +145,27 @@ impl Orchestrator { Ok(()) } - async fn recover_from_trident_state(&self) -> Result<(), Error> { - let node = self.k8s.get_node(&self.config.kubernetes.node_name).await?; - let snapshot = Snapshot::from_node(&node, &self.annotation_keys); + /// Startup recovery. Order matters: a pending post-reboot `commit()` is + /// resumed *before* the first Kubernetes call, since `commit()` is a + /// purely local `tridentd` gRPC call and is exactly the time-sensitive + /// step a k8s outage must not block (a node left uncommitted risks a + /// second reboot silently falling back to the old slot). Only + /// *publishing* the resulting status annotation needs k8s, and that + /// publish is already best-effort/retried (see + /// `best_effort_publish_terminal`), so deferring the k8s read this far + /// costs nothing when k8s is healthy and avoids a crash-loop when it + /// isn't. + async fn recover_from_trident_state(&self) -> Result { let persisted = self.state.load()?; if let Some(pending) = persisted.pending_commit.clone() { - return self.resume_pending_commit(pending).await; + self.resume_pending_commit(pending).await?; + return Ok(LoopControl::Continue); } + let node = self.k8s.get_node(&self.config.kubernetes.node_name).await?; + let snapshot = Snapshot::from_node(&node, &self.annotation_keys); + if let Some(request) = snapshot.request.clone() { if let Some(entry) = persisted.completed.get(&request.operation_id) { if let Some(commit) = entry.commit.clone() { @@ -168,21 +185,15 @@ impl Orchestrator { if !matches { self.publish_status(&operation).await?; } - return Ok(()); + return Ok(LoopControl::Continue); } } } if let Some(request) = snapshot.request { - if matches!( - request.operation, - RequestedOperation::Finalize | RequestedOperation::Rollback - ) { - let status = self.reconstruct_without_state(&request, None, None).await; - self.record_and_publish(status).await?; - } + return self.reconstruct_without_pending_record(&request).await; } - Ok(()) + Ok(LoopControl::Continue) } async fn reconcile_node(&self, node: &Node) -> Result { @@ -265,6 +276,18 @@ impl Orchestrator { self.record_and_publish(status).await?; return Ok(LoopControl::Continue); } + // Same operationId as the outstanding pendingCommit: this is a + // retry/re-issue of the operation already armed and waiting on + // (or ready to resume) its post-reboot commit, not a new + // finalize/rollback to drive from scratch. Falling through to + // handle_finalize/handle_rollback below would re-run + // UpdateFinalize/RollbackFinalize against a boot that may + // already be armed or in flight, and on failure would + // clear_pending_commit and discard a boot the firmware still + // has queued. Resume it the same way startup recovery does + // instead. + self.resume_pending_commit(pending.clone()).await?; + return Ok(LoopControl::Continue); } match request.operation { @@ -373,15 +396,18 @@ impl Orchestrator { Version::parse(FALLBACK_ALWAYS_VERSION) .expect("invariant: FALLBACK_ALWAYS_VERSION is valid semver") }); - let outcome = task::spawn_blocking(move || { - let client = NebraskaClient::new(endpoint, app_id, track, machine_id); - client.check_for_update(¤t_version) + let outcome = task::spawn_blocking({ + let current_version = current_version.clone(); + move || { + let client = NebraskaClient::new(endpoint, app_id, track, machine_id); + client.check_for_update(¤t_version) + } }) .await .context("Nebraska query task panicked")? .context("Nebraska query failed")?; let offered = match outcome { - CheckOutcome::UpToDate | CheckOutcome::UpdateInProgress => { + CheckOutcome::UpToDate => { let status = UpdateStatus::new( &request, Operation::Stage, @@ -396,6 +422,44 @@ impl Orchestrator { self.record_and_publish(status).await?; return Ok(()); } + CheckOutcome::UpdateInProgress => { + // A prior stage attempt reported DownloadStarted to + // Nebraska (below) but never followed up with a terminal + // event - e.g. the agent was killed, or the node rebooted, + // mid-download, before update_stage returned. Nebraska's + // update_in_progress flag for this instance never clears + // on its own: its documented self-heal only fires once the + // instance checks in *at the new version*, which can't + // happen because the update never actually installed. Left + // alone, every later stage attempt would hit this same + // branch forever with no way out. Send the compensating + // Failed event (documented at nebraska::client as clearing + // update_in_progress and re-arming the instance) before + // reporting failure, so a subsequent stage (new + // operationId) has a real chance to succeed instead of + // being permanently wedged. + self.report_nebraska_event( + &request, + NebraskaReport::Failed { + previous: current_version.clone(), + current: current_version.clone(), + }, + ) + .await; + let status = UpdateStatus::new( + &request, + Operation::Stage, + request.operation_id.clone(), + StatusCode::OperationFailed, + "Nebraska reported an update already in progress for this instance; cleared the stuck in-progress state so a retried stage can succeed", + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + return Ok(()); + } CheckOutcome::UpdateAvailable(offer) => offer, }; if request.target_version.as_deref() != Some(offered.version.to_string().as_str()) { @@ -545,7 +609,7 @@ impl Orchestrator { .await; } match result { - Ok(_) => { + Ok(response) if response.reboot_status == RebootStatus::RebootRequired => { let boot_marker = current_boot_marker()?; self.state.set_pending_commit(PendingCommit { request: request.clone(), @@ -569,7 +633,6 @@ impl Orchestrator { match self.reboot() { Ok(()) => Ok(LoopControl::ExitForReboot), Err(err) => { - self.state.clear_pending_commit()?; if let Some(ref v) = current_ver { self.report_nebraska_event( &request, @@ -591,11 +654,36 @@ impl Orchestrator { started, Some(Utc::now()), ); - self.record_and_publish(status).await?; + self.state + .remember_completed_and_clear_pending(status.clone())?; + self.best_effort_publish_terminal(&status).await; Ok(LoopControl::Continue) } } } + Ok(_) => { + // update_finalize returned success but did not report a + // reboot as required - nothing was actually armed (e.g. + // nothing staged to finalize; mirrors handle_rollback's + // ManualRollbackAb check below). Treating any Ok(_) as + // "boot armed" here previously meant only the agent-local + // NotStaged cache guard above stood between a no-op + // finalize and a real reboot + a false-positive Success - + // Trident's own response is the authoritative signal now. + let status = UpdateStatus::new( + &request, + Operation::Finalize, + request.operation_id.clone(), + StatusCode::NotStaged, + "finalize completed without arming a reboot (nothing to finalize)", + from_version, + to_version, + started, + Some(Utc::now()), + ); + self.record_and_publish(status).await?; + Ok(LoopControl::Continue) + } Err(err) => { self.state.clear_pending_commit()?; let status = @@ -698,7 +786,6 @@ impl Orchestrator { match self.reboot() { Ok(()) => Ok(LoopControl::ExitForReboot), Err(err) => { - self.state.clear_pending_commit()?; let status = UpdateStatus::new( &request, Operation::Rollback, @@ -710,7 +797,9 @@ impl Orchestrator { started, Some(Utc::now()), ); - self.record_and_publish(status).await?; + self.state + .remember_completed_and_clear_pending(status.clone())?; + self.best_effort_publish_terminal(&status).await; Ok(LoopControl::Continue) } } @@ -728,10 +817,35 @@ impl Orchestrator { async fn resume_pending_commit(&self, pending: PendingCommit) -> Result<(), Error> { let current_boot = current_boot_marker()?; if current_boot == pending.boot_marker { + // No reboot has happened since finalize/rollback armed this + // boot - the agent restarted (crash, watchdog, crash-loop) + // without the reboot ever taking effect. Re-issue it instead of + // just waiting: previously this branch only logged and + // returned, so a reboot inhibited/delayed past the original + // process exit left the node armed-but-never-rebooting + // forever, showing `finalize: Success` with no commit until an + // external watchdog eventually wiped it. info!( - "pending commit {} is still waiting for the reboot to happen", + "pending commit {} is still waiting for the reboot to happen; re-issuing reboot", pending.operation_id ); + if let Err(err) = self.reboot() { + let now = Utc::now(); + let status = UpdateStatus::new( + &pending.request, + pending.operation, + pending.operation_id.clone(), + StatusCode::AgentInternalError, + format!("armed update is waiting for reboot, but re-issuing it failed: {err}"), + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + Some(now), + ); + self.state + .remember_completed_and_clear_pending(status.clone())?; + self.best_effort_publish_terminal(&status).await; + } return Ok(()); } @@ -745,8 +859,9 @@ impl Orchestrator { Some(err.to_string()), ) .await; - self.state.clear_pending_commit()?; - self.record_and_publish(status).await?; + self.state + .remember_completed_and_clear_pending(status.clone())?; + self.best_effort_publish_terminal(&status).await; return Ok(()); } }; @@ -778,25 +893,31 @@ impl Orchestrator { } } let status = commit_result_to_status(&pending, result); - self.state.clear_pending_commit()?; - self.record_and_publish(status).await + self.state + .remember_completed_and_clear_pending(status.clone())?; + self.best_effort_publish_terminal(&status).await; + Ok(()) } + /// Reconstructs the post-reboot outcome for `request` when `state.json` + /// has no `pendingCommit` for it but a reboot is already known (by the + /// caller) to have happened - either because `resume_pending_commit`'s + /// boot-marker check already confirmed it (its `connect()` failure + /// branch below), or because [`reconstruct_without_pending_record`] + /// independently confirmed the swap via `current_active_version()` + /// before ever calling this. In that situation it's safe to call + /// `commit()` unconditionally: tridentd's own (ServicingKind/ + /// RebootStatus/Result) response distinguishes "already committed" + /// from "target armed but firmware fell back" reliably. Do not call + /// this when a reboot has *not* been confirmed - see + /// [`reconstruct_without_pending_record`] for that (genuinely + /// ambiguous) case, which must never call `commit()` speculatively. async fn reconstruct_without_state( &self, request: &UpdateRequest, from_version: Option, connect_error: Option, ) -> UpdateStatus { - // state.json did not survive the reboot (or was never written, e.g. - // the agent crashed before persisting pendingCommit). Reconstruct - // the answer by - // calling commit() unconditionally rather than guessing from labels - // or the target version alone - tridentd's commit() is self-checking - // and its own (ServicingKind/RebootStatus/Result) response already - // distinguishes "swap happened, run commit" from "reboot hasn't - // happened yet" from "target armed but firmware fell back" far more - // reliably than a bare version-string comparison could. if let Some(status) = reconstruct_precheck_status(request, from_version.clone(), connect_error.as_deref()) { @@ -843,6 +964,85 @@ impl Orchestrator { reconstruct_commit_result_to_status(request, from_version, started, result) } + /// Reconstructs whether a reboot even happened for `request` when + /// `state.json` carries no `pendingCommit` at all for it (missing + /// entirely, or lost across the reboot) - see design doc 2.3's + /// degraded-recovery path. This is the genuinely ambiguous case: + /// unlike `resume_pending_commit`'s boot-marker check, there is no + /// local record proving a reboot occurred, so `commit()` - which is + /// state-changing and can discard an armed-but-unbooted update, or + /// silently report `Success` for a no-op - must never be called + /// speculatively here. + /// + /// Comparing the node's currently-running version against the + /// request's target resolves the common cases without guessing: + /// - **active == target**: the swap already happened (the active + /// version only changes via the post-finalize swap), so it's safe to + /// hand off to [`reconstruct_without_state`] to validate/promote via + /// `commit()`. + /// - **active != target**: not proven that a reboot ever happened, so + /// the request is run fresh instead of guessed at further - always + /// safe (never touches `commit()` on an unconfirmed boot). This + /// cannot yet distinguish "hasn't rebooted" from "did reboot, target + /// failed to boot, firmware fell back" without Trident exposing boot + /// history (`get rollback-chain` / `get last-error`) over gRPC, which + /// isn't available today - a known remaining gap. In the + /// fallen-back case this simply re-drives the finalize/rollback + /// instead of immediately reporting `TargetBootFailed`. + /// + /// `rollback` requests carry no explicit `targetVersion` (the target is + /// implicit: whatever the previous partition was), so this comparison + /// can never match for them and a state.json-missing rollback recovery + /// always falls to "run fresh". That's still safe: `handle_rollback`'s + /// own `stage_response.servicing_kind` check already reports + /// `OperationFailed` if the rollback already happened and nothing is + /// left to roll back to, rather than a false `Success`. + async fn reconstruct_without_pending_record( + &self, + request: &UpdateRequest, + ) -> Result { + if !matches!( + request.operation, + RequestedOperation::Finalize | RequestedOperation::Rollback + ) { + return Ok(LoopControl::Continue); + } + + let now = Utc::now(); + let current_version = match current_active_version() { + Ok(version) => version, + Err(err) => { + let status = UpdateStatus::new( + request, + request.operation.into(), + request.operation_id.clone(), + StatusCode::AgentInternalError, + format!( + "unable to determine current active version to reconstruct recovery state: {err}" + ), + None, + request.target_version.clone(), + now, + Some(now), + ); + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + }; + + if request.target_version.as_deref() == Some(current_version.as_str()) { + let status = self.reconstruct_without_state(request, None, None).await; + self.record_and_publish(status).await?; + return Ok(LoopControl::Continue); + } + + match request.operation { + RequestedOperation::Finalize => self.handle_finalize(request.clone()).await, + RequestedOperation::Rollback => self.handle_rollback(request.clone()).await, + RequestedOperation::Stage => Ok(LoopControl::Continue), + } + } + async fn record_and_publish(&self, status: UpdateStatus) -> Result<(), Error> { let status = status.refreshed_for_write(); self.state.remember_completed(status.clone())?; @@ -1267,11 +1467,14 @@ fn finalize_failure_status( started: DateTime, err: &TridentClientError, ) -> UpdateStatus { + // Pre-reboot status: TargetBootFailed is reserved for the post-reboot + // commit status (see map_trident_commit_failure's docs), so this always + // reports OperationFailed regardless of the error's subkind. UpdateStatus::new( request, Operation::Finalize, request.operation_id.clone(), - map_trident_failure(err), + StatusCode::OperationFailed, format!("finalize failed: {err}"), from_version, to_version, @@ -1280,7 +1483,15 @@ fn finalize_failure_status( ) } -fn map_trident_failure(error: &TridentClientError) -> StatusCode { +/// Maps a `commit()` (or `update_finalize()`/`rollback_finalize()` sharing +/// the same reboot-check error subkinds) failure to a status code, for the +/// **post-reboot `commit` status only**. Per the status-code contract, +/// `TargetBootFailed` means the firmware fell back to the previous slot +/// after a real boot attempt, and is reserved for the `commit` key - +/// callers writing a pre-reboot `finalize`/`rollback` status must use +/// [`StatusCode::OperationFailed`] directly instead of this function, even +/// though the underlying Trident error subkinds are shared plumbing. +fn map_trident_commit_failure(error: &TridentClientError) -> StatusCode { if indicates_target_boot_failed(error) { StatusCode::TargetBootFailed } else { @@ -1377,6 +1588,23 @@ fn reconstruct_commit_result_to_status( started, Some(Utc::now()), ), + // servicing_kind == NoneRequired means commit() found nothing to + // commit (e.g. the node was already on its target with no armed + // update to promote) - reporting Success here would tell AKS-RP a + // real update completed when nothing actually did. + Ok(response) if response.servicing_kind == Some(ServicingKind::NoneRequired) => { + UpdateStatus::new( + request, + Operation::Commit, + request.operation_id.clone(), + StatusCode::OperationFailed, + "state.json missing after reboot; commit() reported nothing to commit", + from_version, + request.target_version.clone(), + started, + Some(Utc::now()), + ) + } Ok(_) => UpdateStatus::new( request, Operation::Commit, @@ -1405,7 +1633,7 @@ fn reconstruct_commit_result_to_status( request, Operation::Commit, request.operation_id.clone(), - map_trident_failure(&err), + map_trident_commit_failure(&err), format!("state.json missing after reboot; commit failed: {err}"), from_version, request.target_version.clone(), @@ -1437,6 +1665,26 @@ fn commit_result_to_status( Some(Utc::now()), ) } + // See reconstruct_commit_result_to_status's comment: a + // NoneRequired servicing_kind means nothing was actually + // committed, which must not be reported as Success even though + // this path is normally only reached with a confirmed pending + // commit (defense in depth against a stale/corrupted state.json + // entry naming a commit that Trident no longer has anything armed + // for). + Ok(response) if response.servicing_kind == Some(ServicingKind::NoneRequired) => { + UpdateStatus::new( + &pending.request, + Operation::Commit, + pending.operation_id.clone(), + StatusCode::OperationFailed, + "commit() reported nothing to commit", + pending.from_version.clone(), + pending.to_version.clone(), + pending.started_utc, + Some(Utc::now()), + ) + } Ok(_) => UpdateStatus::new( &pending.request, Operation::Commit, @@ -1463,7 +1711,7 @@ fn commit_result_to_status( &pending.request, Operation::Commit, pending.operation_id.clone(), - map_trident_failure(&err), + map_trident_commit_failure(&err), format!("commit failed: {err}"), pending.from_version.clone(), pending.to_version.clone(), @@ -1481,11 +1729,13 @@ fn rollback_stage_failure_status( started: DateTime, err: &TridentClientError, ) -> UpdateStatus { + // Pre-reboot status: see finalize_failure_status's comment - + // TargetBootFailed is reserved for the post-reboot commit status. UpdateStatus::new( request, Operation::Rollback, request.operation_id.clone(), - map_trident_failure(err), + StatusCode::OperationFailed, format!("rollback stage failed: {err}"), from_version, None, @@ -1522,11 +1772,13 @@ fn rollback_finalize_failure_status( started: DateTime, err: &TridentClientError, ) -> UpdateStatus { + // Pre-reboot status: see finalize_failure_status's comment - + // TargetBootFailed is reserved for the post-reboot commit status. UpdateStatus::new( request, Operation::Rollback, request.operation_id.clone(), - map_trident_failure(err), + StatusCode::OperationFailed, format!("rollback finalize failed: {err}"), from_version, None, @@ -1695,7 +1947,10 @@ mod tests { } #[tokio::test] - async fn rollback_finalize_reverted_maps_to_reverted_to_previous() { + async fn rollback_finalize_reboot_check_subkind_maps_to_operation_failed() { + // Pre-reboot rollback-finalize failures always report + // OperationFailed now, even with a boot-check subkind - + // TargetBootFailed is reserved for the post-reboot commit status. let config = Arc::new(Mutex::new(MockTridentdConfig { rollback_finalize: Some(Outcome::Failure { subkind: "ab-update-reboot-check", @@ -1714,7 +1969,7 @@ mod tests { &result.unwrap_err(), ); - assert_eq!(status.code, StatusCode::TargetBootFailed); + assert_eq!(status.code, StatusCode::OperationFailed); } // --- stage --- @@ -1878,7 +2133,11 @@ mod tests { } #[tokio::test] - async fn finalize_failure_with_reboot_check_subkind_maps_to_reverted() { + async fn finalize_failure_with_reboot_check_subkind_maps_to_operation_failed() { + // Pre-reboot finalize failures always report OperationFailed now, + // even when the underlying Trident error carries a boot-check + // subkind - TargetBootFailed is reserved for the post-reboot + // commit status (see finalize_failure_status's doc comment). let config = Arc::new(Mutex::new(MockTridentdConfig { finalize: Some(Outcome::Failure { subkind: "ab-update-reboot-check", @@ -1897,7 +2156,7 @@ mod tests { &err, ); - assert_eq!(status.code, StatusCode::TargetBootFailed); + assert_eq!(status.code, StatusCode::OperationFailed); } #[tokio::test] @@ -1966,6 +2225,27 @@ mod tests { assert_eq!(status.operation, Operation::Commit); } + #[tokio::test] + async fn commit_success_with_none_required_servicing_kind_does_not_report_success() { + // A NoneRequired servicing_kind means commit() found nothing to + // commit - reporting Success here would tell AKS-RP a real update + // completed when nothing actually did. + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: Some(ServicingKind::NoneRequired), + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; + + let status = commit_result_to_status(&pending(Operation::Finalize), result); + + assert_ne!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Commit); + } + #[tokio::test] async fn commit_success_but_reboot_required_maps_to_agent_internal_error() { let config = Arc::new(Mutex::new(MockTridentdConfig { @@ -2172,6 +2452,30 @@ mod tests { assert!(status.message.contains("commit() confirmed the swap")); } + #[tokio::test] + async fn reconstruct_commit_result_none_required_servicing_kind_does_not_report_success() { + let config = Arc::new(Mutex::new(MockTridentdConfig { + commit: Some(Outcome::Success { + reboot_status: RebootStatus::RebootNotRequired, + servicing_kind: Some(ServicingKind::NoneRequired), + }), + ..Default::default() + })); + let mut client = connect_mock_client(config).await; + let result = client.commit(MOCK_RPC_TIMEOUT).await; + + let request = request(RequestedOperation::Finalize); + let status = reconstruct_commit_result_to_status( + &request, + Some("1.0.0".to_string()), + Utc::now(), + result, + ); + + assert_ne!(status.code, StatusCode::Success); + assert_eq!(status.operation, Operation::Commit); + } + #[tokio::test] async fn reconstruct_commit_result_reboot_required_maps_to_agent_internal_error() { let config = Arc::new(Mutex::new(MockTridentdConfig { diff --git a/crates/trident-acl-agent/src/annotations/state.rs b/crates/trident-acl-agent/src/annotations/state.rs index f652a8571a..eabf20db60 100644 --- a/crates/trident-acl-agent/src/annotations/state.rs +++ b/crates/trident-acl-agent/src/annotations/state.rs @@ -161,29 +161,65 @@ impl StateStore { Ok(()) } - pub fn remember_completed(&self, status: UpdateStatus) -> Result<(), Error> { + /// Atomically applies `mutate` to the persisted state in a single + /// load->mutate->save cycle. Callers that need to change more than one + /// field (e.g. recording a completed entry while also clearing + /// `pendingCommit`) should compose their change into one call to this + /// method instead of two separate `save()`s - each `save()` is its own + /// crash-safe atomic file replace, but two of them back-to-back still + /// leave a real window where a crash between them can lose whichever + /// half hadn't landed yet. + fn update(&self, mutate: F) -> Result<(), Error> + where + F: FnOnce(&mut PersistentState), + { let mut state = self.load()?; - let entry = state - .completed - .entry(status.operation_id.clone()) - .or_default(); - match status.operation { - Operation::Commit => entry.commit = Some(status), - _ => entry.operation = Some(status), - } + mutate(&mut state); self.save(&state) } + pub fn remember_completed(&self, status: UpdateStatus) -> Result<(), Error> { + self.update(|state| { + let entry = state + .completed + .entry(status.operation_id.clone()) + .or_default(); + match status.operation { + Operation::Commit => entry.commit = Some(status.clone()), + _ => entry.operation = Some(status.clone()), + } + }) + } + pub fn set_pending_commit(&self, pending: PendingCommit) -> Result<(), Error> { - let mut state = self.load()?; - state.pending_commit = Some(pending); - self.save(&state) + self.update(|state| state.pending_commit = Some(pending.clone())) } pub fn clear_pending_commit(&self) -> Result<(), Error> { - let mut state = self.load()?; - state.pending_commit = None; - self.save(&state) + self.update(|state| state.pending_commit = None) + } + + /// Atomically records `status` as a completed entry and clears any + /// pending commit, in a single load->mutate->save cycle. Every + /// post-reboot completion path (a real `resume_pending_commit`, or the + /// `state.json`-missing degraded reconstruction) must use this instead + /// of a separate `remember_completed()` + `clear_pending_commit()` pair: + /// with two separate writes, a crash between them can leave a stale + /// `pendingCommit` with no completed record (re-triggering the same + /// commit attempt) or vice versa (clearing the pending record with the + /// result never persisted, silently losing the outcome). + pub fn remember_completed_and_clear_pending(&self, status: UpdateStatus) -> Result<(), Error> { + self.update(|state| { + let entry = state + .completed + .entry(status.operation_id.clone()) + .or_default(); + match status.operation { + Operation::Commit => entry.commit = Some(status.clone()), + _ => entry.operation = Some(status.clone()), + } + state.pending_commit = None; + }) } } @@ -379,6 +415,26 @@ mod tests { assert!(state.completed.contains_key("op-1")); } + #[test] + fn remember_completed_and_clear_pending_does_both_in_one_write() { + let (_dir, store) = store(); + store + .set_pending_commit(sample_pending()) + .expect("set_pending_commit should succeed"); + + store + .remember_completed_and_clear_pending(sample_status(Operation::Commit)) + .expect("remember_completed_and_clear_pending should succeed"); + + let state = store.load().expect("load should succeed"); + assert!( + state.pending_commit.is_none(), + "pending commit should be cleared" + ); + let entry = state.completed.get("op-1").expect("entry should exist"); + assert!(entry.commit.is_some(), "commit half should be recorded"); + } + #[test] fn save_is_atomic_replace() { let (_dir, store) = store(); From 4092c3c09de59800adf80a4bddb5be2ef767f94d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 18:13:16 +0000 Subject: [PATCH 49/54] trident-acl-agent, osutils: fix remaining frhuelsz nit-review comments - config.rs: interpolate the ENV_PREFIX_* consts into the envy context messages instead of hardcoding the prefix strings, so they cannot drift if a prefix const ever changes. - version.rs: add a doc comment to the public current_active_version(). - osrelease.rs: extract a shared parse_line() helper used by read_key, OsRelease::parse, and ExtensionRelease::parse - previously each carried its own copy of the same KEY=VALUE trim/unquote logic. Added direct unit test coverage for read_key (previously only exercised indirectly via trident-acl-agent's wrapper tests), including a test asserting it agrees with OsRelease::parse on the same input. Left crates/trident-acl-agent/src/cli.rs's --validate-connection as a flag rather than a subcommand: it is an orthogonal early-exit diagnostic mode layered on top of the agent's otherwise-single run mode, not a distinct action that would benefit from clap subcommand structure. Verified: cargo build/test/clippy/fmt clean. --- crates/osutils/src/osrelease.rs | 215 ++++++++++++------- crates/trident-acl-agent/src/core/config.rs | 8 +- crates/trident-acl-agent/src/core/version.rs | 11 + 3 files changed, 149 insertions(+), 85 deletions(-) diff --git a/crates/osutils/src/osrelease.rs b/crates/osutils/src/osrelease.rs index d034909bc2..bcac756dc8 100644 --- a/crates/osutils/src/osrelease.rs +++ b/crates/osutils/src/osrelease.rs @@ -31,6 +31,27 @@ pub fn is_azl3() -> Result { Ok(OsRelease::read()?.get_distro().is_azl3()) } +/// Splits one os-release-style `KEY=VALUE` line into its key and +/// unquoted/trimmed value, or `None` for a blank, comment, or malformed +/// line. Shared by [`read_key`], [`OsRelease::parse`], and +/// [`ExtensionRelease::parse`] so all three os-release-formatted parsers in +/// this file agree on one set of quoting/trimming rules instead of each +/// maintaining its own copy. +fn parse_line(line: &str) -> Option<(&str, String)> { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let (key, raw_value) = line.split_once('=')?; + let value = raw_value + .trim() + .trim_matches('"') + .trim_matches('\'') + .trim() + .to_string(); + Some((key.trim(), value)) +} + /// Reads a single key from an arbitrary os-release-formatted file. Returns /// `None` when the file is unreadable, the key is absent, or its value is /// empty. @@ -38,21 +59,13 @@ pub fn read_key(path: impl AsRef, key: &str) -> Option { let path = path.as_ref(); let contents = fs::read_to_string(path).ok()?; for line in contents.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((line_key, raw_value)) = line.split_once('=') else { + let Some((line_key, value)) = parse_line(line) else { continue; }; - if line_key.trim() != key { + if line_key != key { continue; } - let value = raw_value.trim().trim_matches('"').trim_matches('\'').trim(); - if value.is_empty() { - return None; - } - return Some(value.to_string()); + return (!value.is_empty()).then_some(value); } None } @@ -208,58 +221,43 @@ impl OsRelease { fn parse(data: &str) -> Self { let mut os_release = OsRelease::default(); for line in data.lines() { - if line.is_empty() || line.trim_start().starts_with('#') { - continue; - } - - let Some((key, raw_value)) = line.trim().split_once('=') else { + let Some((key, value)) = parse_line(line) else { continue; }; - - // Fn to trim whitespace and quotes from value, and return as - // Option - let value = || { - Some( - raw_value - .trim() - .trim_matches('\"') - .trim_matches('\'') - .to_string(), - ) - }; + let value = Some(value); match key { - "NAME" => os_release.name = value(), - "ID" => os_release.id = value(), - "ID_LIKE" => os_release.id_like = value(), - "PRETTY_NAME" => os_release.pretty_name = value(), - "CPE_NAME" => os_release.cpe_name = value(), - "VARIANT" => os_release.variant = value(), - "VARIANT_ID" => os_release.variant_id = value(), - "VERSION" => os_release.version = value(), - "VERSION_ID" => os_release.version_id = value(), - "VERSION_CODENAME" => os_release.version_codename = value(), - "BUILD_ID" => os_release.build_id = value(), - "IMAGE_ID" => os_release.image_id = value(), - "IMAGE_VERSION" => os_release.image_version = value(), - "RELEASE_TYPE" => os_release.release_type = value(), - "HOME_URL" => os_release.home_url = value(), - "DOCUMENTATION_URL" => os_release.documentation_url = value(), - "SUPPORT_URL" => os_release.support_url = value(), - "BUG_REPORT_URL" => os_release.bug_report_url = value(), - "PRIVACY_POLICY_URL" => os_release.privacy_policy_url = value(), - "SUPPORT_END" => os_release.support_end = value(), - "LOGO" => os_release.logo = value(), - "ANSI_COLOR" => os_release.ansi_color = value(), - "ANSI_COLOR_REVERSE" => os_release.ansi_color_reverse = value(), - "VENDOR_NAME" => os_release.vendor_name = value(), - "VENDOR_URL" => os_release.vendor_url = value(), - "EXPERIMENT" => os_release.experiment = value(), - "EXPERIMENT_URL" => os_release.experiment_url = value(), - "DEFAULT_HOSTNAME" => os_release.default_hostname = value(), - "ARCHITECTURE" => os_release.architecture = value(), - "SYSEXT_LEVEL" => os_release.sysext_level = value(), - "CONFEXT_LEVEL" => os_release.confext_level = value(), + "NAME" => os_release.name = value, + "ID" => os_release.id = value, + "ID_LIKE" => os_release.id_like = value, + "PRETTY_NAME" => os_release.pretty_name = value, + "CPE_NAME" => os_release.cpe_name = value, + "VARIANT" => os_release.variant = value, + "VARIANT_ID" => os_release.variant_id = value, + "VERSION" => os_release.version = value, + "VERSION_ID" => os_release.version_id = value, + "VERSION_CODENAME" => os_release.version_codename = value, + "BUILD_ID" => os_release.build_id = value, + "IMAGE_ID" => os_release.image_id = value, + "IMAGE_VERSION" => os_release.image_version = value, + "RELEASE_TYPE" => os_release.release_type = value, + "HOME_URL" => os_release.home_url = value, + "DOCUMENTATION_URL" => os_release.documentation_url = value, + "SUPPORT_URL" => os_release.support_url = value, + "BUG_REPORT_URL" => os_release.bug_report_url = value, + "PRIVACY_POLICY_URL" => os_release.privacy_policy_url = value, + "SUPPORT_END" => os_release.support_end = value, + "LOGO" => os_release.logo = value, + "ANSI_COLOR" => os_release.ansi_color = value, + "ANSI_COLOR_REVERSE" => os_release.ansi_color_reverse = value, + "VENDOR_NAME" => os_release.vendor_name = value, + "VENDOR_URL" => os_release.vendor_url = value, + "EXPERIMENT" => os_release.experiment = value, + "EXPERIMENT_URL" => os_release.experiment_url = value, + "DEFAULT_HOSTNAME" => os_release.default_hostname = value, + "ARCHITECTURE" => os_release.architecture = value, + "SYSEXT_LEVEL" => os_release.sysext_level = value, + "CONFEXT_LEVEL" => os_release.confext_level = value, _ => {} } } @@ -309,32 +307,17 @@ impl ExtensionRelease { let mut portable_prefixes = None; for line in data.lines() { - if line.is_empty() || line.trim_start().starts_with('#') { - continue; - } - - let Some((key, raw_value)) = line.trim().split_once('=') else { + let Some((key, value)) = parse_line(line) else { continue; }; - - // Fn to trim whitespace and quotes from value, and return as - // Option - let value = || { - Some( - raw_value - .trim() - .trim_matches('\"') - .trim_matches('\'') - .to_string(), - ) - }; + let value = Some(value); match key { - "SYSEXT_ID" => sysext_id = value(), - "CONFEXT_ID" => confext_id = value(), - "SYSEXT_SCOPE" => sysext_scope = value(), - "CONFEXT_SCOPE" => confext_scope = value(), - "PORTABLE_PREFIXES" => portable_prefixes = value(), + "SYSEXT_ID" => sysext_id = value, + "CONFEXT_ID" => confext_id = value, + "SYSEXT_SCOPE" => sysext_scope = value, + "CONFEXT_SCOPE" => confext_scope = value, + "PORTABLE_PREFIXES" => portable_prefixes = value, _ => {} } } @@ -687,4 +670,74 @@ mod tests { ); assert!(OsRelease::ensure_matching_distro(&os_release1, &os_release2).is_err()); } + + #[test] + fn read_key_finds_a_quoted_value() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("os-release"); + std::fs::write(&path, "ID=azurelinux\nIMAGE_VERSION=\"202608.1.0\"\n") + .expect("failed to write test file"); + + assert_eq!( + read_key(&path, "IMAGE_VERSION"), + Some("202608.1.0".to_string()) + ); + assert_eq!(read_key(&path, "ID"), Some("azurelinux".to_string())); + } + + #[test] + fn read_key_skips_blank_lines_and_comments() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("os-release"); + std::fs::write( + &path, + "\n# a comment\n # indented comment\nID=azurelinux\n", + ) + .expect("failed to write test file"); + + assert_eq!(read_key(&path, "ID"), Some("azurelinux".to_string())); + } + + #[test] + fn read_key_returns_none_for_a_missing_key() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("os-release"); + std::fs::write(&path, "ID=azurelinux\n").expect("failed to write test file"); + + assert_eq!(read_key(&path, "IMAGE_VERSION"), None); + } + + #[test] + fn read_key_returns_none_for_an_empty_value() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("os-release"); + std::fs::write(&path, "IMAGE_VERSION=\n").expect("failed to write test file"); + + assert_eq!(read_key(&path, "IMAGE_VERSION"), None); + } + + #[test] + fn read_key_returns_none_for_a_missing_file() { + assert_eq!(read_key("/nonexistent/os-release", "ID"), None); + } + + #[test] + fn read_key_agrees_with_os_release_parse_on_the_same_input() { + // read_key and OsRelease::parse share parse_line, so both should + // agree on the same file's VERSION_ID. + let data = indoc::indoc! { + r#" + ID=azurelinux + VERSION_ID="3.0" + "#, + }; + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("os-release"); + std::fs::write(&path, data).expect("failed to write test file"); + + assert_eq!( + read_key(&path, "VERSION_ID"), + OsRelease::parse(data).version_id + ); + } } diff --git a/crates/trident-acl-agent/src/core/config.rs b/crates/trident-acl-agent/src/core/config.rs index 043fd8710d..7e12e57159 100644 --- a/crates/trident-acl-agent/src/core/config.rs +++ b/crates/trident-acl-agent/src/core/config.rs @@ -81,16 +81,16 @@ impl AgentConfig { fn from_vars(vars: Vec<(String, String)>) -> Result { let nebraska: RawNebraskaConfig = envy::prefixed(ENV_PREFIX_NEBRASKA) .from_iter(vars.iter().cloned()) - .context("invalid TRIDENT_ACL_AGENT_NEBRASKA_* environment variable")?; + .with_context(|| format!("invalid {ENV_PREFIX_NEBRASKA}* environment variable"))?; let kubernetes: RawKubernetesConfig = envy::prefixed(ENV_PREFIX_KUBERNETES) .from_iter(vars.iter().cloned()) - .context("invalid TRIDENT_ACL_AGENT_KUBERNETES_* environment variable")?; + .with_context(|| format!("invalid {ENV_PREFIX_KUBERNETES}* environment variable"))?; let trident: RawTridentConfig = envy::prefixed(ENV_PREFIX_TRIDENT) .from_iter(vars.iter().cloned()) - .context("invalid TRIDENT_ACL_AGENT_TRIDENT_* environment variable")?; + .with_context(|| format!("invalid {ENV_PREFIX_TRIDENT}* environment variable"))?; let orchestration: RawOrchestrationConfig = envy::prefixed(ENV_PREFIX_ORCHESTRATION) .from_iter(vars.iter().cloned()) - .context("invalid TRIDENT_ACL_AGENT_ORCHESTRATION_* environment variable")?; + .with_context(|| format!("invalid {ENV_PREFIX_ORCHESTRATION}* environment variable"))?; Ok(Self { nebraska: NebraskaConfig { diff --git a/crates/trident-acl-agent/src/core/version.rs b/crates/trident-acl-agent/src/core/version.rs index 1953aec927..38f78f0e76 100644 --- a/crates/trident-acl-agent/src/core/version.rs +++ b/crates/trident-acl-agent/src/core/version.rs @@ -76,6 +76,17 @@ fn env_override(name: &str) -> Option { env::var(name).ok().filter(|v| !v.is_empty()) } +/// Returns the node's currently-running version, for comparison against a +/// request's `targetVersion` (e.g. to short-circuit to `AlreadyAtTarget`, or +/// to decide whether a post-reboot swap actually happened when +/// reconstructing state after a crash - see `annotations::orchestrator`). +/// Reads the key named by `TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY` (default +/// [`DEFAULT_CURRENT_VERSION_KEY`]) from the file named by +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH` (default +/// [`DEFAULT_CURRENT_VERSION_PATH`]), falling back per +/// `TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK` (default +/// [`DEFAULT_CURRENT_VERSION_FALLBACK`]) when that key is absent - see the +/// module-level doc comment above for the three fallback forms. pub fn current_active_version() -> Result { let path = env_override(ENV_CURRENT_VERSION_PATH) .unwrap_or_else(|| DEFAULT_CURRENT_VERSION_PATH.to_string()); From 687b5c785b100ebecf07f69a8dede5015962c721 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 18:51:51 +0000 Subject: [PATCH 50/54] trident-acl-agent: bounded retry for startup recovery Node read recover_from_trident_state's one-shot self.k8s.get_node(...) call (used only when there is no pendingCommit to resume locally) had no retry at all, unlike the watch loop's stream (which already retries/backs off via kube::runtime::watchers default_backoff()) and the terminal-status publish path (best_effort_publish_terminal). A transient k8s hiccup at that exact moment propagated straight through ? into a process exit. Added get_node_with_retry(): a bounded retry (3 attempts, 2s backoff, matching best_effort_publish_terminal's shape) around that single call, returning NodeGone immediately without retrying since that's terminal. Verified: cargo build/test/clippy/fmt clean across the workspace. --- .../src/annotations/orchestrator.rs | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 8bcffb79b6..df6a6158eb 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -42,6 +42,16 @@ use crate::{ const FINAL_STATUS_PATCH_RETRIES: usize = 3; const FINAL_STATUS_PATCH_BACKOFF: Duration = Duration::from_secs(2); +/// Bounded retry for `recover_from_trident_state`'s one-shot Node read (used +/// only when there's no `pendingCommit` to resume locally, so this is not +/// on the earlier, more time-sensitive resume path). Mirrors +/// `FINAL_STATUS_PATCH_RETRIES`/`FINAL_STATUS_PATCH_BACKOFF`'s shape: a +/// short bounded retry absorbs a transient k8s hiccup at startup instead of +/// turning it into an immediate crash-loop, while still giving up and +/// surfacing a real error if k8s stays down. +const RECOVERY_NODE_READ_RETRIES: usize = 3; +const RECOVERY_NODE_READ_BACKOFF: Duration = Duration::from_secs(2); + /// The machine-id source used for every Nebraska request this module makes, /// event reports included. Must match the source used by `handle_stage`'s /// initial `check_for_update` so all requests for a given node present the @@ -163,7 +173,9 @@ impl Orchestrator { return Ok(LoopControl::Continue); } - let node = self.k8s.get_node(&self.config.kubernetes.node_name).await?; + let node = self + .get_node_with_retry(&self.config.kubernetes.node_name) + .await?; let snapshot = Snapshot::from_node(&node, &self.annotation_keys); if let Some(request) = snapshot.request.clone() { @@ -1091,6 +1103,32 @@ impl Orchestrator { } } + /// Reads the agent's own Node object with a bounded retry, so a + /// transient Kubernetes hiccup at startup recovery doesn't propagate the + /// first error straight into a process exit / crash-loop the way a bare + /// `self.k8s.get_node(...).await?` would. Only used by + /// `recover_from_trident_state`'s "no pending commit to resume" branch; + /// the pending-commit resume itself never touches k8s at all (see that + /// function's docs). `NodeGone` is returned immediately without + /// retrying, since it's terminal - the node was deleted, and no amount + /// of retrying changes that. + async fn get_node_with_retry(&self, name: &str) -> Result { + let mut last_err = None; + for attempt in 0..RECOVERY_NODE_READ_RETRIES { + match self.k8s.get_node(name).await { + Ok(node) => return Ok(node), + Err(K8sClientError::NodeGone) => return Err(K8sClientError::NodeGone), + Err(err) => { + last_err = Some(err); + if attempt + 1 < RECOVERY_NODE_READ_RETRIES { + time::sleep(RECOVERY_NODE_READ_BACKOFF).await; + } + } + } + } + Err(last_err.expect("loop runs RECOVERY_NODE_READ_RETRIES >= 1 times")) + } + fn is_node_gone_error(&self, err: &Error) -> bool { matches!( err.downcast_ref::(), From 69b43d53b1c3a5cc43db1b00e69f9ea5d6f18c6e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 18:58:29 +0000 Subject: [PATCH 51/54] trident-acl-agent: document the 4-branch degraded-recovery gap explicitly Expand reconstruct_without_pending_record's doc comment with an explicit desired-vs-implemented-vs-needed breakdown of the design doc's 4-branch degraded-recovery reconstruction (2.3), so the remaining gap (no gRPC-exposed boot history to distinguish a firmware fallback from never having rebooted) is discoverable directly on the function instead of only in review-comment history. Doc-only change, no behavior change. --- .../src/annotations/orchestrator.rs | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index df6a6158eb..2454989042 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -994,13 +994,28 @@ impl Orchestrator { /// `commit()`. /// - **active != target**: not proven that a reboot ever happened, so /// the request is run fresh instead of guessed at further - always - /// safe (never touches `commit()` on an unconfirmed boot). This - /// cannot yet distinguish "hasn't rebooted" from "did reboot, target - /// failed to boot, firmware fell back" without Trident exposing boot - /// history (`get rollback-chain` / `get last-error`) over gRPC, which - /// isn't available today - a known remaining gap. In the - /// fallen-back case this simply re-drives the finalize/rollback - /// instead of immediately reporting `TargetBootFailed`. + /// safe (never touches `commit()` on an unconfirmed boot). + /// + /// KNOWN GAP: the design doc's degraded-recovery path (2.3) has 4 + /// branches; this implements only 2 of them, folding the other 2 + /// together as "run fresh": + /// 1. active == target -> run `commit()` [done] + /// 2. active != target, boot attempted+failed -> report `TargetBootFailed` [missing] + /// 3. active != target, no boot attempted -> run the request fresh [done] + /// 4. anything Trident can't account for -> `AgentInternalError` [handled elsewhere] + /// + /// Branch 2 is missing because distinguishing it from branch 3 needs to + /// know whether a boot was *attempted*, not just which version is + /// currently active - both leave the node on the same (previous) + /// version, since a firmware fallback is invisible from inside the OS + /// that ends up running. The design's answer is Trident's boot history + /// (`get rollback-chain` / `get last-error`), which isn't exposed over + /// gRPC today, only via the CLI. Closing this gap means adding a + /// gRPC-exposed equivalent in `trident` and calling it here to pick + /// branch 2 vs 3. Until then this is always safe (never calls + /// `commit()` on an unconfirmed boot) but not fully correct: a real + /// firmware fallback gets silently retried as a fresh finalize/rollback + /// instead of being reported as `TargetBootFailed`. /// /// `rollback` requests carry no explicit `targetVersion` (the target is /// implicit: whatever the previous partition was), so this comparison From da4cb6c6b83d57bc173495d22eb6b32afe8bb6ef Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 19:46:38 +0000 Subject: [PATCH 52/54] trident-acl-agent: close part of the degraded-recovery boot-detection gap Adds a second, independent signal for confirming a reboot happened when state.json carries no pendingCommit record for an in-flight finalize/ rollback, on top of the existing active-version-vs-target comparison: - osutils::machine_id::boot_time(): reads /proc/stat's btime line (the current boot's absolute wall-clock start time - the same value systemctl show -p KernelTimestamp exposes, read directly with no subprocess/systemd dependency). No persistent storage assumption required, unlike a journald-based approach. - reboot_confirmed_since_arming(): compares that boot time against the finalize/rollback's own terminal status' finished_utc, which is already published to the Node's update-status annotation *before* the reboot is triggered (external to local disk, so it survives even when state.json itself does not). If the current boot started after that timestamp, a reboot has demonstrably happened since, regardless of which version the node landed on - so it's safe to call commit() and let Trident's own response (already handled by reconstruct_commit_result_to_status's indicates_target_boot_failed check) distinguish success from a firmware fallback. - reconstruct_without_pending_record now treats "already at target version" OR "reboot confirmed via boot time" as proof a boot happened, instead of only the version comparison. This closes the specific gap called out in the prior commit's doc comment for the common case where operation_status was successfully published before the reboot - a real firmware fallback in that case is now correctly reported as TargetBootFailed instead of being silently retried as a fresh finalize/rollback. Remaining corner (still requires Trident's own boot history over gRPC, not addressed here): operation_status itself absent or missing finished_utc (e.g. the pre-reboot status publish also failed) - falls back to the existing safe-but-imprecise "run fresh" behavior. Updated reconstruct_without_pending_record's doc comment accordingly. Verified: cargo build/test/clippy/fmt clean across the workspace (908 tests pass, including 3 new osutils::machine_id::boot_time tests and 4 new orchestrator::reboot_confirmed_since_arming tests). --- crates/osutils/src/machine_id.rs | 66 ++++++++ .../src/annotations/orchestrator.rs | 160 ++++++++++++++---- 2 files changed, 189 insertions(+), 37 deletions(-) diff --git a/crates/osutils/src/machine_id.rs b/crates/osutils/src/machine_id.rs index 8d3b32bf8a..c5627e2db1 100644 --- a/crates/osutils/src/machine_id.rs +++ b/crates/osutils/src/machine_id.rs @@ -6,6 +6,7 @@ use uuid::Uuid; const MACHINE_ID_FILE: &str = "/etc/machine-id"; const BOOT_ID_FILE: &str = "/proc/sys/kernel/random/boot_id"; +const PROC_STAT_FILE: &str = "/proc/stat"; #[derive(Debug, Clone, Copy)] pub struct MachineId(u128); @@ -74,3 +75,68 @@ impl MachineId { pub fn boot_id() -> Result { MachineId::boot_id() } + +/// Returns the current boot's start time as Unix epoch seconds, read from +/// `/proc/stat`'s `btime` line - the same absolute wall-clock boot +/// timestamp `systemctl show -p KernelTimestamp` exposes, but read directly +/// with no subprocess/systemd dependency, matching this module's existing +/// style of reading `/proc` files directly (see `boot_id` above). Useful +/// for determining whether a reboot has happened since some earlier +/// wall-clock timestamp, without needing any local state persisted across +/// that reboot: unlike `boot_id`, which only distinguishes "this boot" from +/// "some other boot" (and needs a previously-recorded boot ID to compare +/// against), this can be compared directly against an absolute timestamp +/// recorded anywhere - including one that only survives in an external +/// system (e.g. a Kubernetes annotation), not on local disk. +pub fn boot_time() -> Result { + boot_time_inner(PROC_STAT_FILE) +} + +fn boot_time_inner(path: impl AsRef) -> Result { + let path = path.as_ref(); + let contents = fs::read_to_string(path) + .with_context(|| format!("Failed to read boot time from '{}'", path.display()))?; + let line = contents + .lines() + .find(|line| line.starts_with("btime ")) + .with_context(|| format!("No 'btime' line found in '{}'", path.display()))?; + let raw = line + .split_whitespace() + .nth(1) + .with_context(|| format!("Malformed 'btime' line in '{}': {line:?}", path.display()))?; + raw.parse::().with_context(|| { + format!( + "Failed to parse boot time {raw:?} from '{}'", + path.display() + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn boot_time_inner_parses_btime_line() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("stat"); + std::fs::write(&path, "cpu 1 2 3 4\nbtime 1700000000\nprocesses 100\n") + .expect("failed to write test file"); + + assert_eq!(boot_time_inner(&path).expect("should parse"), 1_700_000_000); + } + + #[test] + fn boot_time_inner_errors_when_btime_missing() { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let path = dir.path().join("stat"); + std::fs::write(&path, "cpu 1 2 3 4\nprocesses 100\n").expect("failed to write test file"); + + assert!(boot_time_inner(&path).is_err()); + } + + #[test] + fn boot_time_inner_errors_for_missing_file() { + assert!(boot_time_inner("/nonexistent/proc-stat-for-test").is_err()); + } +} diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 2454989042..9ddc9b86d7 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -203,7 +203,9 @@ impl Orchestrator { } if let Some(request) = snapshot.request { - return self.reconstruct_without_pending_record(&request).await; + return self + .reconstruct_without_pending_record(&request, snapshot.operation_status.as_ref()) + .await; } Ok(LoopControl::Continue) } @@ -986,47 +988,45 @@ impl Orchestrator { /// silently report `Success` for a no-op - must never be called /// speculatively here. /// - /// Comparing the node's currently-running version against the - /// request's target resolves the common cases without guessing: - /// - **active == target**: the swap already happened (the active - /// version only changes via the post-finalize swap), so it's safe to - /// hand off to [`reconstruct_without_state`] to validate/promote via - /// `commit()`. - /// - **active != target**: not proven that a reboot ever happened, so - /// the request is run fresh instead of guessed at further - always - /// safe (never touches `commit()` on an unconfirmed boot). - /// - /// KNOWN GAP: the design doc's degraded-recovery path (2.3) has 4 - /// branches; this implements only 2 of them, folding the other 2 - /// together as "run fresh": - /// 1. active == target -> run `commit()` [done] - /// 2. active != target, boot attempted+failed -> report `TargetBootFailed` [missing] - /// 3. active != target, no boot attempted -> run the request fresh [done] - /// 4. anything Trident can't account for -> `AgentInternalError` [handled elsewhere] + /// A reboot is considered confirmed (safe to call `commit()` via + /// [`reconstruct_without_state`]) when either: + /// - **active version == target version**: the swap already happened + /// (the active version only changes via the post-finalize swap), or + /// - **the system's current boot started after `operation_status`'s + /// `finished_utc`** (see [`reboot_confirmed_since_arming`]): + /// `operation_status` is the finalize/rollback's own terminal status, + /// already published to the Node's `update-status` annotation + /// *before* the reboot was triggered (see the caller-handled-reboot + /// ordering in module docs), so it survives independent of local + /// disk state. If the node's current boot demonstrably started after + /// that timestamp, a reboot has happened since, whatever version the + /// node ended up on - so it's safe to call `commit()` and let + /// Trident's own response (already handled by + /// `reconstruct_commit_result_to_status`'s `indicates_target_boot_failed` + /// check) distinguish a successful commit from a firmware fallback. /// - /// Branch 2 is missing because distinguishing it from branch 3 needs to - /// know whether a boot was *attempted*, not just which version is - /// currently active - both leave the node on the same (previous) - /// version, since a firmware fallback is invisible from inside the OS - /// that ends up running. The design's answer is Trident's boot history - /// (`get rollback-chain` / `get last-error`), which isn't exposed over - /// gRPC today, only via the CLI. Closing this gap means adding a - /// gRPC-exposed equivalent in `trident` and calling it here to pick - /// branch 2 vs 3. Until then this is always safe (never calls - /// `commit()` on an unconfirmed boot) but not fully correct: a real - /// firmware fallback gets silently retried as a fresh finalize/rollback - /// instead of being reported as `TargetBootFailed`. + /// If neither is true, the request is run fresh instead of guessed at + /// further - always safe (never touches `commit()` on an unconfirmed + /// boot), though it means an *unconfirmable* firmware fallback (e.g. no + /// `operation_status` was ever published, or its `finished_utc` is + /// absent) is retried as a fresh finalize/rollback rather than reported + /// as `TargetBootFailed`. Closing that last corner fully would need + /// Trident's own boot history (`get rollback-chain` / `get last-error`), + /// which isn't exposed over gRPC today, only via the CLI. /// /// `rollback` requests carry no explicit `targetVersion` (the target is - /// implicit: whatever the previous partition was), so this comparison - /// can never match for them and a state.json-missing rollback recovery - /// always falls to "run fresh". That's still safe: `handle_rollback`'s - /// own `stage_response.servicing_kind` check already reports - /// `OperationFailed` if the rollback already happened and nothing is - /// left to roll back to, rather than a false `Success`. + /// implicit: whatever the previous partition was), so the version + /// comparison can never match for them - they rely entirely on the + /// boot-time check above. If that's also inconclusive, a + /// state.json-missing rollback recovery falls to "run fresh", which is + /// still safe: `handle_rollback`'s own `stage_response.servicing_kind` + /// check already reports `OperationFailed` if the rollback already + /// happened and nothing is left to roll back to, rather than a false + /// `Success`. async fn reconstruct_without_pending_record( &self, request: &UpdateRequest, + operation_status: Option<&UpdateStatus>, ) -> Result { if !matches!( request.operation, @@ -1057,7 +1057,8 @@ impl Orchestrator { } }; - if request.target_version.as_deref() == Some(current_version.as_str()) { + let already_at_target = request.target_version.as_deref() == Some(current_version.as_str()); + if already_at_target || reboot_confirmed_since_arming(operation_status) { let status = self.reconstruct_without_state(request, None, None).await; self.record_and_publish(status).await?; return Ok(LoopControl::Continue); @@ -1281,6 +1282,29 @@ fn current_boot_marker() -> Result { machine_id::boot_id() } +/// Returns whether the system has rebooted since `operation_status` (the +/// finalize/rollback's own previously-published terminal status) was +/// recorded as finished, by comparing the current boot's start time +/// (`machine_id::boot_time()`, read from `/proc/stat`'s `btime` - no local +/// state persisted across the reboot required) against that status's +/// `finished_utc`. If the current boot started after the reboot was armed, +/// a reboot has definitely happened since - whatever version the node +/// ended up on. Never speculatively assumes a reboot happened: returns +/// `false` if there's no status to compare against, its `finished_utc` is +/// absent, or the boot time can't be read. +fn reboot_confirmed_since_arming(operation_status: Option<&UpdateStatus>) -> bool { + let Some(finished_utc) = operation_status.and_then(|status| status.finished_utc) else { + return false; + }; + let Ok(boot_time_secs) = machine_id::boot_time() else { + return false; + }; + let Some(boot_time) = DateTime::from_timestamp(boot_time_secs, 0) else { + return false; + }; + boot_time > finished_utc +} + /// Parses `version` (e.g. an `UpdateStatus::from_version`/`to_version` /// field) as a semver [`Version`] for use in a Nebraska event report, /// logging and returning `None` rather than failing if it's absent or not @@ -2614,4 +2638,66 @@ mod tests { None ); } + + // --- reboot_confirmed_since_arming --- + + #[test] + fn reboot_confirmed_since_arming_true_for_ancient_finished_utc() { + // Any real boot time is long after 1970 - a finished_utc from the + // epoch must always read as "a reboot happened since". + let epoch = DateTime::from_timestamp(0, 0).unwrap(); + let status = UpdateStatus::new( + &request(RequestedOperation::Finalize), + Operation::Finalize, + "op-1".to_string(), + StatusCode::Success, + "finalize completed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + epoch, + Some(epoch), + ); + + assert!(reboot_confirmed_since_arming(Some(&status))); + } + + #[test] + fn reboot_confirmed_since_arming_false_for_far_future_finished_utc() { + let far_future = DateTime::from_timestamp(32_503_680_000, 0).unwrap(); // ~year 3000 + let status = UpdateStatus::new( + &request(RequestedOperation::Finalize), + Operation::Finalize, + "op-1".to_string(), + StatusCode::Success, + "finalize completed", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + far_future, + Some(far_future), + ); + + assert!(!reboot_confirmed_since_arming(Some(&status))); + } + + #[test] + fn reboot_confirmed_since_arming_false_when_no_status() { + assert!(!reboot_confirmed_since_arming(None)); + } + + #[test] + fn reboot_confirmed_since_arming_false_when_finished_utc_absent() { + let status = UpdateStatus::new( + &request(RequestedOperation::Finalize), + Operation::Finalize, + "op-1".to_string(), + StatusCode::InProgress, + "finalizing", + Some("1.0.0".to_string()), + Some("2.0.0".to_string()), + Utc::now(), + None, + ); + + assert!(!reboot_confirmed_since_arming(Some(&status))); + } } From 9e9b9f4e4244bbf621c8ef9b88e919227f3e1e7e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 19:54:37 +0000 Subject: [PATCH 53/54] trident-acl-agent: never speculate a commit when operationId was never seen reconstruct_without_pending_record previously computed the version-match check unconditionally, even when operation_status was entirely None (the agent never started processing this operationId at all - it's set to InProgress immediately on dispatch, before anything else). A version match alone in that state isn't proof this specific request caused anything: the node could already be on targetVersion for an unrelated reason (a prior, already-completed operation; a manually re-imaged node), in which case handle_finalize/handle_rollback's own AlreadyAtTarget check is what should produce the status - not a speculative commit() call via the degraded-recovery path. Fixed by short-circuiting to running the request fresh (extracted into a small run_request_fresh() helper, reused by the existing dispatch site) as soon as operation_status is None, before ever computing current_active_version()/the version comparison. The version-match and boot-time checks now only run when there's at least some record (InProgress or terminal) that the agent previously engaged with this operationId. Updated reconstruct_without_pending_record's doc comment to describe this precondition explicitly. Verified: cargo build/test/clippy/fmt clean across the workspace (908 tests pass, unchanged pass count - this is a control-flow-ordering fix, not new observable behavior any current test exercises). --- .../src/annotations/orchestrator.rs | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 9ddc9b86d7..69f054dee0 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -988,8 +988,20 @@ impl Orchestrator { /// silently report `Success` for a no-op - must never be called /// speculatively here. /// - /// A reboot is considered confirmed (safe to call `commit()` via - /// [`reconstruct_without_state`]) when either: + /// If `operation_status` is entirely absent, the agent never even + /// started processing this `operationId` (it's set to `InProgress` + /// immediately on dispatch, before anything else) - there is no + /// evidence this specific request caused anything, so the request is + /// always run fresh rather than considered for `commit()`. A version + /// match alone isn't enough proof here: the node could already be on + /// `targetVersion` for an unrelated reason (a prior, already-completed + /// operation; a manually re-imaged node), in which case + /// `handle_finalize`/`handle_rollback`'s own `AlreadyAtTarget` check + /// produces the correct status without ever needing `commit()`. + /// + /// With an `operation_status` in hand, a reboot is considered confirmed + /// (safe to call `commit()` via [`reconstruct_without_state`]) when + /// either: /// - **active version == target version**: the swap already happened /// (the active version only changes via the post-finalize swap), or /// - **the system's current boot started after `operation_status`'s @@ -1007,12 +1019,12 @@ impl Orchestrator { /// /// If neither is true, the request is run fresh instead of guessed at /// further - always safe (never touches `commit()` on an unconfirmed - /// boot), though it means an *unconfirmable* firmware fallback (e.g. no - /// `operation_status` was ever published, or its `finished_utc` is - /// absent) is retried as a fresh finalize/rollback rather than reported - /// as `TargetBootFailed`. Closing that last corner fully would need - /// Trident's own boot history (`get rollback-chain` / `get last-error`), - /// which isn't exposed over gRPC today, only via the CLI. + /// boot), though it means an *unconfirmable* firmware fallback (e.g. + /// `operation_status` has no `finished_utc`, i.e. it's stuck at + /// `InProgress`) is retried as a fresh finalize/rollback rather than + /// reported as `TargetBootFailed`. Closing that last corner fully would + /// need Trident's own boot history (`get rollback-chain` / `get + /// last-error`), which isn't exposed over gRPC today, only via the CLI. /// /// `rollback` requests carry no explicit `targetVersion` (the target is /// implicit: whatever the previous partition was), so the version @@ -1035,6 +1047,10 @@ impl Orchestrator { return Ok(LoopControl::Continue); } + let Some(operation_status) = operation_status else { + return self.run_request_fresh(request).await; + }; + let now = Utc::now(); let current_version = match current_active_version() { Ok(version) => version, @@ -1058,12 +1074,16 @@ impl Orchestrator { }; let already_at_target = request.target_version.as_deref() == Some(current_version.as_str()); - if already_at_target || reboot_confirmed_since_arming(operation_status) { + if already_at_target || reboot_confirmed_since_arming(Some(operation_status)) { let status = self.reconstruct_without_state(request, None, None).await; self.record_and_publish(status).await?; return Ok(LoopControl::Continue); } + self.run_request_fresh(request).await + } + + async fn run_request_fresh(&self, request: &UpdateRequest) -> Result { match request.operation { RequestedOperation::Finalize => self.handle_finalize(request.clone()).await, RequestedOperation::Rollback => self.handle_rollback(request.clone()).await, From db27ea6da5fc80956d24ddfbaf01a21bc8538d15 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 27 Aug 2026 20:32:14 +0000 Subject: [PATCH 54/54] trident-acl-agent: ignore a stale operation_status for a different operationId reconstruct_without_pending_record trusted whatever UpdateStatus happened to be sitting in the Node's update-status annotation, without checking that it actually belongs to the request currently being reconstructed. Snapshot::from_node parses operation_status purely from the annotation key's current contents - it never cross-references it against the request's own operationId. Concretely: if AKS-RP writes a brand-new finalize (operationId Y), and the update-status annotation still holds a terminal status from an earlier, unrelated, already-completed operation (operationId X, with its own old finished_utc), reboot_confirmed_since_arming would compare the node's boot time against X's finished_utc - which has nothing to do with Y - and could easily conclude "a reboot has happened since Y was armed" even though Y was never processed at all. That would route to commit() for an operation that was never even started, instead of correctly running it fresh. Fixed by filtering operation_status down to Some(status) only when status.operation_id == request.operation_id, folding a stale/mismatched status into the same "run fresh" path as an entirely absent one. Updated reconstruct_without_pending_record's doc comment to describe this precondition. Verified: cargo build/test/clippy/fmt clean across the workspace (908 tests pass, unchanged pass count - this guards against a scenario no current test happens to construct). --- .../src/annotations/orchestrator.rs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/trident-acl-agent/src/annotations/orchestrator.rs b/crates/trident-acl-agent/src/annotations/orchestrator.rs index 69f054dee0..b2d15323e6 100644 --- a/crates/trident-acl-agent/src/annotations/orchestrator.rs +++ b/crates/trident-acl-agent/src/annotations/orchestrator.rs @@ -988,16 +988,19 @@ impl Orchestrator { /// silently report `Success` for a no-op - must never be called /// speculatively here. /// - /// If `operation_status` is entirely absent, the agent never even - /// started processing this `operationId` (it's set to `InProgress` - /// immediately on dispatch, before anything else) - there is no - /// evidence this specific request caused anything, so the request is - /// always run fresh rather than considered for `commit()`. A version - /// match alone isn't enough proof here: the node could already be on - /// `targetVersion` for an unrelated reason (a prior, already-completed - /// operation; a manually re-imaged node), in which case - /// `handle_finalize`/`handle_rollback`'s own `AlreadyAtTarget` check - /// produces the correct status without ever needing `commit()`. + /// If `operation_status` is entirely absent, *or* it exists but belongs + /// to a different `operationId` than `request` (e.g. a still-lingering + /// terminal status annotation from an earlier, unrelated operation that + /// hasn't been overwritten yet), the agent has no evidence this + /// specific request was ever touched (its own status is set to + /// `InProgress` immediately on dispatch, before anything else) - so the + /// request is always run fresh rather than considered for `commit()`. + /// A version match alone isn't enough proof here either: the node + /// could already be on `targetVersion` for an unrelated reason (a + /// prior, already-completed operation; a manually re-imaged node), in + /// which case `handle_finalize`/`handle_rollback`'s own + /// `AlreadyAtTarget` check produces the correct status without ever + /// needing `commit()`. /// /// With an `operation_status` in hand, a reboot is considered confirmed /// (safe to call `commit()` via [`reconstruct_without_state`]) when @@ -1047,6 +1050,8 @@ impl Orchestrator { return Ok(LoopControl::Continue); } + let operation_status = + operation_status.filter(|status| status.operation_id == request.operation_id); let Some(operation_status) = operation_status else { return self.run_request_fresh(request).await; };