diff --git a/AGENTS.md b/AGENTS.md index 9e54f5fb9..40c8af46d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ cargo bench # exercise hotspots when touching pe # GitNexus — Code Intelligence -This project is indexed by GitNexus as **substrate** (46052 symbols, 69611 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **substrate** (49012 symbols, 75513 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. diff --git a/CLAUDE.md b/CLAUDE.md index ab715ef4a..12febe02e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,7 @@ # GitNexus — Code Intelligence -This project is indexed by GitNexus as **substrate** (46052 symbols, 69611 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **substrate** (49012 symbols, 75513 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. diff --git a/Cargo.lock b/Cargo.lock index 92fc7d441..7c5cfbd3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,7 +61,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -152,6 +154,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -217,6 +228,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.6.20" @@ -405,6 +438,12 @@ dependencies = [ "objc2", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "bstr" version = "1.12.1" @@ -422,6 +461,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "byteorder" version = "1.5.0" @@ -479,6 +524,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -581,6 +628,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -606,6 +671,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "config" version = "0.14.1" @@ -897,6 +972,20 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -1070,12 +1159,27 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "encode_unicode" version = "1.0.0" @@ -1140,6 +1244,27 @@ dependencies = [ "regex", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless", + "serde", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -1203,6 +1328,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1215,6 +1351,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1239,6 +1381,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs2" version = "0.4.3" @@ -1249,6 +1401,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1392,9 +1550,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1411,162 +1571,1061 @@ dependencies = [ ] [[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +name = "gix" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0473c64d9ccbcfb9953a133b47c8b9a335b87ac6c52b983ee4b03d49000b0f3f" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-blame", + "gix-commitgraph", + "gix-config", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-index", + "gix-lock", + "gix-merge", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "smallvec", + "thiserror 2.0.18", +] [[package]] -name = "h2" -version = "0.3.27" +name = "gix-actor" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +checksum = "0e5e5b518339d5e6718af108fd064d4e9ba33caf728cf487352873d76411df35" dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", + "bstr", + "gix-date", + "gix-error", + "winnow", ] [[package]] -name = "h2" -version = "0.4.14" +name = "gix-archive" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "651c99be11aac9b303483193ae50b45eb6e094da4f5ed797019b03948f51aad6" dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.4.0", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", + "bstr", + "gix-date", + "gix-error", + "gix-object", + "gix-worktree-stream", ] [[package]] -name = "half" -version = "2.7.1" +name = "gix-attributes" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +checksum = "c233d6eaa098c0ca5ce03236fd7a96e27f1abe72fad74b46003fbd11fe49563c" dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", ] [[package]] -name = "hashbrown" -version = "0.14.5" +name = "gix-bitmap" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "1ecbfc77ec6852294e341ecc305a490b59f2813e6ca42d79efda5099dcab1894" dependencies = [ - "ahash", - "allocator-api2", + "gix-error", ] [[package]] -name = "hashbrown" -version = "0.15.5" +name = "gix-blame" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "c77aaf9f7348f4da3ebfbfbbc35fa0d07155d98377856198dde6f695fd648705" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf288be9b60fe7231de03771faa292be1493d84786f68727e33ad1f91764320" dependencies = [ - "foldhash", + "gix-error", ] [[package]] -name = "hashbrown" -version = "0.17.1" +name = "gix-command" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "ae4bb9fa74c44c93f7238b08255f7f9afc158bafea4b95af665fa535352cd73c" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] [[package]] -name = "hashlink" -version = "0.8.4" +name = "gix-commitgraph" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +checksum = "3196655fd1443f3c58a48c114aa480be3e4e87b393d7292daaa0d543862eb445" dependencies = [ - "hashbrown 0.14.5", + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", ] [[package]] -name = "heck" -version = "0.5.0" +name = "gix-config" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "08939b4c4ed7a663d0e64be9e1e9bdf23a1fb4fcee1febdf449f12229542e50d" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "memchr", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", + "winnow", +] [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "gix-config-value" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "4378c53ec3db049919edf91ff76f56f28886a8b4b4a5a9dc633108d84afc3675" dependencies = [ + "bitflags 2.11.1", + "bstr", + "gix-path", "libc", + "thiserror 2.0.18", ] [[package]] -name = "hermit-abi" -version = "0.5.2" +name = "gix-date" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "b94cdae4eb4b0f4136e3d9b3aa2d2cd03cfb5bb9b636b31263aea2df86d41543" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", + "smallvec", +] [[package]] -name = "hex" -version = "0.4.3" +name = "gix-diff" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "88f3b3475e5d3877d7c30c40827cc2441936ce890efc226e5ba4afe3a7ae33f0" +dependencies = [ + "bstr", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "imara-diff 0.1.8", + "imara-diff 0.2.0", + "thiserror 2.0.18", +] [[package]] -name = "home" -version = "0.5.12" +name = "gix-dir" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +checksum = "5da4604a360988f0ba8efe6f90093ca5a844f4a7f8e1a3dcda501ec44e600ea9" dependencies = [ - "windows-sys 0.61.2", + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", ] [[package]] -name = "host-proxy" -version = "0.2.8" +name = "gix-discover" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65bd3330fe0cb9d40d875bf862fd5e8ad6fa4164ddbc4842fbeb889c3f0b2c6" dependencies = [ - "agent-api-client", - "agent-api-core", - "agent-api-types", - "anyhow", - "async-trait", - "axum 0.7.9", - "chrono", - "futures", - "http-body-util", - "hyper 1.9.0", - "hyper-util", - "hyperlocal 0.9.1", - "serde", - "serde_json", - "tempfile", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tower 0.5.3", - "tower-http 0.5.2", - "tracing", - "tracing-subscriber", - "uuid", + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", ] [[package]] -name = "hostname" -version = "0.3.1" +name = "gix-error" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e207b971746ab724fccdfced2e4e19e854744611904a0195d3aa8fda8a110613" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.46.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "752493cd4b1d5eaaa0138a7493f65c96863fefa990fc021e0e519579e389ab20" +dependencies = [ + "bytes", + "crc32fast", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37598282a6566da6fb52667570c7fe0aedcb122ac886724a9e62a2180523e35" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a964b4aec683eb0bacb87533defa80805bb4768056371a47ab38b00a2d377b72" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e6cd88cc0dc1eafa1fddac0fb719e4e74b6ea58dd016e71125fde4a326bee" +dependencies = [ + "bitflags 2.11.1", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fb896a02d9ab96fa518475a5f30ad3952010f801a8de5840f633f4a6b985dfb" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2664216fc5e89b51e756a4a3ac676315602ce2dac07acf1da959a22038d69b33" +dependencies = [ + "gix-hash", + "hashbrown 0.16.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f915dcf6911e3027537166d34e13f0fe101ed12225178d2ae29cd1272cff26" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-index" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bae54ab14e4e74d5dda60b82ea7afad7c8eb3be68283d6d5f29bd2e6d47fff7" +dependencies = [ + "bitflags 2.11.1", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.16.1", + "itoa", + "libc", + "memmap2", + "rustix 1.1.4", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "21.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054fbd0989700c69dc5aa80bc66944f05df1e15aa7391a9e42aca7366337905f" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-merge" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4606747466512d22c2dffc019142e1941238f543987ea51353c938cca80c500" +dependencies = [ + "bstr", + "gix-command", + "gix-diff", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-quote", + "gix-revision", + "gix-revwalk", + "gix-tempfile", + "gix-trace", + "gix-worktree", + "imara-diff 0.1.8", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-negotiate" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea064c7595eea08fdd01c70748af747d9acc40f727b61f4c8a2145a5c5fc28c" +dependencies = [ + "bitflags 2.11.1", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cafb802bb688a7c1e69ef965612ff5ff859f046bfb616377e4a0ba4c01e43d47" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-path", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", + "winnow", +] + +[[package]] +name = "gix-odb" +version = "0.78.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24833ae9323b4f7079575fb9f961cf9c414b0afbec428a536ab8e7dd93bc002b" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.68.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3484119cd19859d7d7639413c27e192478fa354d3f4ff5f7e3c041e8040f0f4" +dependencies = [ + "clru", + "gix-chunk", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "memmap2", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-packetline" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "362246df440ee691699f0664cbf7006a6ece477db6734222be95e4198e5656e6" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fd1fe596dc393b538e1d5492c5585971a9311475b3255f7b889023df208476" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89611f13544ca5ebeb68a502673814ef57200df60c24a61c2ce7b96f612f08b" +dependencies = [ + "bitflags 2.11.1", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f38666350736b5877c79f57ddae02bde07a4ce186d889adc391e831cddcbe76" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", + "winnow", +] + +[[package]] +name = "gix-quote" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e97b73791a64bc0fa7dd2c5b3e551136115f97750b876ed1c952c7a7dbaf8be" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2159978abb99b7027c8579d15211e262ef0ef2594d5cecb3334fbcbdfe2997c" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", + "winnow", +] + +[[package]] +name = "gix-refspec" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc806ee13f437428f8a1ba4c72ecfaa3f20e14f5f0d4c2bc17d0b33e794aa6ac" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c08f1ec5d1e6a524f8ba291c41f0ccaef64e48ed0e8cf790b3461cae45f6d3d" +dependencies = [ + "bitflags 2.11.1", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4b2b87772b21ca449249e86d32febadba5cba32b0fcce804ab9cefc6f2111c" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "283f4a746c9bde8550be63e6f961ff4651f412ca12666e8f5615f39464960ab9" +dependencies = [ + "bitflags 2.11.1", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf60711c9083b2364b3fac8a352444af76b17201f3682fdebe74fa66d89a772" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d6c598e3fdbc352fba1c5ba7e709e69402fafbc44d9295edad2e3c4738996b" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5c3929c5e6821f651d35e8420f72fea3cfafe9fc1e928a61e718b462c72a5" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "21.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22227f6b203f511ff451c33c89899e87e4f571fc596b06f68e6e613a6508528" +dependencies = [ + "dashmap 6.2.1", + "gix-fs", + "libc", + "parking_lot", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f23569e55f2ffaf958617353b9734a7d52a7c19c439eeaa5e3efc217fd2270e" + +[[package]] +name = "gix-transport" +version = "0.55.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a521e39c6235ce63ed6c001e2dd79818c830b82c3b7b59247ee7b229c39ec9bb" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963dc2afcdb611092aa587c3f9365e749ac0a0892ff27662dbc75f26c953fbec" +dependencies = [ + "bitflags 2.11.1", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.35.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a61ead12e33fa52ae92b207ee27554f646a8e7a3dad8b78da1582ec91eda0a6" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e477b4f07a6e8da4ba791c53c858102959703c60d70f199932010d5b94adb2c" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e26ac2602b43eadfdca0560b81d3341944162a3c9f64ccdeef8fc501ad80dad5" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6bd5830cbc43c9c00918b826467d2afad685b195cb82329cde2b2d116d2c578" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644a1681f96e1be43c2a8384337d9d220e7624f50db54beda70997052aebf707" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e3fb70a1f650a5cec7d5b8d10d6d6fe86daf3cf15bde08ba0c70988a2932c3" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "host-proxy" +version = "0.2.8" +dependencies = [ + "agent-api-client", + "agent-api-core", + "agent-api-types", + "anyhow", + "async-trait", + "axum 0.7.9", + "chrono", + "futures", + "http-body-util", + "hyper 1.9.0", + "hyper-util", + "hyperlocal 0.9.1", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tower 0.5.3", + "tower-http 0.5.2", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "hostname" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ @@ -1708,6 +2767,21 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http 1.4.0", + "hyper 1.9.0", + "hyper-util", + "rustls 0.23.40", + "tokio", + "tokio-rustls 0.26.4", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -1908,6 +2982,25 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "imara-diff" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17d34b7d42178945f775e84bc4c36dde7c1c6cdfea656d3354d009056f2bb3d2" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "imara-diff" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f01d462f766df78ab820dd06f5eb700233c51f0f4c2e846520eaf4ba6aa5c5c" +dependencies = [ + "hashbrown 0.15.5", + "memchr", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1962,6 +3055,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "ioctl-rs" version = "0.1.6" @@ -2025,10 +3128,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-sys 0.61.2", ] [[package]] @@ -2042,6 +3147,80 @@ dependencies = [ "syn", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.98" @@ -2065,6 +3244,35 @@ dependencies = [ "serde", ] +[[package]] +name = "jsonschema" +version = "0.46.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5fe5206f06e589caf25e79fc05ccdf91fca745685fe9fe1a13bbdfb479a631" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex 0.18.0", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "regex-syntax", + "reqwest 0.13.3", + "rustls 0.23.40", + "serde", + "serde_json", + "unicode-general-category", + "uuid-simd", +] + [[package]] name = "kqueue" version = "1.1.1" @@ -2085,6 +3293,15 @@ dependencies = [ "libc", ] +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2172,12 +3389,32 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -2196,6 +3433,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + [[package]] name = "mime" version = "0.3.17" @@ -2353,6 +3596,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -2396,12 +3645,82 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2533,6 +3852,12 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "parking_lot" version = "0.12.5" @@ -2827,6 +4152,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "parking_lot", +] + [[package]] name = "proptest" version = "1.11.0" @@ -3050,6 +4384,43 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "referencing" +version = "0.46.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4e17ef386c5383591d07623d3de49cbc601156e7582973e6db98d66a57de2" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.12.3" @@ -3103,7 +4474,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", - "hyper-rustls", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -3168,6 +4539,45 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.14", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-rustls 0.27.9", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls 0.23.40", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper 1.0.2", + "tokio", + "tokio-rustls 0.26.4", + "tower 0.5.3", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -3277,6 +4687,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -3286,6 +4697,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -3304,6 +4727,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.40", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.13", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3331,6 +4781,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3360,6 +4811,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" + [[package]] name = "same-file" version = "1.0.6" @@ -3481,6 +4938,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_jcs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a60f3fda61525e439ef6d67422118f11e986566997d9021c56867ad814a0aa" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "serde_json" version = "1.0.149" @@ -3630,6 +5098,16 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest", + "sha1", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3781,6 +5259,22 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" @@ -3825,6 +5319,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strip-ansi-escapes" version = "0.2.1" @@ -3945,7 +5445,7 @@ dependencies = [ "chrono", "clap", "config", - "dashmap", + "dashmap 5.5.3", "dirs 5.0.1", "futures", "mockito", @@ -3993,6 +5493,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "substrate-lift" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "clap", + "gix", + "globset", + "jsonschema", + "predicates", + "serde", + "serde_jcs", + "serde_json", + "sha2", + "thiserror 1.0.69", + "toml", + "walkdir", +] + [[package]] name = "substrate-replay" version = "0.1.0" @@ -4248,7 +5767,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "bstr", - "fancy-regex", + "fancy-regex 0.12.0", "lazy_static", "parking_lot", "rustc-hash", @@ -4314,6 +5833,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -4373,6 +5907,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls 0.23.40", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -4734,12 +6278,33 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.2" @@ -4877,6 +6442,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.1" @@ -4906,6 +6481,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.14.1" @@ -5079,6 +6660,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -5888,6 +7478,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 4753b3ff2..e323a0773 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "crates/shim", "crates/shell", "crates/gateway", + "crates/lift", # New Phase 4 crates "crates/world-api", "crates/world", diff --git a/crates/gateway/src/providers/streaming.rs b/crates/gateway/src/providers/streaming.rs index 35b66312f..20edcfb93 100644 --- a/crates/gateway/src/providers/streaming.rs +++ b/crates/gateway/src/providers/streaming.rs @@ -236,11 +236,11 @@ where } *this.logged_message_start = true; } - Some("content_block_delta") => { + Some("content_block_delta") // Mark first token arrival - if this.first_token_time.is_none() { - *this.first_token_time = Some(std::time::Instant::now()); - } + if this.first_token_time.is_none() => + { + *this.first_token_time = Some(std::time::Instant::now()); } Some("message_delta") => { // Track tokens (output_tokens always, input_tokens for OpenAI providers) @@ -296,11 +296,9 @@ where // Build cache info string if caching was used let cache_info = if *this.cache_creation > 0 || *this.cache_read > 0 { - let cache_pct = if total_input > 0 { - (*this.cache_read * 100) / total_input - } else { - 0 - }; + let cache_pct = (*this.cache_read * 100) + .checked_div(total_input) + .unwrap_or(0); format!(" cache:{}%", cache_pct) } else { String::new() diff --git a/crates/gateway/src/server/openai_compat.rs b/crates/gateway/src/server/openai_compat.rs index f9b80178c..b03f5b313 100644 --- a/crates/gateway/src/server/openai_compat.rs +++ b/crates/gateway/src/server/openai_compat.rs @@ -316,10 +316,8 @@ pub fn transform_gateway_response_to_openai( for block in &anthropic_resp.content { match block { - ContentBlock::Known(KnownContentBlock::Text { text, .. }) => { - if !text.is_empty() { - text_parts.push(text.clone()); - } + ContentBlock::Known(KnownContentBlock::Text { text, .. }) if !text.is_empty() => { + text_parts.push(text.clone()); } ContentBlock::Known(KnownContentBlock::ToolUse { id, name, input }) => { let arguments = serde_json::to_string(input).unwrap_or_else(|_| "{}".to_string()); diff --git a/crates/gateway/src/server/openai_responses.rs b/crates/gateway/src/server/openai_responses.rs index 342283e97..41f128996 100644 --- a/crates/gateway/src/server/openai_responses.rs +++ b/crates/gateway/src/server/openai_responses.rs @@ -476,14 +476,12 @@ fn transform_gateway_response_to_openai_response( for block in &gateway_response.content { match block { - ContentBlock::Known(KnownContentBlock::Text { text, .. }) => { - if !text.is_empty() { - pending_text_parts.push(OpenAIResponsesOutputContentPart { - r#type: "output_text".to_string(), - text: text.clone(), - annotations: Vec::new(), - }); - } + ContentBlock::Known(KnownContentBlock::Text { text, .. }) if !text.is_empty() => { + pending_text_parts.push(OpenAIResponsesOutputContentPart { + r#type: "output_text".to_string(), + text: text.clone(), + annotations: Vec::new(), + }); } ContentBlock::Known(KnownContentBlock::ToolUse { id, name, input }) => { flush_pending_text(&mut output, &mut pending_text_parts); diff --git a/crates/lift/Cargo.toml b/crates/lift/Cargo.toml new file mode 100644 index 000000000..0036fe77d --- /dev/null +++ b/crates/lift/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "substrate-lift" +version = "0.1.0" +edition = "2021" +rust-version = "1.89" +description = "Deterministic code-intelligence engine and CLI scaffold" +license = "MIT" +repository = "https://github.com/atomize-hq/substrate" +readme = "README.md" +publish = false +keywords = ["lift", "cli", "code-intelligence"] +categories = ["command-line-utilities", "development-tools"] + +[lib] +name = "substrate_lift" +path = "src/lib.rs" + +[[bin]] +name = "lift" +path = "src/bin/lift.rs" +required-features = ["cli"] +doc = false + +[features] +default = ["cli"] +cli = ["dep:clap"] +compat-v1 = [] +config-lang = [] +rust-lang = [] +python-lang = [] +javascript-lang = [] +typescript-lang = [] +substrate-profile = [] + +[dependencies] +clap = { version = "4.5", features = ["derive"], optional = true } +gix = { version = "0.81.0", default-features = false, features = ["revision", "sha1"] } +globset = "0.4" +jsonschema = "0.46.0" +serde = { version = "1", features = ["derive"] } +serde_jcs = "0.2.0" +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = "1" +toml = "0.8" +walkdir = "2" + +[dev-dependencies] +assert_cmd = "2" +predicates = "3" + +[lints.rust] +unsafe_code = "forbid" +unreachable_pub = "warn" +unused_crate_dependencies = "warn" + +[lints.clippy] +dbg_macro = "warn" +print_stdout = "allow" +print_stderr = "allow" +todo = "warn" +unwrap_used = "warn" diff --git a/crates/lift/README.md b/crates/lift/README.md new file mode 100644 index 000000000..17aa2d6af --- /dev/null +++ b/crates/lift/README.md @@ -0,0 +1,507 @@ +# substrate-lift + +`substrate-lift` is being reshaped around a deterministic code-intelligence engine. + +Before, `lift score` was the center and everything else existed to support it. +Now the center is the engine, and `lift score` is one app on top of it. + +The biggest structural changes are: + +1. The old top-level pure scorer seam moves under `app::score`. +2. Generic seams for `query`, `patch/rewrite`, and `export/index` become first-class. +3. The old generic resolve seam splits into: + - generic fact derivation and provenance + - app-specific materializers like score, contract diff, and context pack +4. Legacy compatibility becomes an edge seam, not a core seam. + +The clean mental model is: + +- Engine seams stop at snapshots, parsed units, graph, topology, matches, facts, derived facts, patches, and exports. +- App seams turn those engine artifacts into user-facing products like score, impact, policy findings, contract diff, context packs, and rewrites. + +## Updated top-level shape + +```mermaid +flowchart TB + subgraph Foundation + K[0. Kernel contracts] + P[1. Pack compiler] + end + + subgraph Engine + R[2. Repo substrate] + L[3. Language platform] + A[4. Concrete adapters] + G[5. Graph and scope] + T[6. Topology and classification] + Q[7. Query and match] + F[8. Facts and detectors] + D[9. Derive and provenance] + W[10. Patch and rewrite planning] + X[11. Export and index] + end + + subgraph Apps + RT[12. App runtime] + S[13a. Score] + I[13b. Impact] + O[13c. Policy] + C[13d. Contract] + CX[13e. Context and Index] + QR[13f. Query and Rewrite] + end + + CLI[14. CLI] + PF[15. Bundled profiles and migration shims] + FX[16. Fixtures, goldens, perf] + + K --> P + K --> R + K --> L + L --> A + R --> G + A --> G + P --> T + R --> T + G --> Q + Q --> F + T --> F + P --> F + F --> D + P --> D + Q --> W + D --> X + G --> X + T --> X + + RT --> S + RT --> I + RT --> O + RT --> C + RT --> CX + RT --> QR + + D --> S + G --> I + D --> O + D --> C + X --> CX + Q --> QR + W --> QR + + RT --> CLI + P --> PF + + FX --> K + FX --> P + FX --> R + FX --> A + FX --> G + FX --> Q + FX --> F + FX --> D + FX --> W + FX --> X + FX --> RT +``` + +## Updated module layout + +```text +crates/lift/ + src/ + lib.rs + bin/lift.rs + + kernel/ + pack/ + repo/ + lang/ + graph/ + topo/ + query/ + facts/ + derive/ + patch/ + export/ + + app/ + mod.rs + runtime.rs + + score/ + mod.rs + vector.rs + materialize.rs + model.rs + scorer.rs + compat_v1.rs + + impact/ + policy/ + contract/ + context/ + index/ + query_app/ + rewrite/ + + cli/ + + schemas/ + profiles/ + recipes/ + rules/ + fixtures/ + examples/ +``` + +## Hard boundary rules + +These rules keep the seams clean. + +1. No app reads the repo directly. Only `repo` does that. +2. No app talks to a language parser directly. Only `lang` and `graph` do that. +3. Adapters never emit findings, scores, or patches. They only emit parsed units, symbols, and local edges. +4. Detectors never emit app results. They only emit facts and evidence. +5. Query never writes patches. Patch planning consumes matches; it does not own matching. +6. Export never computes new analysis. It only serializes already-built artifacts. +7. Compatibility shims live at the app edge, not in the engine core. +8. Packs compile once into immutable runtime objects. The engine consumes compiled packs, never raw config files. + +## Seam breakdown + +### 0. Kernel contracts + +Owns IDs, repo-relative paths, spans, fingerprints, diagnostics, severity, stable JSON helpers, version tags, canonical sort rules, and shared result/error types. + +Exposes `RepoPath`, `FileId`, `SymbolId`, `ComponentId`, `BoundaryId`, `Fingerprint`, `Diagnostic`, and `CanonicalJson`. + +Must not own repo walking, parsing, scoring, policy logic, or app behavior. + +Done when every cross-seam type comes from `kernel`, and no other seam defines duplicate nearly-the-same core types. + +### 1. Pack compiler + +Phase A landed the crate-private profile compiler foundation: builtin, file-backed, and inline profile sources; hand-authored common/profile schemas; deterministic normalization and fingerprints; typed pack refs and diagnostics; and the narrow builtin `builtin:generic/default`. + +Phase B extends that compiler-only surface with embedded boundary taxonomy and component map schemas, focused standalone compilation entrypoints for those two topology pack kinds, builtin topology packs named `builtin:generic/boundaries` and `builtin:generic/components`, and profile topology resolution into a crate-internal `ResolvedProfileTopology`. + +Phase C extends that same explicit compiler pattern to structural `score_model`, `query_pack`, `rule_pack`, and `recipe_pack` inputs and adds deterministic profile bundle resolution into a crate-internal `CompiledPackSet`. + +Exposes `CompiledProfile` plus crate-internal resolved topology and bundle artifacts inside the crate. + +Must not own repo analysis, AST parsing, git access, or app orchestration. + +Done when one standalone profile, topology pack, score model, query pack, rule pack, or recipe pack can be loaded and compiled deterministically from the supported source types, and a compiled profile can resolve its selected pack bundle without requiring a repo snapshot. + +### 2. Repo substrate + +Phase A landed this seam as a crate-private, filesystem-first immutable snapshot substrate. Phase B landed pure path-based diffing over already-materialized `RepoSnapshot` values. Phase C extends the same seam with explicit `GitRev { rev }` materialization, bounded `SymlinkPolicy::Follow`, and typed well-known excludes for the canonical cache/build/vendor directories. + +What is landed today is still narrower than the long-term seam sketch, but it now includes repo root detection from a starting path, immutable snapshot materialization from either the live worktree or an explicit git revision, intrinsic `.git` exclusion plus typed well-known excludes plus caller-supplied ignore globs, bounded symlink follow/skip policy handling, non-UTF8 and large-file policy handling, deterministic inventory assembly, blob storage, content hashing, stable file IDs, snapshot stats, repo diagnostics, an embedded snapshot-manifest schema for fixture contracts, and pure add/remove/modify diff assembly with deterministic diff fingerprints plus an embedded diff-manifest schema for fixture contracts. + +The current implementation is still internal-only. Everything under `src/repo/` is `pub(crate)`, snapshot sources are explicit (`worktree` or `git_rev`), and the seam still does not expose rename detection, public API promotion, or runtime wiring. + +Exposes crate-private `RepoRoot`, `SnapshotRequest`, `RepoSnapshot`, `Inventory`, `BlobStore`, `SnapshotStats`, `RepoDiff`, `DiffEntry`, `DiffKind`, `build_diff`, and repo diagnostics. + +Must not own language parsing, classification, findings, or app semantics. + +Done for Phase C when downstream crate seams can consume immutable snapshots through `RepoSnapshot`, request either worktree or git-revision materialization without fallback, rely on deterministic bounded symlink-follow semantics plus typed well-known excludes, and compute pure diffs through `build_diff` without rereading the worktree. Rename detection, public API promotion, and broader runtime integration remain later work. + +### 3. Language platform + +Phase A landed the crate-private language platform contracts: adapter traits, the single-adapter-per-language registry, normalized parsed-unit models, deterministic request and unit fingerprinting, and fixture-facing parse-manifest schema coverage. Phase B extends that same internal seam with normalized per-file parse caching, cache hit/miss accounting in `ParseStats`, parse-manifest v2/schema fixtures, and deterministic `built_in_registry()` bootstrap preparation. Phase C is metadata plumbing only ahead of seam 4 adapters: the registry now carries adapter capability metadata so query-engine requirements can handshake with registry selection, but cache keys, parse-manifest/schema shape, and runtime parse behavior do not widen here. Production language adapters, public API promotion, and runtime parse orchestration remain later seam work. + +Exposes `LanguageAdapter`, `LanguageRegistry`, `ParsedUnit`, `LocalSymbol`, and `LocalEdge`. + +Must not own repo policy, score math, detector logic, or query recipes. + +Done when a new language can be added by implementing the adapter trait without changing app code. + +### 4. Concrete adapters + +This should be parallelized immediately. + +- `4a. Config adapters`: JSON, TOML, YAML, schema-reference capture, and config-key extraction. +- `4b. Rust adapter`: modules, `use` edges, item symbols, public-surface markers, impl/type references, tests, and Cargo workspace metadata integration. +- `4c. Python adapter`: modules, imports, defs/classes, public module surface conventions, and test markers. +- `4d. JS/TS adapter`: imports/exports, functions/classes/interfaces, public-surface markers, and common test framework markers. + +Exposes parsed units, local symbol tables, local edges, and surface markers. + +Must not own repo-wide graph resolution, policy findings, or score behavior. + +Done when each adapter independently passes fixture repos and parse-failure cases. + +### 5. Graph and scope + +Owns graph assembly, symbol resolution, cross-file edges, stable BFS/DFS policy, depth limits, closure policy, seed normalization, and path-to-symbol mapping. + +Exposes `RepoGraph`, `ResolvedScope`, and `ScopeResolver`. + +Must not own facts, score results, policy findings, or patch generation. + +Done when the same snapshot, seeds, and closure rules always produce the same sorted scope. + +### 6. Topology and classification + +This seam still eventually owns repo-facing component mapping, boundary assignment, public-surface classes, docs/test/ci/migration/security path classes, overlap validation, and component counting rules. + +What has landed so far is narrower: the pack compiler can now compile boundary taxonomy and component map documents and resolve a profile's topology refs into crate-internal compiled artifacts. Repo walking, boundary overlap detection against real inventories, and path classification remain deferred to later topology/classification work. + +The intended runtime-facing outputs for the later seam are `TopologyIndex`, `ComponentMap`, `BoundaryMap`, and `ClassifiedPaths`. + +Must not own AST parsing, score math, or app rendering. + +Done when a repo can be classified with snapshot plus compiled profile and no app logic. That runtime classification handoff is not landed in Phase B. + +### 7. Query and match + +Owns query DSL compilation, normalized structural matching, scoped query execution, capture sets, and match serialization. + +Exposes `QueryEngine`, `CompiledQuery`, `MatchSet`, and `Capture`. + +Must not own findings, score vectors, patches, or repo walking. + +Done when query packs and ad hoc queries both run deterministically over fixture repos. + +### 8. Facts and detectors + +This is the observation seam. + +Owns detector execution, fact emission, evidence emission, detector registration, fact dedupe, and detector diagnostics. + +Exposes `Detector`, `Fact`, `Evidence`, and `FactSet`. + +Must not own final app outputs, score math, patch edits, or CLI messages. + +Detector families: + +- touch +- contract surface +- risk/security +- architecture/boundary +- qa/docs/ops +- migration/backfill +- platform/cross-platform + +Done when detectors emit only facts and evidence, and no detector writes into a score vector, contract report, or context pack directly. + +### 9. Derive and provenance + +Owns generic derivation rules, fact normalization, provenance graphs, conflict resolution, unknown propagation, projection rules, and confidence causes. + +Exposes `DerivedFacts`, `ProvenanceGraph`, and `DeriveEngine`. + +Must not own Lift scoring math, contract-report formatting, or context serialization. + +Done when any derived fact can explain where it came from and what source facts or rules produced it. + +### 10. Patch and rewrite planning + +Owns recipe compilation, patch planning, edit conflict detection, preview hunks, patch application contracts, and formatter hook surfaces. + +Exposes `PatchPlan`, `PatchEdit`, `RewriteRecipe`, and `PatchPlanner`. + +Must not own matching, repo scanning, or score math. + +Phase split: + +- Phase 1: preview-only patch planning +- Phase 2: safe apply +- Phase 3: formatter integration + +Done when a rewrite recipe can turn a `MatchSet` into a deterministic preview plan without mutating the repo. + +### 11. Export and index + +Owns stable export schemas, graph export, topology export, fact export, match export, context-pack export, index export, and canonical serialization. + +Exposes `ExportService`, `ContextPackV1`, `RepoIndexV1`, and `GraphExportV1`. + +Must not own new analysis or user-facing business logic. + +Done when engine artifacts can be exported byte-stably from fixtures. + +### 12. App runtime + +Phase D lands the first runtime consumer boundary, but it is intentionally narrow. + +This seam currently owns the pack-activation boundary between the compiler and future app-facing execution: take a profile pack source, compile it, resolve its selected pack bundle, and hand back a bootstrap artifact that later app entrypoints can consume. + +Today that is bootstrap ingress only, not a full app container. It does not yet provide app registry, request dispatch, dependency injection, shared lifecycle hooks, or generic app result envelopes. + +The current crate-internal surface is `ProfileBootstrap` plus `bootstrap_profile(...)`, which wraps the resolved `CompiledPackSet` for later runtime consumers. + +Must not own score logic, contract semantics, query behavior, CLI parsing, or broader app orchestration. + +Done when every later app entrypoint can share the same pack-bootstrap handoff without forcing the runtime seam to become a dispatch layer. + +### 13. App seams + +This is where products live. + +#### 13a. Score app + +Owns Lift request/response types, vector materialization from facts and hints, pure score models, scorers, and compatibility output. + +Internal sub-seams: + +- `materialize`: facts to Lift vector +- `model`: score model and trigger definitions +- `scorer`: pure scoring math +- `compat_v1`: legacy output and migration helpers + +Must not own repo walking, AST parsing, or raw detector execution. + +Done when `lift score` works for vector, diff, and seed-based estimate modes. + +#### 13b. Impact app + +Owns blast radius summary, scope reports, downstream surfaces, affected components and boundaries, and review-routing data. + +Must not own score math or patch generation. + +Done when `lift impact` can explain what else is likely involved from paths, symbols, or diffs. + +#### 13c. Policy app + +Owns rule findings, severity assignment, architecture violations, compliance/security reports, and policy-focused summaries. + +Must not own scoring or rewrite plans. + +Done when `lift policy` can run packs over a scope or diff and emit findings with evidence. + +#### 13d. Contract app + +Owns public-surface extraction, before/after diff classification, additive vs breaking changes, and machine-readable contract deltas. + +Must not own general score logic. + +Done when `lift contract` can compare two revisions and emit a contract delta report. + +#### 13e. Context and index apps + +Context owns task-bounded packs for humans or agents, scope summaries, and selected files, symbols, and facts. + +Index owns repo-wide export jobs and reusable intelligence artifacts. + +Must not own graph building or serialization internals. + +Done when `lift context` and `lift index` can emit useful deterministic bundles from existing engine artifacts. + +#### 13f. Query and rewrite apps + +Query app owns user-facing structural search workflow, ad hoc query execution, and result rendering. + +Rewrite app owns recipe selection, preview, patch-plan rendering, and optional apply workflow later. + +Must not own parser implementation or patch planning internals. + +Done when `lift query` and `lift rewrite --preview` work without special app-only parsing code. + +### 14. CLI + +The binary is now an umbrella frontend. + +Owns `lift` subcommands, clap args, human output, JSON output, exit codes, and shell completion. + +Commands: + +- `lift score` +- `lift impact` +- `lift policy` +- `lift contract` +- `lift context` +- `lift index` +- `lift query` +- `lift rewrite` + +Must not own any business logic. + +Done when all meaningful behavior lives below the CLI. + +### 15. Bundled profiles and migration shims + +The embedded generic builtin compiler packs now include: + +- `builtin:generic/default` +- `builtin:generic/boundaries` +- `builtin:generic/components` + +These builtin names are crate-internal compiler artifacts. They are not a public API, not repo classification by themselves, and not runtime bootstrap. + +Phase D does add the first runtime consumer boundary: compiled profile selection can now cross into `app::runtime` bootstrap as a resolved pack bundle. That does not make the builtin names a public runtime registry, runtime dispatch surface, or proof that broader app orchestration already exists. + +Bundled Substrate profiles, starter rule/query/recipe content, and migration helpers from old Work Lift inputs are still intentionally deferred until later seams broaden the pack surface and add more concrete runtime consumers. + +Must not own core engine behavior. + +Done when the generic builtin remains a stable narrow foundation for later bundled-profile and migration work. + +### 16. Fixtures, goldens, and perf + +Owns tiny fixture repos, mixed-language repos, legacy score goldens, app goldens, determinism harnesses, parse-failure fixtures, perf benchmarks, and stress repos. + +Must not own runtime logic. + +Done when every seam above has fixture-backed tests and a determinism story. + +## What changed from the previous seam map + +The most important architectural changes from the score-first version are: + +- `compat::v1` is no longer a top-level seam. It moves into `app::score`. +- The old generic scoring core is no longer global. It becomes `app::score::scorer`. +- `query`, `patch`, and `export` are now permanent engine seams. +- Resolve is no longer one generic seam. The engine ends at derived facts; each app materializes its own output from those facts. +- The crate’s center of gravity becomes facts plus provenance, not the Lift score itself. + +That shift keeps the crate from becoming a score tool with extra side features. + +## Parallel work lanes + +```mermaid +flowchart LR + A[Lane A\nKernel + Packs] --> B[Lane B\nRepo + Topology] + A --> C[Lane C\nLanguage platform + Adapters] + B --> D[Lane D\nGraph + Query] + C --> D + D --> E[Lane E\nFacts + Derive] + A --> E + E --> F[Lane F\nApps] + D --> F + F --> G[Lane G\nCLI + Profiles] + H[Lane H\nFixtures + Goldens + Perf] --> A + H --> B + H --> C + H --> D + H --> E + H --> F + H --> G +``` + +## Suggested merge order + +1. `kernel` plus `pack` skeletons +2. `repo` substrate +3. `lang` platform plus config adapters +4. `graph` plus `topo` +5. `query` engine +6. `facts` plus `derive` +7. `app::score` +8. `app::impact` plus `app::policy` +9. `export` plus `app::context` plus `app::index` +10. `app::contract` +11. `patch` plus `app::rewrite` +12. `cli` +13. bundled profiles and migration shims +14. full golden and perf sweep + +That sequence still gets `lift score` early, but avoids designing the entire crate around score-only assumptions. + +## Design rule to protect hardest + +The engine should stop at facts, matches, patches, and exports. + +The moment a lower seam starts knowing what a Lift score, contract delta, or context pack is, the boundaries start collapsing again. Keeping apps above the engine is what gives `lift` room to grow into a genuine code-intelligence toolkit instead of another tightly coupled scoring pipeline. diff --git a/crates/lift/TODOS.md b/crates/lift/TODOS.md new file mode 100644 index 000000000..96a72dd30 --- /dev/null +++ b/crates/lift/TODOS.md @@ -0,0 +1,5 @@ +# lift TODOs + +- Track `QueryEngineKind` serde coupling in `src/lang/capabilities.rs`. + If a new query engine variant is added in `src/pack/compiled/query_pack.rs`, update the manual + serialize/deserialize mapping here and extend the capability coverage tests. diff --git a/crates/lift/examples/README.md b/crates/lift/examples/README.md new file mode 100644 index 000000000..d7f11b3b3 --- /dev/null +++ b/crates/lift/examples/README.md @@ -0,0 +1,5 @@ +# Examples + +This directory is reserved for user-facing example inputs, outputs, and adoption patterns for the `lift` toolchain. + +Examples are intentionally separate from fixtures so test assets and documentation assets can evolve independently. diff --git a/crates/lift/fixtures/README.md b/crates/lift/fixtures/README.md new file mode 100644 index 000000000..39dd3dd49 --- /dev/null +++ b/crates/lift/fixtures/README.md @@ -0,0 +1,21 @@ +# Fixtures + +This directory contains deterministic fixture inputs and golden artifacts for crate seams. + +Current live fixture coverage includes: + +- `fixtures/kernel/` for kernel schema, canonical JSON, and identity determinism coverage +- `fixtures/pack/` for landed pack validation and fingerprint determinism coverage +- `fixtures/repo/` for Phase A repo snapshot manifests and checked-in worktree fixture trees + +Current pack fixtures are organized by intent rather than by runtime feature: + +- `valid/` for standalone pack happy paths and file-backed bundle resolution cases +- `invalid/` for TOML/schema/reference rejection cases plus bundle-resolution edge cases +- `canonical/` for source and semantic fingerprint determinism cases across standalone packs and bundles + +Named pack fixtures now span the Phase C pack families as well as profiles, for example `valid/score/generic_lift_v2.json`, `valid/queries/rust_core.json`, `valid/rules/generic_policy.json`, `valid/recipes/generic_core_recipes.json`, and the `valid/bundle/` and `canonical/bundle_order_*` trees. + +The landed repo seam is narrower and internal-only: crate-private worktree snapshot materialization, deterministic inventory/blob assembly, fingerprinting, ignore-policy handling, snapshot diagnostics, and the snapshot-manifest schema contract used by `fixtures/repo/`. + +The current fixture tree should not be read as proof that git-backed diffs, boundary overlap detection, path classification, query execution, scoring, recipe application, or runtime bootstrap exists. The repo seam proves filesystem-first immutable snapshots only, and the pack seam proves structural compilation and deterministic bundle closure only. diff --git a/crates/lift/fixtures/kernel/canonical/expected.json b/crates/lift/fixtures/kernel/canonical/expected.json new file mode 100644 index 000000000..627d46487 --- /dev/null +++ b/crates/lift/fixtures/kernel/canonical/expected.json @@ -0,0 +1 @@ +{"a":2,"list":[3,1],"z":1} diff --git a/crates/lift/fixtures/kernel/canonical/expected_sha256.txt b/crates/lift/fixtures/kernel/canonical/expected_sha256.txt new file mode 100644 index 000000000..147ed3a99 --- /dev/null +++ b/crates/lift/fixtures/kernel/canonical/expected_sha256.txt @@ -0,0 +1 @@ +6ec5db9c612d797f3db871756ec6675b1885f7dc577328262c4dbf348a8a7c49 diff --git a/crates/lift/fixtures/kernel/canonical/object_a.json b/crates/lift/fixtures/kernel/canonical/object_a.json new file mode 100644 index 000000000..5ed21b134 --- /dev/null +++ b/crates/lift/fixtures/kernel/canonical/object_a.json @@ -0,0 +1,8 @@ +{ + "z": 1, + "a": 2, + "list": [ + 3, + 1 + ] +} diff --git a/crates/lift/fixtures/kernel/canonical/object_b.json b/crates/lift/fixtures/kernel/canonical/object_b.json new file mode 100644 index 000000000..d033efffc --- /dev/null +++ b/crates/lift/fixtures/kernel/canonical/object_b.json @@ -0,0 +1,8 @@ +{ + "list": [ + 3, + 1 + ], + "a": 2, + "z": 1 +} diff --git a/crates/lift/fixtures/kernel/invalid/byte_span_reversed.json b/crates/lift/fixtures/kernel/invalid/byte_span_reversed.json new file mode 100644 index 000000000..2c4c9602f --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/byte_span_reversed.json @@ -0,0 +1,4 @@ +{ + "start_byte": 9, + "end_byte": 4 +} diff --git a/crates/lift/fixtures/kernel/invalid/diagnostic_code_missing_namespace.json b/crates/lift/fixtures/kernel/invalid/diagnostic_code_missing_namespace.json new file mode 100644 index 000000000..86f04cfea --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/diagnostic_code_missing_namespace.json @@ -0,0 +1,5 @@ +{ + "code": "kernel", + "severity": "error", + "message": "missing namespace" +} diff --git a/crates/lift/fixtures/kernel/invalid/fingerprint_bad_hex.json b/crates/lift/fixtures/kernel/invalid/fingerprint_bad_hex.json new file mode 100644 index 000000000..cefbdabd5 --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/fingerprint_bad_hex.json @@ -0,0 +1 @@ +"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdeg" diff --git a/crates/lift/fixtures/kernel/invalid/fingerprint_bad_prefix.json b/crates/lift/fixtures/kernel/invalid/fingerprint_bad_prefix.json new file mode 100644 index 000000000..8a4669b9b --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/fingerprint_bad_prefix.json @@ -0,0 +1 @@ +"sha1:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" diff --git a/crates/lift/fixtures/kernel/invalid/json_pointer_malformed.json b/crates/lift/fixtures/kernel/invalid/json_pointer_malformed.json new file mode 100644 index 000000000..15169823f --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/json_pointer_malformed.json @@ -0,0 +1 @@ +"/touch/~2bad" diff --git a/crates/lift/fixtures/kernel/invalid/related_location_empty_message.json b/crates/lift/fixtures/kernel/invalid/related_location_empty_message.json new file mode 100644 index 000000000..2519f2308 --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/related_location_empty_message.json @@ -0,0 +1,6 @@ +{ + "locator": { + "path": "src/lib.rs" + }, + "message": "" +} diff --git a/crates/lift/fixtures/kernel/invalid/repo_path_absolute.json b/crates/lift/fixtures/kernel/invalid/repo_path_absolute.json new file mode 100644 index 000000000..6c31af845 --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/repo_path_absolute.json @@ -0,0 +1 @@ +"/src/lib.rs" diff --git a/crates/lift/fixtures/kernel/invalid/repo_path_backslash.json b/crates/lift/fixtures/kernel/invalid/repo_path_backslash.json new file mode 100644 index 000000000..232529b71 --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/repo_path_backslash.json @@ -0,0 +1 @@ +"src\\lib.rs" diff --git a/crates/lift/fixtures/kernel/invalid/repo_path_dot_segment.json b/crates/lift/fixtures/kernel/invalid/repo_path_dot_segment.json new file mode 100644 index 000000000..ed7868b1f --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/repo_path_dot_segment.json @@ -0,0 +1 @@ +"src/./lib.rs" diff --git a/crates/lift/fixtures/kernel/invalid/stable_id_bad_hex.json b/crates/lift/fixtures/kernel/invalid/stable_id_bad_hex.json new file mode 100644 index 000000000..97eb2f00e --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/stable_id_bad_hex.json @@ -0,0 +1 @@ +"file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdeg" diff --git a/crates/lift/fixtures/kernel/invalid/stable_id_bad_prefix.json b/crates/lift/fixtures/kernel/invalid/stable_id_bad_prefix.json new file mode 100644 index 000000000..c14bc85c6 --- /dev/null +++ b/crates/lift/fixtures/kernel/invalid/stable_id_bad_prefix.json @@ -0,0 +1 @@ +"file:sha1:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" diff --git a/crates/lift/fixtures/kernel/valid/byte_span.json b/crates/lift/fixtures/kernel/valid/byte_span.json new file mode 100644 index 000000000..34cf5fed9 --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/byte_span.json @@ -0,0 +1,4 @@ +{ + "start_byte": 2, + "end_byte": 7 +} diff --git a/crates/lift/fixtures/kernel/valid/diagnostic_full.json b/crates/lift/fixtures/kernel/valid/diagnostic_full.json new file mode 100644 index 000000000..935c210aa --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/diagnostic_full.json @@ -0,0 +1,22 @@ +{ + "code": "kernel.schema.fixture_example", + "severity": "warning", + "message": "example diagnostic", + "subject": { + "path": "src/lib.rs", + "span": { + "start_byte": 1, + "end_byte": 4 + } + }, + "related": [ + { + "locator": { + "path": "README.md", + "json_pointer": "/sections/0" + }, + "message": "related location" + } + ], + "help": "Update the fixture to match the kernel schema." +} diff --git a/crates/lift/fixtures/kernel/valid/diagnostic_minimal.json b/crates/lift/fixtures/kernel/valid/diagnostic_minimal.json new file mode 100644 index 000000000..7a127df94 --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/diagnostic_minimal.json @@ -0,0 +1,5 @@ +{ + "code": "kernel.repo_path.invalid_absolute", + "severity": "error", + "message": "absolute paths are not allowed" +} diff --git a/crates/lift/fixtures/kernel/valid/fingerprint.json b/crates/lift/fixtures/kernel/valid/fingerprint.json new file mode 100644 index 000000000..2d24ff73c --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/fingerprint.json @@ -0,0 +1 @@ +"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" diff --git a/crates/lift/fixtures/kernel/valid/json_pointer_nested.json b/crates/lift/fixtures/kernel/valid/json_pointer_nested.json new file mode 100644 index 000000000..f3209e1ca --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/json_pointer_nested.json @@ -0,0 +1 @@ +"/touch/crates_touched" diff --git a/crates/lift/fixtures/kernel/valid/json_pointer_root.json b/crates/lift/fixtures/kernel/valid/json_pointer_root.json new file mode 100644 index 000000000..e16c76dff --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/json_pointer_root.json @@ -0,0 +1 @@ +"" diff --git a/crates/lift/fixtures/kernel/valid/locator_path_only.json b/crates/lift/fixtures/kernel/valid/locator_path_only.json new file mode 100644 index 000000000..3f1cca2ed --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/locator_path_only.json @@ -0,0 +1,3 @@ +{ + "path": "src/lib.rs" +} diff --git a/crates/lift/fixtures/kernel/valid/locator_with_pointer.json b/crates/lift/fixtures/kernel/valid/locator_with_pointer.json new file mode 100644 index 000000000..dfb7a5c9d --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/locator_with_pointer.json @@ -0,0 +1,4 @@ +{ + "path": "src/lib.rs", + "json_pointer": "/touch/crates_touched" +} diff --git a/crates/lift/fixtures/kernel/valid/locator_with_span.json b/crates/lift/fixtures/kernel/valid/locator_with_span.json new file mode 100644 index 000000000..3b74e04df --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/locator_with_span.json @@ -0,0 +1,7 @@ +{ + "path": "src/lib.rs", + "span": { + "start_byte": 2, + "end_byte": 7 + } +} diff --git a/crates/lift/fixtures/kernel/valid/repo_path.json b/crates/lift/fixtures/kernel/valid/repo_path.json new file mode 100644 index 000000000..15f0d119f --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/repo_path.json @@ -0,0 +1 @@ +"src/lib.rs" diff --git a/crates/lift/fixtures/kernel/valid/stable_id.json b/crates/lift/fixtures/kernel/valid/stable_id.json new file mode 100644 index 000000000..c8e30a093 --- /dev/null +++ b/crates/lift/fixtures/kernel/valid/stable_id.json @@ -0,0 +1 @@ +"file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" diff --git a/crates/lift/fixtures/lang/README.md b/crates/lift/fixtures/lang/README.md new file mode 100644 index 000000000..897cbae80 --- /dev/null +++ b/crates/lift/fixtures/lang/README.md @@ -0,0 +1,10 @@ +# Lang fixtures + +These fixtures cover the landed crate-private Seam 3 parse-manifest contract across the Phase A v1 +and Phase B v2 manifest envelopes. + +- `valid/` contains manifest-shaped examples for both schema versions. The original v1 fixtures + stay unchanged, and the `_v2` fixtures add the Phase B `cache_hits` / `cache_misses` counters. +- `invalid/` contains schema failures only. Runtime-only invariants such as request fingerprints, unit fingerprints, span ordering, and deterministic sort order are checked in Rust tests instead of the JSON Schema. + +The manifest envelope is fixture-facing only. It adds `version` and `case` around the serialized `ParseSet` fields so fixture files can be named, validated, and evolved deterministically without making `lang` public. diff --git a/crates/lift/fixtures/lang/invalid/adapter_name_invalid.json b/crates/lift/fixtures/lang/invalid/adapter_name_invalid.json new file mode 100644 index 000000000..f7230b40f --- /dev/null +++ b/crates/lift/fixtures/lang/invalid/adapter_name_invalid.json @@ -0,0 +1,40 @@ +{ + "version": 1, + "case": "adapter_name_invalid", + "snapshot_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request": { + "languages": ["json"], + "scope": { + "kind": "snapshot" + } + }, + "units": [ + { + "path": "fixtures/app.fake.json", + "file_id": "file:sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "blob_fingerprint": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "language": "json", + "adapter": "Builtin.fake_config", + "adapter_version": "1.0.0", + "unit_fingerprint": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "symbols": [], + "edges": [], + "surface_markers": [], + "diagnostics": [] + } + ], + "failed": [], + "skipped": [], + "missing_languages": [], + "diagnostics": [], + "stats": { + "considered_files": 1, + "parsed_units": 1, + "failed_units": 0, + "skipped_no_adapter": 0, + "skipped_missing_paths": 0, + "missing_requested_languages": 0, + "diagnostic_count": 0 + } +} diff --git a/crates/lift/fixtures/lang/invalid/reference_target_invalid_shape.json b/crates/lift/fixtures/lang/invalid/reference_target_invalid_shape.json new file mode 100644 index 000000000..14aa231c3 --- /dev/null +++ b/crates/lift/fixtures/lang/invalid/reference_target_invalid_shape.json @@ -0,0 +1,50 @@ +{ + "version": 1, + "case": "reference_target_invalid_shape", + "snapshot_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request": { + "languages": ["json"], + "scope": { + "kind": "snapshot" + } + }, + "units": [ + { + "path": "fixtures/app.fake.json", + "file_id": "file:sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "blob_fingerprint": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "language": "json", + "adapter": "builtin.fake_config", + "adapter_version": "1.0.0", + "unit_fingerprint": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "symbols": [], + "edges": [ + { + "kind": "config_ref", + "source": "FileRoot", + "target": { + "QualifiedName": { + "parts": "settings.feature_flag" + } + } + } + ], + "surface_markers": [], + "diagnostics": [] + } + ], + "failed": [], + "skipped": [], + "missing_languages": [], + "diagnostics": [], + "stats": { + "considered_files": 1, + "parsed_units": 1, + "failed_units": 0, + "skipped_no_adapter": 0, + "skipped_missing_paths": 0, + "missing_requested_languages": 0, + "diagnostic_count": 0 + } +} diff --git a/crates/lift/fixtures/lang/invalid/request_scope_missing_paths.json b/crates/lift/fixtures/lang/invalid/request_scope_missing_paths.json new file mode 100644 index 000000000..b8b2e2f6f --- /dev/null +++ b/crates/lift/fixtures/lang/invalid/request_scope_missing_paths.json @@ -0,0 +1,26 @@ +{ + "version": 1, + "case": "request_scope_missing_paths", + "snapshot_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request": { + "languages": ["rust"], + "scope": { + "kind": "paths" + } + }, + "units": [], + "failed": [], + "skipped": [], + "missing_languages": [], + "diagnostics": [], + "stats": { + "considered_files": 0, + "parsed_units": 0, + "failed_units": 0, + "skipped_no_adapter": 0, + "skipped_missing_paths": 0, + "missing_requested_languages": 0, + "diagnostic_count": 0 + } +} diff --git a/crates/lift/fixtures/lang/invalid/top_level_unknown_field.json b/crates/lift/fixtures/lang/invalid/top_level_unknown_field.json new file mode 100644 index 000000000..8cf7f0863 --- /dev/null +++ b/crates/lift/fixtures/lang/invalid/top_level_unknown_field.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "case": "top_level_unknown_field", + "snapshot_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request": { + "languages": [], + "scope": { + "kind": "snapshot" + } + }, + "units": [], + "failed": [], + "skipped": [], + "missing_languages": [], + "diagnostics": [], + "stats": { + "considered_files": 0, + "parsed_units": 0, + "failed_units": 0, + "skipped_no_adapter": 0, + "skipped_missing_paths": 0, + "missing_requested_languages": 0, + "diagnostic_count": 0 + }, + "unexpected_root": true +} diff --git a/crates/lift/fixtures/lang/proof/toml_consumer_repo/malformed_repo/configs/hostile.toml b/crates/lift/fixtures/lang/proof/toml_consumer_repo/malformed_repo/configs/hostile.toml new file mode 100644 index 000000000..35c941a19 --- /dev/null +++ b/crates/lift/fixtures/lang/proof/toml_consumer_repo/malformed_repo/configs/hostile.toml @@ -0,0 +1,5 @@ +[runner +cmd = "${${${not_toml" + +[runner.env] +PATH = "oops" diff --git a/crates/lift/fixtures/lang/proof/toml_consumer_repo/repeat_run_repo/configs/alpha.toml b/crates/lift/fixtures/lang/proof/toml_consumer_repo/repeat_run_repo/configs/alpha.toml new file mode 100644 index 000000000..7cd7ac345 --- /dev/null +++ b/crates/lift/fixtures/lang/proof/toml_consumer_repo/repeat_run_repo/configs/alpha.toml @@ -0,0 +1,3 @@ +[alpha] +value = 1 +target = "config://shared.endpoint" diff --git a/crates/lift/fixtures/lang/proof/toml_consumer_repo/repeat_run_repo/configs/shared.toml b/crates/lift/fixtures/lang/proof/toml_consumer_repo/repeat_run_repo/configs/shared.toml new file mode 100644 index 000000000..b2f423239 --- /dev/null +++ b/crates/lift/fixtures/lang/proof/toml_consumer_repo/repeat_run_repo/configs/shared.toml @@ -0,0 +1,2 @@ +[shared] +endpoint = "https://example.test" diff --git a/crates/lift/fixtures/lang/proof/toml_consumer_repo/valid_repo/configs/app.toml b/crates/lift/fixtures/lang/proof/toml_consumer_repo/valid_repo/configs/app.toml new file mode 100644 index 000000000..d81a5d23d --- /dev/null +++ b/crates/lift/fixtures/lang/proof/toml_consumer_repo/valid_repo/configs/app.toml @@ -0,0 +1,9 @@ +name = "toml-proof-app" + +[server] +port = 8080 +profile = "config://profiles.primary.database_url" + +[profiles.primary] +database_url = "postgres://localhost/app" +max_connections = 16 diff --git a/crates/lift/fixtures/lang/proof/toml_consumer_repo/valid_repo/configs/feature_flags.toml b/crates/lift/fixtures/lang/proof/toml_consumer_repo/valid_repo/configs/feature_flags.toml new file mode 100644 index 000000000..312a62bea --- /dev/null +++ b/crates/lift/fixtures/lang/proof/toml_consumer_repo/valid_repo/configs/feature_flags.toml @@ -0,0 +1,3 @@ +[flags] +enable_search = true +enable_metrics = false diff --git a/crates/lift/fixtures/lang/valid/paths_scope_parse_manifest.json b/crates/lift/fixtures/lang/valid/paths_scope_parse_manifest.json new file mode 100644 index 000000000..877a0eea3 --- /dev/null +++ b/crates/lift/fixtures/lang/valid/paths_scope_parse_manifest.json @@ -0,0 +1,44 @@ +{ + "version": 1, + "case": "paths_scope_parse_manifest", + "snapshot_fingerprint": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "request_fingerprint": "sha256:ddd3fb14a72282b49bf2c8f8db910e710d70141c609945424290d413fd922199", + "request": { + "languages": ["rust"], + "scope": { + "kind": "paths", + "paths": ["src/lib.rs", "src/missing.rs"] + } + }, + "units": [], + "failed": [], + "skipped": [ + { + "path": "src/missing.rs", + "reason": "path_not_in_snapshot", + "detail": "requested path not present in snapshot" + } + ], + "missing_languages": [ + { + "language": "rust", + "detail": "no adapter registered" + } + ], + "diagnostics": [ + { + "code": "lang.fixture.paths_scope", + "severity": "warning", + "message": "explicit path scope requested one missing file" + } + ], + "stats": { + "considered_files": 0, + "parsed_units": 0, + "failed_units": 0, + "skipped_no_adapter": 0, + "skipped_missing_paths": 1, + "missing_requested_languages": 1, + "diagnostic_count": 1 + } +} diff --git a/crates/lift/fixtures/lang/valid/paths_scope_parse_manifest_v2.json b/crates/lift/fixtures/lang/valid/paths_scope_parse_manifest_v2.json new file mode 100644 index 000000000..4846bacf8 --- /dev/null +++ b/crates/lift/fixtures/lang/valid/paths_scope_parse_manifest_v2.json @@ -0,0 +1,46 @@ +{ + "version": 2, + "case": "paths_scope_parse_manifest_v2", + "snapshot_fingerprint": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "request_fingerprint": "sha256:ddd3fb14a72282b49bf2c8f8db910e710d70141c609945424290d413fd922199", + "request": { + "languages": ["rust"], + "scope": { + "kind": "paths", + "paths": ["src/lib.rs", "src/missing.rs"] + } + }, + "units": [], + "failed": [], + "skipped": [ + { + "path": "src/missing.rs", + "reason": "path_not_in_snapshot", + "detail": "requested path not present in snapshot" + } + ], + "missing_languages": [ + { + "language": "rust", + "detail": "no adapter registered" + } + ], + "diagnostics": [ + { + "code": "lang.fixture.paths_scope", + "severity": "warning", + "message": "explicit path scope requested one missing file" + } + ], + "stats": { + "considered_files": 0, + "parsed_units": 0, + "failed_units": 0, + "skipped_no_adapter": 0, + "skipped_missing_paths": 1, + "missing_requested_languages": 1, + "diagnostic_count": 1, + "cache_hits": 0, + "cache_misses": 0 + } +} diff --git a/crates/lift/fixtures/lang/valid/snapshot_parse_manifest.json b/crates/lift/fixtures/lang/valid/snapshot_parse_manifest.json new file mode 100644 index 000000000..f165d6deb --- /dev/null +++ b/crates/lift/fixtures/lang/valid/snapshot_parse_manifest.json @@ -0,0 +1,125 @@ +{ + "version": 1, + "case": "snapshot_parse_manifest", + "snapshot_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request_fingerprint": "sha256:66244b31776a4ca368e26dc0283239126109987cd51c8ee1848f9f1f61d5c6a0", + "request": { + "languages": ["json", "rust"], + "scope": { + "kind": "snapshot" + } + }, + "units": [ + { + "path": "fixtures/app.fake.json", + "file_id": "file:sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "blob_fingerprint": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "language": "json", + "adapter": "builtin.fake_config", + "adapter_version": "1.0.0", + "unit_fingerprint": "sha256:6b0f6632577c6941c9c292edb8f8573250ffaff67eb1160c30143bfe14a787a6", + "symbols": [ + { + "id": "symbol:sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "kind": "config_key", + "name": "feature_flag", + "path": ["settings", "feature_flag"], + "span": { + "start_byte": 3, + "end_byte": 20 + }, + "visibility": "public" + } + ], + "edges": [ + { + "kind": "config_ref", + "source": "FileRoot", + "target": { + "QualifiedName": { + "parts": ["settings", "feature_flag"] + } + }, + "span": { + "start_byte": 3, + "end_byte": 20 + } + } + ], + "surface_markers": [ + { + "kind": "entry_point", + "symbol": "symbol:sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "span": { + "start_byte": 3, + "end_byte": 20 + }, + "label": "root-config" + } + ], + "diagnostics": [ + { + "code": "lang.fixture.unit_warning", + "severity": "warning", + "message": "fixture unit carries one warning", + "subject": { + "path": "fixtures/app.fake.json", + "span": { + "start_byte": 3, + "end_byte": 20 + } + } + } + ] + } + ], + "failed": [ + { + "path": "fixtures/bad.fake.json", + "file_id": "file:sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "blob_fingerprint": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "language": "json", + "adapter": "builtin.fake_config", + "adapter_version": "1.0.0", + "diagnostics": [ + { + "code": "lang.fixture.parse_failed", + "severity": "error", + "message": "fixture file failed to parse", + "subject": { + "path": "fixtures/bad.fake.json" + } + } + ] + } + ], + "skipped": [ + { + "path": "fixtures/notes.txt", + "reason": "no_matching_adapter", + "detail": "no registered adapter recognized this file" + } + ], + "missing_languages": [ + { + "language": "rust", + "detail": "no adapter registered" + } + ], + "diagnostics": [ + { + "code": "lang.fixture.missing_language", + "severity": "warning", + "message": "requested language rust has no registered adapter" + } + ], + "stats": { + "considered_files": 3, + "parsed_units": 1, + "failed_units": 1, + "skipped_no_adapter": 1, + "skipped_missing_paths": 0, + "missing_requested_languages": 1, + "diagnostic_count": 3 + } +} diff --git a/crates/lift/fixtures/lang/valid/snapshot_parse_manifest_v2.json b/crates/lift/fixtures/lang/valid/snapshot_parse_manifest_v2.json new file mode 100644 index 000000000..1a46823e9 --- /dev/null +++ b/crates/lift/fixtures/lang/valid/snapshot_parse_manifest_v2.json @@ -0,0 +1,127 @@ +{ + "version": 2, + "case": "snapshot_parse_manifest_v2", + "snapshot_fingerprint": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "request_fingerprint": "sha256:66244b31776a4ca368e26dc0283239126109987cd51c8ee1848f9f1f61d5c6a0", + "request": { + "languages": ["json", "rust"], + "scope": { + "kind": "snapshot" + } + }, + "units": [ + { + "path": "fixtures/app.fake.json", + "file_id": "file:sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "blob_fingerprint": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "language": "json", + "adapter": "builtin.fake_config", + "adapter_version": "1.0.0", + "unit_fingerprint": "sha256:6b0f6632577c6941c9c292edb8f8573250ffaff67eb1160c30143bfe14a787a6", + "symbols": [ + { + "id": "symbol:sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "kind": "config_key", + "name": "feature_flag", + "path": ["settings", "feature_flag"], + "span": { + "start_byte": 3, + "end_byte": 20 + }, + "visibility": "public" + } + ], + "edges": [ + { + "kind": "config_ref", + "source": "FileRoot", + "target": { + "QualifiedName": { + "parts": ["settings", "feature_flag"] + } + }, + "span": { + "start_byte": 3, + "end_byte": 20 + } + } + ], + "surface_markers": [ + { + "kind": "entry_point", + "symbol": "symbol:sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "span": { + "start_byte": 3, + "end_byte": 20 + }, + "label": "root-config" + } + ], + "diagnostics": [ + { + "code": "lang.fixture.unit_warning", + "severity": "warning", + "message": "fixture unit carries one warning", + "subject": { + "path": "fixtures/app.fake.json", + "span": { + "start_byte": 3, + "end_byte": 20 + } + } + } + ] + } + ], + "failed": [ + { + "path": "fixtures/bad.fake.json", + "file_id": "file:sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "blob_fingerprint": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "language": "json", + "adapter": "builtin.fake_config", + "adapter_version": "1.0.0", + "diagnostics": [ + { + "code": "lang.fixture.parse_failed", + "severity": "error", + "message": "fixture file failed to parse", + "subject": { + "path": "fixtures/bad.fake.json" + } + } + ] + } + ], + "skipped": [ + { + "path": "fixtures/notes.txt", + "reason": "no_matching_adapter", + "detail": "no registered adapter recognized this file" + } + ], + "missing_languages": [ + { + "language": "rust", + "detail": "no adapter registered" + } + ], + "diagnostics": [ + { + "code": "lang.fixture.missing_language", + "severity": "warning", + "message": "requested language rust has no registered adapter" + } + ], + "stats": { + "considered_files": 3, + "parsed_units": 1, + "failed_units": 1, + "skipped_no_adapter": 1, + "skipped_missing_paths": 0, + "missing_requested_languages": 1, + "diagnostic_count": 3, + "cache_hits": 1, + "cache_misses": 2 + } +} diff --git a/crates/lift/fixtures/pack/canonical/boundary_taxonomy_order_a.json b/crates/lift/fixtures/pack/canonical/boundary_taxonomy_order_a.json new file mode 100644 index 000000000..710edbe65 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/boundary_taxonomy_order_a.json @@ -0,0 +1,21 @@ +{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "generic/boundaries", + "name": "Generic repository boundaries", + "counting": { + "mode": "distinct_minus_one" + }, + "boundaries": [ + { + "id": "application", + "label": "Application surface", + "include": ["apps/**", "services/**", "bin/**", "cmd/**"] + }, + { + "id": "library", + "label": "Shared library surface", + "include": ["crates/**", "packages/**", "libs/**", "src/**"] + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/boundary_taxonomy_order_b.json b/crates/lift/fixtures/pack/canonical/boundary_taxonomy_order_b.json new file mode 100644 index 000000000..0d29855f6 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/boundary_taxonomy_order_b.json @@ -0,0 +1,21 @@ +{ + "boundaries": [ + { + "label": "Shared library surface", + "include": ["src/**", "libs/**", "packages/**", "crates/**"], + "id": "library" + }, + { + "include": ["cmd/**", "bin/**", "services/**", "apps/**"], + "label": "Application surface", + "id": "application" + } + ], + "name": "Generic repository boundaries", + "kind": "boundary_taxonomy", + "counting": { + "mode": "distinct_minus_one" + }, + "id": "generic/boundaries", + "version": 1 +} diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_a/generic_core_recipes.json b/crates/lift/fixtures/pack/canonical/bundle_order_a/generic_core_recipes.json new file mode 100644 index 000000000..37d56cf75 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_a/generic_core_recipes.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "kind": "recipe_pack", + "version": 1, + "id": "acme/core-recipes", + "name": "Acme core recipes", + "recipes": [ + { + "id": "rename_old_api", + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "transforms": [ + { + "op": "replace_capture_text", + "capture": "use", + "text": "new_api" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_a/generic_policy.json b/crates/lift/fixtures/pack/canonical/bundle_order_a/generic_policy.json new file mode 100644 index 000000000..d38ae6e49 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_a/generic_policy.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "acme/policy", + "name": "Acme policy rules", + "rules": [ + { + "id": "architecture.cross_boundary_import", + "severity": "warning", + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "emit": [ + { + "kind": "finding", + "code": "architecture.cross_boundary_import", + "message": "Import crosses boundary" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_a/profile.toml b/crates/lift/fixtures/pack/canonical/bundle_order_a/profile.toml new file mode 100644 index 000000000..862cc2bd0 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_a/profile.toml @@ -0,0 +1,13 @@ +kind = "profile" +version = 1 +id = "acme/bundle-order" +name = "Bundle order" + +[queries] +packs = ["file:queries/rust_core.json"] + +[rules] +packs = ["file:generic_policy.json"] + +[recipes] +packs = ["file:generic_core_recipes.json"] diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_a/queries/rust_core.json b/crates/lift/fixtures/pack/canonical/bundle_order_a/queries/rust_core.json new file mode 100644 index 000000000..53b510980 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_a/queries/rust_core.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "acme/rust-core", + "name": "Acme Rust core queries", + "language": "rust", + "engine": "tree_sitter", + "queries": [ + { + "id": "use_statement", + "pattern": "(use_declaration) @use", + "captures": [{ "name": "use", "required": true }] + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_b/generic_core_recipes.json b/crates/lift/fixtures/pack/canonical/bundle_order_b/generic_core_recipes.json new file mode 100644 index 000000000..78e0978f6 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_b/generic_core_recipes.json @@ -0,0 +1,23 @@ +{ + "kind": "recipe_pack", + "$schema": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "version": 1, + "id": "acme/core-recipes", + "name": "Acme core recipes", + "recipes": [ + { + "transforms": [ + { + "text": "new_api", + "capture": "use", + "op": "replace_capture_text" + } + ], + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "id": "rename_old_api" + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_b/generic_policy.json b/crates/lift/fixtures/pack/canonical/bundle_order_b/generic_policy.json new file mode 100644 index 000000000..81724d087 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_b/generic_policy.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "id": "acme/policy", + "name": "Acme policy rules", + "rules": [ + { + "emit": [ + { + "message": "Import crosses boundary", + "kind": "finding", + "code": "architecture.cross_boundary_import" + } + ], + "query": { + "id": "use_statement", + "pack": "file:queries/rust_core.json" + }, + "severity": "warning", + "id": "architecture.cross_boundary_import" + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_b/profile.toml b/crates/lift/fixtures/pack/canonical/bundle_order_b/profile.toml new file mode 100644 index 000000000..1e20dcd84 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_b/profile.toml @@ -0,0 +1,13 @@ +kind = "profile" +version = 1 +id = "acme/bundle-order" +name = "Bundle order" + +[recipes] +packs = ["file:generic_core_recipes.json"] + +[rules] +packs = ["file:generic_policy.json"] + +[queries] +packs = ["file:queries/rust_core.json"] diff --git a/crates/lift/fixtures/pack/canonical/bundle_order_b/queries/rust_core.json b/crates/lift/fixtures/pack/canonical/bundle_order_b/queries/rust_core.json new file mode 100644 index 000000000..cdde74273 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/bundle_order_b/queries/rust_core.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "language": "rust", + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "id": "acme/rust-core", + "name": "Acme Rust core queries", + "engine": "tree_sitter", + "queries": [ + { + "captures": [{ "required": true, "name": "use" }], + "pattern": "(use_declaration) @use", + "id": "use_statement" + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/component_map_order_a.json b/crates/lift/fixtures/pack/canonical/component_map_order_a.json new file mode 100644 index 000000000..e24907ade --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/component_map_order_a.json @@ -0,0 +1,24 @@ +{ + "kind": "component_map", + "version": 1, + "id": "acme/components", + "name": "Canonical component map", + "counting": { + "mode": "distinct" + }, + "components": [ + { + "id": "backend", + "label": "Backend", + "include": ["api/**", "server/**"], + "exclude": ["api/internal/**"], + "tags": ["server", "service"] + }, + { + "id": "frontend", + "label": "Frontend", + "include": ["web/**", "ui/**"], + "tags": ["ui", "client"] + } + ] +} diff --git a/crates/lift/fixtures/pack/canonical/component_map_order_b.json b/crates/lift/fixtures/pack/canonical/component_map_order_b.json new file mode 100644 index 000000000..ca7274133 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/component_map_order_b.json @@ -0,0 +1,24 @@ +{ + "components": [ + { + "tags": ["client", "ui"], + "include": ["ui/**", "web/**"], + "label": "Frontend", + "id": "frontend" + }, + { + "label": "Backend", + "tags": ["service", "server"], + "exclude": ["api/internal/**"], + "include": ["server/**", "api/**"], + "id": "backend" + } + ], + "counting": { + "mode": "distinct" + }, + "name": "Canonical component map", + "id": "acme/components", + "version": 1, + "kind": "component_map" +} diff --git a/crates/lift/fixtures/pack/canonical/generic_default_order_a.toml b/crates/lift/fixtures/pack/canonical/generic_default_order_a.toml new file mode 100644 index 000000000..a3f923c53 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/generic_default_order_a.toml @@ -0,0 +1,18 @@ +name = "Generic default profile" +id = "generic/default" +version = 1 +kind = "profile" +description = "Default deterministic profile for generic repositories" + +[topology] +boundary_taxonomy = "builtin:generic/boundaries" +component_map = "builtin:generic/components" + +[analysis] +max_scope_depth = 2 +follow_symlinks = false +languages = ["typescript", "javascript", "python", "rust", "yaml", "toml", "json"] + +[apps] +default = "score" +enabled = ["score"] diff --git a/crates/lift/fixtures/pack/canonical/generic_default_order_b.toml b/crates/lift/fixtures/pack/canonical/generic_default_order_b.toml new file mode 100644 index 000000000..eb2c2277f --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/generic_default_order_b.toml @@ -0,0 +1,18 @@ +kind = "profile" +version = 1 +id = "generic/default" +name = "Generic default profile" +description = "Default deterministic profile for generic repositories" + +[apps] +enabled = ["score"] +default = "score" + +[topology] +component_map = "builtin:generic/components" +boundary_taxonomy = "builtin:generic/boundaries" + +[analysis] +languages = ["json", "rust", "python", "javascript", "yaml", "typescript", "toml"] +follow_symlinks = false +max_scope_depth = 2 diff --git a/crates/lift/fixtures/pack/canonical/minimal_explicit_defaults.toml b/crates/lift/fixtures/pack/canonical/minimal_explicit_defaults.toml new file mode 100644 index 000000000..b4cfc6dc6 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/minimal_explicit_defaults.toml @@ -0,0 +1,13 @@ +name = "Acme minimal defaults" +id = "acme/minimal-defaults" +version = 1 +kind = "profile" + +[analysis] +max_scope_depth = 2 +follow_symlinks = false +languages = ["typescript", "javascript", "python", "rust", "yaml", "toml", "json"] + +[apps] +default = "score" +enabled = ["score"] diff --git a/crates/lift/fixtures/pack/canonical/minimal_omitted.toml b/crates/lift/fixtures/pack/canonical/minimal_omitted.toml new file mode 100644 index 000000000..acf4167bb --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/minimal_omitted.toml @@ -0,0 +1,4 @@ +kind = "profile" +version = 1 +id = "acme/minimal-defaults" +name = "Acme minimal defaults" diff --git a/crates/lift/fixtures/pack/canonical/score_model_order_a.json b/crates/lift/fixtures/pack/canonical/score_model_order_a.json new file mode 100644 index 000000000..fdf056ef2 --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/score_model_order_a.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "generic/lift-v2", + "name": "Generic Lift model v2", + "vector_version": 2, + "lift_score": { + "op": "add", + "args": [ + { "op": "field", "path": "/touch/edit_files" }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "estimated_slices": { + "op": "max", + "args": [ + { "op": "const_int", "value": 1 }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "triggers": [], + "confidence": { + "default": "high", + "rules": [] + }, + "missing_input_rules": [] +} diff --git a/crates/lift/fixtures/pack/canonical/score_model_order_b.json b/crates/lift/fixtures/pack/canonical/score_model_order_b.json new file mode 100644 index 000000000..359d66bdc --- /dev/null +++ b/crates/lift/fixtures/pack/canonical/score_model_order_b.json @@ -0,0 +1,28 @@ +{ + "version": 1, + "id": "generic/lift-v2", + "name": "Generic Lift model v2", + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "vector_version": 2, + "estimated_slices": { + "args": [ + { "value": 1, "op": "const_int" }, + { "path": "/touch/components_touched", "op": "field" } + ], + "op": "max" + }, + "lift_score": { + "args": [ + { "path": "/touch/edit_files", "op": "field" }, + { "path": "/touch/components_touched", "op": "field" } + ], + "op": "add" + }, + "missing_input_rules": [], + "confidence": { + "rules": [], + "default": "high" + }, + "triggers": [] +} diff --git a/crates/lift/fixtures/pack/invalid/boundary_taxonomy_schema_violation.json b/crates/lift/fixtures/pack/invalid/boundary_taxonomy_schema_violation.json new file mode 100644 index 000000000..7f6f4b796 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/boundary_taxonomy_schema_violation.json @@ -0,0 +1,15 @@ +{ + "kind": "boundary_taxonomy", + "version": 2, + "id": "acme/boundaries", + "counting": { + "mode": "distinct" + }, + "boundaries": [ + { + "id": "", + "label": 5, + "include": [""] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/boundary_taxonomy_unknown_field.json b/crates/lift/fixtures/pack/invalid/boundary_taxonomy_unknown_field.json new file mode 100644 index 000000000..ed7526ce0 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/boundary_taxonomy_unknown_field.json @@ -0,0 +1,18 @@ +{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "acme/boundaries", + "name": "Unknown field boundary taxonomy", + "unexpected_root": true, + "counting": { + "mode": "distinct_minus_one" + }, + "boundaries": [ + { + "id": "runtime", + "label": "Runtime", + "include": ["services/runtime/**"], + "owner": "platform" + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/bundle_duplicate_pack_id_cross_origin.toml b/crates/lift/fixtures/pack/invalid/bundle_duplicate_pack_id_cross_origin.toml new file mode 100644 index 000000000..b0e0b9c29 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/bundle_duplicate_pack_id_cross_origin.toml @@ -0,0 +1,10 @@ +kind = "profile" +version = 1 +id = "acme/duplicate-bundle" +name = "Duplicate bundle" + +[queries] +packs = ["file:queries/rust_core.json"] + +[rules] +packs = ["file:generic_policy_duplicate.json"] diff --git a/crates/lift/fixtures/pack/invalid/component_map_schema_violation.json b/crates/lift/fixtures/pack/invalid/component_map_schema_violation.json new file mode 100644 index 000000000..f6f3018d3 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/component_map_schema_violation.json @@ -0,0 +1,16 @@ +{ + "kind": "component_map", + "version": 2, + "id": "acme/components", + "counting": { + "mode": "distinct_minus_one" + }, + "components": [ + { + "id": "", + "label": false, + "include": [""], + "tags": [1] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/component_map_unknown_field.json b/crates/lift/fixtures/pack/invalid/component_map_unknown_field.json new file mode 100644 index 000000000..f2a6707ce --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/component_map_unknown_field.json @@ -0,0 +1,19 @@ +{ + "kind": "component_map", + "version": 1, + "id": "acme/components", + "name": "Unknown field component map", + "unexpected_root": true, + "counting": { + "mode": "distinct" + }, + "components": [ + { + "id": "api", + "label": "API", + "include": ["api/**"], + "weight": 5, + "tags": ["public"] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/profile_empty_name.json b/crates/lift/fixtures/pack/invalid/profile_empty_name.json new file mode 100644 index 000000000..b6be3a07c --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_empty_name.json @@ -0,0 +1,6 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/empty-name", + "name": "" +} diff --git a/crates/lift/fixtures/pack/invalid/profile_invalid_pack_ref.json b/crates/lift/fixtures/pack/invalid/profile_invalid_pack_ref.json new file mode 100644 index 000000000..5f40309fe --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_invalid_pack_ref.json @@ -0,0 +1,9 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/invalid-ref", + "name": "Invalid ref profile", + "score": { + "model": "file:/score/model.toml" + } +} diff --git a/crates/lift/fixtures/pack/invalid/profile_invalid_pack_ref_leading_dash.json b/crates/lift/fixtures/pack/invalid/profile_invalid_pack_ref_leading_dash.json new file mode 100644 index 000000000..649b25fa6 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_invalid_pack_ref_leading_dash.json @@ -0,0 +1,9 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/invalid-ref-leading-dash", + "name": "Invalid ref profile", + "rules": { + "packs": ["file:-bad.toml"] + } +} diff --git a/crates/lift/fixtures/pack/invalid/profile_invalid_toml.toml b/crates/lift/fixtures/pack/invalid/profile_invalid_toml.toml new file mode 100644 index 000000000..51ac43bdc --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_invalid_toml.toml @@ -0,0 +1,7 @@ +kind = "profile" +version = 1 +id = "acme/invalid-toml" +name = "Invalid TOML" + +[apps +default = "score" diff --git a/crates/lift/fixtures/pack/invalid/profile_missing_name.json b/crates/lift/fixtures/pack/invalid/profile_missing_name.json new file mode 100644 index 000000000..c9e836791 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_missing_name.json @@ -0,0 +1,5 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/missing-name" +} diff --git a/crates/lift/fixtures/pack/invalid/profile_schema_violation.toml b/crates/lift/fixtures/pack/invalid/profile_schema_violation.toml new file mode 100644 index 000000000..c6bcf78d1 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_schema_violation.toml @@ -0,0 +1,13 @@ +kind = "rule_pack" +version = 2 +id = "Acme/Invalid" +name = "Schema violation" + +[apps] +enabled = ["score", "Bad-App"] +default = "Bad-App" + +[analysis] +languages = ["rust", "BadLang"] +follow_symlinks = "sometimes" +max_scope_depth = "deep" diff --git a/crates/lift/fixtures/pack/invalid/profile_traversal_pack_ref.json b/crates/lift/fixtures/pack/invalid/profile_traversal_pack_ref.json new file mode 100644 index 000000000..d8e4598bf --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_traversal_pack_ref.json @@ -0,0 +1,9 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/traversal-ref", + "name": "Traversal ref profile", + "queries": { + "packs": ["file:rules/../query.toml"] + } +} diff --git a/crates/lift/fixtures/pack/invalid/profile_unknown_field.json b/crates/lift/fixtures/pack/invalid/profile_unknown_field.json new file mode 100644 index 000000000..6fd5607ed --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/profile_unknown_field.json @@ -0,0 +1,7 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/unknown-field", + "name": "Unknown field profile", + "unexpected": true +} diff --git a/crates/lift/fixtures/pack/invalid/query_pack_duplicate_id.json b/crates/lift/fixtures/pack/invalid/query_pack_duplicate_id.json new file mode 100644 index 000000000..4d3c160ac --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/query_pack_duplicate_id.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "acme/duplicate-query", + "name": "Duplicate query ids", + "language": "rust", + "engine": "tree_sitter", + "queries": [ + { + "id": "dup", + "pattern": "(use_declaration) @use", + "captures": [{ "name": "use", "required": true }] + }, + { + "id": "dup", + "pattern": "(extern_crate_item) @extern", + "captures": [{ "name": "extern", "required": true }] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/query_pack_schema_violation.json b/crates/lift/fixtures/pack/invalid/query_pack_schema_violation.json new file mode 100644 index 000000000..b7c9e9bcb --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/query_pack_schema_violation.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "acme/schema-violation", + "name": "Schema violation", + "language": "rust", + "engine": "custom", + "queries": [ + { + "id": "use_statement", + "pattern": "(use_declaration) @use", + "captures": [ + { + "name": "use", + "required": true + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/recipe_pack_bad_transform.json b/crates/lift/fixtures/pack/invalid/recipe_pack_bad_transform.json new file mode 100644 index 000000000..f2cf5c7d8 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/recipe_pack_bad_transform.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "kind": "recipe_pack", + "version": 1, + "id": "acme/bad-transform", + "name": "Bad transform", + "recipes": [ + { + "id": "rename_old_api", + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "transforms": [ + { + "op": "delete_capture_text", + "capture": "use" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/rule_pack_bad_query_ref.json b/crates/lift/fixtures/pack/invalid/rule_pack_bad_query_ref.json new file mode 100644 index 000000000..1ea0d7144 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/rule_pack_bad_query_ref.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "acme/bad-query-ref", + "name": "Bad query ref", + "rules": [ + { + "id": "architecture.bad_query_ref", + "severity": "warning", + "query": { + "pack": "not-a-ref", + "id": "use_statement" + }, + "emit": [ + { + "kind": "finding", + "code": "architecture.bad_query_ref", + "message": "Bad query ref" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/rule_pack_invalid_severity.json b/crates/lift/fixtures/pack/invalid/rule_pack_invalid_severity.json new file mode 100644 index 000000000..a6cf19924 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/rule_pack_invalid_severity.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "acme/bad-severity", + "name": "Bad severity", + "rules": [ + { + "id": "architecture.bad_severity", + "severity": "critical", + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "emit": [ + { + "kind": "finding", + "code": "architecture.bad_severity", + "message": "Bad severity" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/invalid/score_model_bad_expr.json b/crates/lift/fixtures/pack/invalid/score_model_bad_expr.json new file mode 100644 index 000000000..19444fb82 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/score_model_bad_expr.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "acme/bad-expr", + "name": "Bad expr", + "vector_version": 2, + "lift_score": { + "op": "add", + "args": "bad" + }, + "estimated_slices": { + "op": "const_int", + "value": 1 + }, + "triggers": [], + "confidence": { + "default": "high", + "rules": [] + }, + "missing_input_rules": [] +} diff --git a/crates/lift/fixtures/pack/invalid/score_model_duplicate_trigger.json b/crates/lift/fixtures/pack/invalid/score_model_duplicate_trigger.json new file mode 100644 index 000000000..f1bfc72e3 --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/score_model_duplicate_trigger.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "acme/duplicate-trigger", + "name": "Duplicate trigger", + "vector_version": 2, + "lift_score": { + "op": "const_int", + "value": 1 + }, + "estimated_slices": { + "op": "const_int", + "value": 1 + }, + "triggers": [ + { "id": "dup", "when": { "op": "const_bool", "value": true } }, + { "id": "dup", "when": { "op": "const_bool", "value": false } } + ], + "confidence": { + "default": "high", + "rules": [] + }, + "missing_input_rules": [] +} diff --git a/crates/lift/fixtures/pack/invalid/score_model_schema_violation.json b/crates/lift/fixtures/pack/invalid/score_model_schema_violation.json new file mode 100644 index 000000000..5656574fe --- /dev/null +++ b/crates/lift/fixtures/pack/invalid/score_model_schema_violation.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "acme/schema-violation", + "name": "Schema violation", + "vector_version": 2, + "lift_score": { + "op": "mystery" + }, + "estimated_slices": { + "op": "const_int", + "value": 1 + }, + "triggers": [], + "confidence": { + "default": "high", + "rules": [] + }, + "missing_input_rules": [] +} diff --git a/crates/lift/fixtures/pack/valid/bundle/generic_core_recipes.json b/crates/lift/fixtures/pack/valid/bundle/generic_core_recipes.json new file mode 100644 index 000000000..cc89e2c67 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/bundle/generic_core_recipes.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "kind": "recipe_pack", + "version": 1, + "id": "acme/core-recipes", + "name": "Acme core recipes", + "recipes": [ + { + "id": "rename_old_api", + "summary": "Rename old_api to new_api", + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "transforms": [ + { + "op": "replace_capture_text", + "capture": "use", + "text": "new_api" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/bundle/generic_policy.json b/crates/lift/fixtures/pack/valid/bundle/generic_policy.json new file mode 100644 index 000000000..ec6cf804a --- /dev/null +++ b/crates/lift/fixtures/pack/valid/bundle/generic_policy.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "acme/policy", + "name": "Acme policy rules", + "rules": [ + { + "id": "architecture.cross_boundary_import", + "summary": "Flags imports that cross configured boundaries", + "severity": "warning", + "scope": { + "languages": ["rust"], + "path_classes": ["public_api"] + }, + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "emit": [ + { + "kind": "finding", + "code": "architecture.cross_boundary_import", + "message": "Import crosses boundary" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/bundle/profile_advanced_dedupe.toml b/crates/lift/fixtures/pack/valid/bundle/profile_advanced_dedupe.toml new file mode 100644 index 000000000..0431ba42f --- /dev/null +++ b/crates/lift/fixtures/pack/valid/bundle/profile_advanced_dedupe.toml @@ -0,0 +1,16 @@ +kind = "profile" +version = 1 +id = "acme/advanced-dedupe" +name = "Advanced dedupe profile" + +[score] +model = "file:score/generic_lift_v2.json" + +[queries] +packs = ["file:queries/rust_core.json"] + +[rules] +packs = ["file:generic_policy.json"] + +[recipes] +packs = ["file:generic_core_recipes.json"] diff --git a/crates/lift/fixtures/pack/valid/bundle/profile_advanced_file_backed.toml b/crates/lift/fixtures/pack/valid/bundle/profile_advanced_file_backed.toml new file mode 100644 index 000000000..bd76eea73 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/bundle/profile_advanced_file_backed.toml @@ -0,0 +1,13 @@ +kind = "profile" +version = 1 +id = "acme/advanced-bundle" +name = "Advanced bundle profile" + +[score] +model = "file:score/generic_lift_v2.json" + +[rules] +packs = ["file:generic_policy.json"] + +[recipes] +packs = ["file:generic_core_recipes.json"] diff --git a/crates/lift/fixtures/pack/valid/bundle/queries/rust_core.json b/crates/lift/fixtures/pack/valid/bundle/queries/rust_core.json new file mode 100644 index 000000000..7c1a314e7 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/bundle/queries/rust_core.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "acme/rust-core", + "name": "Acme Rust core queries", + "language": "rust", + "engine": "tree_sitter", + "queries": [ + { + "id": "use_statement", + "summary": "Matches Rust use declarations", + "pattern": "(use_declaration) @use", + "captures": [ + { "name": "use", "required": true } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/bundle/score/generic_lift_v2.json b/crates/lift/fixtures/pack/valid/bundle/score/generic_lift_v2.json new file mode 100644 index 000000000..1321be824 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/bundle/score/generic_lift_v2.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "acme/lift-v2", + "name": "Acme Lift model v2", + "vector_version": 2, + "lift_score": { + "op": "add", + "args": [ + { "op": "field", "path": "/touch/edit_files" }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "estimated_slices": { + "op": "max", + "args": [ + { "op": "const_int", "value": 1 }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "triggers": [], + "confidence": { + "default": "high", + "rules": [] + }, + "missing_input_rules": [] +} diff --git a/crates/lift/fixtures/pack/valid/generic_boundaries.json b/crates/lift/fixtures/pack/valid/generic_boundaries.json new file mode 100644 index 000000000..509492ddb --- /dev/null +++ b/crates/lift/fixtures/pack/valid/generic_boundaries.json @@ -0,0 +1,42 @@ +{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "generic/boundaries", + "name": "Generic repository boundaries", + "description": "Coarse boundary taxonomy for generic repositories.", + "counting": { + "mode": "distinct_minus_one" + }, + "boundaries": [ + { + "id": "application", + "label": "Application surface", + "include": ["apps/**", "services/**", "bin/**", "cmd/**"] + }, + { + "id": "library", + "label": "Shared library surface", + "include": ["crates/**", "packages/**", "libs/**", "src/**"] + }, + { + "id": "interface", + "label": "Interface and contract surface", + "include": ["api/**", "proto/**", "schemas/**", "contracts/**"] + }, + { + "id": "tests", + "label": "Test surface", + "include": ["tests/**", "test/**", "e2e/**", "integration-tests/**"] + }, + { + "id": "operations", + "label": "Operations surface", + "include": ["infra/**", "ops/**", "deploy/**", ".github/**", "scripts/**"] + }, + { + "id": "docs", + "label": "Documentation surface", + "include": ["docs/**"] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/generic_components.json b/crates/lift/fixtures/pack/valid/generic_components.json new file mode 100644 index 000000000..c3ce3b18c --- /dev/null +++ b/crates/lift/fixtures/pack/valid/generic_components.json @@ -0,0 +1,42 @@ +{ + "kind": "component_map", + "version": 1, + "id": "generic/components", + "name": "Generic repository components", + "description": "Generic component map for common repository layouts.", + "counting": { + "mode": "distinct" + }, + "components": [ + { + "id": "frontend", + "label": "Frontend", + "include": ["web/**", "ui/**", "frontend/**", "apps/web/**", "packages/ui/**"], + "tags": ["client", "ui"] + }, + { + "id": "backend", + "label": "Backend", + "include": ["api/**", "backend/**", "server/**", "services/**", "apps/api/**"], + "tags": ["server", "service"] + }, + { + "id": "shared", + "label": "Shared libraries", + "include": ["crates/**", "packages/shared/**", "packages/config/**", "libs/**", "src/**"], + "tags": ["shared", "library"] + }, + { + "id": "tooling", + "label": "Tooling", + "include": ["scripts/**", "tools/**", ".github/**", "infra/**"], + "tags": ["tooling", "ops"] + }, + { + "id": "tests", + "label": "Tests", + "include": ["tests/**", "test/**", "e2e/**", "integration-tests/**"], + "tags": ["test"] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/profile_file_backed.toml b/crates/lift/fixtures/pack/valid/profile_file_backed.toml new file mode 100644 index 000000000..d42a057c2 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/profile_file_backed.toml @@ -0,0 +1,30 @@ +kind = "profile" +version = 1 +id = "acme/file-backed" +name = "File backed profile" +description = "Uses local file refs when loaded from disk" + +[apps] +enabled = ["score"] +default = "score" + +[analysis] +languages = ["rust", "toml"] +follow_symlinks = true +max_scope_depth = 3 + +[topology] +boundary_taxonomy = "file:topology/boundary-taxonomy.json" +component_map = "file:topology/component-map.json" + +[score] +model = "file:score/generic_lift_v2.json" + +[rules] +packs = ["file:rules/generic_policy.json"] + +[queries] +packs = ["file:queries/rust_core.json"] + +[recipes] +packs = ["file:recipes/generic_core_recipes.json"] diff --git a/crates/lift/fixtures/pack/valid/profile_full.json b/crates/lift/fixtures/pack/valid/profile_full.json new file mode 100644 index 000000000..649812455 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/profile_full.json @@ -0,0 +1,32 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/file-backed", + "name": "File backed profile", + "description": "Uses local file refs when loaded from disk", + "apps": { + "enabled": ["score"], + "default": "score" + }, + "analysis": { + "languages": ["rust", "toml"], + "follow_symlinks": true, + "max_scope_depth": 3 + }, + "topology": { + "boundary_taxonomy": "file:topology/boundary-taxonomy.json", + "component_map": "file:topology/component-map.json" + }, + "score": { + "model": "file:score/generic_lift_v2.json" + }, + "rules": { + "packs": ["file:rules/generic_policy.json"] + }, + "queries": { + "packs": ["file:queries/rust_core.json"] + }, + "recipes": { + "packs": ["file:recipes/generic_core_recipes.json"] + } +} diff --git a/crates/lift/fixtures/pack/valid/profile_minimal.json b/crates/lift/fixtures/pack/valid/profile_minimal.json new file mode 100644 index 000000000..5a8849584 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/profile_minimal.json @@ -0,0 +1,6 @@ +{ + "kind": "profile", + "version": 1, + "id": "acme/minimal", + "name": "Acme minimal" +} diff --git a/crates/lift/fixtures/pack/valid/queries/rust_core.json b/crates/lift/fixtures/pack/valid/queries/rust_core.json new file mode 100644 index 000000000..55d17fe30 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/queries/rust_core.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "rust/core", + "name": "Core Rust tree-sitter queries", + "language": "rust", + "engine": "tree_sitter", + "queries": [ + { + "id": "use_statement", + "summary": "Matches Rust use declarations", + "pattern": "(use_declaration) @use", + "captures": [ + { "name": "use", "required": true } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/recipes/generic_core_recipes.json b/crates/lift/fixtures/pack/valid/recipes/generic_core_recipes.json new file mode 100644 index 000000000..4df4dce09 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/recipes/generic_core_recipes.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "kind": "recipe_pack", + "version": 1, + "id": "generic/core-recipes", + "name": "Generic rewrite recipes", + "recipes": [ + { + "id": "rename_old_api", + "summary": "Rename old_api to new_api", + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "transforms": [ + { + "op": "replace_capture_text", + "capture": "use", + "text": "new_api" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/rules/generic_policy.json b/crates/lift/fixtures/pack/valid/rules/generic_policy.json new file mode 100644 index 000000000..212d1706d --- /dev/null +++ b/crates/lift/fixtures/pack/valid/rules/generic_policy.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "generic/policy", + "name": "Generic policy rules", + "rules": [ + { + "id": "architecture.cross_boundary_import", + "summary": "Flags imports that cross configured boundaries", + "severity": "warning", + "scope": { + "languages": ["rust"], + "path_classes": ["public_api"] + }, + "query": { + "pack": "file:queries/rust_core.json", + "id": "use_statement" + }, + "emit": [ + { + "kind": "finding", + "code": "architecture.cross_boundary_import", + "message": "Import crosses boundary" + } + ] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/score/generic_lift_v2.json b/crates/lift/fixtures/pack/valid/score/generic_lift_v2.json new file mode 100644 index 000000000..cc6b77e85 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/score/generic_lift_v2.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "generic/lift-v2", + "name": "Generic Lift model v2", + "vector_version": 2, + "lift_score": { + "op": "add", + "args": [ + { "op": "field", "path": "/touch/edit_files" }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "estimated_slices": { + "op": "max", + "args": [ + { "op": "const_int", "value": 1 }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "triggers": [ + { + "id": "many_files", + "when": { + "op": "gt", + "lhs": { "op": "field", "path": "/touch/edit_files" }, + "rhs": { "op": "const_int", "value": 12 } + } + } + ], + "confidence": { + "default": "high", + "rules": [ + { + "id": "missing_touch_inputs", + "when": { "op": "is_null", "path": "/touch/edit_files" }, + "set": "low", + "causes": ["missing_inputs"] + } + ] + }, + "missing_input_rules": [ + { + "field": "/touch/edit_files", + "when": { "op": "is_null", "path": "/touch/edit_files" } + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/topology/boundary-taxonomy.json b/crates/lift/fixtures/pack/valid/topology/boundary-taxonomy.json new file mode 100644 index 000000000..f0ef59240 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/topology/boundary-taxonomy.json @@ -0,0 +1,23 @@ +{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "acme/boundaries", + "name": "Acme boundaries", + "description": "Boundary taxonomy for file-backed topology resolution tests.", + "counting": { + "mode": "distinct_minus_one" + }, + "boundaries": [ + { + "id": "runtime", + "label": "Runtime", + "include": ["services/runtime/**"], + "exclude": ["services/runtime/generated/**"] + }, + { + "id": "public_api", + "label": "Public API", + "include": ["api/**", "openapi/**"] + } + ] +} diff --git a/crates/lift/fixtures/pack/valid/topology/component-map.json b/crates/lift/fixtures/pack/valid/topology/component-map.json new file mode 100644 index 000000000..9c4749036 --- /dev/null +++ b/crates/lift/fixtures/pack/valid/topology/component-map.json @@ -0,0 +1,25 @@ +{ + "kind": "component_map", + "version": 1, + "id": "acme/components", + "name": "Acme components", + "description": "Component map for file-backed topology resolution tests.", + "counting": { + "mode": "distinct" + }, + "components": [ + { + "id": "api", + "label": "API", + "include": ["api/**", "openapi/**"], + "tags": ["public", "service"] + }, + { + "id": "runtime", + "label": "Runtime", + "include": ["services/runtime/**"], + "exclude": ["services/runtime/generated/**"], + "tags": ["internal"] + } + ] +} diff --git a/crates/lift/fixtures/repo/README.md b/crates/lift/fixtures/repo/README.md new file mode 100644 index 000000000..826c8459e --- /dev/null +++ b/crates/lift/fixtures/repo/README.md @@ -0,0 +1,15 @@ +Repo fixtures cover the landed crate-private repo seam: Phase A snapshot materialization plus Phase B pure diff fixtures over already-materialized snapshots. + +Static trees in this directory are used for: +- root detection semantics +- deterministic snapshot manifests +- paired-snapshot pure diff manifests under `fixtures/repo/diff/**` +- explicit ignore behavior +- default inclusion of common cache/build directories +- canonical typed well-known exclude directory examples under `fixtures/repo/trees/well_known_excludes` + +Some policy-sensitive cases are created dynamically inside the tests: +- non-UTF8 filesystem paths +- symlinks on platforms that support them +- large-file thresholds +- traversal-order and post-snapshot mutation cases diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_added_with_before_blob.json b/crates/lift/fixtures/repo/diff/invalid/manifest_added_with_before_blob.json new file mode 100644 index 000000000..58f913aae --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_added_with_before_blob.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "added-with-before-blob", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:add93cd80ced9cb76a2fcf7943913c51d4718ea317930e9fe5c1b5095285bbe3", + "entries": [ + { + "path": "docs/new.txt", + "kind": "added", + "before_blob_fingerprint": "sha256:7ec5d58e058c0ba12b07aee16397bb31edebce652614d76d4f970125791ffba7", + "after_blob_fingerprint": "sha256:0f15384d18789b1ebf3043dc7b6bc27273c8576373fbeb6f3e15854b588141c0" + } + ], + "diff_fingerprint": "sha256:cb27191bfd752f18e43d1060bc36c14d6a1ed93d21287bb15cf5fa73d9fc036c" +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_bad_kind.json b/crates/lift/fixtures/repo/diff/invalid/manifest_bad_kind.json new file mode 100644 index 000000000..e5fc9509e --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_bad_kind.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "invalid-kind", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:add93cd80ced9cb76a2fcf7943913c51d4718ea317930e9fe5c1b5095285bbe3", + "entries": [ + { + "path": "docs/new.txt", + "kind": "Added", + "before_blob_fingerprint": null, + "after_blob_fingerprint": "sha256:0f15384d18789b1ebf3043dc7b6bc27273c8576373fbeb6f3e15854b588141c0" + } + ], + "diff_fingerprint": "sha256:cb27191bfd752f18e43d1060bc36c14d6a1ed93d21287bb15cf5fa73d9fc036c" +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_bad_repo_path.json b/crates/lift/fixtures/repo/diff/invalid/manifest_bad_repo_path.json new file mode 100644 index 000000000..4a59481df --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_bad_repo_path.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "invalid-repo-path", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:add93cd80ced9cb76a2fcf7943913c51d4718ea317930e9fe5c1b5095285bbe3", + "entries": [ + { + "path": "/docs/new.txt", + "kind": "added", + "before_blob_fingerprint": null, + "after_blob_fingerprint": "sha256:0f15384d18789b1ebf3043dc7b6bc27273c8576373fbeb6f3e15854b588141c0" + } + ], + "diff_fingerprint": "sha256:cb27191bfd752f18e43d1060bc36c14d6a1ed93d21287bb15cf5fa73d9fc036c" +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_before_after_shape_invalid.json b/crates/lift/fixtures/repo/diff/invalid/manifest_before_after_shape_invalid.json new file mode 100644 index 000000000..2d9046360 --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_before_after_shape_invalid.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "case": "before-after-shape-invalid", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:4f85fa23bafa0b79706c1755e4e9535706d089fcff0422e38e6f32ad95253d93", + "entries": [ + { + "path": "src/lib.rs", + "kind": "modified", + "before_blob_fingerprint": { + "unexpected": "shape" + }, + "after_blob_fingerprint": "sha256:1e6d8613a8d2a26ff893263f8dbb2db84526dcec10d2a767bdba7eba24df52da" + } + ], + "diff_fingerprint": "sha256:bfc6f540059bf988db38ec259916a75111e90764f5b0b33881a9d942710fdcd0" +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_missing_diff_fingerprint.json b/crates/lift/fixtures/repo/diff/invalid/manifest_missing_diff_fingerprint.json new file mode 100644 index 000000000..abe5c2dfa --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_missing_diff_fingerprint.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "case": "missing-diff-fingerprint", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:1e54dff245aff0dcf803692c63b8e43765b7ab342303b0d0df7458e6c3547dbb", + "entries": [ + { + "path": "build/output.txt", + "kind": "removed", + "before_blob_fingerprint": "sha256:36603704a81ec200acfe816b7f89b3b0986c7bc8c2c95a9d986290c01befab94", + "after_blob_fingerprint": null + } + ] +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_modified_missing_after_blob.json b/crates/lift/fixtures/repo/diff/invalid/manifest_modified_missing_after_blob.json new file mode 100644 index 000000000..457f7310f --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_modified_missing_after_blob.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "modified-missing-after-blob", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:4f85fa23bafa0b79706c1755e4e9535706d089fcff0422e38e6f32ad95253d93", + "entries": [ + { + "path": "src/lib.rs", + "kind": "modified", + "before_blob_fingerprint": "sha256:7ec5d58e058c0ba12b07aee16397bb31edebce652614d76d4f970125791ffba7", + "after_blob_fingerprint": null + } + ], + "diff_fingerprint": "sha256:bfc6f540059bf988db38ec259916a75111e90764f5b0b33881a9d942710fdcd0" +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_modified_missing_before_blob.json b/crates/lift/fixtures/repo/diff/invalid/manifest_modified_missing_before_blob.json new file mode 100644 index 000000000..dd1bbfa4c --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_modified_missing_before_blob.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "modified-missing-before-blob", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:4f85fa23bafa0b79706c1755e4e9535706d089fcff0422e38e6f32ad95253d93", + "entries": [ + { + "path": "src/lib.rs", + "kind": "modified", + "before_blob_fingerprint": null, + "after_blob_fingerprint": "sha256:1e6d8613a8d2a26ff893263f8dbb2db84526dcec10d2a767bdba7eba24df52da" + } + ], + "diff_fingerprint": "sha256:bfc6f540059bf988db38ec259916a75111e90764f5b0b33881a9d942710fdcd0" +} diff --git a/crates/lift/fixtures/repo/diff/invalid/manifest_removed_with_after_blob.json b/crates/lift/fixtures/repo/diff/invalid/manifest_removed_with_after_blob.json new file mode 100644 index 000000000..d6d3d275d --- /dev/null +++ b/crates/lift/fixtures/repo/diff/invalid/manifest_removed_with_after_blob.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "removed-with-after-blob", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:1e54dff245aff0dcf803692c63b8e43765b7ab342303b0d0df7458e6c3547dbb", + "entries": [ + { + "path": "build/output.txt", + "kind": "removed", + "before_blob_fingerprint": "sha256:36603704a81ec200acfe816b7f89b3b0986c7bc8c2c95a9d986290c01befab94", + "after_blob_fingerprint": "sha256:0f15384d18789b1ebf3043dc7b6bc27273c8576373fbeb6f3e15854b588141c0" + } + ], + "diff_fingerprint": "sha256:6a46252b0593da0f6575ed70a1d16e9b05eac76ed61c764b277522fe6a44dd1e" +} diff --git a/crates/lift/fixtures/repo/diff/valid/added_file.json b/crates/lift/fixtures/repo/diff/valid/added_file.json new file mode 100644 index 000000000..0d0ca5fe2 --- /dev/null +++ b/crates/lift/fixtures/repo/diff/valid/added_file.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "added-file", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:add93cd80ced9cb76a2fcf7943913c51d4718ea317930e9fe5c1b5095285bbe3", + "entries": [ + { + "path": "docs/new.txt", + "kind": "added", + "before_blob_fingerprint": null, + "after_blob_fingerprint": "sha256:0f15384d18789b1ebf3043dc7b6bc27273c8576373fbeb6f3e15854b588141c0" + } + ], + "diff_fingerprint": "sha256:cb27191bfd752f18e43d1060bc36c14d6a1ed93d21287bb15cf5fa73d9fc036c" +} diff --git a/crates/lift/fixtures/repo/diff/valid/empty_diff.json b/crates/lift/fixtures/repo/diff/valid/empty_diff.json new file mode 100644 index 000000000..18c5bd1b2 --- /dev/null +++ b/crates/lift/fixtures/repo/diff/valid/empty_diff.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "case": "empty-diff", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "entries": [], + "diff_fingerprint": "sha256:f643d29d0e0be167c8fa0aa33831c05b11f58fc2d92bc9d72aab45c36443d300" +} diff --git a/crates/lift/fixtures/repo/diff/valid/modified_file.json b/crates/lift/fixtures/repo/diff/valid/modified_file.json new file mode 100644 index 000000000..a690f67b0 --- /dev/null +++ b/crates/lift/fixtures/repo/diff/valid/modified_file.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "modified-file", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:4f85fa23bafa0b79706c1755e4e9535706d089fcff0422e38e6f32ad95253d93", + "entries": [ + { + "path": "src/lib.rs", + "kind": "modified", + "before_blob_fingerprint": "sha256:7ec5d58e058c0ba12b07aee16397bb31edebce652614d76d4f970125791ffba7", + "after_blob_fingerprint": "sha256:1e6d8613a8d2a26ff893263f8dbb2db84526dcec10d2a767bdba7eba24df52da" + } + ], + "diff_fingerprint": "sha256:bfc6f540059bf988db38ec259916a75111e90764f5b0b33881a9d942710fdcd0" +} diff --git a/crates/lift/fixtures/repo/diff/valid/removed_file.json b/crates/lift/fixtures/repo/diff/valid/removed_file.json new file mode 100644 index 000000000..869b80a14 --- /dev/null +++ b/crates/lift/fixtures/repo/diff/valid/removed_file.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "removed-file", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:1e54dff245aff0dcf803692c63b8e43765b7ab342303b0d0df7458e6c3547dbb", + "entries": [ + { + "path": "build/output.txt", + "kind": "removed", + "before_blob_fingerprint": "sha256:36603704a81ec200acfe816b7f89b3b0986c7bc8c2c95a9d986290c01befab94", + "after_blob_fingerprint": null + } + ], + "diff_fingerprint": "sha256:6a46252b0593da0f6575ed70a1d16e9b05eac76ed61c764b277522fe6a44dd1e" +} diff --git a/crates/lift/fixtures/repo/diff/valid/rename_as_add_remove.json b/crates/lift/fixtures/repo/diff/valid/rename_as_add_remove.json new file mode 100644 index 000000000..a96371864 --- /dev/null +++ b/crates/lift/fixtures/repo/diff/valid/rename_as_add_remove.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "case": "rename-as-add-remove", + "base_fingerprint": "sha256:915b9a0916f0d7b9389d2abea1bd34bcd19128d4381c7c0b12c46010076ac174", + "head_fingerprint": "sha256:3669519358efb85699d3b38038f1ebd1cd63d32c68661f46184ac825d2a11630", + "entries": [ + { + "path": "src/lib.rs", + "kind": "removed", + "before_blob_fingerprint": "sha256:7ec5d58e058c0ba12b07aee16397bb31edebce652614d76d4f970125791ffba7", + "after_blob_fingerprint": null + }, + { + "path": "src/renamed.rs", + "kind": "added", + "before_blob_fingerprint": null, + "after_blob_fingerprint": "sha256:7ec5d58e058c0ba12b07aee16397bb31edebce652614d76d4f970125791ffba7" + } + ], + "diff_fingerprint": "sha256:9dddb52cd86542af3dbd945675fc4cbd5a78983698b4021b11989247fe211b9b" +} diff --git a/crates/lift/fixtures/repo/invalid/manifest_bad_repo_path.json b/crates/lift/fixtures/repo/invalid/manifest_bad_repo_path.json new file mode 100644 index 000000000..69b52d10f --- /dev/null +++ b/crates/lift/fixtures/repo/invalid/manifest_bad_repo_path.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "case": "bad-repo-path", + "source_kind": "worktree", + "options": { + "symlink_policy": "skip", + "well_known_excludes": [], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [ + { + "path": "/absolute.txt", + "file_id": "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "blob_fingerprint": "sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "size_bytes": 1 + } + ], + "snapshot_fingerprint": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "stats": { + "file_count": 1, + "total_bytes": 1, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/fixtures/repo/invalid/manifest_bad_well_known_exclude.json b/crates/lift/fixtures/repo/invalid/manifest_bad_well_known_exclude.json new file mode 100644 index 000000000..430167fcc --- /dev/null +++ b/crates/lift/fixtures/repo/invalid/manifest_bad_well_known_exclude.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "case": "bad-well-known-exclude", + "source_kind": "worktree", + "options": { + "symlink_policy": "skip", + "well_known_excludes": ["cargo_home"], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [], + "snapshot_fingerprint": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "stats": { + "file_count": 0, + "total_bytes": 0, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/fixtures/repo/invalid/manifest_git_rev_missing_source_rev.json b/crates/lift/fixtures/repo/invalid/manifest_git_rev_missing_source_rev.json new file mode 100644 index 000000000..90412a04b --- /dev/null +++ b/crates/lift/fixtures/repo/invalid/manifest_git_rev_missing_source_rev.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "case": "git-rev-missing-source-rev", + "source_kind": "git_rev", + "options": { + "symlink_policy": "skip", + "well_known_excludes": [], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [], + "snapshot_fingerprint": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "stats": { + "file_count": 0, + "total_bytes": 0, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/fixtures/repo/invalid/manifest_missing_stats.json b/crates/lift/fixtures/repo/invalid/manifest_missing_stats.json new file mode 100644 index 000000000..dfec629a9 --- /dev/null +++ b/crates/lift/fixtures/repo/invalid/manifest_missing_stats.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "case": "missing-stats", + "source_kind": "worktree", + "options": { + "symlink_policy": "skip", + "well_known_excludes": [], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [], + "snapshot_fingerprint": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +} diff --git a/crates/lift/fixtures/repo/invalid/manifest_worktree_with_source_rev.json b/crates/lift/fixtures/repo/invalid/manifest_worktree_with_source_rev.json new file mode 100644 index 000000000..38f2bb839 --- /dev/null +++ b/crates/lift/fixtures/repo/invalid/manifest_worktree_with_source_rev.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "case": "worktree-with-source-rev", + "source_kind": "worktree", + "source_rev": "HEAD", + "options": { + "symlink_policy": "skip", + "well_known_excludes": [], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [], + "snapshot_fingerprint": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "stats": { + "file_count": 0, + "total_bytes": 0, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/fixtures/repo/trees/basic_worktree/.venv/bin/python b/crates/lift/fixtures/repo/trees/basic_worktree/.venv/bin/python new file mode 100644 index 000000000..e5a0d9b48 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/basic_worktree/.venv/bin/python @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/crates/lift/fixtures/repo/trees/basic_worktree/Cargo.toml b/crates/lift/fixtures/repo/trees/basic_worktree/Cargo.toml new file mode 100644 index 000000000..e4d28057c --- /dev/null +++ b/crates/lift/fixtures/repo/trees/basic_worktree/Cargo.toml @@ -0,0 +1,3 @@ +[package] +name = "fixture-basic-worktree" +version = "0.1.0" diff --git a/crates/lift/fixtures/repo/trees/basic_worktree/build/output.txt b/crates/lift/fixtures/repo/trees/basic_worktree/build/output.txt new file mode 100644 index 000000000..615d610f5 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/basic_worktree/build/output.txt @@ -0,0 +1 @@ +compiled output diff --git a/crates/lift/fixtures/repo/trees/basic_worktree/dist/bundle.js b/crates/lift/fixtures/repo/trees/basic_worktree/dist/bundle.js new file mode 100644 index 000000000..bb874b013 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/basic_worktree/dist/bundle.js @@ -0,0 +1 @@ +console.log("bundle"); diff --git a/crates/lift/fixtures/repo/trees/basic_worktree/node_modules/pkg/index.js b/crates/lift/fixtures/repo/trees/basic_worktree/node_modules/pkg/index.js new file mode 100644 index 000000000..65f9b92af --- /dev/null +++ b/crates/lift/fixtures/repo/trees/basic_worktree/node_modules/pkg/index.js @@ -0,0 +1 @@ +module.exports = "fixture"; diff --git a/crates/lift/fixtures/repo/trees/basic_worktree/src/lib.rs b/crates/lift/fixtures/repo/trees/basic_worktree/src/lib.rs new file mode 100644 index 000000000..ffba6852b --- /dev/null +++ b/crates/lift/fixtures/repo/trees/basic_worktree/src/lib.rs @@ -0,0 +1,3 @@ +pub fn fixture_value() -> &'static str { + "fixture" +} diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/.venv/bin/python b/crates/lift/fixtures/repo/trees/well_known_excludes/.venv/bin/python new file mode 100644 index 000000000..e5a0d9b48 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/.venv/bin/python @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/Cargo.toml b/crates/lift/fixtures/repo/trees/well_known_excludes/Cargo.toml new file mode 100644 index 000000000..a026c09b2 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/Cargo.toml @@ -0,0 +1,3 @@ +[package] +name = "well-known-excludes" +version = "0.1.0" diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/build/output.txt b/crates/lift/fixtures/repo/trees/well_known_excludes/build/output.txt new file mode 100644 index 000000000..643b668fd --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/build/output.txt @@ -0,0 +1 @@ +build output diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/dist/bundle.js b/crates/lift/fixtures/repo/trees/well_known_excludes/dist/bundle.js new file mode 100644 index 000000000..bb874b013 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/dist/bundle.js @@ -0,0 +1 @@ +console.log("bundle"); diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/node_modules/pkg/index.js b/crates/lift/fixtures/repo/trees/well_known_excludes/node_modules/pkg/index.js new file mode 100644 index 000000000..65f9b92af --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/node_modules/pkg/index.js @@ -0,0 +1 @@ +module.exports = "fixture"; diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/src/lib.rs b/crates/lift/fixtures/repo/trees/well_known_excludes/src/lib.rs new file mode 100644 index 000000000..320a05275 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/src/lib.rs @@ -0,0 +1,3 @@ +pub fn fixture_value() -> &'static str { + "kept" +} diff --git a/crates/lift/fixtures/repo/trees/well_known_excludes/venv/bin/python b/crates/lift/fixtures/repo/trees/well_known_excludes/venv/bin/python new file mode 100644 index 000000000..e5a0d9b48 --- /dev/null +++ b/crates/lift/fixtures/repo/trees/well_known_excludes/venv/bin/python @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/crates/lift/fixtures/repo/valid/manifest_follow_policy.json b/crates/lift/fixtures/repo/valid/manifest_follow_policy.json new file mode 100644 index 000000000..8aa9df230 --- /dev/null +++ b/crates/lift/fixtures/repo/valid/manifest_follow_policy.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "case": "follow-policy-valid", + "source_kind": "worktree", + "options": { + "symlink_policy": "follow", + "well_known_excludes": ["rust_target", "web_dist"], + "exclude_globs": ["generated"], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [ + { + "path": "src/lib.rs", + "file_id": "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "blob_fingerprint": "sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "size_bytes": 7 + } + ], + "snapshot_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "stats": { + "file_count": 1, + "total_bytes": 7, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/fixtures/repo/valid/manifest_git_rev.json b/crates/lift/fixtures/repo/valid/manifest_git_rev.json new file mode 100644 index 000000000..c12b2e84d --- /dev/null +++ b/crates/lift/fixtures/repo/valid/manifest_git_rev.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "case": "git-rev-valid", + "source_kind": "git_rev", + "source_rev": "HEAD~1", + "options": { + "symlink_policy": "skip", + "well_known_excludes": ["node_modules"], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [ + { + "path": "src/lib.rs", + "file_id": "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "blob_fingerprint": "sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "size_bytes": 7 + } + ], + "snapshot_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "stats": { + "file_count": 1, + "total_bytes": 7, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/fixtures/repo/valid/manifest_minimal.json b/crates/lift/fixtures/repo/valid/manifest_minimal.json new file mode 100644 index 000000000..e6716e783 --- /dev/null +++ b/crates/lift/fixtures/repo/valid/manifest_minimal.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "case": "minimal-valid", + "source_kind": "worktree", + "options": { + "symlink_policy": "skip", + "well_known_excludes": [], + "exclude_globs": [], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + }, + "files": [ + { + "path": "src/lib.rs", + "file_id": "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "blob_fingerprint": "sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "size_bytes": 7 + } + ], + "snapshot_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "stats": { + "file_count": 1, + "total_bytes": 7, + "skipped_by_ignore": 0, + "skipped_symlinks": 0, + "skipped_non_utf8_paths": 0, + "skipped_large_files": 0, + "skipped_unsupported_file_kinds": 0 + } +} diff --git a/crates/lift/lift_seam0_spec.md b/crates/lift/lift_seam0_spec.md new file mode 100644 index 000000000..551af2228 --- /dev/null +++ b/crates/lift/lift_seam0_spec.md @@ -0,0 +1,753 @@ +# `lift` seam 0 spec + +Yes, seam 0 should now be treated as a strict kernel contract seam, not a bag of helpers. + +It should stay as an internal module under `crates/lift/src/kernel`, not as a separate workspace crate yet. The repo is already a multi-member Cargo workspace targeting Rust 1.89, and the current Lift system is already contract-first: the existing Lift Vector schema is Draft 2020-12 with `additionalProperties: false`, required root keys, and pinned v1 semantics; the current `--emit-json` contract also requires sorted unique arrays, additive evolution only, and deterministic `triggers` and `missing_inputs` for the same input and config. That makes seam 0 the right place to define the shared wire primitives and determinism rules that every later seam will inherit. + +Two standards-level choices are locked here: + +- internal field addressing uses JSON Pointer as the canonical path primitive, because RFC 6901 gives a standard, machine-precise way to identify values inside JSON-like documents +- canonical JSON bytes for hashing and fingerprints use RFC 8785 JCS, because it standardizes deterministic property ordering and canonical JSON serialization + +The compat note is important: current Work Lift v1 emits dotted paths like `touch.crates_touched` in `missing_inputs`, so seam 0 must not make dotted field paths canonical. Keep JSON Pointer canonical in the kernel, and let `app::score::compat_v1` translate between `"/touch/crates_touched"` and `"touch.crates_touched"` at the boundary. + +## 1. Mission + +Seam 0 owns the portable primitive contracts used across the entire crate. + +That means: + +- deterministic IDs +- repo-relative paths +- byte spans and locators +- JSON field pointers +- fingerprints +- diagnostics +- canonical JSON serialization +- shared typed kernel errors +- versioning conventions for schemas and wire objects + +It does not own: + +- repo walking +- AST parsing +- graph edges +- facts +- derived facts +- Lift vectors +- scoring +- query matching +- patch plans +- CLI rendering + +The kernel must stay useful even if score disappeared. + +## 2. Directory and module shape + +```text +crates/lift/ + src/kernel/ + mod.rs + error.rs + path.rs + json_pointer.rs + id.rs + span.rs + locator.rs + fingerprint.rs + canonical_json.rs + diagnostic.rs + schema.rs + + schemas/kernel/ + primitives.v1.json +``` + +`mod.rs` should re-export only the stable public surface. + +```rust +pub mod canonical_json; +pub mod diagnostic; +pub mod error; +pub mod fingerprint; +pub mod id; +pub mod json_pointer; +pub mod locator; +pub mod path; +pub mod schema; +pub mod span; + +pub use canonical_json::{canonical_json_bytes, canonical_json_string}; +pub use diagnostic::{Diagnostic, DiagnosticCode, RelatedLocation, Severity}; +pub use error::{KernelError, KernelResult}; +pub use fingerprint::{sha256_bytes, sha256_canonical_json, Fingerprint}; +pub use id::{ + BoundaryId, ComponentId, EvidenceId, FactId, FileId, MatchId, QueryId, RecipeId, RuleId, + StableId, SymbolId, +}; +pub use json_pointer::{FieldPath, JsonPointer}; +pub use locator::Locator; +pub use path::RepoPath; +pub use span::ByteSpan; +``` + +Kernel-level crate rules: + +```rust +#![forbid(unsafe_code)] +#![deny(missing_docs)] +``` + +## 3. Public Rust surface + +This is the exact shape to lock first. + +```rust +pub type ByteOffset = u64; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RepoPath(String); +// Canonical form: +// - UTF-8 only +// - repo-root-relative +// - POSIX separators only ('/') +// - no leading slash +// - no trailing slash +// - no empty segments +// - no '.' or '..' segments +// - no backslashes + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct JsonPointer(String); +// RFC 6901 canonical pointer syntax. +// Empty string "" means root. +// Leading '/' required for non-root pointers. + +pub type FieldPath = JsonPointer; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Fingerprint(String); +// Format: "sha256:<64 lower-hex chars>" + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct StableId(String); +// Format: ":sha256:<64 lower-hex chars>" + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct FileId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SymbolId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ComponentId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct BoundaryId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct FactId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct EvidenceId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RuleId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct QueryId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RecipeId(StableId); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct MatchId(StableId); + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct ByteSpan { + pub start_byte: u64, + pub end_byte: u64, +} +// Half-open interval [start_byte, end_byte) + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct Locator { + pub path: RepoPath, + #[serde(skip_serializing_if = "Option::is_none")] + pub span: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub json_pointer: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Severity { + Error, + Warning, + Info, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct DiagnosticCode(String); +// Pattern: dot-separated, lowercase namespaced code +// Example: "kernel.repo_path.invalid_absolute" + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RelatedLocation { + pub locator: Locator, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Diagnostic { + pub code: DiagnosticCode, + pub severity: Severity, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub related: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} +``` + +### Constructors and helpers + +These should exist in seam 0 and nowhere else: + +```rust +impl RepoPath { + pub fn parse(input: &str) -> KernelResult; + pub fn as_str(&self) -> &str; + pub fn join(&self, child: &RepoPath) -> KernelResult; + pub fn parent(&self) -> Option; +} + +impl JsonPointer { + pub fn parse(input: &str) -> KernelResult; + pub fn root() -> Self; + pub fn as_str(&self) -> &str; + pub fn push_token(&self, token: &str) -> Self; +} + +impl StableId { + pub fn parse(input: &str) -> KernelResult; + pub fn from_identity(kind: &'static str, identity_lemma: &str) -> Self; + pub fn kind(&self) -> &str; + pub fn as_str(&self) -> &str; +} + +impl Fingerprint { + pub fn parse(input: &str) -> KernelResult; + pub fn as_str(&self) -> &str; +} + +impl ByteSpan { + pub fn new(start_byte: u64, end_byte: u64) -> KernelResult; + pub fn len(&self) -> u64; + pub fn is_empty(&self) -> bool; +} + +pub fn canonical_json_string(value: &T) -> KernelResult; +pub fn canonical_json_bytes(value: &T) -> KernelResult>; +pub fn sha256_bytes(bytes: &[u8]) -> Fingerprint; +pub fn sha256_canonical_json(value: &T) -> KernelResult; +``` + +## 4. Error surface + +```rust +pub type KernelResult = Result; + +#[derive(Debug, thiserror::Error)] +pub enum KernelError { + #[error("invalid repo path: {reason}")] + InvalidRepoPath { input: String, reason: String }, + + #[error("invalid JSON pointer")] + InvalidJsonPointer { input: String }, + + #[error("invalid stable id")] + InvalidStableId { input: String, expected_kind: Option<&'static str> }, + + #[error("invalid fingerprint")] + InvalidFingerprint { input: String }, + + #[error("invalid byte span: start {start_byte} > end {end_byte}")] + InvalidByteSpan { start_byte: u64, end_byte: u64 }, + + #[error("canonical JSON failure: {reason}")] + CanonicalJson { reason: String }, + + #[error("schema validation failure: {reason}")] + SchemaViolation { reason: String }, +} +``` + +Rule: seam 0 exposes typed errors, not `anyhow::Error`. + +## 5. Schema inventory + +The repo already uses Draft 2020-12 for Lift’s current JSON contracts, so seam 0 stays on the same dialect and keeps hand-authored schemas under `schemas/kernel/`. + +Ship one source-of-truth schema file: + +- `schemas/kernel/primitives.v1.json` + +Later app schemas can `$ref` into its `$defs`. + +### `schemas/kernel/primitives.v1.json` + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/kernel/primitives.v1.json", + "title": "Lift Kernel Primitives v1", + "$defs": { + "repo_path": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!.*\\\\)(?!.*//)(?!.*(?:^|/)\\.{1,2}(?:/|$))(?!.*/$).+$", + "description": "UTF-8 repo-root-relative POSIX path." + }, + "json_pointer": { + "type": "string", + "description": "RFC 6901 JSON Pointer. Empty string denotes root." + }, + "stable_id": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*:sha256:[0-9a-f]{64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "byte_span": { + "type": "object", + "additionalProperties": false, + "required": ["start_byte", "end_byte"], + "properties": { + "start_byte": { "type": "integer", "minimum": 0 }, + "end_byte": { "type": "integer", "minimum": 0 } + } + }, + "locator": { + "type": "object", + "additionalProperties": false, + "required": ["path"], + "properties": { + "path": { "$ref": "#/$defs/repo_path" }, + "span": { "$ref": "#/$defs/byte_span" }, + "json_pointer": { "$ref": "#/$defs/json_pointer" } + } + }, + "diagnostic_code": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(\\.[a-z][a-z0-9_]*)+$" + }, + "severity": { + "type": "string", + "enum": ["error", "warning", "info"] + }, + "related_location": { + "type": "object", + "additionalProperties": false, + "required": ["locator", "message"], + "properties": { + "locator": { "$ref": "#/$defs/locator" }, + "message": { "type": "string", "minLength": 1 } + } + }, + "diagnostic": { + "type": "object", + "additionalProperties": false, + "required": ["code", "severity", "message"], + "properties": { + "code": { "$ref": "#/$defs/diagnostic_code" }, + "severity": { "$ref": "#/$defs/severity" }, + "message": { "type": "string", "minLength": 1 }, + "subject": { "$ref": "#/$defs/locator" }, + "related": { + "type": "array", + "items": { "$ref": "#/$defs/related_location" } + }, + "help": { "type": "string", "minLength": 1 } + } + } + } +} +``` + +### Runtime-only invariants not fully expressible in schema + +These must be enforced in Rust validators: + +- `ByteSpan.end_byte >= ByteSpan.start_byte` +- `Locator.json_pointer` must parse as RFC 6901 +- `RepoPath` must be normalized, not just regex-valid +- `StableId.kind()` must match the typed newtype constructor +- arrays that are logical sets must be sorted ascending and deduped before serialization +- canonical JSON hashing must reject non-canonicalizable values + +## 6. Determinism rules + +These are non-negotiable. + +### 6.1 Repo paths + +`RepoPath` canonical form: + +- UTF-8 only +- relative to repo root +- forward slashes only +- no leading slash +- no trailing slash +- no `.` segment +- no `..` segment +- no empty segment +- no backslash +- equality is exact byte equality of the normalized UTF-8 string + +Example valid: + +- `src/lib.rs` +- `crates/lift/src/kernel/path.rs` +- `docs/project_management/system/README.md` + +Example invalid: + +- `/src/lib.rs` +- `.\src\lib.rs` +- `src//lib.rs` +- `src/./lib.rs` +- `src/../lib.rs` +- `src/lib.rs/` + +### 6.2 Field paths + +Internal canonical field references use `JsonPointer` only. + +Examples: + +- `""` +- `/touch/crates_touched` +- `/contract/behavior_deltas` +- `/risk/unknowns_high` + +Compat rule: + +- `app::score::compat_v1` maps internal pointers to current dotted Lift field strings on output. + +### 6.3 Stable IDs + +Stable ID wire format is fixed: + +`:sha256:<64-lower-hex>` + +Examples: + +- `file:sha256:7f4d...` +- `symbol:sha256:2c9a...` +- `component:sha256:8b18...` + +Kernel rule: + +- seam 0 defines the output format and digest algorithm +- each producing seam defines the identity lemma it hashes +- identity lemmas must be deterministic and documented in that seam’s spec + +Recommended lemma prelude: + +- `lift-kernel\0v1\0\0` + +### 6.4 Fingerprints + +Fingerprint wire format is fixed: + +- `sha256:<64-lower-hex>` + +Allowed fingerprint inputs: + +- raw bytes +- canonical JSON bytes + +Canonical JSON must be RFC 8785 JCS over an I-JSON-compatible payload. + +### 6.5 Diagnostics + +Diagnostics are logical records, not free-form logs. + +Sort order must be deterministic: + +1. severity rank: error, warning, info +2. subject path +3. subject span start +4. subject span end +5. code +6. message + +### 6.6 Maps and set-like arrays + +- public serializable maps must use deterministic ordering (`BTreeMap` / `BTreeSet` or equivalent) +- arrays that are conceptually sets must be emitted sorted ascending and unique + +This aligns with the current Lift v1 `emit-json` contract, which already requires sorted unique arrays for `triggers` and `missing_inputs`. + +## 7. Versioning conventions + +Seam 0 should define the rule, but not ship a generic `VersionTag` JSON object in v1. + +Conventions: + +- top-level wire objects use explicit fields like `api_version`, `schema_version`, `model_version`, `profile_version` +- these version fields are integers `>= 1` +- semver strings are used only for crate, adapter, or tool versions +- v1 evolution rule for kernel primitives: + - additive only + - never rename or remove existing fields in `primitives.v1.json` + - incompatible changes require `primitives.v2.json` + +This mirrors the existing Lift v1 evolution posture: additive contracts and frozen v1 semantics. + +## 8. Acceptance criteria + +Seam 0 is done only when all of these are true. + +### Contract completeness + +- every public serializable kernel type has a matching `$defs` entry in `schemas/kernel/primitives.v1.json` +- every `$defs` entry has at least one round-trip fixture +- public examples exist for: + - valid repo path + - invalid repo path + - valid JSON pointer + - valid stable ID + - valid fingerprint + - valid locator + - valid diagnostic + +### Determinism + +- canonical JSON serialization is byte-identical across repeated runs for the same value +- map insertion order does not affect canonical JSON output +- stable IDs are identical across repeated runs for the same kind plus lemma +- diagnostics sort deterministically +- no public kernel serialization uses `HashMap` iteration order + +### Validation behavior + +- invalid repo paths are rejected at construction time +- invalid JSON Pointers are rejected at construction time +- invalid stable IDs are rejected at parse time +- invalid byte spans are rejected at construction time +- typed ID constructors reject wrong kinds + +### Boundary cleanliness + +- kernel has zero dependencies on: + - repo walking + - git + - tree-sitter + - query engine + - Lift score types + - CLI +- kernel exposes typed errors only +- kernel performs no I/O, no env access, no time access, no randomness + +### Schema parity + +- Rust serde round-trips match schema expectations +- fixture JSON validates against `primitives.v1.json` +- parity tests fail if a public Rust field changes without a schema update + +### Compatibility readiness + +- there is a tested helper in `app::score::compat_v1` that translates: + - `JsonPointer` to dotted Lift field path strings + - dotted Lift field path strings to `JsonPointer` +- seam 0 itself contains no Lift-specific dotted field path logic + +## 9. Invariants + +These are must-not-break constraints. + +- `RepoPath` is always normalized. +- `RepoPath` is never absolute. +- `JsonPointer` is the only canonical field-path type inside the engine. +- `ByteSpan` is always half-open and valid. +- `StableId` and `Fingerprint` are lowercase only. +- stable IDs never encode host-specific absolute paths directly. +- kernel types are plain data; no hidden global state. +- kernel serialization is deterministic. +- kernel schemas are additive within a version. +- kernel never depends on any app seam. +- human-friendly rendering concerns do not enter kernel contracts. +- seam 0 does not know what a Lift Vector, Contract Delta, or Patch Plan is. + +## 10. Falsification questions + +These are the PR-review questions that can invalidate the seam. + +- Can two equivalent repo paths produce different `RepoPath` values? +- Can a backslash path like `src\\lib.rs` enter the kernel without rejection? +- Can a `.` or `..` path segment survive normalization? +- Can the same logical JSON value hash differently because object keys were inserted in a different order? +- Can a stable ID depend on memory address, hash-map iteration order, temp directory, or wall clock? +- Can a dotted Lift field path appear anywhere in kernel types? +- Can a `Locator` represent a source location without a valid normalized `RepoPath`? +- Can `ByteSpan { start > end }` be constructed? +- Can a diagnostic be emitted without a stable machine code? +- Can a kernel public type reference a score-specific, policy-specific, or query-specific enum? +- Can an incompatible field change land in `primitives.v1.json` without a version bump? +- Can kernel serialization differ across Linux, macOS, and Windows for the same fixture payload? +- Can public kernel APIs require `anyhow` or stringly-typed errors to interpret failures? +- Can non-deterministic collection ordering leak into any public JSON object? + +If any answer is yes, seam 0 is not clean enough. + +## 11. In scope and out of scope + +### In scope + +- normalized repo paths +- JSON pointers +- stable ID envelope format +- fingerprint envelope format +- byte spans +- locators +- diagnostics +- canonical JSON +- typed kernel errors +- hand-authored JSON schema +- determinism rules +- schema parity tests +- compat hook for dotted Lift field paths outside kernel + +### Out of scope + +- non-UTF8 repo path support in v1 +- line and column display ranges +- AST node shapes +- graph edges +- source excerpts +- query DSL +- patch hunks +- Lift score math +- policy findings +- app response envelopes +- repo walking +- git integration +- editor and LSP-specific UTF-16 position types +- binary serialization formats +- caching strategy + +Non-UTF8 paths stay out of scope for v1. If the repo substrate later encounters them, it should surface a diagnostic and skip them until there is a deliberate v2 design. + +## 12. Risks and mitigations + +### Risk 1: seam 0 becomes a junk drawer + +Mitigation: no app types, no repo I/O, no parsing, no scoring. + +### Risk 2: too many abstractions too early + +Mitigation: keep only primitive contracts here. No `EntityRef`, `FactSet`, or `MatchSet`. + +### Risk 3: canonical JSON implementation drift + +Mitigation: treat RFC 8785 as normative and add byte-for-byte golden tests. + +### Risk 4: dotted Lift paths leak into the engine + +Mitigation: make JSON Pointer canonical in kernel and quarantine dotted-path logic to `app::score::compat_v1`. The current Lift contract is the only place that should still speak dotted paths. + +### Risk 5: path normalization behaves differently across platforms + +Mitigation: `RepoPath` is a platform-independent logical path type, not an OS path wrapper. + +### Risk 6: stable IDs churn too much + +Mitigation: freeze the envelope format in seam 0, but let each producing seam define and test its own lemma. + +### Risk 7: schema and Rust types drift apart + +Mitigation: hand-authored schema plus parity tests in CI. + +### Risk 8: temptation to move seam 0 into a separate crate too early + +Mitigation: keep it inside `crates/lift/src/kernel` until at least two independent workspace crates need the same public API. + +## 13. Specific implementation items + +### A. Rust modules + +- `path.rs` +- `json_pointer.rs` +- `id.rs` +- `span.rs` +- `locator.rs` +- `fingerprint.rs` +- `canonical_json.rs` +- `diagnostic.rs` +- `error.rs` +- `schema.rs` + +### B. Schema + +- add `schemas/kernel/primitives.v1.json` +- add fixture JSON samples under `fixtures/kernel/` + +### C. Tests + +- constructor validation tests +- serde round-trip tests +- schema validation tests +- canonical JSON golden tests +- stable ID determinism tests +- diagnostic ordering tests +- cross-platform path fixture tests + +### D. Compat hook + +- implement `JsonPointer` to dotted Lift path translation in `app::score::compat_v1` +- golden-test it against current v1 field names + +### E. Docs + +- seam 0 contract doc +- examples for every primitive +- how to add a new ID kind note +- how to bump kernel schema version note + +## 14. Recommended decision set + +These decisions are locked now: + +- seam 0 stays internal to `crates/lift` +- JSON Pointer is the canonical internal field-path type +- canonical JSON hashing uses RFC 8785 JCS +- `RepoPath` is UTF-8-only in v1 +- kernel owns byte spans only, not line and column display types +- public kernel JSON contracts live in one hand-authored Draft 2020-12 schema file +- dotted Lift field paths are compat-only, not engine-native +- stable ID envelope is fixed in seam 0; identity lemmas are defined by producing seams + +That yields a seam 0 that is both small and durable. + +The next useful artifact after this is seam 1 in the same format so the sessions implementing the crate can snap together cleanly. diff --git a/crates/lift/lift_seam1_spec_reviewed.md b/crates/lift/lift_seam1_spec_reviewed.md new file mode 100644 index 000000000..e5f9c3903 --- /dev/null +++ b/crates/lift/lift_seam1_spec_reviewed.md @@ -0,0 +1,4014 @@ + + +# substrate-lift seam 1 spec — pack compiler (reviewed against landed seam 0) + +## 0. Ground truth from the landed crate + +This spec is intentionally anchored to the crate as it exists today, not to the earlier idealized design. + +Observed state in the landed crate: + +- `src/kernel/**` is real, tested, schema-backed, and publicly re-exported from `lib.rs`. +- `src/pack/**` is now real code, not a stub, but it currently covers only profile + topology compilation. +- `pack` is still `pub(crate)`, not public API. +- `PackKind`, `PackError`, and `CompiledProfile` already reserve the score/query/rule/recipe surface, but the raw/compiled modules and standalone compile entrypoints for those pack kinds do not exist yet. +- `schemas/pack/` currently contains `common.v1.json`, `profile.v1.json`, `boundary_taxonomy.v1.json`, and `component_map.v1.json`; the advanced pack schemas are still missing. +- `fixtures/pack/**` plus `tests/pack_schema.rs`, `tests/pack_compile.rs`, `tests/pack_fingerprints.rs`, and `tests/pack_topology.rs` now exercise the landed profile + topology compiler behavior. +- `src/app/runtime.rs`, `src/query/mod.rs`, and `src/topo/mod.rs` are still placeholders, so there is still no real downstream consumer loop. +- feature flags already include `substrate-profile`, `compat-v1`, and language flags, but no pack-specific feature flag yet. +- the CLI is still only a scaffold; no pack-loading path exists yet. + +That means seam 1 should be defined as an **internal, contract-heavy compiler seam** that turns declarative pack inputs into immutable runtime objects, while preserving the existing public/API posture of the crate. + +The key consequence is this: + +> seam 1 should **not** make `pack` public yet, and it should **not** reach into repo walking, language parsing, scoring, or CLI behavior. + +--- + +## 1. Mission + +Seam 1 owns the **pack compiler**. + +It is responsible for: + +- loading declarative pack documents from disk or embedded builtins; +- validating them against hand-authored schemas; +- applying pack-local defaults and canonicalization; +- compiling them into immutable runtime objects; +- resolving profile references to other packs; +- computing deterministic source and semantic fingerprints; +- producing deterministic typed pack errors and pack diagnostics. + +It is **not** responsible for: + +- repo snapshots or git; +- AST parsing or tree-sitter; +- graph construction; +- detectors/facts; +- derived facts; +- Lift scoring math; +- query execution; +- patch application; +- CLI argument parsing. + +A useful rule: + +> seam 1 ends at **compiled declarative intent**. +> It does not execute the intent. + +--- + +## 2. Why seam 1 is a separate seam + +The landed seam 0 established a pattern: + +- typed data contracts; +- embedded schemas; +- deterministic serialization/fingerprinting; +- strict validation at construction time. + +Seam 1 should extend that pattern for all declarative-on-disk engine inputs. + +It should be the only seam that knows: + +- where a profile file lives; +- how a rule pack is parsed; +- how a recipe pack references a query pack; +- how a boundary taxonomy document is validated; +- how built-in profiles are embedded and named. + +Later seams should consume `Compiled*` objects, not raw JSON/TOML. + +--- + +## 3. Boundary with existing code + +### Existing seam-0 primitives seam 1 should reuse + +Use directly from `kernel`: + +- `Fingerprint` +- `JsonPointer` +- `FieldPath` +- `StableId` +- `BoundaryId` +- `ComponentId` +- `RuleId` +- `QueryId` +- `RecipeId` +- `Severity` +- canonical JSON helpers + +### Existing seam-0 primitives seam 1 should **not** force-fit + +Do **not** use `Locator` as the primary pack-location primitive. + +Reason: `Locator` requires a `RepoPath`, and seam 1 must be able to report diagnostics for: + +- built-in embedded packs, +- arbitrary file-backed packs before repo snapshot semantics exist, +- inline test packs. + +So seam 1 should introduce its own source-origin type and only convert to repo-relative kernel locators later when possible. + +This is a real adjustment based on the landed seam 0 shape. + +--- + +## 4. Exact module shape + +```text +src/pack/ + mod.rs + error.rs + schema.rs + source.rs + names.rs + refs.rs + diagnostics.rs + expr.rs + compiler.rs + builtin.rs + + raw/ + mod.rs + common.rs + profile.rs + boundary_taxonomy.rs + component_map.rs + score_model.rs + rule_pack.rs + query_pack.rs + recipe_pack.rs + + compiled/ + mod.rs + header.rs + profile.rs + topology.rs + score_model.rs + rule_pack.rs + query_pack.rs + recipe_pack.rs +``` + +`src/pack/mod.rs` should re-export only the internal stable seam-1 surface: + +```rust +pub(crate) mod builtin; +pub(crate) mod compiler; +pub(crate) mod diagnostics; +pub(crate) mod error; +pub(crate) mod expr; +pub(crate) mod names; +pub(crate) mod refs; +pub(crate) mod schema; +pub(crate) mod source; + +pub(crate) mod raw; +pub(crate) mod compiled; + +pub(crate) use compiled::{ + CompiledBoundaryTaxonomy, CompiledComponentMap, CompiledPackHeader, CompiledPackSet, + CompiledProfile, CompiledQueryPack, CompiledRecipePack, CompiledRulePack, + CompiledScoreModel, +}; +pub(crate) use compiler::PackCompiler; +pub(crate) use error::{PackError, PackResult}; +pub(crate) use names::PackName; +pub(crate) use refs::PackRef; +pub(crate) use source::{PackFormat, PackOrigin, PackSource}; +``` + +### Hard rule + +`pack` remains `pub(crate)` in seam 1. + +No new public crate API should be added yet. + +--- + +## 5. New dependencies seam 1 is allowed to add + +Add only what seam 1 actually needs: + +```toml +[dependencies] +toml = "0.8" +globset = "0.4" +``` + +Optional but **not required** for seam 1: + +- `serde_path_to_error` +- `camino` +- `regex` + +### Dependency rules + +Seam 1 must **not** depend on: + +- git tooling, +- tree-sitter, +- repo snapshot code, +- app-specific code, +- `anyhow`. + +--- + +## 6. Exact internal Rust contracts + +## 6.1 Common pack kinds + +```rust +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PackKind { + Profile, + BoundaryTaxonomy, + ComponentMap, + ScoreModel, + RulePack, + QueryPack, + RecipePack, +} +``` + +## 6.2 Pack name + +Pack-level names are **not** kernel stable IDs. + +Reason: seam 0 did not land a `PackId` type, and seam 1 does not need one to be correct. + +```rust +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct PackName(String); +``` + +Canonical form: + +- ASCII lowercase only +- segments separated by `/` +- each segment pattern: `[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*` +- no empty segments +- no leading or trailing slash + +Examples: + +- `generic` +- `generic/default` +- `substrate/default` +- `rust/core` + +Invalid: + +- `Generic` +- `/generic` +- `generic/` +- `generic//default` +- `generic default` + +## 6.3 Pack references + +```rust +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct PackFileRef(String); + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) enum PackRef { + Builtin(PackName), + File(PackFileRef), +} +``` + +Canonical string forms: + +- `builtin:` +- `file:` + +Examples: + +- `builtin:generic/default` +- `file:rules/security.v1.json` +- `file:profiles/default.toml` + +Rules: + +- `file:` refs must use forward slashes; +- `file:` refs must be logical relative paths, never absolute filesystem paths; +- `file:` refs must not contain empty segments, `.` segments, `..` segments, or backslashes; +- `file:` refs are resolved relative to the referring document directory only when the referring pack itself is file-backed; +- builtin and inline sources may only use `builtin:` references in v1; +- seam 1 does not support HTTP, registry, or environment references in v1. + +Canonicalization rule: + +- `file:rules/security.v1.json` is valid +- `file:./rules/security.v1.json` is invalid +- `file:../rules/security.v1.json` is invalid +- `file:/rules/security.v1.json` is invalid + +`PackFileRef` is a pack-local logical path type. It is intentionally separate from `kernel::RepoPath`, because it is relative to the referring pack document, not repo root. + +## 6.4 Pack format and origin + +```rust +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum PackFormat { + Json, + Toml, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum PackOrigin { + Builtin { logical_name: String }, + File { display_path: String }, + Inline { logical_name: String }, +} + +#[derive(Clone, Debug)] +pub(crate) enum PackSource { + Builtin { + logical_name: &'static str, + format: PackFormat, + bytes: &'static [u8], + }, + File { + path: std::path::PathBuf, + format_hint: Option, + }, + Inline { + logical_name: String, + format: PackFormat, + bytes: Vec, + }, +} +``` + +`PackOrigin` is diagnostic/reporting identity. + +It must **never** participate in semantic fingerprinting or stable entry IDs. + +## 6.5 Pack diagnostics + +Because kernel `Locator` is repo-relative, seam 1 needs its own diagnostic location type. + +```rust +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct PackLocation { + pub origin: PackOrigin, + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct PackRelatedLocation { + pub location: PackLocation, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct PackDiagnostic { + pub code: DiagnosticCode, + pub severity: Severity, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub related: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} +``` + +Code pattern should mirror kernel style: + +```text +pack.profile.duplicate_include +pack.schema.invalid_version +pack.refs.unknown_reference +pack.rule_pack.duplicate_rule_id +``` + +Determinism rule: + +- `PackDiagnostic` ordering should mirror kernel diagnostic ordering rules as closely as possible; +- diagnostic codes must use `kernel::DiagnosticCode`, not free-form strings. + +## 6.6 Pack errors + +```rust +pub(crate) type PackResult = Result; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PackError { + #[error("pack I/O failure: {reason}")] + Io { origin: String, reason: String }, + + #[error("unsupported pack format")] + UnsupportedFormat { origin: String }, + + #[error("schema validation failure")] + SchemaViolation { + origin: String, + schema_id: &'static str, + diagnostics: Vec, + }, + + #[error("invalid pack name")] + InvalidPackName { input: String }, + + #[error("invalid pack reference")] + InvalidPackRef { input: String }, + + #[error("duplicate pack id")] + DuplicatePackId { kind: PackKind, id: String }, + + #[error("duplicate entry id")] + DuplicateEntryId { + pack_kind: PackKind, + pack_id: String, + entry_kind: &'static str, + entry_id: String, + }, + + #[error("unknown pack reference")] + UnknownPackReference { + referring_pack: String, + reference: String, + }, + + #[error("pack reference kind mismatch")] + RefKindMismatch { + reference: String, + expected: PackKind, + actual: PackKind, + }, + + #[error("glob compile failure")] + GlobCompile { pattern: String, reason: String }, + + #[error("expression compile failure")] + ExpressionCompile { + pack_id: String, + path: JsonPointer, + reason: String, + }, +} +``` + +Hard rule: seam 1 uses typed errors, not `anyhow`. + +--- + +## 7. Raw document schemas and exact pack shapes + +Seam 1 should add these schema files: + +```text +schemas/pack/common.v1.json +schemas/pack/profile.v1.json +schemas/pack/boundary_taxonomy.v1.json +schemas/pack/component_map.v1.json +schemas/pack/score_model.v1.json +schemas/pack/rule_pack.v1.json +schemas/pack/query_pack.v1.json +schemas/pack/recipe_pack.v1.json +``` + +`src/pack/schema.rs` should embed each one with constants, exactly like seam 0 does for `kernel`. + +## 7.1 Common schema + +`schemas/pack/common.v1.json` should define shared `$defs`: + +- `pack_name` +- `pack_ref` +- `pack_kind` +- `glob` +- `glob_list` +- `language_id` +- `severity` +- `json_pointer` +- `expr` +- `query_ref` +- `reserved_path_class` + +### `pack_name` + +Pattern: + +```text +^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*(?:/[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*)*$ +``` + +### `pack_ref` + +`oneOf`: + +- `^builtin:[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*(?:/[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*)*$` +- `^file:[a-z0-9_][a-z0-9._/-]*$` + +Semantic compile validation must still reject: + +- absolute paths +- empty segments +- `.` segments +- `..` segments +- backslashes + +### `language_id` + +Enum in v1: + +- `json` +- `toml` +- `yaml` +- `rust` +- `python` +- `javascript` +- `typescript` + +### `reserved_path_class` + +Enum in v1: + +- `test` +- `docs` +- `ci` +- `migration` +- `security` +- `public_api` +- `generated` +- `vendor` + +### `expr` + +Shared expression AST used first by score-model compilation and optionally by rule packs. + +Allowed operators in v1: + +- `const_int` +- `const_bool` +- `field` +- `exists` +- `is_null` +- `add` +- `sub` +- `mul` +- `div` +- `max` +- `min` +- `eq` +- `ne` +- `gt` +- `ge` +- `lt` +- `le` +- `and` +- `or` +- `not` + +Field references in expressions must use canonical kernel JSON Pointer strings. + +## 7.2 Profile schema + +Profiles are the top-level pack category. + +### File format + +- canonical on disk format: **TOML** +- schema validation target: JSON representation after TOML parse +- v1 profiles are restricted to TOML values representable losslessly as canonical JSON + +### Raw shape + +```toml +kind = "profile" +version = 1 +id = "generic/default" +name = "Generic default profile" +description = "Default deterministic profile for generic repositories" + +[apps] +enabled = ["score"] +default = "score" + +[analysis] +languages = ["json", "toml", "yaml", "rust", "python", "javascript", "typescript"] +follow_symlinks = false +max_scope_depth = 2 +``` + +### Required fields + +- `kind = "profile"` +- `version = 1` +- `id` +- `name` + +### Optional sections + +- `description` +- `apps` +- `analysis` +- `topology` +- `score` +- `rules` +- `queries` +- `recipes` + +### Compiled shape + +```rust +pub(crate) struct CompiledProfile { + pub header: CompiledPackHeader, + pub apps: CompiledProfileApps, + pub analysis: CompiledAnalysisDefaults, + pub topology: CompiledProfileTopology, + pub score: CompiledProfileScore, + pub includes: CompiledProfileIncludes, + pub diagnostics: Vec, +} + +pub(crate) struct CompiledProfileApps { + pub enabled: std::collections::BTreeSet, + pub default: AppName, +} + +pub(crate) struct CompiledAnalysisDefaults { + pub languages: std::collections::BTreeSet, + pub follow_symlinks: bool, + pub max_scope_depth: u8, +} + +pub(crate) struct CompiledProfileTopology { + pub boundary_taxonomy: Option, + pub component_map: Option, + pub classes: CompiledPathClasses, +} + +pub(crate) struct CompiledProfileScore { + pub model: Option, +} + +pub(crate) struct CompiledProfileIncludes { + pub rule_packs: std::collections::BTreeSet, + pub query_packs: std::collections::BTreeSet, + pub recipe_packs: std::collections::BTreeSet, +} +``` + +## 7.3 Boundary taxonomy schema + +### Raw shape + +```json +{ + "$schema": "https://schemas.substrate.dev/lift/pack/boundary_taxonomy.v1.json", + "kind": "boundary_taxonomy", + "version": 1, + "id": "generic/boundaries", + "name": "Generic boundary taxonomy", + "counting": { "mode": "distinct_minus_one" }, + "boundaries": [ + { + "id": "runtime", + "label": "Runtime", + "include": ["services/runtime/**"], + "exclude": [] + }, + { + "id": "public_api", + "label": "Public API", + "include": ["api/**", "openapi/**"], + "exclude": [] + } + ] +} +``` + +### Compiled shape + +```rust +pub(crate) struct CompiledBoundaryTaxonomy { + pub header: CompiledPackHeader, + pub counting_mode: BoundaryCountingMode, + pub boundaries: std::collections::BTreeMap, + pub diagnostics: Vec, +} + +pub(crate) struct CompiledBoundary { + pub local_id: String, + pub id: BoundaryId, + pub label: String, + pub include_patterns: Vec, + pub exclude_patterns: Vec, + pub include_matcher: globset::GlobSet, + pub exclude_matcher: globset::GlobSet, +} +``` + +### Identity lemma + +```text +boundary id lemma = "pack\0boundary_taxonomy\0\0boundary\0" +``` + +### Important limitation + +Seam 1 compiles glob patterns but does **not** prove global boundary non-overlap. + +That must be checked later against an actual repo snapshot in the topology/classification seam. + +## 7.4 Component map schema + +### Raw shape + +```json +{ + "$schema": "https://schemas.substrate.dev/lift/pack/component_map.v1.json", + "kind": "component_map", + "version": 1, + "id": "generic/components", + "name": "Generic component map", + "counting": { "mode": "distinct" }, + "components": [ + { + "id": "api", + "label": "API", + "include": ["api/**", "openapi/**"], + "exclude": [], + "tags": ["public"] + } + ] +} +``` + +### Compiled shape + +```rust +pub(crate) struct CompiledComponentMap { + pub header: CompiledPackHeader, + pub counting_mode: ComponentCountingMode, + pub components: std::collections::BTreeMap, + pub diagnostics: Vec, +} + +pub(crate) struct CompiledComponent { + pub local_id: String, + pub id: ComponentId, + pub label: String, + pub include_patterns: Vec, + pub exclude_patterns: Vec, + pub tags: std::collections::BTreeSet, + pub include_matcher: globset::GlobSet, + pub exclude_matcher: globset::GlobSet, +} +``` + +### Identity lemma + +```text +component id lemma = "pack\0component_map\0\0component\0" +``` + +## 7.5 Score model schema + +Seam 1 should compile score models structurally but should not yet perform score evaluation. + +### Raw shape + +```json +{ + "$schema": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "kind": "score_model", + "version": 1, + "id": "generic/lift-v2", + "name": "Generic Lift model v2", + "vector_version": 2, + "lift_score": { + "op": "add", + "args": [ + { "op": "field", "path": "/touch/edit_files" }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "estimated_slices": { + "op": "max", + "args": [ + { "op": "const_int", "value": 1 }, + { "op": "field", "path": "/touch/components_touched" } + ] + }, + "triggers": [ + { + "id": "many_files", + "when": { + "op": "gt", + "lhs": { "op": "field", "path": "/touch/edit_files" }, + "rhs": { "op": "const_int", "value": 12 } + } + } + ], + "confidence": { + "default": "high", + "rules": [ + { + "id": "missing_touch_inputs", + "when": { "op": "is_null", "path": "/touch/edit_files" }, + "set": "low", + "causes": ["missing_inputs"] + } + ] + }, + "missing_input_rules": [ + { + "field": "/touch/edit_files", + "when": { "op": "is_null", "path": "/touch/edit_files" } + } + ] +} +``` + +### Compiled shape + +```rust +pub(crate) struct CompiledScoreModel { + pub header: CompiledPackHeader, + pub vector_version: u32, + pub lift_score: CompiledExpr, + pub estimated_slices: CompiledExpr, + pub triggers: Vec, + pub confidence: CompiledConfidenceModel, + pub missing_input_rules: Vec, + pub diagnostics: Vec, +} +``` + +### Important boundary + +Seam 1 validates: + +- expression shape, +- expression operator names, +- JSON Pointer syntax, +- duplicate trigger IDs, +- duplicate confidence rule IDs, +- duplicate missing-input field entries. + +Seam 1 does **not** validate: + +- whether every field path exists in a specific vector version implementation, +- whether the score model is semantically “good”, +- whether score behavior matches compatibility expectations. + +That deeper semantic validation belongs in `app::score`. + +## 7.6 Query pack schema + +### Raw shape + +```json +{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "rust/core", + "name": "Core Rust tree-sitter queries", + "language": "rust", + "engine": "tree_sitter", + "queries": [ + { + "id": "use_statement", + "summary": "Matches Rust use declarations", + "pattern": "(use_declaration) @use", + "captures": [ + { "name": "use", "required": true } + ] + } + ] +} +``` + +### Compiled shape + +```rust +pub(crate) struct CompiledQueryPack { + pub header: CompiledPackHeader, + pub language: LanguageId, + pub engine: QueryEngineKind, + pub queries: std::collections::BTreeMap, + pub diagnostics: Vec, +} +``` + +### Identity lemma + +```text +query id lemma = "pack\0query_pack\0\0query\0" +``` + +### Important boundary + +Seam 1 validates only structural query-pack correctness. + +Actual grammar/query compilation belongs in the later query seam. + +## 7.7 Rule pack schema + +### Raw shape + +```json +{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "generic/policy", + "name": "Generic policy rules", + "rules": [ + { + "id": "architecture.cross_boundary_import", + "summary": "Flags imports that cross configured boundaries", + "severity": "warning", + "scope": { + "languages": ["rust"], + "path_classes": ["public_api"] + }, + "query": { + "pack": "builtin:rust/core", + "id": "use_statement" + }, + "emit": [ + { + "kind": "finding", + "code": "architecture.cross_boundary_import", + "message": "Import crosses boundary" + } + ] + } + ] +} +``` + +### Compiled shape + +```rust +pub(crate) struct CompiledRulePack { + pub header: CompiledPackHeader, + pub rules: std::collections::BTreeMap, + pub diagnostics: Vec, +} +``` + +### Identity lemma + +```text +rule id lemma = "pack\0rule_pack\0\0rule\0" +``` + +### Important boundary + +Seam 1 validates: + +- rule IDs unique, +- referenced query pack refs are syntactically valid, +- referenced local query IDs are well-formed strings, +- severity enums valid, +- emit action shape valid. + +Seam 1 does **not** validate that the referenced query actually exists unless compiling a fully resolved profile bundle. + +## 7.8 Recipe pack schema + +### Raw shape + +```json +{ + "$schema": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "kind": "recipe_pack", + "version": 1, + "id": "generic/core-recipes", + "name": "Generic rewrite recipes", + "recipes": [ + { + "id": "rename_old_api", + "summary": "Rename old_api to new_api", + "query": { + "pack": "builtin:rust/core", + "id": "use_statement" + }, + "transforms": [ + { + "op": "replace_capture_text", + "capture": "use", + "text": "new_api" + } + ] + } + ] +} +``` + +### Compiled shape + +```rust +pub(crate) struct CompiledRecipePack { + pub header: CompiledPackHeader, + pub recipes: std::collections::BTreeMap, + pub diagnostics: Vec, +} +``` + +### Identity lemma + +```text +recipe id lemma = "pack\0recipe_pack\0\0recipe\0" +``` + +--- + +## 8. Compiled headers and bundle shape + +### Compiled pack header + +```rust +pub(crate) struct CompiledPackHeader { + pub kind: PackKind, + pub id: PackName, + pub version: u32, + pub name: String, + pub description: Option, + pub schema_id: &'static str, + pub origin: PackOrigin, + pub source_fingerprint: Fingerprint, + pub semantic_fingerprint: Fingerprint, +} +``` + +Rules: + +- `source_fingerprint` = SHA-256 of source bytes exactly as loaded. +- `semantic_fingerprint` = SHA-256 of normalized canonical JSON after parse/defaults/canonicalization. +- formatting-only changes may change `source_fingerprint` but must not change `semantic_fingerprint`. + +### Resolved compiled bundle + +The main seam-1 product should be a resolved bundle object: + +```rust +pub(crate) struct CompiledPackSet { + pub profile: CompiledProfile, + pub boundary_taxonomy: Option, + pub component_map: Option, + pub score_model: Option, + pub rule_packs: std::collections::BTreeMap, + pub query_packs: std::collections::BTreeMap, + pub recipe_packs: std::collections::BTreeMap, + pub diagnostics: Vec, + pub semantic_fingerprint: Fingerprint, +} +``` + +`CompiledPackSet` is the exact object later seams should consume. + +--- + +## 9. Compiler phases and exact behavior + +```mermaid +flowchart LR + A[PackSource] --> B[Load bytes] + B --> C[Infer/confirm format] + C --> D[Parse TOML/JSON] + D --> E[Schema validate] + E --> F[Apply defaults + normalize] + F --> G[Canonical JSON + semantic fingerprint] + G --> H[Pack-local compile] + H --> I[Resolve profile references] + I --> J[Build CompiledPackSet] +``` + +## 9.1 Phase 1 — load + +Allowed sources: + +- embedded builtins, +- file paths, +- inline test bytes. + +Determinism rules: + +- file bytes read exactly once per source; +- builtins must be embedded by `include_str!` / `include_bytes!`; +- no environment-variable interpolation; +- no remote network fetch. + +## 9.2 Phase 2 — parse + +Format rules: + +- profiles parse as TOML only in v1; +- all other packs parse as JSON only in v1; +- format mismatch is an error, not heuristic fallback. + +## 9.3 Phase 3 — schema validate + +Validation order: + +1. parse syntax, +2. validate `kind`, +3. validate `version`, +4. validate full document schema, +5. run semantic compile checks. + +Schema validation errors should be turned into `PackDiagnostic`s grouped under `PackError::SchemaViolation`. + +## 9.4 Phase 4 — normalize + +Normalization rules: + +- apply default values; +- convert TOML document to canonical JSON value; +- reject TOML constructs that cannot be represented losslessly in canonical JSON; +- sort set-like arrays ascending and dedupe; +- trim no strings silently; +- preserve exact user strings for summaries/messages; +- canonicalize empty option arrays to `[]` and optional scalars to `null` only when represented in JSON form; +- do not reorder arrays that are semantically ordered sequences. + +Set-like arrays in v1: + +- enabled apps +- languages +- include lists +- exclude lists +- tags +- confidence causes + +Ordered arrays in v1: + +- trigger rule sequence +- confidence rule sequence +- missing input rule sequence +- query captures sequence +- recipe transforms sequence +- rule emit sequence + +## 9.5 Phase 5 — pack-local compile + +Pack-local compile responsibilities: + +- parse `PackName` and `PackRef` values; +- compile globs using `globset`; +- compile expression AST into internal enums; +- allocate entry stable IDs; +- detect duplicate local IDs; +- compute semantic fingerprint. + +## 9.6 Phase 6 — profile resolution + +Profile resolution responsibilities: + +- load all referenced packs recursively from the selected profile; +- reject duplicate pack IDs across every resolved bundle slot; +- reject duplicate pack IDs with different semantic fingerprints; +- allow duplicate references to the same normalized resolved source and dedupe them; +- reject cross-origin duplicate pack IDs in v1, even when semantic fingerprints match; +- ensure referenced pack kinds match expected slots; +- resolve rule/query/recipe cross-pack references when building a bundle; +- include transitive pack references reachable from resolved rule and recipe packs, not only refs declared directly on the root profile. + +### Resolution rule + +Only `CompiledPackSet` performs cross-pack reference validation. + +Individual pack compilation may leave external refs unresolved. + +That separation keeps seam 1 composable and testable. + +Resolution cache rule: + +- traversal must be deterministic; +- the memoization key must be the normalized resolved source identity; +- builtin/file/inline origins are distinct identities in v1 and must never silently collapse into one another. + +--- + +## 10. Determinism rules + +These are seam-1 invariants. + +1. Same source bytes + same embedded schema set + same built-in registry = same compiled output. +2. No pack ID or semantic fingerprint may depend on absolute filesystem paths. +3. `source_fingerprint` may depend on bytes; `semantic_fingerprint` must depend only on normalized document semantics. +4. All maps exposed by compiled forms must be `BTreeMap`-ordered. +5. All set-like arrays must be canonicalized to sorted unique form. +6. Duplicate-ID errors must name the same conflicting IDs regardless of load order. +7. Built-in registry iteration order must not affect outputs. +8. Pack compilation must not touch wall clock, hostname, env vars, randomness, or network. +9. `PackOrigin` must not leak into semantic fingerprints or stable entry IDs. + +--- + +## 11. Acceptance criteria + +The original broad Seam 1 intent is executed as ordered execution horizons within one seam family. + +- `Phase A` = Seam 1 +- `Phase B` = Seam 1.25 +- `Phase C` = Seam 1.5 +- `Phase D` = Seam 1.75 + +Seam 1 is not done as a one-shot mega-seam. It is complete only when each ordered horizon reaches its own promotion gate. + +### 11.1 Phase A — compiler spine + profile foundation + +- `src/pack/mod.rs` no longer contains only a stub. +- `schemas/pack/common.v1.json` and `schemas/pack/profile.v1.json` exist and are embedded via `src/pack/schema.rs`. +- `PackCompiler` can compile one standalone profile from builtin, file, and inline sources. +- the same profile compiled twice yields identical semantic fingerprint. +- TOML key order changes do not change profile semantic fingerprint. +- malformed `PackRef` strings are rejected, including absolute paths and dot-segment traversal refs. +- `builtin:generic/default` exists as a deliberately narrow builtin that does not require late-phase pack families. + +#### 11.1.a Step 0 — scope challenge + +Phase A is the first real implementation slice, so it needs the same scope discipline as the eng review, not just a wish list. + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase A decision | +|---|---|---| +| schema embedding | `src/kernel/schema.rs` | mirror the same embed-and-access pattern, do not invent a second loader style | +| canonical JSON + semantic hashing | `src/kernel/canonical_json.rs`, `src/kernel/fingerprint.rs` | reuse for semantic fingerprints | +| machine-readable diagnostics | `src/kernel/diagnostic.rs`, `src/kernel/json_pointer.rs` | keep pack diagnostics typed and pointer-addressable | +| crate-internal seam boundary | `src/lib.rs`, `src/pack/mod.rs` | preserve `pub(crate) mod pack;`, no public API promotion | +| downstream runtime bootstrap | `src/app/runtime.rs` stub | explicitly defer to Phase D | + +Scope decision for Phase A: + +- keep Phase A profile-only +- do not add topology pack compilation yet +- do not add `expr.rs`, score-model, query, rule, or recipe compilation yet +- do not touch CLI loading, repo walking, or runtime bootstrap yet +- do not add new feature flags or any public crate API + +Complexity check: + +- Phase A still touches more than 8 files, but that is acceptable after reduction because this seam is almost entirely contracts, schema files, and deterministic compiler plumbing +- the scope reduction already happened here: topology, advanced pack families, and the runtime consumer were pushed to later horizons instead of pretending they fit in one first slice + +Distribution check: + +- Phase A does not introduce a new distributable artifact +- no release pipeline or install path changes are required in this phase + +#### 11.1.b Phase A architecture + +Phase A execution shape: + +```text +PackSource + ├── Builtin { logical_name, format, bytes } + ├── File { path, format_hint } + └── Inline { logical_name, format, bytes } + | + v +PackCompiler::load_source + | + v +infer format -> parse TOML/JSON -> schema validate + | + v +normalize profile document -> canonical JSON -> semantic fingerprint + | + v +compile refs/defaults/header -> CompiledProfile + | + +--> typed PackDiagnostic / PackError on every failure path +``` + +Module boundary for Phase A only: + +```text +src/lib.rs + └── pub(crate) mod pack + ├── names.rs + ├── refs.rs + ├── source.rs + ├── diagnostics.rs + ├── error.rs + ├── schema.rs + ├── compiler.rs + ├── builtin.rs + ├── raw/ + │ ├── mod.rs + │ ├── common.rs + │ └── profile.rs + └── compiled/ + ├── mod.rs + ├── header.rs + └── profile.rs +``` + +#### 11.1.c Exact implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| A1. seam contracts | `src/pack/mod.rs`, `names.rs`, `refs.rs`, `source.rs`, `diagnostics.rs`, `error.rs` | crate-internal seam surface exists with typed contracts | `mod.rs` is no longer a stub and all core types compile | +| A2. schema embedding | `src/pack/schema.rs`, `schemas/pack/common.v1.json`, `schemas/pack/profile.v1.json` | profile/common schemas embedded like kernel schemas | compiler can fetch both schemas without disk I/O | +| A3. raw + compiled profile types | `src/pack/raw/{mod.rs,common.rs,profile.rs}`, `src/pack/compiled/{mod.rs,header.rs,profile.rs}` | deterministic raw/compiled profile shapes | raw parse and compiled construction compile cleanly | +| A4. compiler spine | `src/pack/compiler.rs` | builtin/file/inline profile compile pipeline | load, parse, validate, normalize, fingerprint, compile all work for profiles | +| A5. narrow builtin | `src/pack/builtin.rs` | `builtin:generic/default` exists with only Phase-A guarantees | builtin profile compiles without any topology or advanced pack dependency | +| A6. fixtures + integration tests | `fixtures/pack/`, `tests/pack_schema.rs`, `tests/pack_compile.rs`, `tests/pack_fingerprints.rs` | contract and determinism coverage for every Phase-A branch | all required acceptance paths have tests | +| A7. docs sweep | `schemas/README.md`, `profiles/README.md`, `fixtures/README.md`, root `README.md` if needed | docs match the landed Phase-A behavior | no README still describes pack as purely reserved | + +Implementation order: + +1. Land A1. +2. Land A2. +3. Land A3. +4. Land A4. +5. Land A5. +6. Launch A6 and A7 once A4/A5 interfaces stop moving. + +#### 11.1.d Test review — required coverage for Phase A + +CODE PATH COVERAGE +=========================== +[+] `PackName` / `PackFileRef` / `PackRef` + ├── [REQUIRED] valid builtin ref parse + ├── [REQUIRED] valid file ref parse + ├── [REQUIRED] absolute-path rejection + ├── [REQUIRED] dot-segment rejection + └── [REQUIRED] builtin/inline source rejects `file:` refs + +[+] Profile source loading + ├── [REQUIRED] builtin source happy path + ├── [REQUIRED] file source happy path + ├── [REQUIRED] inline source happy path + ├── [REQUIRED] unsupported format hard-fails deterministically + └── [REQUIRED] unreadable file yields `PackError::Io` + +[+] Profile parse + schema validation + ├── [REQUIRED] valid TOML profile compiles + ├── [REQUIRED] invalid TOML produces typed diagnostics + ├── [REQUIRED] schema violation produces ordered diagnostics + └── [REQUIRED] omitted optional fields normalize deterministically + +[+] Fingerprinting + normalization + ├── [REQUIRED] same profile twice => same semantic fingerprint + ├── [REQUIRED] TOML key reorder => same semantic fingerprint + ├── [REQUIRED] source fingerprint differs while semantic fingerprint matches + └── [REQUIRED] builtin/file/inline semantic equivalence holds for the same logical document + +USER FLOW COVERAGE +=========================== +[+] Internal engine flow + ├── [REQUIRED] select builtin profile -> compile -> inspect compiled header + ├── [REQUIRED] select file-backed profile -> compile -> inspect compiled profile + └── [REQUIRED] invalid profile -> diagnostic surfaced without panic + +───────────────────────────────── +COVERAGE GOAL: 16/16 required paths covered before Phase A promotion + Code paths: 13/13 + Internal flows: 3/3 +QUALITY BAR: no `★` smoke-only tests for acceptance paths +GAPS ALLOWED AT PROMOTION: 0 +───────────────────────────────── + +Required test files: + +- `tests/pack_schema.rs` for schema embedding, parse failures, and diagnostic shape +- `tests/pack_compile.rs` for builtin/file/inline compile paths and reference validation +- `tests/pack_fingerprints.rs` for semantic/source fingerprint invariants +- extend `tests/compile_matrix.rs` only if the new pack compiler needs to participate in existing crate-level matrix assertions + +Recommended validation commands: + +```bash +cargo fmt --all +cargo clippy -p substrate-lift --all-targets -- -D warnings +cargo test -p substrate-lift --test pack_schema -- --nocapture +cargo test -p substrate-lift --test pack_compile -- --nocapture +cargo test -p substrate-lift --test pack_fingerprints -- --nocapture +``` + +#### 11.1.e Failure modes for Phase A + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| `PackFileRef` parse | absolute or traversal path accepted | yes | yes, `InvalidPackRef` | typed compile failure | +| schema validation | malformed profile accepted past validation | yes | yes, `SchemaViolation` with diagnostics | typed compile failure with locations | +| normalization | semantically equal TOML docs hash differently | yes | test-enforced invariant | failing determinism test | +| builtin registry | builtin profile drifts from embedded schema | yes | yes, compile-time or test-time hard fail | typed compile failure | +| diagnostic ordering | same invalid input yields unstable diagnostic ordering | yes | test-enforced invariant | flaky contract surface | + +Critical gap rule: + +- Phase A does not promote if any failure mode is both untested and capable of surfacing as a silent semantic drift + +#### 11.1.f NOT in scope for Phase A + +- boundary taxonomy compilation, component-map compilation, and topology ref resolution +- score-model expression AST and glob compilation +- query, rule, and recipe pack compilation +- transitive bundle resolution across pack families +- runtime bootstrap adapter in `src/app/runtime.rs` +- CLI flags or config wiring for pack loading +- any public export of the `pack` module + +#### 11.1.g Worktree parallelization strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| A1 seam contracts | `src/pack/` | — | +| A2 schema embedding | `src/pack/`, `schemas/pack/` | A1 | +| A3 raw + compiled profile types | `src/pack/raw/`, `src/pack/compiled/`, `src/pack/` | A1, A2 | +| A4 compiler spine | `src/pack/` | A1, A2, A3 | +| A5 builtin registry | `src/pack/`, `fixtures/pack/valid/` | A4 | +| A6 fixtures + tests | `fixtures/pack/`, `tests/` | A4, A5 | +| A7 docs sweep | `README.md`, `schemas/README.md`, `profiles/README.md`, `fixtures/README.md` | A2, A5 | + +Parallel lanes: + +- Lane A: A1 -> A2 -> A3 -> A4 -> A5 (sequential, shared `src/pack/`) +- Lane B: A6 (sidecar after A5, lives in `fixtures/pack/` and `tests/`) +- Lane C: A7 (sidecar after A5, lives in README surfaces) + +Execution order: + +- Run Lane A first until A5 is stable. +- Once the builtin profile shape and compiler surface stop moving, launch Lane B and Lane C in parallel worktrees. +- Merge B and C back into A before the Phase-A promotion gate. + +Conflict flags: + +- A1 through A5 all share `src/pack/`, so they are one lane only +- Lane B must not edit `src/pack/`; if missing test hooks require core changes, fold that work back into Lane A +- Lane C is safe only if doc updates stay in README surfaces and do not reopen contract debates + +#### 11.1.h Phase A completion summary + +- Step 0: scope accepted after reduction to profile-only compiler spine +- Architecture: Phase-A data flow and module diagram written +- Test Review: 16 required paths named, 0 gaps allowed at promotion +- Failure modes: 0 silent-drift gaps allowed +- NOT in scope: written +- What already exists: written +- Parallelization: 3 lanes, 2 sidecar lanes after 1 sequential core lane + +Phase-A promotion gate: + +1. `src/pack/mod.rs` is no longer a stub and remains crate-private. +2. `common.v1.json` and `profile.v1.json` are embedded via `src/pack/schema.rs`. +3. builtin, file, and inline profile sources all compile through the same `PackCompiler` spine. +4. `PackRef` rejects absolute and dot-segment traversal forms deterministically. +5. semantic fingerprints are stable across equivalent TOML documents. +6. every required Phase-A test above is present and passing. +7. no later-phase module or runtime bootstrap work leaked into this slice. + +### 11.2 Phase B / Phase 2 — topology packs + +Phase B starts from the landed Phase-A spine as it exists in this branch today. + +This phase is not "more pack stuff" in the abstract. It is the first slice that turns the +profile's deferred `topology` refs into real compiled artifacts, while still refusing to +cross the seam into repo walking, path classification, or runtime bootstrap. + +- boundary taxonomy and component map schemas exist and are embedded. +- `PackCompiler` can compile one standalone boundary taxonomy and one standalone component map. +- duplicate local boundary/component IDs are rejected deterministically. +- topology pack refs resolve from a selected profile without repo walking or classification logic. + +#### 11.2.a Step 0 — scope challenge + +Phase B must treat Phase A as the baseline, not reopen it. + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase B decision | +|---|---|---| +| pack source loading | `src/pack/source.rs`, `src/pack/compiler.rs` | extend the same builtin/file/inline load path, do not invent a second source abstraction | +| typed pack refs | `src/pack/refs.rs` | keep `PackRef` and `PackFileRef` as the only topology reference syntax | +| profile topology selection | `src/pack/raw/profile.rs`, `src/pack/compiled/profile.rs` | reuse the existing `topology.boundary_taxonomy` and `topology.component_map` slots, do not redesign profile shape | +| deterministic diagnostics | `src/pack/diagnostics.rs`, `src/pack/error.rs` | continue using `PackDiagnostic`, `PackError`, and sorted diagnostics | +| semantic fingerprints | `src/kernel/canonical_json.rs`, `src/kernel/fingerprint.rs` | keep canonical-JSON hashing for topology pack semantics | +| typed stable IDs | `src/kernel/id.rs` | use `BoundaryId` and `ComponentId` via explicit identity lemmas, do not add new ad-hoc ID types | +| glob dependency and failure shape | `Cargo.toml`, `PackError::GlobCompile` | finally use `globset` for real compiled matchers in topology packs | +| profile compiler spine | `src/pack/compiler.rs::compile_profile` | preserve the existing load -> parse -> schema validate -> normalize -> fingerprint pattern instead of creating a second compiler style | + +Scope decision for Phase B: + +- add only the topology pack families: boundary taxonomy and component map; +- keep profile compilation behavior stable unless a change is required to resolve topology refs cleanly; +- add focused standalone JSON compilation entrypoints for topology packs, not a generic "compile any pack kind" dispatcher yet; +- add one crate-internal resolved topology result for a selected profile; +- allow `builtin:generic/default` to grow topology refs only after the referenced builtin topology packs exist; +- do not add score-model, query-pack, rule-pack, recipe-pack, expression compilation, or runtime bootstrap work. + +Complexity check: + +- this phase still touches more than 8 files, but that is acceptable because the work is still concentrated inside one compiler seam plus schema/tests/docs surfaces; +- the real smell is not file count, it is accidental semantic spillover into classification or bundle orchestration; +- if a change requires touching `src/topo/**`, `src/repo/**`, or `src/app/runtime.rs`, it is probably Phase D work and should be deferred. + +Distribution check: + +- Phase B does not introduce a new distributable artifact. +- No CLI entrypoint, release pipeline, or install path changes belong here. +- If topology packs become externally loadable in this slice, that is scope creep. The seam is still crate-internal. + +#### 11.2.b Phase B architecture + +Phase B execution shape: + +```text +PackSource(JSON) + | + v +PackCompiler::load_source + | + v +infer/confirm JSON -> parse JSON -> schema validate + | + +--> normalize boundary taxonomy -> canonical JSON -> semantic fingerprint + | | + | +--> compile typed BoundaryId entries + globset matchers + | + +--> normalize component map -> canonical JSON -> semantic fingerprint + | + +--> compile typed ComponentId entries + globset matchers + +CompiledProfile::topology refs + | + v +resolve_profile_topology(profile) + | + +--> resolve boundary_taxonomy ref -> compile/fetch exact pack kind + +--> resolve component_map ref -> compile/fetch exact pack kind + | + v +ResolvedProfileTopology +``` + +The exact new crate-internal output should be explicit: + +```rust +pub(crate) struct ResolvedProfileTopology { + pub boundary_taxonomy: Option, + pub component_map: Option, + pub semantic_fingerprint: Fingerprint, +} +``` + +Phase-B entrypoints should be explicit and narrow: + +```rust +impl PackCompiler { + pub(crate) fn compile_boundary_taxonomy( + &self, + source: PackSource, + ) -> PackResult; + + pub(crate) fn compile_component_map( + &self, + source: PackSource, + ) -> PackResult; + + pub(crate) fn resolve_profile_topology( + &self, + profile: &CompiledProfile, + ) -> PackResult; +} +``` + +Rules: + +- `CompiledProfile` still represents the selected intent and keeps deferred `PackRef` slots. +- `ResolvedProfileTopology` is the Phase-B handoff object. It is narrower than `CompiledPackSet` and exists only to close the topology gap cleanly. +- `ResolvedProfileTopology.semantic_fingerprint` is derived from canonical ordered topology semantics, not source path strings. +- `compile_profile` remains TOML-first and profile-only in this slice. Topology packs get sibling entrypoints, not a generic polymorphic compiler layer. +- `CompiledProfileTopology.classes` remains a placeholder in Phase B. Topology compilation is about typed artifacts and resolution, not runtime path classification yet. +- Phase B does not introduce bundle-wide rule/query/recipe closure. That remains Phase C. + +Module boundary for Phase B only: + +```text +src/pack/ + raw/ + boundary_taxonomy.rs + component_map.rs + compiled/ + topology.rs + compiler.rs + schema.rs + builtin.rs + mod.rs +``` + +Implementation shape inside those modules: + +```text +src/pack/compiler.rs + ├── compile_boundary_taxonomy() + ├── compile_component_map() + ├── resolve_profile_topology() + ├── parse_json_bytes() + ├── validate_boundary_taxonomy_document() + └── validate_component_map_document() + +src/pack/compiled/topology.rs + ├── CompiledBoundaryTaxonomy + ├── CompiledBoundary + ├── CompiledComponentMap + ├── CompiledComponent + └── ResolvedProfileTopology +``` + +#### 11.2.c Resolution rules to lock now + +Phase B needs a concrete answer for "how does a profile topology ref become a real pack?" + +Lock these rules: + +1. File-backed profiles resolve `file:` topology refs lexically relative to the profile file's parent directory only. +2. Builtin and inline profiles may resolve only `builtin:` topology refs in v1. +3. Resolution is explicit, never by directory scan, glob search, or repo-root discovery. +4. Boundary-taxonomy slots accept only `PackKind::BoundaryTaxonomy`. +5. Component-map slots accept only `PackKind::ComponentMap`. +6. Wrong-kind refs fail with `PackError::RefKindMismatch`. +7. Missing refs fail with `PackError::UnknownPackReference`. +8. Absolute resolved file paths may be used to load bytes, but they must not affect stable IDs or semantic fingerprints. +9. Inline profiles reject `file:` topology refs for the same reason Phase A rejected file-backed includes from inline sources, there is no stable base directory to resolve against. +10. `ResolvedProfileTopology.semantic_fingerprint` is derived from the resolved compiled artifacts in slot order: boundary taxonomy first, component map second, omitting absent slots. +11. Phase B must not cache compiled topology packs globally yet. Reuse may happen within one resolution call, but process-wide registries belong to the later bundle-resolution seam. + +#### 11.2.d Phase B implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| B1. topology schemas | `src/pack/schema.rs`, `schemas/pack/{boundary_taxonomy.v1.json,component_map.v1.json}` | embed topology schemas using the existing kernel-style pattern | compiler can fetch both schema blobs without disk I/O and schema constants are exported beside the Phase-A profile/common constants | +| B2. raw + compiled topology contracts | `src/pack/raw/{mod.rs,boundary_taxonomy.rs,component_map.rs}`, `src/pack/compiled/{mod.rs,topology.rs}` | deterministic raw/compiled topology types land with identity lemmas reflected in code | raw deserialize and compiled structs build with typed IDs, sorted maps/sets, and diagnostics fields | +| B3. standalone topology compile | `src/pack/compiler.rs` | one boundary taxonomy and one component map compile from builtin/file/inline JSON sources | parse, validate, normalize, fingerprint, glob compile, and duplicate detection all work through focused entrypoints | +| B4. profile topology resolution | `src/pack/compiler.rs`, `src/pack/compiled/topology.rs` | the two profile topology slots resolve into `ResolvedProfileTopology` | selected profile can resolve zero, one, or two topology refs deterministically, with kind mismatch and missing-ref failures typed | +| B5. builtin topology packs | `src/pack/builtin.rs`, `fixtures/pack/valid/` if builtin payloads are mirrored there | `builtin:generic/boundaries` and `builtin:generic/components` exist, and `builtin:generic/default` may reference them | builtin profile topology resolution succeeds without file I/O and builtin names are frozen in tests | +| B6. fixtures + tests | `fixtures/pack/`, `tests/pack_schema.rs`, `tests/pack_compile.rs`, `tests/pack_fingerprints.rs`, `tests/pack_topology.rs` | every Phase-B branch is covered with deterministic assertions | all required Phase-B acceptance paths have tests and no acceptance path is smoke-only | +| B7. docs sweep | `README.md`, `schemas/README.md`, `profiles/README.md`, `fixtures/README.md` | docs match the landed Phase-B contract | README surfaces describe topology compilation and resolution accurately without implying repo classification exists | + +Implementation order: + +1. Land B1. +2. Land B2. +3. Land B3. +4. Launch B4 and B5 once the standalone topology compile surface in B3 stops moving. +5. Land B6 after B4 and B5 are both stable. +6. Land B7 as the last sidecar once builtin names, fixture names, and schema file names are frozen. + +#### 11.2.e Test review — required coverage for Phase B + +Phase B should not promote on "we compiled a couple JSON files." + +CODE PATH COVERAGE +=========================== +[+] Boundary taxonomy standalone compile + ├── [REQUIRED] valid JSON -> typed `BoundaryId` map + compiled include/exclude `GlobSet` + ├── [REQUIRED] malformed JSON -> typed `PackError::ParseFailure` + ├── [REQUIRED] schema violation -> ordered `PackDiagnostic`s + ├── [REQUIRED] duplicate local boundary id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] invalid include glob -> `PackError::GlobCompile` + └── [REQUIRED] invalid exclude glob -> `PackError::GlobCompile` + +[+] Component map standalone compile + ├── [REQUIRED] valid JSON -> typed `ComponentId` map + sorted tag set + compiled `GlobSet` + ├── [REQUIRED] malformed JSON -> typed `PackError::ParseFailure` + ├── [REQUIRED] schema violation -> ordered `PackDiagnostic`s + ├── [REQUIRED] duplicate local component id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] invalid include glob -> `PackError::GlobCompile` + └── [REQUIRED] invalid exclude glob -> `PackError::GlobCompile` + +[+] Fingerprinting + normalization + ├── [REQUIRED] topology schema key reordering does not change semantic fingerprint + ├── [REQUIRED] formatting-only changes can change source fingerprint while semantic fingerprint stays stable + ├── [REQUIRED] equivalent builtin/file/inline topology documents hash to the same semantic fingerprint + └── [REQUIRED] resolved topology fingerprint is stable when slot order and semantics are unchanged + +[+] Profile topology resolution + ├── [REQUIRED] file-backed profile resolves `file:` refs relative to the profile directory only + ├── [REQUIRED] builtin profile resolves builtin refs only + ├── [REQUIRED] inline profile rejects `file:` refs + ├── [REQUIRED] missing topology ref -> `PackError::UnknownPackReference` + ├── [REQUIRED] wrong-kind topology ref -> `PackError::RefKindMismatch` + └── [REQUIRED] zero, one, or two topology slots resolve deterministically + +USER FLOW COVERAGE +=========================== +[+] Internal engine flow + ├── [REQUIRED] compile standalone boundary taxonomy -> inspect compiled boundaries and header + ├── [REQUIRED] compile standalone component map -> inspect compiled components and tags + ├── [REQUIRED] compile profile -> resolve topology -> inspect `ResolvedProfileTopology` + └── [REQUIRED] invalid topology input surfaces typed diagnostics without panic + +───────────────────────────────── +COVERAGE GOAL: 20/20 required Phase-B paths covered before promotion + Code paths: 16/16 + Internal flows: 4/4 +QUALITY BAR: no `★` smoke-only tests for acceptance paths +GAPS ALLOWED AT PROMOTION: 0 +───────────────────────────────── + +Required test files: + +- extend `tests/pack_schema.rs` for topology schema embedding and version constants +- extend `tests/pack_compile.rs` for standalone topology compile, parse/schema failures, and duplicate/glob failure paths +- extend `tests/pack_fingerprints.rs` for topology semantic/source fingerprint invariants +- add `tests/pack_topology.rs` for resolved-profile-topology success paths, slot-kind checks, and file-vs-builtin resolution rules +- extend `tests/compile_matrix.rs` only if topology resolution becomes part of an existing crate-level matrix assertion + +Recommended validation commands: + +```bash +cargo fmt --all +cargo clippy -p substrate-lift --all-targets -- -D warnings +cargo test -p substrate-lift --test pack_schema -- --nocapture +cargo test -p substrate-lift --test pack_compile -- --nocapture +cargo test -p substrate-lift --test pack_fingerprints -- --nocapture +cargo test -p substrate-lift --test pack_topology -- --nocapture +``` + +#### 11.2.f Failure modes for Phase B + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| topology JSON parse | malformed bytes or invalid UTF-8 accepted past parse | yes | yes, `PackError::ParseFailure` with diagnostics | typed compile failure | +| topology schema validation | missing `kind`, wrong `version`, or wrong field shape compiles | yes | yes, `PackError::SchemaViolation` with sorted JSON-pointer diagnostics | typed compile failure with locations | +| boundary/component identity allocation | duplicate local IDs silently overwrite prior entries | yes | yes, `PackError::DuplicateEntryId` | typed compile failure | +| glob matcher build | invalid include/exclude pattern is deferred until later repo analysis | yes | yes, `PackError::GlobCompile` during compile | typed compile failure before any resolved topology exists | +| profile topology lookup | referenced pack not found or resolved relative to the wrong base directory | yes | yes, `PackError::UnknownPackReference` | typed topology-resolution failure | +| profile topology kind matching | boundary slot points at a component map or vice versa and still compiles | yes | yes, `PackError::RefKindMismatch` | typed topology-resolution failure | +| builtin topology registry | builtin pack names drift from the profile refs or payload kind | yes | yes, compile/test-time hard fail | failing builtin resolution test | +| semantic hashing | absolute source path leaks into topology semantic fingerprint | yes | test-enforced invariant | failing determinism test | +| resolved topology assembly | one slot succeeds and the other silently drops diagnostics or data | yes | yes, hard fail the full resolution step | no partial-success object handed downstream | + +Critical gap rule: + +- Phase B does not promote if any failure mode is both untested and capable of surfacing as silent topology drift. + +#### 11.2.g NOT in scope for Phase B + +- repo inventory or filesystem classification against real files +- boundary overlap detection against an actual snapshot +- component-to-boundary relationship inference +- path-class derivation beyond carrying compiled topology artifacts forward +- score-model, query-pack, rule-pack, and recipe-pack compilation +- full `CompiledPackSet` bundle orchestration +- runtime bootstrap or CLI loading + +#### 11.2.h Worktree parallelization strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| B1 topology schemas | `src/pack/`, `schemas/pack/` | — | +| B2 raw + compiled topology contracts | `src/pack/raw/`, `src/pack/compiled/`, `src/pack/` | B1 | +| B3 standalone topology compile | `src/pack/compiler`, `src/pack/` | B1, B2 | +| B4 profile topology resolution | `src/pack/compiler`, `src/pack/compiled/` | B3 | +| B5 builtin topology packs | `src/pack/builtin`, `fixtures/pack/valid/` | B3 | +| B6 fixtures + tests | `fixtures/pack/`, `tests/` | B4, B5 | +| B7 docs sweep | `README surfaces`, `schemas/`, `profiles/`, `fixtures/` docs | B5 | + +Parallel lanes: + +- Lane A: B1 -> B2 -> B3 (sequential core lane, shared `src/pack/` compiler contracts) +- Lane B: B4 (starts after B3, owns topology resolution logic in `src/pack/compiler` and `src/pack/compiled/`) +- Lane C: B5 (starts after B3, owns builtin topology payloads in `src/pack/builtin`) +- Lane D: B6 (starts after B4 + B5, owns fixtures and tests) +- Lane E: B7 (starts after B5, docs-only sidecar) + +Execution order: + +- Run Lane A first until standalone topology compilation is stable. +- Then launch Lane B and Lane C in parallel worktrees. +- Once B and C merge, launch Lane D. +- Lane E can launch as soon as builtin pack names and schema filenames are frozen, but it must not reopen contract design. + +Conflict flags: + +- Lane A is strictly sequential. B1 through B3 all share compiler contract surfaces. +- Lanes B and C are parallel only if B5 stays inside `src/pack/builtin` and fixture/doc surfaces. If B5 needs to reopen `src/pack/compiler.rs`, fold that change back into Lane A or B. +- Lane D must not change `src/pack/` except for missing test hooks discovered during implementation. If tests require compiler changes, stop and route them back to Lane B before continuing. +- Lane E is safe only if it remains a docs-only sweep. If docs work uncovers unresolved semantics, fix the semantics first in the owning lane rather than patching around them in prose. + +#### 11.2.i Phase B completion summary + +Phase-B completion summary: + +- Step 0: scope accepted as topology-only compiler expansion +- Architecture: Phase-B data flow and module boundary diagram written +- Test Review: 20 required paths named, 0 gaps allowed at promotion +- Failure modes: 0 silent-drift gaps allowed +- NOT in scope: written +- What already exists: written +- Parallelization: 5 lanes, 2 parallel core-adjacent lanes after 1 sequential setup lane + +Phase-B promotion gate: + +1. `boundary_taxonomy.v1.json` and `component_map.v1.json` are embedded through `src/pack/schema.rs`. +2. `PackCompiler` can compile standalone boundary-taxonomy and component-map packs from builtin, file, and inline JSON sources through focused Phase-B entrypoints. +3. `BoundaryId` and `ComponentId` are allocated deterministically from the identity lemmas already defined in this spec. +4. invalid topology globs fail during compile, not later during repo analysis. +5. a selected `CompiledProfile` can resolve zero, one, or two topology refs into a crate-internal `ResolvedProfileTopology`. +6. `builtin:generic/boundaries` and `builtin:generic/components` exist before any builtin profile depends on them. +7. every required Phase-B test above is present and passing. +8. no Phase-B change requires touching `src/topo/**`, `src/repo/**`, or `src/app/runtime.rs`. + +### 11.3 Phase C / Phase 3 — advanced pack families + bundle resolution + +Phase C starts from the landed Phase-B spine as it exists in this branch today. + +This is the first phase where the spec has to stop sounding elegant and start being +operationally sharp. The goal is not "add the remaining enum variants." The goal is +to close the already-landed contract gap honestly: profiles can already name +score/query/rule/recipe packs, but the compiler still cannot turn those refs into one +deterministic crate-internal `CompiledPackSet`. + +Phase C is where that becomes real. + +#### 11.3.a Step 0 — scope challenge + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase C decision | +|---|---|---| +| deferred advanced refs on profiles | `src/pack/raw/profile.rs`, `src/pack/compiled/profile.rs` | keep `score.model`, `rules.packs`, `queries.packs`, and `recipes.packs` as the authoritative Phase-C entry surface; do not invent a second bundle-root config shape | +| pack kind universe | `src/pack/raw/common.rs::PackKind` | fill in the already-landed `ScoreModel`, `QueryPack`, `RulePack`, and `RecipePack` variants instead of redesigning the pack kind model | +| typed error vocabulary | `src/pack/error.rs` | reuse `DuplicatePackId`, `DuplicateEntryId`, `UnknownPackReference`, `RefKindMismatch`, and `ExpressionCompile`; do not create bundle-only ad hoc errors | +| common schema seed | `schemas/pack/common.v1.json` | extend the existing `expr` and `query_ref` defs; do not fork a second "advanced common" schema | +| deterministic compile pipeline | `src/pack/compiler.rs` | reuse the landed load -> parse -> schema validate -> normalize -> canonicalize -> compile pipeline for every new pack kind | +| topology resolution precedent | `src/pack/compiler.rs`, `src/pack/compiled/topology.rs` | build Phase-C bundle resolution on the same normalized-source identity, fingerprint, and kind-checking rules already used for topology resolution | +| crate boundary | `src/lib.rs`, `src/pack/mod.rs` | keep `pack` crate-private and keep Phase C inside `src/pack/**` plus schemas, fixtures, tests, and docs | + +Scope decision for Phase C: + +- add only `score_model`, `query_pack`, `rule_pack`, and `recipe_pack` structural compilation plus bundle resolution into `CompiledPackSet`; +- keep all four advanced pack families JSON-only in v1; +- add one crate-internal bundle-resolution entrypoint, `PackCompiler::resolve_profile_pack_set(&CompiledProfile)`, rather than a generic "compile any pack" dispatcher; +- keep standalone compile entrypoints for each advanced pack kind so the seam stays composable and testable in isolation; +- allow standalone rule and recipe packs to carry unresolved external query refs, then close those refs only when building a `CompiledPackSet`; +- add a Phase-C builtin score model only if a builtin profile actually references it; +- do not add builtin query/rule/recipe packs just to make the registry look bigger; +- keep runtime bootstrap, score evaluation, tree-sitter query compilation, rule execution, and recipe execution out of this phase. + +Complexity and blast radius for Phase C: + +- Phase C necessarily touches more than 8 files because the missing surface spans schemas, raw contracts, compiled contracts, compiler entrypoints, fixtures, and tests. +- That is acceptable here because the blast radius stays inside `src/pack/**`, `schemas/pack/**`, `fixtures/pack/**`, dedicated test files, and README surfaces. +- The scope reduction already happened by keeping the first real consumer loop in Phase D. +- If this phase starts slipping, `rule_pack` and `recipe_pack` are still the first slip candidates, not bundle resolution itself. + +Distribution check: + +- Phase C does not introduce a new distributable artifact. +- There is no new binary, package, container image, or install surface in this phase. +- If implementation starts pulling in CLI discovery, repo auto-loading, or external pack distribution, that is scope creep and should be pushed to later seams. + +#### 11.3.b Phase C architecture + +Phase C execution shape: + +```text +selected profile source + | + v +compile_profile(source) + | + +--> compiled profile + | + +--> resolve_profile_topology(profile) [reuse landed Phase-B path] + | + v +resolve_profile_pack_set(profile) + | + +--> resolve optional score model + +--> resolve direct query packs + +--> resolve direct rule packs + | | + | +--> enqueue transitive query refs + | + +--> resolve direct recipe packs + | | + | +--> enqueue transitive query refs + | + +--> detect kind mismatches / duplicate pack ids + +--> canonicalize ordered bundle semantics + | + v +CompiledPackSet + ├── profile + ├── boundary_taxonomy? + ├── component_map? + ├── score_model? + ├── query_packs + ├── rule_packs + ├── recipe_packs + ├── diagnostics + └── semantic_fingerprint +``` + +Phase-C entrypoints should be explicit and narrow: + +```rust +impl PackCompiler { + pub(crate) fn compile_score_model( + &self, + source: PackSource, + ) -> PackResult; + + pub(crate) fn compile_query_pack( + &self, + source: PackSource, + ) -> PackResult; + + pub(crate) fn compile_rule_pack( + &self, + source: PackSource, + ) -> PackResult; + + pub(crate) fn compile_recipe_pack( + &self, + source: PackSource, + ) -> PackResult; + + pub(crate) fn resolve_profile_pack_set( + &self, + profile: &CompiledProfile, + ) -> PackResult; +} +``` + +Module boundary for Phase C: + +```text +src/pack/ + expr.rs + compiler.rs + schema.rs + builtin.rs [optional score-model builtin only] + + raw/ + score_model.rs + query_pack.rs + rule_pack.rs + recipe_pack.rs + + compiled/ + score_model.rs + query_pack.rs + rule_pack.rs + recipe_pack.rs + mod.rs [exports CompiledPackSet] +``` + +Architecture rules to lock before coding: + +1. `resolve_profile_pack_set` consumes a compiled profile, not raw source bytes. +2. `profile.score.model` may resolve only to `PackKind::ScoreModel`. +3. `profile.queries.packs` may resolve only to `PackKind::QueryPack`. +4. `profile.rules.packs` may resolve only to `PackKind::RulePack`. +5. `profile.recipes.packs` may resolve only to `PackKind::RecipePack`. +6. Rule and recipe defs may reference query packs structurally through `query_ref`, and those refs expand the bundle closure. +7. Standalone pack compilation leaves external refs unresolved on purpose. +8. Bundle traversal order must be deterministic and key-based, never filesystem-discovery-order-based. +9. The memoization key for resolution remains the normalized resolved source identity. +10. The same normalized source may be deduped within one resolution call. +11. Cross-origin duplicate pack IDs still fail in v1, even if semantic fingerprints match. +12. `CompiledPackSet.semantic_fingerprint` must depend only on canonical ordered bundle semantics, never on origin display strings or discovery order. + +Bundle-resolution algorithm to implement, in order: + +1. Start from one already-compiled profile. +2. Reuse Phase-B topology resolution unchanged for topology slots. +3. Seed a deterministic queue from `score.model`, `queries.packs`, `rules.packs`, and `recipes.packs`. +4. Resolve each ref by expected kind only, using existing builtin/file/inline source rules. +5. Compile each newly discovered pack with its focused pack-kind entrypoint. +6. When compiling rule or recipe packs, enqueue transitive `query_ref` dependencies. +7. Memoize by normalized resolved source identity inside one call. +8. Reject cross-origin duplicate pack IDs even if payload semantics happen to match. +9. Assemble the final bundle into ordered `BTreeMap` / `BTreeSet` surfaces. +10. Compute the bundle semantic fingerprint only from canonical ordered bundle semantics. + +#### 11.3.c Code quality and contract hygiene + +This phase is at high risk of becoming clever in the wrong places. Do not do that. + +Code-quality rules for Phase C: + +- do not add a generic pack dispatcher, registry trait, or plugin abstraction just because four pack kinds now exist; +- keep every advanced pack family on the same explicit compiler pattern Phase A and B already use; +- keep `src/pack/compiler.rs` as the single integration choke point for bundle closure; +- keep raw contracts in `src/pack/raw/**`, compiled contracts in `src/pack/compiled/**`, and schema embedding in `src/pack/schema.rs`; +- do not duplicate query-ref parsing or expression validation logic across pack families; that belongs in `expr.rs` plus shared schema defs; +- keep diagnostics machine-readable and pointer-addressable, never stringly typed or pack-family-specific in format; +- keep `CompiledPackSet` a compiler output object, not a runtime service locator or app-logic container; +- do not introduce any dependency from new Phase-C modules into `src/app/**`, `src/query/**`, `src/topo/**`, `src/repo/**`, `src/graph/**`, or `src/facts/**`. + +ASCII diagram maintenance requirements: + +- `src/pack/compiler.rs` should gain one inline ASCII comment showing the Phase-C bundle-resolution flow once the implementation lands. +- `src/pack/compiled/mod.rs` should gain a short bundle-shape diagram if the exported compiled surface becomes non-obvious. +- If any nearby diagrams in pack modules become stale during Phase C, they must be updated in the same change. + +#### 11.3.d Exact implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| C1. expression compiler + common-schema finalization | `src/pack/expr.rs`, `src/pack/schema.rs`, `schemas/pack/common.v1.json` | one explicit expression compiler exists for score-model expressions and shared query-ref validation | unsupported ops, bad JSON pointers, malformed node shapes, and query-ref shape errors fail with typed errors | +| C2. score-model contracts | `src/pack/raw/score_model.rs`, `src/pack/compiled/score_model.rs`, `schemas/pack/score_model.v1.json`, `src/pack/compiler.rs` | standalone score-model compile works from builtin/file/inline JSON | duplicate trigger, confidence, and missing-input entries fail deterministically | +| C3. query-pack contracts | `src/pack/raw/query_pack.rs`, `src/pack/compiled/query_pack.rs`, `schemas/pack/query_pack.v1.json`, `src/pack/compiler.rs` | standalone query-pack compile works with typed query IDs and capture metadata | duplicate query IDs, unsupported engines, and invalid capture shapes fail deterministically | +| C4. rule-pack + recipe-pack contracts | `src/pack/raw/{rule_pack.rs,recipe_pack.rs}`, `src/pack/compiled/{rule_pack.rs,recipe_pack.rs}`, `schemas/pack/{rule_pack.v1.json,recipe_pack.v1.json}`, `src/pack/compiler.rs` | standalone rule/recipe compile works with structural query refs and typed emit/transform definitions | invalid severity, bad emit shape, bad transform shape, and duplicate local IDs fail deterministically | +| C5. bundle resolution | `src/pack/compiler.rs`, `src/pack/compiled/mod.rs` | one `CompiledPackSet` is built from a selected profile using deterministic closure rules | optional score model, direct includes, transitive query closure, duplicate detection, and bundle fingerprinting all work through one owned path | +| C6. fixtures + optional builtins | `fixtures/pack/**`, `src/pack/builtin.rs` | valid, invalid, and canonical fixtures exist for every advanced pack family and bundle-resolution edge case | builtin score model is added only if a builtin profile needs it, and no decorative builtins land | +| C7. tests + docs sweep | `tests/{pack_schema.rs,pack_compile.rs,pack_fingerprints.rs,pack_bundle.rs,pack_topology.rs}`, `README.md`, `schemas/README.md`, `profiles/README.md`, `rules/README.md`, `recipes/README.md`, `fixtures/README.md` | Phase-C behavior is fully covered and documented | docs describe structural compile + bundle resolution accurately without implying runtime execution exists | + +Implementation order: + +1. Land C1 first so every advanced pack shares one expression and common-schema contract. +2. Land C2 and C3 next, because rule and recipe packs depend on those shapes being stable. +3. Land C4 once query-pack shape is frozen. +4. Land C5 only after standalone compile for all four advanced families stops moving. +5. Land C6 after C5 so fixtures mirror the final contract instead of churning during core implementation. +6. Land C7 last, after fixture names, bundle semantics, and schema filenames are stable enough to document honestly. + +#### 11.3.e Test review — required coverage for Phase C + +This is the promotion gate for Phase C. + +Write the QA-facing artifact to: + +- `~/.gstack/projects/atomize-hq-substrate/{user}-{branch}-phase-c-test-plan-{datetime}.md` + +Required coverage diagram: + +```text +[+] Expression compiler + ├── [REQUIRED] valid arithmetic / comparison / boolean trees compile + ├── [REQUIRED] unsupported operator -> `PackError::ExpressionCompile` + ├── [REQUIRED] invalid JSON Pointer -> `PackError::ExpressionCompile` + └── [REQUIRED] arity / field-shape mismatches fail deterministically + +[+] Score model compile + ├── [REQUIRED] builtin/file/inline happy path + ├── [REQUIRED] duplicate trigger id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] duplicate confidence rule id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] duplicate missing-input field -> `PackError::DuplicateEntryId` + └── [REQUIRED] semantic/source fingerprint split stays stable + +[+] Query pack compile + ├── [REQUIRED] valid query pack happy path + ├── [REQUIRED] duplicate query id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] unsupported engine or bad shape -> typed schema/compile failure + └── [REQUIRED] capture ordering / required flags preserve semantics deterministically + +[+] Rule pack compile + ├── [REQUIRED] valid rule pack with structural query ref + ├── [REQUIRED] duplicate rule id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] invalid severity / emit shape fails deterministically + └── [REQUIRED] standalone rule-pack compile does not pretend external query refs are resolved + +[+] Recipe pack compile + ├── [REQUIRED] valid recipe pack with structural query ref + ├── [REQUIRED] duplicate recipe id -> `PackError::DuplicateEntryId` + ├── [REQUIRED] invalid transform op / shape fails deterministically + └── [REQUIRED] transform sequence ordering is preserved semantically + +[+] Bundle resolution + ├── [REQUIRED] profile score-model ref resolves into `CompiledPackSet` + ├── [REQUIRED] direct query/rule/recipe includes resolve by expected kind + ├── [REQUIRED] rule-pack query refs pull transitive query-pack closure + ├── [REQUIRED] recipe-pack query refs pull transitive query-pack closure + ├── [REQUIRED] missing ref -> `PackError::UnknownPackReference` + ├── [REQUIRED] wrong-kind ref -> `PackError::RefKindMismatch` + ├── [REQUIRED] duplicate against profile/topology/advanced slots -> `PackError::DuplicatePackId` + ├── [REQUIRED] cross-origin duplicate pack id -> `PackError::DuplicatePackId` + ├── [REQUIRED] same resolved source referenced twice dedupes deterministically + └── [REQUIRED] bundle semantic fingerprint is stable across discovery order +``` + +Required test file plan: + +- extend `tests/pack_schema.rs` for advanced-pack schema embedding and fixture validation; +- extend `tests/pack_compile.rs` for standalone score/query/rule/recipe compile success and failure paths; +- extend `tests/pack_fingerprints.rs` for score-model and bundle-order determinism invariants; +- keep `tests/pack_topology.rs` as a regression suite, because Phase C bundle resolution reuses Phase-B topology rules; +- add `tests/pack_bundle.rs` for closure, duplicate, wrong-kind, and dedupe coverage; +- touch `tests/compile_matrix.rs` only if `CompiledPackSet` becomes part of an existing crate-level compile assertion. + +Required validation commands before Phase-C promotion: + +```bash +cargo fmt --all +cargo clippy -p substrate-lift --all-targets -- -D warnings +cargo test -p substrate-lift --test pack_schema -- --nocapture +cargo test -p substrate-lift --test pack_compile -- --nocapture +cargo test -p substrate-lift --test pack_fingerprints -- --nocapture +cargo test -p substrate-lift --test pack_topology -- --nocapture +cargo test -p substrate-lift --test pack_bundle -- --nocapture +``` + +#### 11.3.f Performance and determinism review + +Phase C is still primarily a correctness seam, but there are two performance traps worth killing now: + +- repeated bundle traversal over already-resolved refs, which turns deterministic closure into quadratic sludge; +- accidental dependence on insertion order, directory iteration order, or origin display strings, which turns correctness bugs into flaky performance bugs. + +Performance rules for Phase C: + +- keep one per-call memo map keyed by normalized resolved source identity; +- keep one per-call seen-id map keyed by `PackName` plus origin provenance for duplicate detection; +- use ordered maps and sets for all bundle assembly surfaces; +- do not add a process-global cache or registry in this phase; +- do not recompile the same resolved source twice inside one bundle-resolution call; +- do not perform repo walking, query execution, or scoring work in this phase; +- treat any bundle-semantic fingerprint drift caused by load order as a correctness bug, not a perf optimization tradeoff. + +Phase-C performance is acceptable only if bundle closure work scales with the number of unique referenced packs, not with repeated revisits to the same pack graph. + +#### 11.3.g Failure modes for Phase C + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| `expr.rs` compiler | invalid expression tree compiles or fails ambiguously | yes | yes, `PackError::ExpressionCompile` with path-aware diagnostics | typed compile failure at the pack-local path | +| score-model compile | duplicate trigger, confidence, or missing-input entry silently overwrites prior state | yes | yes, `PackError::DuplicateEntryId` | typed compile failure | +| query-pack compile | malformed capture or unsupported engine shape leaks into compiled output | yes | yes, schema or typed compile failure | no partial compiled query pack | +| rule or recipe standalone compile | external query refs look resolved even though closure has not happened yet | yes | yes, unresolved refs stay structural until bundle resolution | explicit deferred state, never false success | +| bundle resolution | rule or recipe pack carries a query ref of the wrong pack kind | yes | yes, `PackError::RefKindMismatch` during bundle resolution | typed bundle-resolution failure | +| bundle resolution | transitive query closure is incomplete | yes | yes, hard fail bundle resolution | no partially-resolved `CompiledPackSet` | +| bundle resolution | duplicate IDs across profile, topology, score, query, rule, or recipe slots | yes | yes, `PackError::DuplicatePackId` | typed bundle-resolution failure | +| bundle fingerprinting | semantic fingerprint depends on discovery order or origin display strings | yes | test-enforced invariant | failing determinism test | + +Critical gap rule: + +- Phase C does not promote if any failure mode above is both untested and capable of producing silent semantic drift. + +#### 11.3.h NOT in scope for Phase C + +- tree-sitter query compilation or execution in `src/query/**` +- score evaluation against real work-lift vectors +- rule execution, finding emission, or any runtime policy loop +- recipe application or patch generation +- `src/app/runtime.rs` bootstrap work +- repo walking, topology classification, or any `src/topo/**` implementation +- CLI discovery or pack-management commands +- remote registries, environment interpolation, or other non-local pack sources + +#### 11.3.i Worktree-parallel execution strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| C1 expression compiler + common schema | `src/pack/expr.rs`, `src/pack/schema.rs`, `schemas/pack/` | — | +| C2 score-model contracts | `src/pack/raw/`, `src/pack/compiled/`, `src/pack/compiler.rs`, `schemas/pack/` | C1 | +| C3 query-pack contracts | `src/pack/raw/`, `src/pack/compiled/`, `src/pack/compiler.rs`, `schemas/pack/` | C1 | +| C4 rule-pack + recipe-pack contracts | `src/pack/raw/`, `src/pack/compiled/`, `src/pack/compiler.rs`, `schemas/pack/` | C1, C3 | +| C5 bundle resolution | `src/pack/compiler.rs`, `src/pack/compiled/`, `tests/pack_bundle.rs` | C2, C3, C4, landed Phase-B topology code | +| C6 fixtures | `fixtures/pack/` | C5 contract shape stable | +| C7 tests | `tests/`, `fixtures/pack/` | C5, C6 | +| C8 docs sweep | `README.md`, `schemas/README.md`, `profiles/README.md`, `rules/README.md`, `recipes/README.md`, `fixtures/README.md` | C5 contract text stable | + +Parallel lanes: + +- Lane A: C1 -> C2 -> C3 -> C4 -> C5 (sequential core lane, one owner for `src/pack/compiler.rs`) +- Lane B: C6 (fixture sidecar after C5 freezes schema names, builtin names, and bundle behavior) +- Lane C: C7 (test sidecar after C5 and C6 are stable enough to stop renaming fixtures) +- Lane D: C8 (docs-only sidecar after C5 contract text stops moving) + +Execution order: + +- Run Lane A first until C5 lands and the bundle contract stops moving. +- Once C5 is stable, launch Lane B and Lane D in parallel worktrees. +- Launch Lane C once fixture filenames from Lane B are frozen. +- Merge Lane D first if it stays docs-only, then merge B, then merge C so the final test pass sees the finished fixture set and docs already describe the actual landed behavior. + +Conflict flags: + +- `src/pack/compiler.rs` is the integration choke point for Phase C and should have one owner at a time. +- `schemas/pack/common.v1.json` is another conflict hotspot because C1 touches defs shared by every later slice. +- Lane B must not reopen `src/pack/compiler.rs`; if fixture work uncovers missing hooks, route that back to Lane A before continuing. +- Lane C may overlap `fixtures/pack/` only for test consumption, not fixture redesign. If tests require fixture reshaping, finish that in Lane B first. +- Lane D is safe only if it remains a docs-only sweep. If docs work uncovers unresolved semantics, fix the semantics first in the owning lane rather than papering over them in prose. + +#### 11.3.j Phase C completion summary + +- Step 0: scope accepted as advanced structural packs plus deterministic bundle resolution only +- Architecture: Phase-C data flow, module boundary, and bundle-resolution algorithm written +- Code quality: explicit compiler ownership and no-generic-dispatcher rules locked +- Test Review: coverage diagram written, 0 promotion gaps allowed +- Performance Review: per-call memoization and deterministic ordering rules locked +- Failure modes: 0 silent-drift gaps allowed +- NOT in scope: written +- What already exists: written +- Parallelization: 4 lanes, 3 sidecar lanes after 1 sequential core lane +- Test artifact: write to `~/.gstack/projects/atomize-hq-substrate/{user}-{branch}-phase-c-test-plan-{datetime}.md` + +Phase-C promotion gate: + +1. all four advanced pack schemas are embedded through `src/pack/schema.rs`; +2. standalone compile entrypoints exist for score, query, rule, and recipe packs; +3. `resolve_profile_pack_set` produces a deterministic `CompiledPackSet`; +4. transitive query closure from rule and recipe packs is enforced; +5. duplicate pack IDs, wrong-kind refs, invalid expressions, and malformed advanced-pack payloads all fail with typed errors; +6. every required Phase-C validation command above passes; +7. no Phase-C implementation change touches `src/app/runtime.rs`, `src/query/**`, `src/topo/**`, `src/repo/**`, `src/graph/**`, or `src/facts/**`. + +### 11.4 Phase D / Phase 4 — pack activation boundary + +Phase C already proved the hard compiler-side claim: + +- `PackCompiler::resolve_profile_pack_set` can produce a deterministic `CompiledPackSet`; +- advanced pack refs now close transitively through query/rule/recipe relationships; +- the remaining unresolved risk is no longer "can something consume packs at all?" +- the remaining unresolved risk is "what is the one allowed handoff contract out of `pack`?" + +That means Phase D should stop being a vague "first consumer loop" and become an explicit **pack activation boundary**. + +The first real consumer still matters, but it is now an **exit criterion**, not the phase identity. + +#### 11.4.a Step 0 — scope challenge + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase D decision | +|---|---|---| +| standalone source loading + profile compile | `src/pack/compiler.rs::compile_profile` | reuse directly, do not duplicate parsing or validation in runtime | +| topology and advanced-pack closure | `src/pack/compiler.rs::resolve_profile_topology`, `resolve_profile_pack_set` | reuse directly as the only closure path | +| builtin/file/inline source model | `src/pack/source.rs`, `src/pack/builtin.rs` | keep as the only accepted Phase-D ingress surface | +| compiled bundle representation | `src/pack/compiled/mod.rs::CompiledPackSet` | treat as the compiler-owned payload wrapped by the runtime bootstrap contract | +| runtime seam placeholder | `src/app/runtime.rs` | replace the stub with the narrow bootstrap adapter, not a full app runtime | +| current bundle-resolution regression suite | `tests/pack_bundle.rs` | keep as the compiler-resolution regression suite; add bootstrap-specific assertions in a dedicated runtime suite instead of bloating bundle tests | +| score compatibility helper | `src/app/score/compat_v1.rs` | leave untouched, score semantics remain out of scope | + +Scope decision for Phase D: + +- define one crate-internal handoff contract in `src/app/runtime.rs`; +- prove one crate-internal runtime ingress can obtain that contract from pack sources; +- keep typed `PackError` and `PackDiagnostic` surfaces intact; +- keep CLI loading, repo discovery, score evaluation, query execution, and app dispatch out of scope; +- keep the blast radius mostly inside `src/app/runtime.rs`, one new integration suite, and README text. + +Complexity check for Phase D: + +- if implementation needs more than these core surfaces, `src/app/runtime.rs`, `tests/runtime_bootstrap.rs`, and a small README/doc sweep, treat that as a smell and stop to justify it in review before coding; +- touching `src/pack/compiler.rs` is acceptable only for tiny visibility or helper extraction corrections discovered during implementation, not for reopening Phase-C behavior; +- touching `src/cli/**`, `src/repo/**`, `src/query/**`, `src/topo/**`, or `src/app/score/**` means the seam boundary has already been violated. + +Distribution check: + +- Phase D does not introduce a new distributable artifact; +- there is no new binary, package, container image, or install surface here; +- if implementation starts needing CLI knobs or publish-time config just to exercise bootstrap, that is proof the seam is overreaching. + +#### 11.4.b Phase D architecture + +Phase D should introduce a narrow runtime bootstrap contract, not a second compiler. + +Recommended contract: + +```rust +pub(crate) struct ProfileBootstrap { + pub bundle: CompiledPackSet, +} + +impl ProfileBootstrap { + pub(crate) fn from_pack_set(bundle: CompiledPackSet) -> Self; +} + +pub(crate) fn bootstrap_profile(source: PackSource) -> PackResult; +``` + +Contract rules: + +- `ProfileBootstrap` lives in `src/app/runtime.rs`, not in `src/pack/**`; +- `bootstrap_profile` is a thin orchestration entrypoint: + - construct or borrow one `PackCompiler`, + - compile one profile from `PackSource`, + - resolve one `CompiledPackSet`, + - wrap it in `ProfileBootstrap`, + - return `PackResult` unchanged; +- `ProfileBootstrap::from_pack_set` must be pure and filesystem-free, so later consumers can be tested from already-compiled bundles; +- later app seams may depend on `ProfileBootstrap`, but they may not call pack parsing or bundle closure directly unless they are extending the runtime boundary itself; +- Phase D must **not** introduce a new runtime-specific stringly error enum for bootstrap failures. + +Phase D execution shape: + +```text +PackSource + | + v +app::runtime::bootstrap_profile + | + +--> PackCompiler::compile_profile + | | + | v + | CompiledProfile + | + +--> PackCompiler::resolve_profile_pack_set + | + v + CompiledPackSet + | + v +ProfileBootstrap::from_pack_set + | + v +ProfileBootstrap + | + +--> later app seams read compiled intent only + +--> no raw document re-open + +--> no CLI / repo / score semantics +``` + +Dependency boundary for Phase D: + +```text +future app::* seams + | + v +app::runtime + | + v +pack + | + v +kernel +``` + +Forbidden reverse or sideways edges: + +- `pack -> app::runtime` +- `app::runtime -> cli` +- `app::runtime -> repo` +- `app::runtime -> query` +- `app::runtime -> topo` +- `app::runtime -> app::score` +- `app::runtime -> raw pack parsing helpers` + +This is intentionally boring. + +That is the point. + +The boundary should be obvious in 30 seconds to a new contributor. + +#### 11.4.c Code quality and contract hygiene + +Phase D is where a narrow seam is most likely to turn into a junk drawer. Guard against that explicitly. + +Code-quality rules for Phase D: + +- keep `ProfileBootstrap` a dumb typed wrapper over `CompiledPackSet`, not a runtime service locator; +- keep `bootstrap_profile` as orchestration only, no normalization, schema work, or ad hoc bundle mutation in `src/app/runtime.rs`; +- keep typed pack failures as-is. No string translation, no `Box`, no crate-specific wrapper enum just for runtime bootstrap; +- do not create a second bootstrap config type parallel to `PackSource`. `PackSource` is already the ingress contract; +- do not add feature flags, registries, or trait abstractions for future runtime seams in this phase; +- if `CompiledPackSet` access from `app::runtime` needs a visibility tweak, keep it as the minimal compiler-owned change and document why. Do not move the type out of `src/pack/compiled/mod.rs`; +- keep any helper added for tests in the runtime test suite or behind `#[cfg(test)]`; do not let test ergonomics reshape the runtime contract. + +ASCII diagram maintenance requirements: + +- if `src/app/runtime.rs` gains enough logic to stop being self-evident, add a short inline ASCII flow comment showing `PackSource -> compiler -> bundle -> bootstrap`; +- if any nearby README or plan diagrams imply runtime does more than activation/bootstrap after Phase D lands, update them in the same change. + +#### 11.4.d Exact implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| D1. bootstrap contract | `src/app/runtime.rs` | `ProfileBootstrap` replaces `ReservedForFutureSeam` and exposes `from_pack_set` | the runtime surface is one obvious wrapper with no hidden compiler logic | +| D2. thin load path | `src/app/runtime.rs` | `bootstrap_profile(source: PackSource) -> PackResult` delegates to `PackCompiler::compile_profile` plus `resolve_profile_pack_set` | no raw pack payload is parsed, normalized, or reopened in runtime | +| D3. pure compiled-bundle handoff | `src/app/runtime.rs`, optional tiny `src/pack/compiled/mod.rs` doc/visibility touch | already-compiled bundles can be converted into `ProfileBootstrap` with no filesystem dependency | source files can disappear after compilation and the bootstrap object still behaves the same | +| D4. runtime crate-local suite | `src/lib.rs`, `src/runtime_bootstrap_tests.rs`, optional tiny helper reuse from `tests/pack_bundle.rs` | one dedicated suite proves builtin, file, inline, compile-failure, resolution-failure, and pure-handoff paths through the real crate module graph | runtime bootstrap behavior is fully named and regression-safe without turning bundle tests into a second runtime suite | +| D5. docs sweep | `README.md`, optional `src/app/mod.rs` comment touch | docs describe `app::runtime` as the pack activation boundary, not a full app runtime | no prose claims runtime dispatch, scoring, or query execution already exist | + +Implementation order: + +1. Land D1 first so the runtime seam has a real contract instead of a placeholder. +2. Land D2 immediately after D1. The runtime API is not real until it delegates through the compiler path end to end. +3. Land D3 before the runtime integration suite freezes, because the pure-handoff guarantee is the hardest Phase-D invariant. +4. Land D4 once the contract text and helper shape stop moving. +5. Land D5 last, after the runtime contract wording is stable enough to document honestly. + +Do not manufacture a second non-test consumer module just to make the seam sound bigger. In Phase D, `app::runtime::bootstrap_profile` is itself the allowed runtime ingress. + +#### 11.4.e Test review — required coverage for Phase D + +Phase D is the promotion gate for "no raw fallback" and "one allowed handoff contract." + +Write the QA-facing artifact to: + +- `~/.gstack/projects/atomize-hq-substrate/{user}-{branch}-phase-d-test-plan-{datetime}.md` + +Required coverage diagram: + +```text +PHASE D COVERAGE +================ +[+] src/app/runtime.rs + | + ├── bootstrap_profile(PackSource::Builtin) + │ └── must succeed and return ProfileBootstrap over compiled bundle + | + ├── bootstrap_profile(PackSource::File) + │ └── must succeed for the Phase-C advanced bundle fixture + | + ├── bootstrap_profile(PackSource::Inline) + │ └── must succeed for a deterministic inline profile case + | + ├── bootstrap_profile -> compile_profile failure + │ └── malformed profile input must surface PackError unchanged + | + ├── bootstrap_profile -> resolve_profile_pack_set failure + │ ├── missing pack ref must surface UnknownPackReference unchanged + │ └── wrong-kind ref must surface RefKindMismatch unchanged + | + └── ProfileBootstrap::from_pack_set + ├── must accept an already-compiled bundle with no filesystem access + └── must still work after the original source files are deleted + +[+] Internal runtime flow + | + ├── select builtin profile -> compile -> resolve -> bootstrap + ├── select file-backed advanced profile -> compile -> resolve -> bootstrap + └── invalid bundle closure -> loud typed failure, no fallback path +``` + +Required tests: + +- `bootstrap_builtin_profile_returns_profile_bootstrap` +- `bootstrap_file_backed_advanced_profile_returns_profile_bootstrap` +- `bootstrap_inline_profile_returns_profile_bootstrap` +- `bootstrap_propagates_profile_compile_errors_without_translation` +- `bootstrap_propagates_bundle_resolution_errors_without_translation` +- `profile_bootstrap_from_pack_set_is_pure_after_source_cleanup` + +Required test file plan: + +- add `#[cfg(test)] mod runtime_bootstrap_tests;` in `src/lib.rs`; +- add `src/runtime_bootstrap_tests.rs`; +- keep `tests/pack_bundle.rs` as the compiler-resolution regression suite and extend it only for tiny shared fixture helpers if that keeps the runtime suite explicit; +- keep `tests/pack_compile.rs` and `tests/pack_topology.rs` in the validation set so Phase D cannot silently regress upstream compiler behavior while chasing runtime coverage. + +Phase-D validation commands: + +- `cargo fmt --all` +- `cargo clippy -p substrate-lift --all-targets -- -D warnings` +- `cargo test -p substrate-lift --test pack_schema -- --nocapture` +- `cargo test -p substrate-lift --test pack_compile -- --nocapture` +- `cargo test -p substrate-lift --test pack_topology -- --nocapture` +- `cargo test -p substrate-lift --test pack_bundle -- --nocapture` +- `cargo test -p substrate-lift runtime_bootstrap_tests -- --nocapture` + +Critical promotion rule: + +- Phase D does not promote if the raw-fallback regression path is untested. + +#### 11.4.f Performance and determinism review + +Phase D is not the place to invent caching, but it is the place to keep runtime activation honest. + +Performance and determinism rules for Phase D: + +- `bootstrap_profile` may allocate a `PackCompiler` and run one compile plus one bundle-resolution call. That is acceptable. Do not add a process-global cache or registry in this seam; +- do not re-run `compile_profile` or `resolve_profile_pack_set` more than once inside a single bootstrap call; +- do not clone the bundle repeatedly just to satisfy helper ergonomics. Wrap once, then move; +- keep `ProfileBootstrap::from_pack_set` deterministic and side-effect-free; +- treat any runtime behavior that depends on original source-file availability after bundle creation as a correctness bug, not a performance tradeoff; +- if a performance concern appears, the only acceptable Phase-D optimization is avoiding duplicate work inside the bootstrap call itself. Everything broader belongs to a later runtime seam. + +Phase-D performance is acceptable only if activation cost is still exactly one compile path plus one bundle-resolution path, with no hidden reparse and no hidden second closure walk. + +#### 11.4.g Failure modes for Phase D + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| `bootstrap_profile` orchestration | runtime bootstrap re-parses raw pack docs after bundle resolution | yes, critical | yes, route only through compiler entrypoints | silent semantic drift if omitted | +| `bootstrap_profile` error path | runtime bootstrap translates `PackError` into opaque strings | yes | yes, return `PackResult` unchanged | typed failure instead of opaque runtime error | +| `ProfileBootstrap::from_pack_set` | bootstrap object depends on live source files after compilation | yes, critical | N/A, pure-constructor invariant | post-cleanup runtime breakage if omitted | +| Phase-D seam boundary | implementation quietly expands into CLI loading or repo discovery | yes, gate by touched files | yes, reject out-of-scope file touches | scope creep and coupled runtime | +| future app consumers | later app seams bypass runtime and call pack compiler directly | docs + review gate | yes, document runtime as the only allowed handoff path | boundary erosion | +| runtime shaping | Phase D becomes score-first and pulls semantics into runtime | yes, touched-files gate | yes, keep `src/app/score/**` untouched | second control plane starts forming | + +Phase D does not promote if any failure mode above is both untested and capable of producing silent raw-fallback behavior. + +#### 11.4.h NOT in scope for Phase D + +- CLI flags or CLI profile-loading UX +- repo-relative profile discovery +- score-model evaluation +- tree-sitter query compilation +- rule execution +- recipe execution +- app registry, dispatch, DI container, or app result envelopes beyond the bootstrap contract +- public crate API changes + +#### 11.4.i Worktree-parallel execution strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| D1 bootstrap contract | `src/app/runtime.rs` | — | +| D2 thin load path | `src/app/runtime.rs` | D1 | +| D3 pure compiled-bundle handoff | `src/app/runtime.rs`, optional tiny `src/pack/compiled/mod.rs` touch | D1, D2 | +| D4 runtime integration suite | `tests/runtime_bootstrap.rs`, optional tiny helper reuse in `tests/pack_bundle.rs` | D2, D3 | +| D5 docs sweep | `README.md`, optional `src/app/mod.rs` comment touch | D2 contract wording stable | + +Parallel lanes: + +- Lane A: D1 -> D2 -> D3 (sequential core lane, one owner for `src/app/runtime.rs`) +- Lane B: D4 (test sidecar after D3 freezes the runtime contract and pure-handoff invariant) +- Lane C: D5 (docs-only sidecar after D2 makes the runtime boundary wording stable) + +Execution order: + +- Run Lane A first until the runtime contract and pure-handoff invariant stop moving. +- Once D2 is stable, launch Lane C in parallel because it is docs-only if it stays inside README and module comments. +- Once D3 is stable, launch Lane B in a separate worktree for the runtime integration suite. +- Merge Lane C first if it stays prose-only, then merge Lane A if still open, then merge Lane B last so the final test run sees the settled runtime contract. + +Conflict flags: + +- `src/app/runtime.rs` is the Phase-D choke point and should have one owner at a time. +- If Lane B discovers missing hooks that require reopening `src/app/runtime.rs`, stop the sidecar and route the change back to Lane A. +- `tests/pack_bundle.rs` is not a safe second ownership lane. Reuse helpers only if they remain tiny and the runtime suite stays the primary owner of bootstrap assertions. +- Lane C is safe only if it remains docs-only. If docs work uncovers unresolved runtime semantics, fix the semantics first in Lane A. +- Any attempt to open `src/pack/compiler.rs` for more than a tiny helper or visibility fix collapses the lanes back into one sequential owner, because that means Phase D is no longer just activation. + +#### 11.4.j Phase D completion summary + +- Step 0: scope accepted as the pack activation boundary only +- Architecture: one explicit runtime handoff contract written +- Code quality: no second compiler, no stringly runtime error layer, no bootstrap junk drawer +- Test Review: coverage diagram written, runtime regression suite required, 0 raw-fallback gaps allowed +- Performance Review: one compile path plus one bundle-resolution path only, no hidden reparse +- Failure modes: 0 silent raw-fallback gaps allowed +- NOT in scope: written +- What already exists: written +- Parallelization: 3 lanes, 2 sidecar lanes after 1 sequential core lane +- Test artifact: write to `~/.gstack/projects/atomize-hq-substrate/{user}-{branch}-phase-d-test-plan-{datetime}.md` + +Phase D is complete when: + +1. `src/app/runtime.rs` defines `ProfileBootstrap` and a thin `bootstrap_profile` entrypoint; +2. the only compiler and bundle closure used by Phase D is `PackCompiler::compile_profile` plus `resolve_profile_pack_set`; +3. bootstrap failures return typed pack errors without string translation; +4. one dedicated integration suite proves builtin, file-backed, and inline bootstrap success paths; +5. one regression test proves `ProfileBootstrap::from_pack_set` remains pure after source cleanup, so Phase D cannot hide a raw-document fallback; +6. no Phase-D implementation change touches `src/cli/**`, `src/repo/**`, `src/query/**`, `src/topo/**`, `src/app/score/**`, or public crate exports. + +### 11.5 Boundary cleanliness + +- `pack` depends only on: + - `kernel`, + - `serde`, + - `serde_json`, + - `serde_jcs`, + - `thiserror`, + - `toml`, + - `globset`, + - `jsonschema`. +- the crate manifest still carries workspace `sha2`, but Phase C should continue routing hashing through `kernel::{sha256_bytes, sha256_canonical_json}` rather than calling hashing primitives directly from new pack code. +- `pack` does not depend on: + - `repo`, + - `lang`, + - `graph`, + - `facts`, + - `derive`, + - any `app::*`, + - `cli`. + +### 11.6 Fixture coverage + +Fixtures across the staged rollout must exist for: + +- valid profile TOML +- invalid profile TOML +- valid boundary taxonomy +- invalid boundary taxonomy +- valid component map +- invalid component map +- valid score model +- invalid score model +- valid query pack +- invalid query pack +- valid rule pack +- invalid rule pack +- valid recipe pack +- invalid recipe pack +- valid bundle root profile that resolves score/query/rule/recipe closure +- wrong-kind advanced-pack reference +- profile with missing referenced pack +- profile with wrong-kind reference +- transitive rule-pack -> query-pack closure +- transitive recipe-pack -> query-pack closure +- same-source duplicate ref dedupe +- cross-origin duplicate pack ID rejection +- semantic fingerprint stability across key order +- source fingerprint difference with semantic fingerprint equality +- bundle semantic fingerprint stability across discovery order +- traversal rejection for invalid `file:` refs +- canonical-equivalent ref rejection where normalization would otherwise be ambiguous + +### 11.7 Built-ins + +- `builtin:generic/default` exists in tests or fixtures and only promises the phase-appropriate pack surface already landed. +- Phase C should add `builtin:generic/lift-v2` only if a builtin profile actually references that score model. +- builtin query/rule/recipe packs are optional in Phase C and should land only for a concrete profile or Phase-D consumer path. +- if `substrate-profile` is enabled, at least one built-in Substrate profile is loadable once the staged dependencies it references exist. +- if `substrate-profile` is disabled, generic built-ins still work. + +--- + +## 12. Invariants + +1. `pack` remains crate-private in seam 1. +2. Seam 1 compiles declarative inputs; it does not execute them. +3. Pack schemas are the source of truth for raw document shape. +4. Compiled pack types are immutable data objects. +5. Profile resolution is deterministic and duplicate-safe. +6. Cross-pack reference validation occurs only in resolved bundle compilation. +7. Formatting changes must not change semantic fingerprints. +8. Built-in registry contents are explicit and finite. +9. Stable entry IDs are derived from pack ID + local entry ID, never absolute paths. +10. Pack compiler diagnostics are typed and machine-readable. +11. Seam 1 does not require a repo snapshot. +12. Seam 1 does not require any language adapter. +13. Seam 1 does not know how to score, match, rewrite, or classify code. + +--- + +## 13. Falsification questions + +If any answer below is “yes”, seam 1 is not clean enough. + +1. Can pack compilation succeed with a malformed `builtin:` or `file:` reference? +2. Can the same semantic profile compile to different fingerprints because TOML keys were reordered? +3. Can a built-in pack and file-backed pack with the same `id` silently override each other? +4. Can stable rule/query/recipe/component/boundary IDs depend on source file path? +5. Can bundle output depend on filesystem directory iteration order? +6. Can a profile reference a pack of the wrong kind without a deterministic error? +7. Can a pack-local compile perform repo scanning or language parsing? +8. Can a schema-valid but semantically invalid glob slip through without compile failure? +9. Can unresolved external refs be silently dropped when building a `CompiledPackSet`? +10. Can pack compiler diagnostics require human log parsing instead of machine-readable fields? +11. Can pack origin strings affect semantic fingerprints? +12. Can duplicate bundle-slot IDs produce non-deterministic error reporting? +13. Can a rule/query/recipe pack compile depend on CLI feature flags? +14. Can the seam be used only inside Substrate because generic built-ins are missing? +15. Can pack compiler code depend on `repo`, `lang`, `graph`, or `app::*` modules? + +--- + +## 14. Risks and mitigations + +### Risk 1 — seam 1 becomes a second app/config layer + +If pack files start encoding runtime behavior that should belong in app seams, the compiler becomes a junk drawer. + +Mitigation: +- keep pack categories declarative; +- no execution logic in seam 1; +- app-specific semantics remain in app seams. + +### Risk 2 — over-validating future semantics too early + +If seam 1 tries to prove full query correctness, score-model field compatibility, or recipe safety now, it will depend on seams that do not exist yet. + +Mitigation: +- seam 1 performs structural compilation only; +- deeper semantic checks happen in later seams. + +### Risk 3 — leaking filesystem-specific paths into stable outputs + +Mitigation: +- keep `PackOrigin` out of IDs and semantic fingerprints; +- derive stable entry IDs from pack ID + local ID only. + +### Risk 4 — glob overlap is mistaken for a seam-1 responsibility + +Mitigation: +- seam 1 compiles globs; +- topology seam later checks actual overlaps against real repo inventory. + +### Risk 5 — too many file formats + +Mitigation: +- profile uses TOML only; +- all other packs use JSON only in v1. + +### Risk 6 — schema/readme drift + +The current `schemas/README.md` still says commit 1 reserves the directory only, which is already outdated after seam 0. + +Mitigation: +- update `schemas/README.md`, `profiles/README.md`, `rules/README.md`, and `recipes/README.md` as part of seam 1. + +### Risk 7 — `pack` becomes public too early + +Mitigation: +- keep `pack` crate-private until runtime/app seams stabilize. + +--- + +## 15. In scope / out of scope + +### In scope for seam 1 + +- pack source loading +- schema embedding +- TOML profile parsing +- JSON pack parsing +- pack refs +- pack names +- pack-local diagnostics +- semantic fingerprints +- source fingerprints +- glob compilation +- expression AST compilation +- profile resolution +- bundle construction into `CompiledPackSet` +- duplicate/kind-mismatch validation +- built-in generic profile support +- optional built-in Substrate profile support behind feature flag + +### Out of scope for seam 1 + +- repo-relative path matching against actual files +- query execution against syntax trees +- score evaluation +- rule execution +- recipe application +- patch generation +- contract diffing +- broad app-runtime integration beyond the narrow Phase-D bootstrap adapter +- CLI subcommands for pack management +- environment-variable interpolation +- remote registries or HTTP loading +- YAML-based pack documents +- auto-discovery of `lift.toml` from a repo root + +A note on that last point: + +> Seam 1 may compile a selected profile file, but automatic repo-level profile discovery belongs later, likely at app runtime / repo substrate integration. + +--- + +## 16. Specific implementation items + +### 16.1 Phase A code changes + +Replace the Phase-A stub with only the modules required by the profile foundation: + +- `src/pack/mod.rs` +- `src/pack/error.rs` +- `src/pack/schema.rs` +- `src/pack/source.rs` +- `src/pack/names.rs` +- `src/pack/refs.rs` +- `src/pack/diagnostics.rs` +- `src/pack/compiler.rs` +- `src/pack/builtin.rs` +- `src/pack/raw/mod.rs` +- `src/pack/raw/common.rs` +- `src/pack/raw/profile.rs` +- `src/pack/compiled/mod.rs` +- `src/pack/compiled/header.rs` +- `src/pack/compiled/profile.rs` + +Explicitly deferred from Phase A: + +- `src/pack/expr.rs` +- topology-pack raw/compiled modules +- score-model, query-pack, rule-pack, and recipe-pack raw/compiled modules +- `src/app/runtime.rs` bootstrap work + +### 16.2 Phase A schema files + +Add and embed only: + +- `schemas/pack/common.v1.json` +- `schemas/pack/profile.v1.json` + +Explicitly defer all other pack schemas to their later horizons. + +### 16.3 Phase A fixture layout + +```text +fixtures/pack/ + valid/ + profile_generic_default.toml + profile_generic_default.json + profile_generic_default.inline.json + invalid/ + profile_bad_ref.toml + profile_absolute_ref.toml + profile_traversal_ref.toml + profile_schema_violation.toml + profile_invalid_toml.toml + canonical/ + profile_a.toml + profile_b.toml + expected_profile_semantic_sha256.txt + expected_profile_source_sha256.txt +``` + +### 16.4 Phase A tests + +Add integration tests such as: + +- `tests/pack_schema.rs` +- `tests/pack_compile.rs` +- `tests/pack_fingerprints.rs` + +And only extend `tests/compile_matrix.rs` if the pack compiler becomes part of an existing crate-level assertion path. + +### 16.5 Phase A README updates + +Update: + +- `README.md` if the root crate docs mention pack as purely reserved +- `schemas/README.md` +- `profiles/README.md` +- `fixtures/README.md` + +So they describe actual Phase-A behavior rather than only reserved directories. + +### 16.6 Phase B code changes + +Add or extend only the modules needed for topology packs: + +- `src/pack/mod.rs` +- `src/pack/compiler.rs` +- `src/pack/schema.rs` +- `src/pack/builtin.rs` +- `src/pack/raw/mod.rs` +- `src/pack/raw/boundary_taxonomy.rs` +- `src/pack/raw/component_map.rs` +- `src/pack/compiled/mod.rs` +- `src/pack/compiled/topology.rs` + +Explicitly defer from Phase B: + +- `src/pack/expr.rs` +- score-model raw/compiled modules +- query-pack raw/compiled modules +- rule-pack raw/compiled modules +- recipe-pack raw/compiled modules +- `src/topo/**` repo-backed classification logic +- `src/app/runtime.rs` bootstrap work + +### 16.7 Phase B schema files + +Add and embed: + +- `schemas/pack/boundary_taxonomy.v1.json` +- `schemas/pack/component_map.v1.json` + +Do not pull score-model, query-pack, rule-pack, or recipe-pack schemas into this phase. + +### 16.8 Phase B fixture layout + +```text +fixtures/pack/ + valid/ + boundary_taxonomy_generic.json + component_map_generic.json + profile_topology_file_backed.toml + invalid/ + boundary_taxonomy_duplicate_id.json + boundary_taxonomy_bad_glob.json + component_map_duplicate_id.json + component_map_bad_glob.json + profile_topology_missing_ref.toml + profile_topology_wrong_kind.toml + canonical/ + boundary_taxonomy_order_a.json + boundary_taxonomy_order_b.json + component_map_order_a.json + component_map_order_b.json +``` + +### 16.9 Phase B tests + +Extend or add: + +- `tests/pack_schema.rs` +- `tests/pack_compile.rs` +- `tests/pack_fingerprints.rs` +- `tests/pack_topology.rs` + +And only touch `tests/compile_matrix.rs` if topology resolution becomes part of an existing crate-level compile assertion. + +### 16.10 Phase B README updates + +Update: + +- `README.md` +- `schemas/README.md` +- `profiles/README.md` +- `fixtures/README.md` + +So they describe actual Phase-B topology behavior once those contracts land, without implying that repo classification or runtime consumption already exists. + +### 16.11 Phase C code changes + +Add or extend only the modules needed for advanced structural packs and bundle resolution: + +- `src/pack/mod.rs` +- `src/pack/compiler.rs` +- `src/pack/schema.rs` +- `src/pack/expr.rs` +- `src/pack/builtin.rs` if a builtin score model is actually referenced +- `src/pack/raw/mod.rs` +- `src/pack/raw/score_model.rs` +- `src/pack/raw/query_pack.rs` +- `src/pack/raw/rule_pack.rs` +- `src/pack/raw/recipe_pack.rs` +- `src/pack/compiled/mod.rs` +- `src/pack/compiled/score_model.rs` +- `src/pack/compiled/query_pack.rs` +- `src/pack/compiled/rule_pack.rs` +- `src/pack/compiled/recipe_pack.rs` + +Explicitly defer from Phase C: + +- `src/app/runtime.rs` +- `src/query/**` +- `src/topo/**` +- `src/graph/**` +- `src/facts/**` +- `src/derive/**` +- CLI pack discovery or management work + +### 16.12 Phase C schema files + +Add and embed: + +- `schemas/pack/score_model.v1.json` +- `schemas/pack/query_pack.v1.json` +- `schemas/pack/rule_pack.v1.json` +- `schemas/pack/recipe_pack.v1.json` + +Also tighten `schemas/pack/common.v1.json` only as needed for shared `expr` and `query_ref` constraints. + +Do not create a new schema generation layer, a new common-schema version, or runtime-only schemas in this phase. + +### 16.13 Phase C fixture layout + +```text +fixtures/pack/ + valid/ + score/ + generic_lift_v2.json + queries/ + rust_core.json + rules/ + generic_policy.json + recipes/ + generic_core_recipes.json + bundle/ + profile_advanced_file_backed.toml + invalid/ + score_model_bad_expr.json + score_model_duplicate_trigger.json + query_pack_duplicate_id.json + rule_pack_bad_query_ref.json + rule_pack_invalid_severity.json + recipe_pack_bad_transform.json + bundle_duplicate_pack_id_cross_origin.toml + canonical/ + score_model_order_a.json + score_model_order_b.json + bundle_order_a/ + bundle_order_b/ +``` + +### 16.14 Phase C tests + +Extend or add: + +- `tests/pack_schema.rs` +- `tests/pack_compile.rs` +- `tests/pack_fingerprints.rs` +- `tests/pack_topology.rs` as a regression suite +- `tests/pack_bundle.rs` + +And only touch `tests/compile_matrix.rs` if `CompiledPackSet` becomes part of an existing crate-level compile assertion. + +### 16.15 Phase C README updates + +Update: + +- `README.md` +- `schemas/README.md` +- `profiles/README.md` if builtin/default profile semantics widen +- `rules/README.md` +- `recipes/README.md` +- `fixtures/README.md` + +So they describe structural advanced-pack compilation and bundle resolution accurately, without implying that query execution, score evaluation, recipe execution, or runtime bootstrap already exist. + +### 16.16 Phase D code changes + +Touch only: + +- `src/app/runtime.rs` +- `src/app/mod.rs` only if the runtime module comments or exports need a narrow correction +- `README.md` + +Optional shared touch points: + +- `src/pack/compiled/mod.rs` only if a tiny visibility or doc tweak is needed for `CompiledPackSet` consumption +- `src/pack/compiler.rs` only if a tiny helper extraction or visibility correction is required to preserve the "one compile + one resolve" bootstrap path without reopening Phase-C behavior +- `src/error.rs` only if the crate-level error story needs to acknowledge typed pack bootstrap failures without wrapping them + +Explicitly defer from Phase D: + +- `src/cli/**` +- `src/repo/**` +- `src/query/**` +- `src/topo/**` +- `src/app/score/**` beyond existing placeholders +- any public export changes in `src/lib.rs` + +### 16.17 Phase D tests + +Add or extend: + +- `src/lib.rs` +- `src/runtime_bootstrap_tests.rs` +- `tests/pack_bundle.rs` only if small helper reuse keeps the bootstrap tests explicit + +Keep in the required validation set: + +- `tests/pack_compile.rs` +- `tests/pack_topology.rs` + +Required assertions: + +- builtin, file-backed, and inline profile bootstrap happy paths +- unchanged propagation of `PackError` variants from compile and bundle-resolution failures +- proof that `ProfileBootstrap::from_pack_set` does not need original source files once the bundle is compiled + +Required planning artifact: + +- write the Phase-D test plan to `~/.gstack/projects/atomize-hq-substrate/{user}-{branch}-phase-d-test-plan-{datetime}.md` + +### 16.18 Phase D README updates + +Update: + +- `README.md` +- `src/app/mod.rs` comment block only if the app seam overview would otherwise misstate runtime ownership +- `fixtures/README.md` only if Phase-D tests add a new runtime-focused fixture or helper note + +So they describe `app::runtime` as the first pack-activation boundary, not as an already-complete app runtime or a score-first entrypoint. + +--- + +## 17. Suggested merge order inside Phase A + +```mermaid +flowchart LR + A[1. seam contracts] --> B[2. common/profile schema embedding] + B --> C[3. raw and compiled profile types] + C --> D[4. PackCompiler profile pipeline] + D --> E[5. builtin generic default] + E --> F[6. fixtures and tests] + E --> G[7. README sweep] +``` + +Parallelization note: + +- steps 1 through 5 are a single sequential lane because they all touch `src/pack/` +- steps 6 and 7 are the first safe parallel sidecars once step 5 lands + +That order gives you a useful `CompiledProfile` path early without pretending the rest of the pack control plane is already in scope. + +--- + +## 18. Recommended decisions to lock now + +1. `pack` stays crate-private in seam 1. +2. Profiles use TOML; all other packs use JSON in v1. +3. `PackOrigin` is separate from kernel `Locator`. +4. Pack semantic fingerprints are canonical-JSON based. +5. Pack source origins never affect semantic fingerprints or stable IDs. +6. Stable entry IDs use existing kernel typed IDs and seam-1-defined identity lemmas. +7. Cross-pack reference resolution happens only when building `CompiledPackSet`. +8. Seam 1 validates structure and references, not downstream execution semantics. +9. Generic built-ins are required; Substrate built-ins are optional and feature-gated. +10. The seam-1 deliverable consumed by later seams is `CompiledPackSet`. + +## AUTOPLAN PHASE 0 — INTAKE AND CEO STEP 0 + +### Working Summary + +Here's what I'm working with: + +- plan under review: `crates/lift/lift_seam1_spec_reviewed.md` +- branch: `feat/lift` +- base branch: `main` +- actual crate state checked from code, not assumed from the doc: + - `src/kernel/**` is real and tested + - `src/pack/mod.rs` is still a stub + - `src/lib.rs` keeps `pack` crate-private + - `schemas/kernel/primitives.v1.json` plus kernel embedding/tests exist + - `schemas/README.md` is stale + - `profiles/`, `rules/`, `recipes/`, and `fixtures/` are still mostly reserved placeholders +- UI scope: no + - this spec uses words like `component` and `form`, but they refer to pack/compiler data, not user-facing screens or flows + - `/plan-design-review` is skipped for this run + +### 0A. Premise Challenge + +Premise 1: seam 1 should be an internal, crate-private compiler seam. + +- verdict: mostly valid +- why: the crate surface is still scaffold-heavy, and keeping `pack` private preserves freedom to reshape contracts without freezing public API too early + +Premise 2: seam 1 should compile all declarative pack categories in one seam, including profiles, taxonomy, component maps, score models, queries, rules, recipes, built-ins, and bundle resolution. + +- verdict: challenged +- why: this jumps from a `src/pack/mod.rs` stub to most of the control plane in one seam +- risk: the crate can finish a huge amount of internal infrastructure and still not move one end-to-end user workflow + +Premise 3: seam 1 success can be defined primarily by schema coverage, determinism, fingerprints, and seam purity. + +- verdict: challenged +- why: those are engineering quality bars, not product outcome bars +- risk: the plan can be "correct" and still strategically premature if nothing downstream consumes `CompiledPackSet` in a real flow + +Premise 4: seam 1 needs a separate `PackOrigin` / `PackLocation` stack because kernel `Locator` is repo-relative. + +- verdict: challenged, but narrower than Premise 2 +- why: the need is plausible, but the doc currently treats it as settled before demonstrating a concrete seam-1 failure with `RepoPath` + `Locator` +- safer framing: prove the gap with a builtin/inline diagnostic case, then introduce the minimum new type needed + +Premise 5: `builtin:generic/default` should already span eight apps, many languages, topology classes, scoring, rules, queries, and recipes. + +- verdict: challenged +- why: broad generic defaults are usually where first-run trust dies +- risk: "works everywhere a little" is a bad wedge for an engine that needs deterministic credibility + +Premise 6: runtime integration, CLI discovery, and repo-level profile discovery can all wait for later seams. + +- verdict: partly valid, partly dangerous +- valid: full public CLI and repo auto-discovery do not need to ship now +- dangerous: if absolutely no downstream consumer is exercised in this seam, the abstraction hardens without product pressure + +### 0B. Existing Code Leverage + +| Sub-problem | Existing code | Reuse posture | +|---|---|---| +| Canonical JSON normalization | `src/kernel/canonical_json.rs` | Reuse directly | +| SHA-256 fingerprinting | `src/kernel/fingerprint.rs` | Reuse directly | +| Typed stable IDs for boundaries/components/rules/queries/recipes | `src/kernel/id.rs` | Reuse directly | +| JSON Pointer and field paths | `src/kernel/json_pointer.rs` | Reuse directly | +| Severity and diagnostic structure patterns | `src/kernel/diagnostic.rs` | Reuse patterns directly | +| Schema embedding pattern | `src/kernel/schema.rs` | Reuse directly | +| Schema validation test style | `tests/kernel_identity_schema.rs` | Reuse directly | +| Repo-relative path handling | `src/kernel/path.rs` and `src/kernel/locator.rs` | Reuse where possible, only add new origin types for builtin/inline gaps | +| CLI entry surface | `src/bin/lift.rs`, `src/cli/args.rs` | Reuse scaffold only, but do not expand CLI scope in this seam | +| Pack compiler modules | `src/pack/mod.rs` | No existing runtime logic | +| Topology / query / score downstream consumers | `src/topo/mod.rs`, `src/query/mod.rs`, `src/app/score/**` | Present as seam placeholders, not real consumers yet | + +What already exists: + +- The kernel already solved the hard parts around deterministic identity, canonical serialization, typed IDs, JSON-pointer addressing, and schema embedding. +- The strongest implementation leverage is to extend those patterns, not to invent a second parallel contract system unless the builtin/inline use case proves it necessary. +- The weakest part of the current crate is not contract machinery. It is absence of a real downstream consumer path. + +### 0C. Dream State Mapping + +```text + CURRENT STATE THIS PLAN (as written) 12-MONTH IDEAL + kernel seam landed, full declarative pack control plane profile- or repo-intent in, + pack is a stub, compiled into immutable pack objects trusted lift outputs out, + app/runtime/query/topology with strong determinism and schemas with authoring, debugging, + seams are mostly placeholders but no real consumer in seam 1 migration, and distribution loops +``` + +Dream-state read: + +- the 12-month ideal is not "beautiful compiler internals" +- it is "builders can author or infer intent, run one command, and trust the result" +- this plan moves toward that ideal on determinism and contracts, but away from it on adoption pressure because it still ends before a visible user outcome + +### 0C-bis. Implementation Alternatives + +APPROACH A: Full control-plane seam + Summary: Ship the spec largely as written, including all pack kinds, bundle resolution, built-ins, fixtures, schema inventory, and README sweep. + Effort: XL + Risk: High + Pros: + - locks down a unified pack contract early + - gives later seams a very complete compiled bundle surface + - maximizes determinism and schema rigor immediately + Cons: + - huge blast radius from a stub starting point + - no real product-pressure loop in this seam + - highest chance of hardening the wrong abstraction + Reuses: + - kernel determinism primitives and schema/testing patterns + +APPROACH B: Profile-first compiler with one exercised consumer boundary + Summary: Build the compiler spine, but narrow seam 1 to the minimal pack categories needed to prove one downstream consumer path, likely profile + one topology or score-related dependency chain. + Effort: M + Risk: Medium + Pros: + - preserves the internal compiler seam idea + - forces one real exit criterion beyond schema correctness + - keeps follow-on seams honest about what must actually work + Cons: + - defers some elegant "complete platform" design + - may require explicitly stating which pack kinds are phase-1A vs later + - makes the current doc less symmetrical + Reuses: + - all kernel primitives, schema embedding pattern, existing test harness structure + +APPROACH C: Contract-foundation only + Summary: Limit seam 1 to names, refs, source/origin, error surface, schema embedding, and profile parsing, leaving rule/query/recipe/component/score packs for later seams. + Effort: S + Risk: Low + Pros: + - smallest diff + - least chance of over-design + - easy to verify against current crate maturity + Cons: + - risks pushing real integration uncertainty downstream + - may under-deliver relative to the README's seam map + - could create a second round of pack contract churn soon after + Reuses: + - kernel almost entirely, with minimal new compiler code + +RECOMMENDATION: Choose Approach B because it keeps the seam honest. It still builds the compiler, but it ties success to one exercised consumer path instead of rewarding seam purity alone. + +### 0D. Selective Expansion Analysis + +Mode selected: `SELECTIVE EXPANSION` + +Why this mode: + +- this is an enhancement to an existing seam plan, not a greenfield product concept +- the spec already has strong structure; the problem is calibration, not ambition +- the right move is to hold the compiler direction, then cherry-pick only the expansions that are truly load-bearing + +Complexity check: + +- the current plan touches well over 8 files and introduces more than 2 new modules/types +- by the CEO skill's own rubric, that is a smell and needs challenge + +Minimum set of changes that still achieves the stated goal: + +- real `src/pack/` module tree +- pack names/refs/errors/source model +- schema embedding for the first truly needed pack kinds +- deterministic compile path with semantic/source fingerprints +- fixture-backed tests proving determinism and cross-pack resolution for the narrowed scope +- README/schema directory cleanup only for surfaces actually shipped in seam 1 + +Expansion scan: + +| Candidate | Decision | Why | +|---|---|---| +| Keep all 8 pack kinds in seam 1 | challenged | too much control-plane surface before one real consumer exists | +| Require one exercised consumer of `CompiledPackSet` in seam 1 exit criteria | accepted | this closes the biggest product-risk gap inside the same blast radius | +| Prove whether `PackLocation` is necessary before freezing it | accepted | explicit-over-clever and avoids duplicate contract stacks | +| Narrow `builtin:generic/default` to a smaller first-use slice | accepted | prevents noisy broad defaults from faking maturity | +| Add remote/distributed registry support now | deferred | product-important, but outside the 1-day blast-radius rule for this seam | +| Add CLI pack management now | deferred | useful later, but not necessary to validate the seam itself | + +### 0E. Temporal Interrogation + +```text + HOUR 1 (foundations): The implementer needs to know which pack kinds are truly in seam 1 and whether one consumer path is mandatory. + HOUR 2-3 (core logic): They will hit ambiguity around whether to mint a new location/diagnostic contract or reuse kernel patterns. + HOUR 4-5 (integration): They will discover that "compiled" can still mean late semantic failure unless one downstream consumer is actually exercised. + HOUR 6+ (polish/tests): They will wish the plan had said exactly which fixture matrices are required for determinism vs wrong-kind vs duplicate-id vs semantic-equality/source-difference. +``` + +### 0F. Mode Selection Confirmation + +- selected mode: `SELECTIVE EXPANSION` +- selected implementation approach under this mode: `APPROACH B` +- reason: highest completeness without rewarding a control-plane land grab + +## CEO DUAL VOICES — OUTSIDE REVIEW + +### CLAUDE SUBAGENT (CEO — strategic independence) + +- `[High]` Wrong problem framing: the spec is too infrastructure-first for a crate that still lacks one downstream user-visible flow. Fix: narrow seam 1 around the first shippable loop, not the full compiler surface. +- `[High]` Premises are more assumed than proven around `PackOrigin` / `PackLocation`. Fix: prove kernel `Locator` fails on a real seam-1 case before introducing a second location stack. +- `[Medium-High]` Six-month regret: you can ship a beautiful pack subsystem with no practical consumer. Fix: require one downstream workflow to consume `CompiledPackSet` in practice. +- `[Medium]` Alternatives were not explored enough. Fix: compare the current full-control-plane path with narrower rollout options. +- `[Medium]` Competitive risk is underweighted. Fix: anchor the roadmap to one distinctive workflow, not the compiler abstraction itself. + +### CODEX SAYS (CEO — strategy challenge) + +- The doc optimizes an internal compiler, not a user outcome. +- The seam scope is platform-first to a reckless degree, effectively most of the control plane at once. +- The plan keeps packs central but unusable as a product surface by leaving runtime integration, CLI, and discovery out of scope. +- There is no distribution story beyond `builtin:` and `file:`. +- `builtin:generic/default` is too broad to be trustworthy as a default wedge. +- "Compiled" still allows late semantic failure, which weakens the product promise. +- Authoring ergonomics, migration, and debugging loops are under-specified. +- The broader business wedge may need to be more opinionated and less universal than this plan assumes. + +### CEO DUAL VOICES — CONSENSUS TABLE + +═══════════════════════════════════════════════════════════════ + Dimension Claude Codex Consensus + ──────────────────────────────────── ─────── ─────── ───────── + 1. Premises valid? No No CONFIRMED challenge + 2. Right problem to solve? No No CONFIRMED challenge + 3. Scope calibration correct? No No CONFIRMED challenge + 4. Alternatives sufficiently explored?No No CONFIRMED challenge + 5. Competitive/product risks covered?No No CONFIRMED challenge + 6. 6-month trajectory sound? No No CONFIRMED challenge +═══════════════════════════════════════════════════════════════ + +Confirmed means both outside voices challenged the current direction on the same axis. + +## Decision Audit Trail + +| # | Phase | Decision | Classification | Principle | Rationale | Rejected | +|---|-------|----------|----------------|-----------|-----------|----------| +| 1 | Phase 0 | Treat UI scope as absent and skip design review | Mechanical | P3 Pragmatic | This is a backend compiler seam. The spec's "component" language refers to pack data, not screens or UX. | Running design review on a non-UI spec | +| 2 | Phase 0 | Constrain review blast radius to `crates/lift` plus the seam file, not the full branch diff | Mechanical | P3 Pragmatic | The branch diff against `main` is enormous and unrelated. The user explicitly redirected this run to the corrected seam file in this crate. | Letting unrelated repo churn distort the review | +| 3 | Phase 1 | Use `SELECTIVE EXPANSION` mode | Mechanical | P3 Pragmatic / P5 Explicit | The plan is an existing seam enhancement that needs recalibration, not either a hotfix or a moonshot. | `HOLD SCOPE` because it ignores strategic risk, `EXPANSION` because it rewards more surface area | +| 4 | Phase 1 | Prefer Approach B over A or C | Taste | P1 Completeness / P5 Explicit | It is the cleanest way to keep the compiler seam intact while forcing one real consumer path. | A: full control-plane seam, C: too narrow and likely to defer real integration uncertainty | + +### Premise Gate Result + +User confirmation: + +- keep the broad Seam 1 intent +- do not force it into one mega-seam +- phase it across ordered execution horizons + +Auto-decision applied after the user's confirmation: + +- choose the four-step execution horizon +- use `Phase A/B/C/D` labels in the CEO artifact while preserving the seam numbering alias (`1 / 1.25 / 1.5 / 1.75`) +- reason: more explicit boundaries, cleaner promotion criteria, lower chance of hidden mega-seams + +## Phase 1 CEO Review Sections + +### Section 1: Architecture Review + +What I examined: + +- the seam spec's declared module shape in `src/pack/` +- the current crate reality in `src/lib.rs`, `src/pack/mod.rs`, `src/app/runtime.rs`, `src/topo/mod.rs`, `src/query/mod.rs`, and `src/app/score/mod.rs` +- the crate README seam map and where seam 1 fits in the larger architecture + +Primary finding: + +- the one-shot seam was over-coupled to future seams because it tried to define nearly every declarative surface before a consumer existed +- the phased rollout fixes the biggest architectural problem by making the control plane arrive in layers instead of as one indivisible abstraction dump + +Required architecture diagram: + +```text +CURRENT +======= +kernel ---> pack(stub) + | | + | └── no runtime consumer + | + ├── app/runtime (stub) + ├── topo (stub) + ├── query (stub) + └── app/score (mostly stub, compat helper only) + +PHASED TARGET +============= +kernel + | + v +pack/core (Phase A: names, refs, sources, errors, profile compile) + | + +--> pack/topology (Phase B: boundary_taxonomy, component_map) + | + +--> pack/advanced (Phase C: score/query/rule/recipe structural compile + bundle resolution) + | + v +app/runtime/profile_bootstrap (Phase D: first crate-internal consumer) +``` + +System beauty check: + +- the strongest architectural move is not adding more pack kinds +- it is making each phase end at a contract that a later seam can actually consume without reopening raw-file semantics + +### Section 2: Error & Rescue Map + +What I examined: + +- the compiler phases in section 9 +- the typed `PackError` surface in section 6.6 +- the current kernel error/diagnostic pattern used in `src/kernel/error.rs` and `src/kernel/diagnostic.rs` + +Main result: + +- the planned error surface is directionally strong because it uses typed errors instead of `anyhow` +- the gap was not error taxonomy breadth +- the gap was when those errors become observable to a real downstream consumer + +Error & Rescue Registry: + +| METHOD/CODEPATH | WHAT CAN GO WRONG | EXCEPTION CLASS | RESCUED? | RESCUE ACTION | USER/CONSUMER SEES | +|---|---|---|---|---|---| +| `PackCompiler::load_source` | file missing / unreadable | `PackError::Io` | Y | bubble with `origin` context | typed compile failure | +| `PackCompiler::infer_format` | unsupported format hint / extension mismatch | `PackError::UnsupportedFormat` | Y | deterministic hard fail | typed compile failure | +| `PackCompiler::parse_profile_toml` | TOML syntax invalid | `PackError::SchemaViolation` or parse-mapped diagnostic | Y | emit structured diagnostics | typed compile failure with diagnostics | +| `PackCompiler::parse_json_pack` | invalid JSON | `PackError::SchemaViolation` or parse-mapped diagnostic | Y | emit structured diagnostics | typed compile failure with diagnostics | +| `PackCompiler::compile_globs` | malformed glob | `PackError::GlobCompile` | Y | fail compile early | typed compile failure | +| `PackCompiler::compile_expr` | bad operator / malformed AST | `PackError::ExpressionCompile` | Y | fail compile early with pointer path | typed compile failure | +| `PackCompiler::resolve_profile_refs` | missing pack / wrong kind / duplicate ID | `UnknownPackReference` / `RefKindMismatch` / `DuplicatePackId` | Y | fail bundle build deterministically | typed compile failure | +| `app::runtime::profile_bootstrap` | receives broken compiled set | propagated `PackError` + diagnostics | Y | surface diagnostics, do not silently default | bootstrap failure, no raw fallback | + +CEO judgment: + +- no major taxonomy gap remains after phasing +- the real rescue requirement is "no raw document fallback" once the Phase D consumer exists + +### Section 3: Security & Threat Model + +What I examined: + +- pack loading surfaces (`builtin`, `file`, `inline`) +- explicit out-of-scope rules around env interpolation, HTTP loading, remote registries, and CLI management + +Findings: + +- this seam is relatively low external attack surface because it is intentionally internal and local +- the main security risks are contract confusion and unsafe future expansion, not internet-facing exposure + +Threat table: + +| Threat | Likelihood | Impact | Mitigation | +|---|---|---|---| +| path confusion via `file:` refs | Medium | Medium | keep forward-slash-only, relative-only ref validation | +| hidden remote fetch / env interpolation creep | Low | High | keep explicitly out of scope in this staged rollout | +| wrong-kind pack wired into later runtime | Medium | Medium | deterministic `RefKindMismatch` during bundle resolution | +| silent fallback from compiled packs to raw docs in runtime bootstrap | Medium | High | forbid raw fallback in Phase D consumer | + +No new high-severity security blocker surfaced, but the runtime bootstrap must fail closed, not degrade into implicit raw config handling. + +### Section 4: Data Flow & Interaction Edge Cases + +What I examined: + +- the compile pipeline from `PackSource` to `CompiledPackSet` +- the new phased consumer loop + +Required data-flow diagram: + +```text +INPUT SOURCE ──▶ LOAD BYTES ──▶ PARSE ──▶ SCHEMA VALIDATE ──▶ NORMALIZE + │ │ │ │ │ + │ │ │ │ ├── duplicate set items + │ │ │ │ ├── key reorder + │ │ │ │ └── semantic-equality/source-difference + │ │ │ ├── wrong kind/version + │ │ │ └── invalid raw shape + │ │ ├── TOML/JSON syntax invalid + │ │ └── format mismatch + │ ├── file missing + │ └── unsupported format + v +SEMANTIC FINGERPRINT ──▶ PACK-LOCAL COMPILE ──▶ BUNDLE RESOLUTION ──▶ PROFILE BOOTSTRAP + │ │ │ + ├── bad refs ├── duplicate ID ├── typed diagnostics surfaced + ├── bad glob ├── missing pack └── no raw fallback + └── bad expr └── wrong kind +``` + +Unhandled edge-case gap fixed by the phased plan: + +- originally there was no explicit handoff from compiled data to a consumer +- now Phase D names the first adapter boundary, which closes the biggest "what happens after compile?" ambiguity + +### Section 5: Code Quality Review + +What I examined: + +- current crate patterns: small focused kernel modules, typed wrappers, deterministic tests, explicit docs +- proposed new pack module surface + +Findings: + +- the spec matches current crate taste better after phasing because it stops trying to introduce a huge abstraction family in one commit-sized seam +- the highest code-quality risk is parallel-contract drift between kernel and pack + +Code quality decisions: + +- keep reusing kernel typed IDs, canonical JSON, and schema embedding patterns +- require proof before introducing `PackLocation` as a separate contract +- keep `pack` crate-private throughout the staged rollout + +### Section 6: Test Review + +What I examined: + +- current crate tests in `tests/kernel_identity_schema.rs`, `tests/compile_matrix.rs`, `tests/cli_help.rs` +- proposed fixture inventory in section 16.3 +- phased rollout acceptance/promotion gates + +New test surface introduced by the phased plan: + +```text +NEW DATA FLOWS +============== +- builtin/file/inline source -> compile profile +- profile -> optional topology refs +- full pack family -> CompiledPackSet +- CompiledPackSet -> ProfileBootstrap + +NEW CODEPATHS +============= +- PackName parse / PackRef parse +- format inference and parse split +- schema embed + validation +- normalization + fingerprint split +- duplicate id detection +- wrong-kind / duplicate-id detection +- runtime bootstrap consumption path + +NEW ERROR/RESCUE PATHS +====================== +- missing file ref +- malformed TOML/JSON +- schema violations +- malformed glob +- malformed expr +- missing pack ref +- wrong-kind ref +- cyclic refs +- runtime bootstrap compile failure +``` + +CEO test judgment: + +- the original spec had decent fixture ambition but weak promotion logic +- the phased plan now needs per-phase promotion gates, which the CEO plan artifact provides +- no further CEO-level test-scope issue remains after that change + +### Section 7: Performance Review + +What I examined: + +- the compile pipeline shape +- the absence of repo walking, parser execution, or scoring math in this seam + +Findings: + +- performance risk is low-to-medium in this staged rollout because the work is bounded to file parse, schema validation, normalization, glob compilation, and bundle resolution +- likely slowest path is repeated bundle resolution across many packs, not any single profile parse + +No major performance blocker at plan stage. Deterministic correctness is still the right priority over micro-optimization here. + +### Section 8: Observability & Debuggability Review + +What I examined: + +- planned diagnostic model and typed errors +- the need to make later failures explainable when a downstream consumer first appears + +Findings: + +- observability is acceptable if pack diagnostics remain machine-readable and are surfaced intact through `ProfileBootstrap` +- the main future operability risk is hiding diagnostic origins once runtime bootstrap arrives + +Recommendation applied: + +- Phase D consumer should surface compiled diagnostics directly rather than translating them into opaque runtime strings + +### Section 9: Deployment & Rollout Review + +What I examined: + +- seam rollout order +- current crate stubs vs staged introduction + +Findings: + +- there is no production rollout risk in the normal app-deployment sense because this is an internal crate seam +- the real rollout risk is promotion drift, where later phases assume earlier contracts are frozen when they are not + +Required rollout flow: + +```text +Phase A land -> validate deterministic profile compile + | + v +Phase B land -> validate topology packs without repo logic + | + v +Phase C land -> validate full structural pack family and bundle resolution + | + v +Phase D land -> validate first runtime consumer uses compiled packs only +``` + +### Section 10: Long-Term Trajectory Review + +What I examined: + +- README seam map +- the corrected spec's long-term destination +- outside-voice strategic concerns + +Findings: + +- long-term trajectory is now sound if the staged sequence is obeyed +- reversibility is `4/5` after phasing, because crate-private pack contracts can still change between stages +- the biggest 1-year risk remains broad generic defaults that pretend to understand arbitrary repos + +Dream state delta: + +- before this review, the plan moved strongly toward a beautiful internal compiler but weakly toward product validation +- after the phased correction, it still aims at the same end state, but now each stage has a chance to prove that the abstraction deserves to survive + +### Section 11: Design & UX Review + +Skipped, no UI scope detected. + +## NOT in scope + +- remote/distributed pack registry semantics, deferred because they are product-important but outside this seam's execution radius +- public CLI pack management, deferred because the compiler surface should be proven internally first +- repo-level auto-discovery of profile files, deferred because it adds runtime/product semantics before the compiler stabilizes +- score evaluation math, deferred because this rollout is about structural compilation, not app scoring behavior +- query execution, rule execution, and recipe application, deferred because they belong to later engine/app seams + +## Failure Modes Registry + +| CODEPATH | FAILURE MODE | RESCUED? | TEST? | USER/CONSUMER SEES? | LOGGED? | +|---|---|---|---|---|---| +| source load | missing file ref | Y | Planned | typed compile error | Yes via diagnostics | +| parse | malformed TOML/JSON | Y | Planned | typed compile error | Yes via diagnostics | +| normalize | semantic/source fingerprint mismatch bug | N | Planned | internal test failure | Yes via tests | +| compile | bad glob / bad expr | Y | Planned | typed compile error | Yes via diagnostics | +| bundle resolution | wrong-kind / missing / duplicate ID | Y | Planned | typed compile error | Yes via diagnostics | +| runtime bootstrap | raw fallback sneaks in | N | Planned critical | potentially silent drift if untested | Must be asserted in tests | + +Critical gap assessment: + +- the only true critical gap is raw-fallback drift in the first consumer loop +- this is why the Phase D consumer must consume `ProfileBootstrap` only, never raw documents + +## Scope Expansion Decisions + +- Accepted: + - preserve broad Seam 1 intent + - phase delivery across ordered execution horizons + - require one real downstream consumer by the end of the staged series + - narrow the first generic built-in + - prove `PackLocation` before freezing it +- Deferred: + - remote/distributed registry semantics + - CLI pack management + - broader authoring ergonomics and migration workflows +- Skipped: + - one-shot mega-seam delivery of the entire control plane + +## Diagrams Produced + +- system architecture +- compiler data flow +- staged rollout flow + +## Stale Diagram Audit + +- current crate README seam map remains directionally accurate +- this review adds no contradiction to that map, but it does require the Seam 1 implementation plan to become explicitly phased instead of one-shot + +### Completion Summary + +```text + +====================================================================+ + | MEGA PLAN REVIEW — COMPLETION SUMMARY | + +====================================================================+ + | Mode selected | SELECTIVE EXPANSION | + | System Audit | seam was too broad for current crate state | + | Step 0 | phased broad intent accepted | + | Section 1 (Arch) | 1 major issue fixed by phased rollout | + | Section 2 (Errors) | 8 error paths mapped, 0 unresolved gaps | + | Section 3 (Security)| 0 high-severity blockers | + | Section 4 (Data/UX) | 1 major handoff ambiguity fixed | + | Section 5 (Quality) | 1 major contract-drift risk contained | + | Section 6 (Tests) | Diagram produced, promotion gates added | + | Section 7 (Perf) | 0 major blockers | + | Section 8 (Observ) | 1 runtime-surfacing requirement | + | Section 9 (Deploy) | staged promotion risk noted | + | Section 10 (Future) | Reversibility: 4/5, debt items: 3 | + | Section 11 (Design) | SKIPPED (no UI scope) | + +--------------------------------------------------------------------+ + | NOT in scope | written (5 items) | + | What already exists | written | + | Dream state delta | written | + | Error/rescue registry| 8 methods, 0 critical gaps | + | Failure modes | 6 total, 1 critical gap | + | TODOS.md updates | 0 proposed yet, backlog tracked in review | + | Scope proposals | 5 accepted, 3 deferred, 1 skipped | + | CEO plan | written, reviewed, concerns persisted | + | Outside voice | ran (codex + claude subagent) | + | Lake Score | 4/4 recommendations chose complete option | + | Diagrams produced | 3 | + | Stale diagrams found | 0 | + | Unresolved decisions | 0 | + +====================================================================+ +``` + +## CEO Reviewer Concerns + +The CEO plan artifact survived three rounds of adversarial review and improved materially, but these concerns remain open and should stay visible: + +- the fractional seam numbering needs one explicit line defining that these are ordered execution horizons within the original broad Seam 1, not separate unrelated seams +- `ProfileBootstrap`, `CompiledPackSet`, `kernel Locator`, and `bundle resolution` should stay concretely defined wherever the execution artifact is promoted +- if Phase C starts behaving like a hidden mega-seam, the first slip candidates should be `rule_pack` and `recipe_pack`, not the first consumer loop +- each phase should preserve a rollback note stating which contracts are stable and which may still change without invalidating prior acceptance + +**Phase 1 complete.** Codex: 8 concerns. Claude subagent: 5 issues. Consensus: 6/6 confirmed challenges, 0 disagreements. Passing to Phase 3. + +## Phase 2 Design Review + +Skipped, no UI scope. + +## Phase 3 Engineering Review + +### Step 0: Scope Challenge + +What I examined in code: + +- `src/lib.rs` keeps `pack` crate-private and all likely consumers (`topo`, `query`, `app::runtime`) are still placeholders +- `src/pack/mod.rs` is still a stub +- seam-0 kernel already provides deterministic diagnostics, diagnostic codes, JSON pointers, fingerprints, stable IDs, and schema embedding patterns +- `README.md` states the engine should consume compiled packs rather than raw config files + +Engineering conclusion: + +- broad intent is still acceptable +- but the plan had to become authoritative as phased contract text, not just a review note +- that correction is now reflected in the normative acceptance criteria and reference rules above + +### CLAUDE SUBAGENT (eng — independent review) + +- `[High]` `file:` resolution was underspecified enough to create correctness and security risk. Fixed by introducing `PackFileRef` semantics, traversal bans, and source-kind limits. +- `[Medium]` Pack diagnostics were weaker than kernel diagnostics. Fixed by switching to `DiagnosticCode` and typed related locations. +- `[Medium]` TOML normalization to canonical JSON was underspecified. Fixed by explicitly requiring lossless JSON-compatible TOML in v1. +- `[Medium]` Recursive resolution lacked a memoization identity. Fixed by adding normalized-source identity rules and cross-origin duplicate rejection. +- `[Medium]` Fixture coverage missed nil/empty/equivalence stress cases. Fixed by expanding fixture expectations and the eng test artifact. + +### CODEX SAYS (eng — architecture challenge) + +- The plan still behaved like a mega-seam because phased rollout lived in commentary while acceptance criteria still demanded the whole control plane at once. Fixed by rewriting section 11 into phased acceptance horizons. +- `file:` refs were still too loose at both contract and schema level. Fixed by tightening `PackRef` rules and common-schema guidance. +- Duplicate-pack dedupe lacked coherent provenance. Fixed by rejecting cross-origin duplicate IDs in v1 and only deduping the same normalized resolved source. +- Dependency closure for rule/recipe pack query refs was ambiguous. Fixed by making transitive closure explicit in bundle resolution. +- The generic builtin was still too broad. Fixed by narrowing the profile example and builtin acceptance text to phase-appropriate scope. + +### ENG DUAL VOICES — CONSENSUS TABLE + +═══════════════════════════════════════════════════════════════ + Dimension Claude Codex Consensus + ──────────────────────────────────── ─────── ─────── ───────── + 1. Architecture sound? No No CONFIRMED challenge + 2. Test coverage sufficient? No No CONFIRMED challenge + 3. Performance risks addressed? Partial Partial CONFIRMED medium risk + 4. Security threats covered? Partial Yes DISAGREE (low-medium) + 5. Error paths handled? No No CONFIRMED challenge + 6. Deployment risk manageable? Yes Yes CONFIRMED manageable +═══════════════════════════════════════════════════════════════ + +### Section 1: Architecture Review + +Required ASCII dependency graph: + +```text +kernel + ├── canonical_json + ├── fingerprint + ├── json_pointer + ├── diagnostic + ├── id + └── schema + | + v +pack/core + ├── names + ├── refs + ├── source + ├── diagnostics + ├── error + ├── schema + └── compiler + | + +--> pack/topology + | ├── boundary_taxonomy + | └── component_map + | + +--> pack/advanced + | ├── score_model + | ├── query_pack + | ├── rule_pack + | └── recipe_pack + | + v +app::runtime::profile_bootstrap +``` + +Architecture findings: + +- the strongest coupling boundary is now explicit: `app::runtime` consumes compiled output through a bootstrap adapter instead of raw documents +- the highest residual architecture risk is still Phase C scope size +- if Phase C slips, `rule_pack` and `recipe_pack` are the first candidates to defer within the broad intent, not the runtime consumer + +### Section 2: Code Quality Review + +What I examined: + +- current kernel style in `src/kernel/*` +- current placeholder modules in `src/pack/mod.rs`, `src/app/runtime.rs`, `src/topo/mod.rs`, `src/query/mod.rs` +- the revised contract text in sections 6, 7, 9, and 11 + +Findings: + +- the plan now better matches the crate's established taste: typed wrappers, explicit validation, deterministic contracts +- the biggest avoided quality failure was creating a second weaker diagnostic system; that is now corrected +- the next quality trap to guard against is letting `ProfileBootstrap` become a junk drawer for later app semantics + +### Section 3: Test Review + +CODE PATH COVERAGE +=========================== +[+] `PackName` / `PackFileRef` / `PackRef` + ├── [GAP] valid builtin ref parse + ├── [GAP] valid file ref parse + ├── [GAP] absolute-path rejection + ├── [GAP] dot-segment rejection + └── [GAP] builtin/inline source rejects `file:` refs + +[+] Profile compile pipeline + ├── [GAP] builtin profile happy path + ├── [GAP] file-backed profile happy path + ├── [GAP] inline profile happy path + ├── [GAP] TOML key reorder semantic stability + ├── [GAP] source fingerprint differs while semantic fingerprint matches + └── [GAP] omitted optional sections normalize deterministically + +[+] Advanced/bundle resolution + ├── [GAP] wrong-kind slot reference + ├── [GAP] duplicate detection against profile/topology/advanced slots + ├── [GAP] cross-origin duplicate pack rejection + ├── [GAP] transitive rule/recipe -> query dependency closure + └── [GAP] large shared-reference graph traversal stability + +[+] Runtime bootstrap + ├── [GAP] compiled pack set -> bootstrap adapter happy path + ├── [GAP] missing ref fails loudly + ├── [GAP] wrong-kind ref fails loudly + └── [GAP] raw fallback forbidden + +USER FLOW COVERAGE +=========================== +[+] Internal builder flow + ├── [GAP] select builtin profile -> compile -> bootstrap + ├── [GAP] select file-backed profile -> resolve deps -> bootstrap + └── [GAP] diagnostic surfacing path for invalid profile pack + +───────────────────────────────── +COVERAGE: 0/19 paths tested in the current seam plan artifact + Code paths: 0/16 + Internal flows: 0/3 +QUALITY: ★★★: 0 ★★: 0 ★: 0 +GAPS: 19 paths need tests +───────────────────────────────── + +Interpretation: + +- this is a planning-stage review, so zero existing tests is expected +- the important output is that every required path is now named and written into the eng test-plan artifact: + - `/Users/spensermcconnell/.gstack/projects/atomize-hq-substrate/spensermcconnell-feat-lift-eng-review-test-plan-20260416-092133.md` + +### Section 4: Performance Review + +Findings: + +- the main performance risk is recursive resolution under shared fan-out, not parsing a single pack +- the plan now names deterministic traversal and memoization identity, which removes the biggest hidden complexity +- no meaningful caching plan is needed beyond normalized-source memoization inside bundle resolution + +### "NOT in scope" section + +Already written above. No change from the CEO pass. + +### "What already exists" section + +Already written above. Eng review confirms that seam-0 kernel is still the main leverage surface. + +### Failure modes + +For each new codepath identified in the test review diagram: + +| CODEPATH | FAILURE MODE | TEST COVERS? | ERROR HANDLING EXISTS? | USER/CONSUMER SEES | +|---|---|---|---|---| +| `PackFileRef::parse` | absolute or traversal path accepted | Planned | Planned | compile failure | +| profile TOML normalization | semantically equal docs hash differently | Planned | N/A, test-enforced | test failure | +| bundle resolution | cross-origin duplicates silently dedupe | Planned | Planned | compile failure | +| transitive dependency closure | referenced query pack not loaded | Planned | Planned | compile failure | +| runtime bootstrap | raw fallback bypasses compiled output | Planned critical | must exist | silent drift if omitted | + +Critical gap: + +- raw fallback in runtime bootstrap remains the one issue that becomes dangerous if left untested + +### Worktree parallelization strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|------|-----------------|------------| +| Phase A compiler spine | `src/pack/`, `schemas/pack/`, `fixtures/pack/`, `tests/` | — | +| Phase B topology packs | `src/pack/`, `schemas/pack/`, `fixtures/pack/`, `tests/` | Phase A | +| Phase C advanced pack families | `src/pack/`, `schemas/pack/`, `fixtures/pack/`, `tests/` | Phase A, Phase B | +| Phase D runtime bootstrap | `src/app/runtime*`, `src/pack/`, `tests/` | Phase C | +| README / reserved-dir docs sweep | `README.md`, `schemas/README.md`, `profiles/README.md`, `rules/README.md`, `recipes/README.md`, `fixtures/README.md` | Phase A semantics stabilized | + +Parallel lanes: + +- Lane A: Phase A -> Phase B -> Phase C -> Phase D (sequential, shared `src/pack/`) +- Lane B: docs sweep (can run after Phase A semantics settle) + +Execution order: + +- Launch Lane A as the main implementation lane. +- Launch Lane B in parallel after Phase A lands and the contract text stops moving. + +Conflict flags: + +- Any attempt to parallelize Phase A/B/C across separate worktrees will collide on `src/pack/` and `schemas/pack/` +- docs sweep is the only safe sidecar lane + +### Completion Summary + +- Step 0: Scope Challenge — broad intent accepted, converted from mega-seam into staged execution horizons +- Architecture Review: 3 issues found and resolved in plan text +- Code Quality Review: 2 issues found and resolved in plan text +- Test Review: diagram produced, 19 gaps identified for implementation +- Performance Review: 1 issue found and resolved via explicit memoization identity +- NOT in scope: written +- What already exists: written +- TODOS.md updates: 0 proposed, deferred backlog tracked in review artifacts instead +- Failure modes: 1 critical gap flagged +- Outside voice: ran (`codex` + Claude subagent) +- Parallelization: 2 lanes, 1 parallel / 1 sequential +- Lake Score: 5/5 recommendations chose the complete option + +### Additional Decision Audit Trail Rows + +| 5 | Phase 3 | Make phased horizons normative in section 11, not advisory in appended review notes | Mechanical | P5 Explicit | If the phases matter, they must live in acceptance criteria, not commentary. | Leaving the original mega-seam acceptance criteria untouched | +| 6 | Phase 3 | Introduce `PackFileRef` semantics and traversal bans | Mechanical | P1 Completeness / P5 Explicit | This closes a determinism and safety hole with a small explicit contract. | Keeping raw `String` file refs | +| 7 | Phase 3 | Reuse `DiagnosticCode` in pack diagnostics | Mechanical | P4 DRY / P5 Explicit | The kernel already solved typed machine-readable diagnostics. | Free-form pack diagnostic codes | +| 8 | Phase 3 | Reject cross-origin duplicate pack IDs in v1 | Taste | P5 Explicit | Simpler and safer than inventing provenance-merging rules now. | Silent semantic dedupe across builtin/file origins | +| 9 | Phase 3 | Keep Phase C broad but make `rule_pack` and `recipe_pack` the first slip candidates if needed | Taste | P3 Pragmatic | Preserves broad intent while protecting the first consumer loop. | Splitting the plan into even more numbered seams immediately | +| 10 | Phase 3 | Build Phase C on the already-landed deferred profile refs instead of inventing a second bundle-root config surface | Mechanical | P4 DRY / P5 Explicit | `score.model` plus the `rules/queries/recipes.packs` sections already exist in the landed profile contract. Phase C should honor them directly. | Adding a parallel bundle manifest just for advanced packs | +| 11 | Phase 3 | Make `resolve_profile_pack_set(&CompiledProfile)` the single cross-pack closure gate | Mechanical | P5 Explicit | One owned bundle-resolution boundary keeps standalone compile composable and makes kind and duplicate checks deterministic. | Spreading cross-pack validation across individual pack compilers | +| 12 | Phase 3 | Keep generic builtins narrow in Phase C, score-model only if a builtin profile actually needs it | Taste | P3 Pragmatic | The builtin registry should prove real usage, not swell to look complete. | Adding builtin query/rule/recipe packs without a consumer | +| 13 | Phase 3 follow-up | Reframe Phase D from `first consumer loop` to `pack activation boundary` | Taste | P5 Explicit / P3 Pragmatic | After Phase C landed, the unresolved risk is the handoff contract out of `pack`, not whether any consumer can be fabricated. | Preserving the vague consumer-loop wording | +| 14 | Phase 3 follow-up | Keep `ProfileBootstrap` as a thin wrapper over `CompiledPackSet` with unchanged `PackError` propagation | Mechanical | P5 Explicit / P4 DRY | This creates one obvious runtime ingress without inventing a second compiler or a stringly runtime error layer. | Adding a parallel runtime config model or opaque bootstrap strings | + +**Phase 3 complete.** Codex: 6 concerns. Claude subagent: 5 issues. Consensus: 4/6 confirmed, 1 disagreement, 1 medium-risk partial alignment. + +## GSTACK REVIEW REPORT + +| Review | Trigger | Why | Runs | Status | Findings | +|--------|---------|-----|------|--------|----------| +| CEO Review | `/plan-ceo-review` | Scope & strategy | 2 | issues_open | mode: SELECTIVE_EXPANSION, 1 critical gap | +| Codex Review | `/codex review` | Independent 2nd opinion | 0 | — | — | +| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 2 | issues_open | 25 issues, 1 critical gap | +| Design Review | `/plan-design-review` | UI/UX gaps | 0 | — | — | + +- **CROSS-MODEL:** CEO and Eng outside voices both converged on the same structural problem, the original spec behaved like a mega-seam and had to be made normatively phased. +- **UNRESOLVED:** 0 +- **VERDICT:** APPROVED VIA `/autoplan`, but CEO and ENG remain `issues_open` by design because the implementation still must preserve the no-raw-fallback bootstrap rule and the staged promotion gates. diff --git a/crates/lift/lift_seam2_spec_reviewed.md b/crates/lift/lift_seam2_spec_reviewed.md new file mode 100644 index 000000000..61507a387 --- /dev/null +++ b/crates/lift/lift_seam2_spec_reviewed.md @@ -0,0 +1,2742 @@ + + + +# substrate-lift seam 2 spec — repo substrate (reviewed against landed seam 0 + seam 1 + seam 2 Phase A + Phase B) + +## 0. Ground truth from the landed crate + +This spec is intentionally anchored to the crate as it exists today, not to the earlier idealized design. + +Observed state in the landed crate: + +- `src/kernel/**` is real, tested, schema-backed, and publicly re-exported from `lib.rs`. +- `src/pack/**` is real code and now compiles profiles, topology packs, score models, query packs, rule packs, recipe packs, and deterministic bundle resolution into `CompiledPackSet`. +- `src/repo/**` is real, tested, schema-backed, and crate-private. `src/repo/mod.rs` now re-exports the landed Phase-A seam surface. +- `lib.rs` still exposes `repo` only as `pub(crate) mod repo;`, so seam 2 is not public API yet. +- `src/app/runtime.rs` currently stops at `ProfileBootstrap { bundle: CompiledPackSet }`; there is no repo-facing runtime loop yet. +- `kernel::RepoPath`, `FileId`, `Fingerprint`, `DiagnosticCode`, and `Severity` already exist and should be reused rather than redefined. +- `RepoRoot`, `RepoRootDetectionOptions`, `SnapshotRequest`, `SnapshotOptions`, `RepoSnapshot`, `Inventory`, `BlobStore`, `RepoDiagnostic`, and `SnapshotStats` are now landed and covered by targeted repo tests. +- `pack::CompiledAnalysisDefaults` already exists and currently contains: + - `languages: BTreeSet` + - `follow_symlinks: bool` + - `max_scope_depth: u8` +- there is **no** landed pack-level repo config surface yet for ignore globs, non-UTF8 policy, large-file policy, VCS selection, or snapshot mode. +- `Cargo.toml` already contains both `globset` and `walkdir`; it does **not** yet contain a git backend crate. +- `schemas/repo/snapshot_manifest.v1.json` is landed and embedded through `src/repo/schema.rs`. +- `fixtures/repo/**` plus `tests/repo_root.rs`, `tests/repo_snapshot.rs`, `tests/repo_ignore.rs`, `tests/repo_fingerprints.rs`, `tests/repo_purity.rs`, and `tests/repo_schema.rs` are landed and passing. +- `src/repo/diff.rs`, `schemas/repo/diff_manifest.v1.json`, `fixtures/repo/diff/**`, `tests/repo_diff.rs`, and repo-schema parity coverage for both repo manifests are now landed. +- the compile-matrix test already asserts the crate still builds with `--no-default-features`. + +Validated on this branch before updating this spec: + +- `cargo test -p substrate-lift --test repo_root --test repo_snapshot --test repo_ignore --test repo_fingerprints --test repo_purity --test repo_diff --test repo_schema -- --nocapture` +- `cargo test -p substrate-lift --test compile_matrix -- --nocapture` + +That means seam 2 is now an **internal, filesystem-first, immutable snapshot seam** with Phase A and Phase B landed, while still staying cleanly downstream of `kernel` and cleanly upstream of `lang`, `topo`, `graph`, and app orchestration. + +The key consequence is this: + +> seam 2 should **not** make `repo` public yet, and it should **not** depend on `pack`, `lang`, `graph`, app code, or CLI code. + +A second consequence, based on the current seam-1 plus landed seam-2 Phase A/Phase B reality: + +> seam 2 Phase B did **not** require retroactive changes to `profile.v1.toml` or `CompiledAnalysisDefaults`. +> It diffs already-materialized `RepoSnapshot` values directly, and later runtime/orchestration code can keep any pack-to-snapshot option mapping outside `repo`. + +--- + +## 1. Mission + +Seam 2 owns the **repo substrate**. + +It is responsible for: + +- detecting a repository root from a starting path; +- materializing an immutable repo snapshot from a selected source; +- enumerating a deterministic inventory of included files; +- deriving repo-relative `RepoPath` values for those files; +- reading and storing blob bytes for later seams; +- computing content digests and deterministic snapshot fingerprints; +- applying intrinsic and caller-supplied ignore rules; +- surfacing typed repo errors and deterministic repo diagnostics; +- computing pure path-based diffs between two already-materialized snapshots. + +It is **not** responsible for: + +- parsing source languages; +- interpreting config semantics; +- classifying components/boundaries/docs/tests; +- query matching; +- fact/detector execution; +- Lift scoring; +- pack/profile loading; +- CLI argument parsing. + +A useful rule: + +> seam 2 ends at **immutable repository materialization**. +> It does not interpret the contents. + +--- + +## 2. Why seam 2 is a separate seam + +The landed seam 1 established another important pattern beyond seam 0: + +- raw on-disk inputs are compiled once; +- runtime consumers operate on immutable compiled artifacts; +- later seams do not keep reaching back into live sources. + +Seam 2 should apply the same idea to filesystem and revision state. + +Later seams should consume `RepoSnapshot`, not raw directories or live files. + +That gives the engine: + +- deterministic inventory ordering; +- deterministic content digests; +- purity after snapshot construction; +- no accidental live re-read drift while analysis is running. + +The most important design constraint for seam 2 is therefore: + +> `RepoSnapshot` must be **fully materialized and immutable**. +> Once created, later seams must not read the live filesystem again through repo APIs. + +That requirement is stronger than “convenient” — it is necessary if determinism is going to survive parallelism, retries, or source-tree changes during a run. + +--- + +## 3. Boundary with existing code + +### Existing seam-0 primitives seam 2 should reuse + +Use directly from `kernel`: + +- `RepoPath` +- `FileId` +- `Fingerprint` +- `DiagnosticCode` +- `Severity` +- `sha256_bytes` +- `sha256_canonical_json` + +### Existing seam-0 primitives seam 2 should **not** force-fit + +Do **not** use `Locator` as the primary repo-local diagnostic location contract. + +Reason: + +- seam 2 must report root-detection and filesystem-entry issues before a stable `RepoPath` always exists; +- root-detection failures may involve absolute or relative host paths, not repo-relative logical paths; +- non-UTF8 entries may be diagnosable even when they cannot become a `RepoPath`. + +Like seam 1’s `PackLocation`, seam 2 should define its own lightweight repo-local location type and only attach `RepoPath` when one exists. + +### Dependency direction + +Seam 2 should depend only on: + +- `kernel` +- `std` +- a small traversal utility if needed +- `globset` for caller-supplied ignore compilation + +Seam 2 should **not** depend on: + +- `pack` +- `lang` +- `graph` +- `topo` +- `facts` +- `derive` +- `app` +- `cli` +- `anyhow` + +A future runtime/orchestration seam may map `CompiledAnalysisDefaults` into `SnapshotOptions`, but that translation must live outside `repo`. + +--- + +## 4. Canonical phase map + +Because the current crate now has landed Phase A and Phase B, but still has no git backend crate and no landed repo-facing runtime loop, seam 2 should continue in **three** phases. + +This section is canonical. Later sections should reference these phases, not restate a competing phase model. + +### Phase A — filesystem-first snapshot substrate + +Phase A landed: + +- repo root detection; +- worktree snapshot materialization; +- deterministic inventory; +- immutable in-memory blob store; +- intrinsic `.git` exclusion; +- caller-supplied exclude globs; +- repo diagnostics; +- snapshot fingerprints; +- fixture manifests and tests. + +Phase A does **not** land: + +- git revision materialization; +- path-based diffing; +- symlink following; +- well-known cache/build/vendor exclude presets; +- `.gitignore` interpretation; +- pack-driven repo config. + +### Phase B — pure diff over already-materialized snapshots + +Phase B landed: + +- pure path-based `RepoDiff`; +- diff fixture schema; +- diff fixtures and tests. + +Phase B does **not** land: + +- git revision materialization; +- new filesystem walking behavior; +- symlink-following behavior; +- rename detection. + +### Phase C — expanded materialization semantics + +Phase C lands: + +- `SnapshotSource::GitRev { rev }`; +- `SymlinkPolicy::Follow`; +- a typed well-known exclude policy for common cache/build/vendor directories; +- optional provider abstraction if and only if a second backend now exists. + +Examples for the Phase C well-known exclude policy: + +- `target/` +- `node_modules/` +- `.venv/` +- `venv/` +- `__pycache__/` +- `dist/` +- `build/` + +This keeps Phase A honest, Phase B pure, and Phase C as the only place where snapshot-construction semantics expand beyond the minimal worktree substrate. + +--- + +## 5. Exact module shape by phase + +```text +src/repo/ + mod.rs + error.rs + diagnostics.rs + root.rs + ignore.rs + inventory.rs + blob.rs + snapshot.rs + schema.rs +``` + +Phase B added: + +```text +src/repo/ + diff.rs +``` + +Phase C may add: + +```text +src/repo/ + provider.rs +``` + +### Phase A `src/repo/mod.rs` + +`src/repo/mod.rs` should re-export only the stable internal **Phase A** seam surface: + +```rust +//! Internal repo substrate seam. + +pub(crate) mod blob; +pub(crate) mod diagnostics; +pub(crate) mod error; +pub(crate) mod ignore; +pub(crate) mod inventory; +pub(crate) mod root; +pub(crate) mod schema; +pub(crate) mod snapshot; + +pub(crate) use blob::{BlobRecord, BlobStore}; +pub(crate) use diagnostics::{RepoDiagnostic, RepoLocation, RepoRelatedLocation}; +pub(crate) use error::{RepoError, RepoResult}; +pub(crate) use ignore::{ + CompiledIgnoreSet, LargeFilePolicy, NonUtf8PathPolicy, SnapshotOptions, SymlinkPolicy, +}; +pub(crate) use inventory::{Inventory, InventoryEntry}; +pub(crate) use root::{RepoRoot, RepoRootDetectionOptions, RootMarker}; +pub(crate) use snapshot::{RepoSnapshot, SnapshotRequest, SnapshotSource, SnapshotStats}; +``` + +Phase B extended `mod.rs` with: + +```rust +pub(crate) mod diff; +pub(crate) use diff::{build_diff, DiffEntry, DiffKind, RepoDiff}; +``` + +Phase C may extend `mod.rs` with: + +```rust +pub(crate) mod provider; +pub(crate) use provider::{FsRepoProvider, RepoProvider}; +``` + +### Hard rules + +- `repo` remains `pub(crate)` in seam 2. +- No new public crate API should be added yet. +- No other seam should shell out to git or walk the filesystem directly. +- Later seams should consume `RepoSnapshot`, not raw `PathBuf` trees. + +--- + +## 6. Allowed new dependencies + +The current crate already has `globset`, which seam 2 should reuse for caller-supplied exclude globs. + +Phase A is allowed to add **at most one** new runtime dependency for deterministic directory walking if the implementation benefits from it: + +```toml +walkdir = "2" +``` + +Phase A should **not** add: + +- `ignore` +- `git2` +- `gix` +- `anyhow` + +Reason: + +- `.gitignore` semantics are intentionally out of scope in phase A; +- git revision materialization is deferred to phase C; +- the seam should keep typed error contracts. + +If the implementation chooses not to add `walkdir`, a small internal recursive walker is acceptable, but traversal order must still be deterministic. + +--- + +## 7. Exact internal Rust contracts + +## 7.1 Repo root detection + +```rust +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct RootMarker(String); +// Canonical form: +// - non-empty +// - exact entry name only +// - no path separators +// - UTF-8 only +// +// Examples: +// ".git" +// "work-lift.toml" +// "Cargo.toml" + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoRootDetectionOptions { + pub markers: std::collections::BTreeSet, + pub ceiling_dir: Option, +} +// Default: +// markers = { RootMarker::parse(".git")? } +// ceiling_dir = None + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoRoot { + absolute_path: std::path::PathBuf, +} +``` + +Required constructors/helpers: + +```rust +impl RootMarker { + pub(crate) fn parse(input: &str) -> RepoResult; + pub(crate) fn as_str(&self) -> &str; +} + +impl RepoRootDetectionOptions { + pub(crate) fn git_default() -> Self; +} + +impl RepoRoot { + pub(crate) fn from_dir(path: &std::path::Path) -> RepoResult; + pub(crate) fn as_path(&self) -> &std::path::Path; + pub(crate) fn display(&self) -> String; +} +``` + +### Root-detection semantics + +- input start path may be a file or directory; +- if file, detection starts from the parent directory; +- search ascends toward filesystem root until: + - a marker match is found, or + - the optional `ceiling_dir` is reached, or + - the filesystem root is reached; +- the **nearest** matching ancestor wins; +- marker order must not affect the chosen root; +- a marker match is “an entry with the exact marker name exists in that directory”; +- `.git` must match as either a file or directory; +- detected root is stored as an absolute directory path; +- the absolute root path is for I/O only and must not enter snapshot fingerprints. + +### Why `RootMarker` exists + +Do not use bare strings directly across the seam. + +A dedicated `RootMarker` type gives seam 2 a place to validate “exact entry name, not a glob, not a path fragment,” which avoids leaking caller sloppiness into root detection. + +--- + +## 7.2 Snapshot requests and options + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SnapshotSource { + Worktree, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum SymlinkPolicy { + Skip, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum NonUtf8PathPolicy { + Error, + Skip, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum LargeFilePolicy { + Error, + Skip, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SnapshotOptions { + pub symlink_policy: SymlinkPolicy, + pub exclude_globs: Vec, + pub non_utf8_path_policy: NonUtf8PathPolicy, + pub max_file_bytes: Option, + pub large_file_policy: LargeFilePolicy, +} +// Default: +// symlink_policy = SymlinkPolicy::Skip +// exclude_globs = Vec::new() +// non_utf8_path_policy = NonUtf8PathPolicy::Error +// max_file_bytes = None +// large_file_policy = LargeFilePolicy::Error + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SnapshotRequest { + pub root: RepoRoot, + pub source: SnapshotSource, + pub options: SnapshotOptions, +} +``` + +### Reality-based constraint + +Phase A should **not** add new pack/profile schema fields just to support these options. + +Instead: + +- seam 2 accepts `SnapshotOptions` directly; +- later runtime/orchestration code may translate currently-landed `CompiledAnalysisDefaults.follow_symlinks` into: + - `SymlinkPolicy::Skip` when `false` + - a later Phase C follow policy when `true` + +This avoids forcing seam 1 churn before seam 2 exists. + +### Phase C extension shape + +Phase C may extend these contracts with: + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SnapshotSource { + Worktree, + GitRev { rev: String }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum SymlinkPolicy { + Skip, + Follow, +} +``` + +Phase C is also the home for a typed well-known exclude policy. The exact enum naming can be chosen in Phase C, but the semantics should cover the common cache/build/vendor directories listed in the canonical phase map. + +--- + +## 7.3 Repo diagnostics + +Seam 2 should follow the seam-1 diagnostics pattern: structured, sortable, and machine-readable. + +```rust +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct RepoLocation { + pub display_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_path: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct RepoRelatedLocation { + pub location: RepoLocation, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub(crate) struct RepoDiagnostic { + pub code: crate::kernel::DiagnosticCode, + pub severity: crate::kernel::Severity, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub related: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} +``` + +Required ordering rules: + +- diagnostics are sortable and emitted deterministically; +- sort key: + 1. severity rank: `error`, `warning`, `info` + 2. subject display path + 3. subject repo path + 4. diagnostic code + 5. message + +Representative codes: + +- `repo.root.start_path_missing` +- `repo.root.not_found` +- `repo.snapshot.non_utf8_path` +- `repo.snapshot.symlink_skipped` +- `repo.snapshot.unsupported_file_kind` +- `repo.snapshot.file_too_large` + +### Important distinction from seam 0 + +`RepoLocation.display_path` is **not** stable across hosts and is **not** fingerprint input. + +It exists for diagnostics only. + +--- + +## 7.4 Inventory and blobs + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct InventoryEntry { + pub file_id: crate::kernel::FileId, + pub path: crate::kernel::RepoPath, + pub blob_fingerprint: crate::kernel::Fingerprint, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Inventory { + pub entries: std::collections::BTreeMap, + pub fingerprint: crate::kernel::Fingerprint, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BlobRecord { + pub file_id: crate::kernel::FileId, + pub path: crate::kernel::RepoPath, + pub blob_fingerprint: crate::kernel::Fingerprint, + pub size_bytes: u64, + bytes: std::sync::Arc<[u8]>, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct BlobStore { + by_path: std::collections::BTreeMap, +} +``` + +Required methods: + +```rust +impl Inventory { + pub(crate) fn get(&self, path: &crate::kernel::RepoPath) -> Option<&InventoryEntry>; + pub(crate) fn len(&self) -> usize; + pub(crate) fn is_empty(&self) -> bool; + pub(crate) fn iter(&self) -> impl Iterator; +} + +impl BlobStore { + pub(crate) fn contains(&self, path: &crate::kernel::RepoPath) -> bool; + pub(crate) fn get(&self, path: &crate::kernel::RepoPath) -> Option<&BlobRecord>; + pub(crate) fn read_bytes(&self, path: &crate::kernel::RepoPath) -> RepoResult<&[u8]>; + pub(crate) fn len(&self) -> usize; +} +``` + +### Inventory membership rule + +Inventory contains **only regular files** that survive all filtering/policy checks. + +It does **not** contain: + +- directories +- symlinks when `SymlinkPolicy::Skip` +- unsupported file kinds +- excluded `.git/**` +- files excluded by caller-supplied globs +- files rejected by non-UTF8 or large-file policy + +### File identity lemma + +Seam 0 required each producing seam to document stable-ID lemmas. + +For seam 2: + +```text +FileId lemma: +repo\0file\0v1\0 +``` + +`file_id` must therefore be independent of: + +- absolute root path +- blob digest +- mtime +- inode +- traversal order + +That makes `FileId` stable for a logical path across snapshots, while `blob_fingerprint` captures content changes. + +--- + +## 7.5 Repo snapshots + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SnapshotStats { + pub file_count: u64, + pub total_bytes: u64, + pub skipped_by_ignore: u64, + pub skipped_symlinks: u64, + pub skipped_non_utf8_paths: u64, + pub skipped_large_files: u64, + pub skipped_unsupported_file_kinds: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoSnapshot { + pub source: SnapshotSource, + pub inventory: Inventory, + pub blob_store: BlobStore, + pub fingerprint: crate::kernel::Fingerprint, + pub diagnostics: Vec, + pub stats: SnapshotStats, + + root: RepoRoot, +} +``` + +Required methods: + +```rust +impl RepoSnapshot { + pub(crate) fn root(&self) -> &RepoRoot; + pub(crate) fn entry(&self, path: &crate::kernel::RepoPath) -> Option<&InventoryEntry>; + pub(crate) fn read_bytes(&self, path: &crate::kernel::RepoPath) -> RepoResult<&[u8]>; +} +``` + +### Critical snapshot purity invariant + +Once `RepoSnapshot` is constructed: + +- reading bytes must come from `blob_store`, never from the live filesystem; +- inventory iteration must not touch the filesystem; +- deleting or mutating the source tree after snapshot construction must not change: + - `inventory` + - `blob_store` + - `fingerprint` + - `stats` + - `diagnostics` + +This is the seam-2 equivalent of seam 1’s “compiled pack set stays pure after source cleanup.” + +### Snapshot fingerprint semantics + +In phase A: + +```text +RepoSnapshot.fingerprint == Inventory.fingerprint +``` + +The fingerprint input document must be: + +```json +{ + "version": 1, + "files": [ + { + "path": "src/lib.rs", + "blob_fingerprint": "sha256:...", + "size_bytes": 1234 + } + ] +} +``` + +Rules: + +- files are sorted ascending by `RepoPath`; +- absolute root path is excluded; +- diagnostics are excluded; +- stats are excluded; +- file IDs are excluded; +- timestamps, inodes, and permissions are excluded. + +That keeps the snapshot fingerprint purely content-and-path based. + +--- + +## 7.6 Diffs (Phase B) + +Phase B adds a pure diff builder over already-materialized snapshots. + +```rust +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum DiffKind { + Added, + Modified, + Removed, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DiffEntry { + pub path: crate::kernel::RepoPath, + pub kind: DiffKind, + pub before: Option, + pub after: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoDiff { + pub base_fingerprint: crate::kernel::Fingerprint, + pub head_fingerprint: crate::kernel::Fingerprint, + pub entries: Vec, + pub fingerprint: crate::kernel::Fingerprint, +} +``` + +Required function: + +```rust +pub(crate) fn build_diff(base: &RepoSnapshot, head: &RepoSnapshot) -> RepoDiff; +``` + +### Diff semantics + +Diff is path-based only. + +For each path in the union of base and head paths: + +- only in head => `Added` +- only in base => `Removed` +- in both and `blob_fingerprint` differs => `Modified` +- in both and `blob_fingerprint` matches => omitted + +### Important v1 rule + +There is **no rename detection** in seam 2 v1. + +A rename appears as: + +- one `Removed` entry for the old path +- one `Added` entry for the new path + +This is deliberate. Rename heuristics are notoriously unstable across backends and thresholds. Path-based add/remove/modify is simpler and deterministic. + +### Diff fingerprint semantics + +```json +{ + "version": 1, + "base_fingerprint": "sha256:...", + "head_fingerprint": "sha256:...", + "entries": [ + { + "path": "src/lib.rs", + "kind": "modified", + "before": "sha256:...", + "after": "sha256:..." + } + ] +} +``` + +Entries must be sorted ascending by path. + +--- + +## 7.7 Provider contract (Phase C) + +The provider abstraction is deferred until Phase C. + +Reason: + +- Phase A has only one concrete backend, the filesystem worktree; +- Phase B is pure diffing over already-built snapshots; +- introducing the provider trait before a second backend exists is premature abstraction. + +If Phase C lands a second backend, the seam may add: + +```rust +pub(crate) trait RepoProvider { + fn detect_root( + &self, + start: &std::path::Path, + options: &RepoRootDetectionOptions, + ) -> RepoResult; + + fn materialize( + &self, + request: &SnapshotRequest, + ) -> RepoResult; +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct FsRepoProvider; +``` + +Hard rule: + +- do not add `RepoProvider` in Phase A just to reserve shape. + +--- + +## 7.8 Ignore compilation + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledIgnoreSet { + // internal compiled matcher +} +``` + +Required constructor: + +```rust +impl CompiledIgnoreSet { + pub(crate) fn compile(exclude_globs: &[String]) -> RepoResult; + pub(crate) fn is_ignored(&self, repo_path: &crate::kernel::RepoPath, is_dir: bool) -> bool; +} +``` + +### Ignore semantics + +Phase A ignore behavior is intentionally narrow: + +- intrinsic exclude: + - `.git` + - anything under `.git/` +- caller-supplied exclude globs compiled with `globset` +- matching is evaluated against repo-relative POSIX `RepoPath` strings +- if a directory path matches an exclude, its subtree is skipped +- no automatic exclusion of: + - `target` + - `node_modules` + - `.venv` + - `dist` +- no `.gitignore`, `.ignore`, or global git-excludes interpretation + +That last point is important: + +> seam 2 phase A models “the materialized worktree under a repo root,” not “whatever git would currently treat as tracked.” + +--- + +## 8. Exact error surface + +```rust +pub(crate) type RepoResult = Result; + +#[derive(Debug, thiserror::Error, Clone, Eq, PartialEq)] +pub(crate) enum RepoError { + #[error("repo start path does not exist")] + StartPathNotFound { path: String }, + + #[error("repo root could not be detected")] + RootNotFound { + start_path: String, + markers: Vec, + }, + + #[error("repo root is not a directory")] + RootNotDirectory { path: String }, + + #[error("repo I/O failure")] + Io { + op: &'static str, + path: String, + reason: String, + }, + + #[error("invalid root marker")] + InvalidRootMarker { input: String }, + + #[error("invalid repo-relative path derived from filesystem entry")] + InvalidRepoPath { + display_path: String, + reason: String, + }, + + #[error("encountered non-utf8 filesystem path")] + NonUtf8Path { display_path: String }, + + #[error("ignore glob compile failure")] + IgnoreGlobCompile { pattern: String, reason: String }, + + #[error("snapshot source is not implemented")] + UnsupportedSnapshotSource { source: &'static str }, + + #[error("snapshot option is not implemented")] + UnsupportedSnapshotOption { option: &'static str }, + + #[error("file exceeds configured max_file_bytes")] + FileTooLarge { + display_path: String, + size_bytes: u64, + max_file_bytes: u64, + }, + + #[error("blob not present in snapshot")] + MissingBlob { path: crate::kernel::RepoPath }, +} +``` + +### Notes + +- typed errors only; no `anyhow`; +- filesystem path strings in errors are display-only and may be host-specific; +- `RepoError` is for hard failure; +- `RepoDiagnostic` is for successful snapshot warnings/info. + +--- + +## 9. JSON schema inventory + +Unlike seam 1, seam 2 does **not** need user-authored runtime config schemas in its first landing. + +The only JSON schemas seam 2 has added so far are **fixture manifest schemas** that lock deterministic test expectations. + +### Phase A required schema + +```text +schemas/repo/snapshot_manifest.v1.json +``` + +### Phase B landed schema + +```text +schemas/repo/diff_manifest.v1.json +``` + +`src/repo/schema.rs` should follow the same embed-and-access pattern already used by `src/kernel/schema.rs` and `src/pack/schema.rs`. + +### Phase A manifest shape + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/repo/snapshot_manifest.v1.json", + "title": "Lift Repo Snapshot Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "case", + "source_kind", + "options", + "files", + "snapshot_fingerprint", + "stats" + ], + "properties": { + "version": { "const": 1 }, + "case": { "type": "string", "minLength": 1 }, + "source_kind": { "type": "string", "enum": ["worktree"] }, + "options": { + "type": "object", + "additionalProperties": false, + "required": [ + "symlink_policy", + "exclude_globs", + "non_utf8_path_policy", + "max_file_bytes", + "large_file_policy" + ], + "properties": { + "symlink_policy": { "type": "string", "enum": ["skip", "follow"] }, + "exclude_globs": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "uniqueItems": true + }, + "non_utf8_path_policy": { "type": "string", "enum": ["error", "skip"] }, + "max_file_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "large_file_policy": { "type": "string", "enum": ["error", "skip"] } + } + }, + "files": { + "type": "array", + "items": { "$ref": "#/$defs/file_record" } + }, + "snapshot_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "stats": { "$ref": "#/$defs/stats" } + }, + "$defs": { + "file_record": { + "type": "object", + "additionalProperties": false, + "required": ["path", "file_id", "blob_fingerprint", "size_bytes"], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + } + } + }, + "stats": { + "type": "object", + "additionalProperties": false, + "required": [ + "file_count", + "total_bytes", + "skipped_by_ignore", + "skipped_symlinks", + "skipped_non_utf8_paths", + "skipped_large_files", + "skipped_unsupported_file_kinds" + ], + "properties": { + "file_count": { "type": "integer", "minimum": 0 }, + "total_bytes": { "type": "integer", "minimum": 0 }, + "skipped_by_ignore": { "type": "integer", "minimum": 0 }, + "skipped_symlinks": { "type": "integer", "minimum": 0 }, + "skipped_non_utf8_paths": { "type": "integer", "minimum": 0 }, + "skipped_large_files": { "type": "integer", "minimum": 0 }, + "skipped_unsupported_file_kinds": { "type": "integer", "minimum": 0 } + } + } + } +} +``` + +### Runtime invariants not fully expressible in schema + +- `files` must be sorted ascending by `path` +- `file_id` must be a `file:` stable ID, not just any stable ID +- `snapshot_fingerprint` must equal the canonical fingerprint of the sorted file list +- `stats.file_count == files.len()` +- `stats.total_bytes == sum(files[].size_bytes)` + +### Phase B diff manifest shape + +Phase B added `schemas/repo/diff_manifest.v1.json` with: + +- `version` +- `case` +- `base_fingerprint` +- `head_fingerprint` +- `entries` +- `diff_fingerprint` + +where each entry includes: + +- `path` +- `kind` +- `before_blob_fingerprint | null` +- `after_blob_fingerprint | null` + +--- + +## 10. Determinism and invariants + +These must be treated as normative. + +### 10.1 Repo-root invariants + +1. `RepoRoot` is always an existing absolute directory path. +2. Root-marker order never changes the chosen root. +3. The nearest matching ancestor always wins. +4. Root absolute path never enters snapshot fingerprints. + +### 10.2 Snapshot invariants + +1. `RepoSnapshot` is immutable after construction. +2. Snapshot inventory contains only regular files. +3. Snapshot inventory paths are valid kernel `RepoPath` values. +4. Inventory keys are unique and sorted. +5. Blob reads never hit the live filesystem after snapshot creation. +6. Snapshot fingerprint depends only on sorted `(path, blob_fingerprint, size_bytes)`. +7. Snapshot fingerprint does not depend on: + - absolute root path + - traversal order + - mtime + - inode + - permissions + - host path separators +8. `.git/**` is always excluded in phase A. +9. `target`, `node_modules`, `.venv`, and similar directories are **not** implicitly excluded by seam 2. +10. Ignore matching uses repo-relative POSIX paths only. +11. Non-UTF8 handling follows the configured policy and is never silently normalized into an invalid `RepoPath`. +12. `BlobStore.len() == Inventory.len()`. + +### 10.3 Diff invariants + +1. Diff is computed only from already-materialized snapshots. +2. Diff omits unchanged files. +3. Diff entries are sorted by path. +4. Diff kind is only one of: + - `Added` + - `Modified` + - `Removed` +5. Rename detection does not exist in v1. + +### 10.4 Boundary invariants + +1. Seam 2 does not depend on `pack`. +2. Seam 2 does not depend on any language adapters. +3. Seam 2 does not know about components, boundaries, docs, tests, or CI classes. +4. Seam 2 does not know about Lift scoring or app result shapes. +5. Seam 2 contains all repo-walking logic; later seams must not re-walk the filesystem. + +--- + +## 11. Acceptance criteria + +Seam 2 is done only when all of these are true. + +## 11.1 Phase A acceptance criteria + +### Contract completeness + +- all seam-2 runtime types above exist under `src/repo/**` +- `src/repo/mod.rs` re-exports the stable internal seam surface +- no new public crate API is added in `lib.rs` + +### Root detection + +- starting from a nested file under a fixture repo finds the nearest root marker +- starting from a nested directory under a fixture repo finds the nearest root marker +- `ceiling_dir` can stop ascent before a higher matching ancestor +- marker order does not change the chosen root +- `.git` marker works when `.git` is a directory +- `.git` marker works when `.git` is a file +- missing start path returns `RepoError::StartPathNotFound` +- no marker match returns `RepoError::RootNotFound` + +### Snapshot materialization + +- worktree snapshot materializes deterministic inventory and blob bytes +- `.git/**` is excluded even without caller globs +- files excluded by caller globs are absent from inventory and blob store +- directory-path glob matches skip the full subtree +- ignore matching uses repo-relative POSIX paths only +- `target`, `node_modules`, `.venv`, `venv`, `__pycache__`, `dist`, and `build` are **not** implicitly excluded in Phase A without caller globs +- `RepoPath` values in inventory are valid and normalized +- `Inventory.len() == BlobStore.len() == SnapshotStats.file_count` + +### Purity + +- after snapshot creation, mutating or deleting source files does not change: + - `RepoSnapshot::entry` + - `RepoSnapshot::read_bytes` + - `RepoSnapshot::fingerprint` + +### Determinism + +- copying the same fixture tree to two different absolute temp roots yields identical snapshot fingerprints +- walk order in the underlying filesystem does not affect inventory ordering or fingerprint +- changing only one file changes only: + - that file’s blob fingerprint + - the snapshot fingerprint +- diagnostics sort deterministically + +### Errors and policies + +- non-UTF8 and large-file policies behave exactly as configured +- invalid ignore globs return `RepoError::IgnoreGlobCompile` +- skipped symlink / non-UTF8 / large-file / unsupported-kind diagnostics sort deterministically when emitted +- `BlobStore::read_bytes` for a missing path returns `RepoError::MissingBlob` + +### Schema and fixtures + +- `schemas/repo/snapshot_manifest.v1.json` exists and is embedded through `src/repo/schema.rs` +- fixture snapshot manifests validate against the embedded schema +- manifest-based tests verify snapshot fingerprint and inventory records +- manifest-based tests verify runtime invariants that JSON Schema cannot express: + - file ordering + - `stats.file_count == files.len()` + - `stats.total_bytes == sum(files[].size_bytes)` + - snapshot fingerprint matches the canonical sorted file list + +### Build posture + +- `cargo check -p substrate-lift --no-default-features` still passes +- seam 2 adds no dependency on CLI-only code + +## 11.2 Phase B acceptance criteria + +- `build_diff(base, head)` returns sorted add/remove/modify entries only +- unchanged files are omitted +- rename shows up as remove + add +- `schemas/repo/diff_manifest.v1.json` exists and validates diff fixtures +- diff fingerprints are identical across repeated runs for the same snapshot pair + +## 11.3 Phase C acceptance criteria + +- `SnapshotSource::GitRev { .. }` materializes deterministic snapshots for fixtures +- `SymlinkPolicy::Follow` is bounded, explicitly tested, and cannot escape repo constraints +- if `RepoProvider` is introduced, it lands only because more than one backend now exists +- the Phase C well-known exclude policy is typed, explicit, and tested against the canonical directory examples from the phase map +- Phase C snapshot materialization preserves the same purity and determinism guarantees as Phase A +- Phase C adds no silent fallback from `GitRev` materialization to worktree semantics + +--- + +## 12. Falsification questions + +These are the PR-review questions that can invalidate the seam. + +1. Can the same fixture tree produce different snapshot fingerprints when copied to different absolute roots? +2. Can `RepoSnapshot::read_bytes` change after the source file changes on disk? +3. Can anything under `.git/` leak into inventory? +4. Can filesystem walk order change the emitted inventory order or fingerprint? +5. Can a file with the same repo path but different content keep the same blob fingerprint? +6. Can `FileId` depend on blob digest, absolute root path, mtime, or traversal order? +7. Can a later seam still read files directly from disk instead of from `RepoSnapshot`? +8. Can seam 2 silently honor `.gitignore` or global git ignores in phase A? +9. Can `target/` or `node_modules/` be excluded without an explicit caller glob or later policy layer? +10. Can a non-UTF8 path be converted into a bogus `RepoPath` instead of erroring or being skipped per policy? +11. Can a symlink escape the repo root in phase A without an explicit, tested policy? +12. Can `SnapshotSource::GitRev` appear to work in one environment but fall back to worktree semantics in another? +13. Can the diff builder emit rename semantics or heuristic similarity-based behavior? +14. Can seam 2 require `pack`, `lang`, or app types to compile? +15. Can the snapshot fingerprint depend on permission bits, inode numbers, or timestamps? +16. Can diagnostics or stats alter the snapshot fingerprint? +17. Can `BlobStore.len()` differ from `Inventory.len()` in a successful snapshot? +18. Can root-marker ordering change the detected root? + +If any answer is “yes,” seam 2 is not clean enough. + +--- + +## 13. In scope / out of scope + +## 13.1 In scope for phase A + +- root detection from a start path +- exact-name marker matching +- worktree snapshot materialization +- immutable blob store +- deterministic inventory ordering +- kernel `RepoPath` derivation +- content digests +- snapshot fingerprinting +- intrinsic `.git` exclusion +- caller-supplied exclude globs +- typed repo errors +- repo diagnostics +- fixture snapshot manifests and tests + +## 13.2 In scope for phase B + +- pure path-based diffing between snapshots +- diff fixture manifests and tests + +## 13.3 In scope for phase C + +- git revision materialization +- symlink-following semantics +- typed well-known exclude policy for common cache/build/vendor directories +- provider abstraction only if a second backend now exists + +## 13.4 Out of scope for Phase A + +- `.gitignore`, `.ignore`, or global git-exclude interpretation +- implicit policy-driven excludes like `target`, `node_modules`, `.venv`, `venv`, `__pycache__`, `dist`, `build` +- git revision materialization +- path-based diffing +- symlink following +- provider abstraction + +## 13.5 Out of scope for seam 2 entirely in the current plan + +- `.gitignore`, `.ignore`, or global git-exclude interpretation +- component or boundary classification +- docs/tests/CI classification +- language detection or parsing +- query execution +- Lift scoring +- rename detection +- permission-bit or executable-bit tracking +- line endings or text encoding inference +- mmap or on-disk blob caching +- public CLI/runtime integration +- pack schema changes for repo options +- sparse include-path snapshots +- live incremental FS watching + +One explicit choice worth locking: + +> seam 2 does **not** attempt to model “what git considers tracked.” +> It models “a deterministic materialized file tree under a detected repo root.” + +--- + +## 14. Risks and mitigations + +### Risk 1: memory growth from fully materialized snapshots + +Because purity requires storing blob bytes, large repos can consume memory quickly. + +Mitigation: + +- phase A includes `max_file_bytes` and `LargeFilePolicy`; +- Phase A records `file_count` and `total_bytes` so whole-snapshot cost is visible from day one; +- hard limits on total snapshot bytes or file count are deferred until real Phase A usage proves they are needed; +- scope is explicit: immutable materialization first, optimization later; +- future work can add chunked or spilled blob storage without changing the snapshot contract. + +### Risk 2: lack of `.gitignore` support makes worktree snapshots noisy + +Build artifacts may appear unless explicitly excluded. + +Mitigation: + +- make the non-support explicit in the seam; +- keep `.git` as the only intrinsic exclude; +- allow caller-supplied exclude globs now; +- defer tracked-revision snapshots to phase C; +- defer well-known cache/build/vendor excludes to Phase C. + +### Risk 3: symlink following is tricky and easy to make unsafe + +Symlink loops or escapes can break boundedness. + +Mitigation: + +- phase A defaults to `SymlinkPolicy::Skip`; +- only add `Follow` in Phase C with explicit in-repo-bounds tests. + +### Risk 4: non-UTF8 paths collide with seam-0 `RepoPath` + +Mitigation: + +- make non-UTF8 a first-class policy with `Error` / `Skip`; +- never silently coerce to invalid `RepoPath`. + +### Risk 5: backend details leak upward + +If later apps start depending on provider quirks, the seam becomes brittle. + +Mitigation: + +- keep downstream consumers on `RepoSnapshot` only; +- keep provider/backend details below the seam; +- do not introduce the provider abstraction until Phase C proves a second backend exists. + +### Risk 6: git integration grows too early + +Pulling in `git2`/`gix` before the phase-A contracts are stable will slow seam 2 down. + +Mitigation: + +- phase A is filesystem-first; +- Phase C adds git materialization only after snapshot contracts are stable. + +### Risk 7: the seam is correct but does more work than needed + +Snapshotting can stay functionally correct while still doing redundant sorting, hashing, or copying. + +Mitigation: + +- Phase A should materialize in one pass over the selected tree; +- inventory ordering should be produced with one deterministic sort boundary, not repeated resorting across layers; +- once the snapshot exists, no live re-read is allowed; +- benchmark thresholds are deferred until later hardening after the seam exists. + +--- + +## 15. Specific implementation items + +## 15.1 Rust modules + +Phase A landed: + +- `src/repo/error.rs` +- `src/repo/diagnostics.rs` +- `src/repo/root.rs` +- `src/repo/ignore.rs` +- `src/repo/inventory.rs` +- `src/repo/blob.rs` +- `src/repo/snapshot.rs` +- `src/repo/schema.rs` + +`src/repo/mod.rs` is now the real seam re-export surface. + +Phase B added: + +- `src/repo/diff.rs` + +Phase C may add: + +- `src/repo/provider.rs` + +## 15.2 Schemas + +Phase A landed: + +- `schemas/repo/snapshot_manifest.v1.json` + +Phase B added: + +- `schemas/repo/diff_manifest.v1.json` + +`src/repo/schema.rs` should continue embedding schema constants in the same style already used by seam 0 and seam 1. + +## 15.3 Fixtures + +Phase A landed baseline: + +```text +fixtures/repo/ + README.md + valid/ + manifest_minimal.json + invalid/ + manifest_bad_repo_path.json + manifest_missing_stats.json + trees/ + basic_worktree/ +``` + +The landed Phase-A fixture plus temp-tree test helpers already exercise the important content cases: + +- nearest `.git` marker wins +- file-start and dir-start root detection +- `ceiling_dir` stops ascent +- marker order does not change root selection +- `.git` marker as directory +- `.git` marker as file +- snapshot ignores `.git` +- explicit glob exclusion +- directory-match subtree skip semantics +- repo-relative POSIX path matching semantics +- no implicit exclusion of `target` / `node_modules` / `.venv` / `venv` / `__pycache__` / `dist` / `build` +- same tree under two different temp roots -> same fingerprint +- source tree deleted after snapshot -> snapshot still readable +- non-UTF8 path policy behavior (platform-guarded if necessary) +- large-file policy behavior + +Phase B added: + +- `fixtures/repo/diff/**` + +Phase C adds: + +- `fixtures/repo/git_rev/**` +- `fixtures/repo/symlink_follow/**` +- `fixtures/repo/well_known_excludes/**` + +## 15.4 Tests + +Phase A landed tests analogous to the kernel/pack pattern: + +- `tests/repo_root.rs` +- `tests/repo_snapshot.rs` +- `tests/repo_ignore.rs` +- `tests/repo_fingerprints.rs` +- `tests/repo_purity.rs` +- `tests/repo_schema.rs` + +Phase B added: + +- `tests/repo_diff.rs` + +Phase C adds: + +- `tests/repo_git_rev.rs` +- `tests/repo_symlink_policy.rs` +- `tests/repo_materialization_policies.rs` + +### Important test posture + +Like the existing pack tests, these can include the internal seam directly: + +```rust +#[path = "../src/repo/mod.rs"] +mod repo; +``` + +No public crate exports are required just to test seam 2. + +## 15.5 README / housekeeping cleanup + +With seam 2 Phase A and Phase B landed, update: + +- top-level `README.md` seam breakdown text to mark repo substrate as landed snapshot plus pure diffing +- `fixtures/repo/README.md` to mention `fixtures/repo/diff/**` alongside snapshot fixtures +- `schemas/README.md` to mention both `schemas/repo/snapshot_manifest.v1.json` and `schemas/repo/diff_manifest.v1.json` + +--- + +## 15.6 Phase A execution plan, eng-review locked + +This section is the implementation-grade companion to the earlier contract sections. + +The earlier sections answer "what Phase A is." This section answers "how Phase A lands without inventing new decisions mid-implementation." + +### 15.6.a Step 0, scope challenge + +Phase A is the first real repo slice, so it needs the same scope discipline as the eng review, not just good contracts. + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase A decision | +|---|---|---| +| crate-private seam boundary | `src/lib.rs`, current `pub(crate) mod repo;` | keep `repo` crate-private, do not promote public API yet | +| stable repo-relative path validation | `src/kernel/path.rs` | derive every inventory path through `RepoPath`, do not add a second path normalization layer | +| typed stable IDs | `src/kernel/id.rs` | derive `FileId` from the documented path lemma, do not add repo-local ad hoc IDs | +| content hashing and canonical fingerprinting | `src/kernel/fingerprint.rs` | reuse `sha256_bytes` and `sha256_canonical_json` directly | +| typed diagnostics and deterministic ordering pattern | `src/kernel/diagnostic.rs`, `src/pack/diagnostics.rs` | mirror the same sortable diagnostic style for repo failures | +| schema embedding pattern | `src/pack/schema.rs`, `tests/pack_schema.rs`, `tests/kernel_identity_schema.rs` | mirror the same embed-on-disk plus disk-vs-embedded validation pattern | +| no-default-features build posture | `tests/compile_matrix.rs` | Phase A must preserve `cargo check -p substrate-lift --no-default-features` | +| downstream runtime boundary | `src/app/runtime.rs` | keep runtime bootstrap out of Phase A, repo stops at immutable snapshot production | + +Scope decision for the Phase A implementation PR: + +- keep Phase A filesystem-first and worktree-only +- keep Phase A internal to `src/repo/**` +- take the already-allowed `walkdir = "2"` dependency instead of writing a custom recursive walker +- do not add `SnapshotSource::GitRev`, `RepoDiff`, `RepoProvider`, or runtime wiring +- do not add pack/profile schema changes just to thread repo options through seam 1 +- do not add well-known cache/build/vendor excludes yet + +Complexity check: + +- Phase A touches more than 8 files, but that is expected and acceptable after reduction because most of the work is seam contracts, schema/fixture inventory, and deterministic tests +- the real reduction already happened in the phase map: git materialization, diffing, symlink following, well-known excludes, and provider abstraction all stay out + +Distribution check: + +- Phase A does not introduce a new binary, package, or published artifact +- no release workflow or install-path change is part of this PR + +### 15.6.b Phase A architecture + +Execution shape: + +```text +start path + | + v +detect_repo_root(start_path, RepoRootDetectionOptions) + | + v +SnapshotRequest { root, source: Worktree, options } + | + +--> compile caller globs into CompiledIgnoreSet + | + v +walk root tree deterministically (`walkdir`) + | + +--> skip `.git/**` intrinsically + +--> skip explicit-glob matches + +--> reject or skip non-UTF8 paths by policy + +--> reject or skip symlinks by policy + +--> reject or skip large files by policy + +--> reject unsupported file kinds with typed diagnostics + | + v +read regular-file bytes once + | + +--> blob_fingerprint = sha256_bytes(bytes) + +--> file_id = FileId::from_identity("repo\\0file\\0v1\\0") + | + v +build BlobRecord + InventoryEntry + | + v +one deterministic sort boundary by RepoPath + | + v +snapshot fingerprint from sorted `(path, blob_fingerprint, size_bytes)` + | + v +RepoSnapshot { root, source, inventory, blobs, diagnostics, stats, fingerprint } +``` + +Module boundary for Phase A: + +```text +src/lib.rs + └── pub(crate) mod repo + ├── mod.rs + ├── error.rs + ├── diagnostics.rs + ├── root.rs + ├── ignore.rs + ├── inventory.rs + ├── blob.rs + ├── snapshot.rs + └── schema.rs + +tests/ + ├── repo_root.rs + ├── repo_snapshot.rs + ├── repo_ignore.rs + ├── repo_fingerprints.rs + ├── repo_purity.rs + └── repo_schema.rs +``` + +Architectural rule: + +> `snapshot.rs` owns the one-way assembly flow. Later seams read from `RepoSnapshot`. They do not reopen the filesystem or reinterpret the traversal. + +### 15.6.c Exact implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| A1. seam surface + typed failures | `src/repo/mod.rs`, `error.rs`, `diagnostics.rs`, `root.rs` | stub replaced by a real crate-private seam surface with typed root-detection and repo-diagnostic contracts | `src/repo/mod.rs` is no longer a placeholder and root detection compiles with typed errors | +| A2. traversal + ignore policy | `src/repo/ignore.rs`, `Cargo.toml` | explicit Phase-A traversal policy, caller glob compilation, `.git` intrinsic exclusion, `walkdir` dependency choice landed | invalid globs fail deterministically and traversal policy types compile | +| A3. inventory + blob contracts | `src/repo/inventory.rs`, `blob.rs` | immutable inventory/blob-store contracts and lookup helpers exist | `Inventory` and `BlobStore` compile with the documented methods and invariants | +| A4. snapshot builder | `src/repo/snapshot.rs`, `root.rs`, `ignore.rs`, `inventory.rs`, `blob.rs` | deterministic worktree materialization pipeline exists end to end | root detect -> walk -> filter -> read -> hash -> sort -> fingerprint -> `RepoSnapshot` all work through one codepath | +| A5. schema embedding | `src/repo/schema.rs`, `schemas/repo/snapshot_manifest.v1.json` | embedded repo snapshot schema matches the manifest on disk | schema constants compile and disk-vs-embedded validation passes | +| A6. fixtures + integration tests | `fixtures/repo/**`, `tests/repo_*.rs`, `tests/compile_matrix.rs` | every required Phase-A branch and invariant is covered | all Phase-A acceptance paths have deterministic tests, including no-default-features posture | +| A7. docs sweep | `README.md`, `fixtures/README.md`, `schemas/README.md` | docs describe the landed repo seam honestly | no README still describes repo as only reserved future work | + +Implementation order: + +1. Land A1. +2. Land A2. +3. Land A3. +4. Land A4. +5. Land A5. +6. Launch A6 and A7 only after A4/A5 interfaces stop moving. + +### 15.6.d Test review, required coverage for Phase A + +CODE PATH COVERAGE +=========================== +[+] Repo root detection + ├── [REQUIRED] start path is a directory, nearest marker ancestor wins + ├── [REQUIRED] start path is a file, detection begins at the parent directory + ├── [REQUIRED] `ceiling_dir` stops ascent deterministically + ├── [REQUIRED] marker order does not change the selected root + ├── [REQUIRED] `.git` marker works as either file or directory + └── [REQUIRED] missing start path / missing root emits typed failure + +[+] Ignore compilation and path-policy handling + ├── [REQUIRED] invalid caller glob hard-fails with typed error + ├── [REQUIRED] `.git/**` is always excluded without caller input + ├── [REQUIRED] explicit glob exclusion removes matching subtree + ├── [REQUIRED] `target/`, `node_modules/`, `.venv/`, `dist/`, `build/` are not implicitly excluded + ├── [REQUIRED] non-UTF8 path with `Error` fails deterministically + ├── [REQUIRED] non-UTF8 path with `Skip` omits the path and records deterministic diagnostics + ├── [REQUIRED] symlink with `Skip` never enters inventory + └── [REQUIRED] large file with `Error` / `Skip` follows policy exactly + +[+] Snapshot materialization + ├── [REQUIRED] inventory contains only regular files that survive policy checks + ├── [REQUIRED] each surviving file produces matching `InventoryEntry` + `BlobRecord` + ├── [REQUIRED] `BlobStore.len()` equals `Inventory.len()` on success + ├── [REQUIRED] identical trees under different temp roots yield the same fingerprint + ├── [REQUIRED] traversal order does not affect inventory order or fingerprint + ├── [REQUIRED] source tree deletion after snapshot does not change `read_bytes` + └── [REQUIRED] missing blob/path lookup returns typed repo failure, never panic + +[+] Schema and manifest coverage + ├── [REQUIRED] embedded `snapshot_manifest.v1.json` matches the on-disk schema + ├── [REQUIRED] valid fixture manifests validate and deserialize + └── [REQUIRED] invalid fixture manifests fail validation deterministically + +USER FLOW COVERAGE +=========================== +[+] Internal engine flow + ├── [REQUIRED] detect root -> build snapshot -> inspect stats/inventory/fingerprint + ├── [REQUIRED] apply caller exclusion -> snapshot omits subtree and preserves determinism + ├── [REQUIRED] policy violation -> typed diagnostics/error without partial corrupt snapshot + └── [REQUIRED] snapshot created once -> later reads come only from `BlobStore`, not disk + +───────────────────────────────── +COVERAGE GOAL: 28/28 required paths covered before Phase A promotion + Code paths: 24/24 + Internal flows: 4/4 +QUALITY BAR: no smoke-only tests for acceptance paths +GAPS ALLOWED AT PROMOTION: 0 +───────────────────────────────── + +Required test files: + +- `tests/repo_root.rs` for root detection semantics and root-marker invariants +- `tests/repo_snapshot.rs` for worktree materialization happy path and policy behavior +- `tests/repo_ignore.rs` for caller glob compilation and intrinsic `.git` exclusion +- `tests/repo_fingerprints.rs` for cross-root determinism and traversal-order invariants +- `tests/repo_purity.rs` for "snapshot survives source mutation/deletion" behavior +- `tests/repo_schema.rs` for embedded-schema parity and fixture-manifest validation +- extend `tests/compile_matrix.rs` only as needed to keep `--no-default-features` coverage intact + +Recommended validation commands: + +```bash +cargo fmt --all +cargo clippy -p substrate-lift --all-targets -- -D warnings +cargo test -p substrate-lift --test repo_root -- --nocapture +cargo test -p substrate-lift --test repo_snapshot -- --nocapture +cargo test -p substrate-lift --test repo_ignore -- --nocapture +cargo test -p substrate-lift --test repo_fingerprints -- --nocapture +cargo test -p substrate-lift --test repo_purity -- --nocapture +cargo test -p substrate-lift --test repo_schema -- --nocapture +cargo test -p substrate-lift --test compile_matrix -- --nocapture +``` + +### 15.6.e Failure modes for Phase A + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| root detection | wrong ancestor chosen because marker order leaks into selection | yes | yes, deterministic nearest-match semantics | typed root-detection failure or correct root | +| ignore compilation | invalid glob accepted and silently broadens or narrows scope | yes | yes, typed compile failure for the request | snapshot build rejected before traversal | +| path normalization | non-UTF8 host path coerced into bogus `RepoPath` | yes | yes, policy-driven error/skip only | typed failure or deterministic omission | +| symlink handling | symlink enters inventory in Phase A despite `Skip` policy | yes | yes, deterministic skip diagnostic | file absent from snapshot, never followed | +| snapshot purity | `read_bytes` reopens live disk after materialization | yes | test-enforced invariant | deterministic snapshot read, even if source tree changes | +| fingerprinting | traversal order or absolute temp root alters snapshot fingerprint | yes | test-enforced invariant | stable fingerprint across equivalent trees | +| blob/inventory consistency | inventory entry exists without backing blob bytes | yes | yes, constructor-level invariant | typed `MissingBlob` style failure, never panic | +| large-file policy | oversize file leaks through despite `Error` / `Skip` policy | yes | yes, typed policy failure or deterministic omission | clear error or clear omission, never silent inclusion | + +Critical gap rule: + +- Phase A does not promote if any failure mode is both untested and capable of causing silent snapshot drift + +### 15.6.f NOT in scope for the Phase A implementation PR + +- `SnapshotSource::GitRev { rev }` +- `RepoDiff`, rename detection, or any other Phase-B diff behavior +- `SymlinkPolicy::Follow` +- provider abstraction or git backend crates +- `.gitignore`, `.ignore`, or global git-exclude interpretation +- typed well-known excludes for cache/build/vendor directories +- runtime or CLI translation from `CompiledAnalysisDefaults` into `SnapshotOptions` +- public API promotion of `repo` + +### 15.6.g Worktree parallelization strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| A1 seam surface + root detection | `src/repo/`, `src/lib.rs` | — | +| A2 traversal + ignore policy | `src/repo/`, `Cargo.toml` | A1 | +| A3 inventory + blob contracts | `src/repo/` | A1 | +| A4 snapshot builder | `src/repo/` | A2, A3 | +| A5 schema embedding | `src/repo/`, `schemas/repo/` | A4 | +| A6 fixtures + tests | `fixtures/repo/`, `tests/` | A4, A5 | +| A7 docs sweep | `README.md`, `fixtures/README.md`, `schemas/README.md` | A5 | + +Parallel lanes: + +- Lane A: A1 -> A2/A3 -> A4 -> A5 (core lane; A2 and A3 are the only safe micro-split because both fold back into shared `src/repo/` before A4) +- Lane B: A6 (sidecar after A5, lives in `fixtures/repo/` and `tests/`) +- Lane C: A7 (sidecar after A5, lives in README surfaces only) + +Execution order: + +- Run Lane A first until the snapshot contract, schema ID, and typed errors stop moving. +- Once A5 lands, launch Lane B and Lane C in parallel worktrees. +- Merge B and C back into A before the Phase-A promotion gate. + +Conflict flags: + +- A1 through A5 all touch `src/repo/`, so the core implementation is one lane in practice +- Lane B must not reopen `src/repo/`; if missing test hooks require core changes, fold that work back into Lane A +- Lane C is safe only if doc updates stay in README surfaces and do not reopen contract decisions + +### 15.6.h Phase A completion summary + +- Step 0: scope accepted as filesystem-first immutable snapshot substrate +- Architecture: root-detect -> ignore-compile -> walk -> hash -> sort -> snapshot diagram written +- Test Review: 28 required paths named, 0 gaps allowed at promotion +- Failure modes: 0 silent-snapshot-drift gaps allowed +- NOT in scope: written +- What already exists: written +- Parallelization: 3 lanes, 2 sidecar lanes after 1 sequential core lane + +Phase-A promotion gate: + +1. `src/repo/mod.rs` is no longer a stub and remains crate-private. +2. root detection obeys nearest-match, file-start, ceiling-dir, and marker-order invariants. +3. worktree snapshot materialization produces immutable blobs plus deterministic inventory ordering. +4. `.git/**` is intrinsically excluded, caller globs behave deterministically, and no implicit cache/build/vendor excludes leak in. +5. embedded snapshot schema matches disk, and fixture manifests validate. +6. all required Phase-A tests above are present and passing. +7. `cargo check -p substrate-lift --no-default-features` still passes. + +## 15.7 Phase B execution plan, eng-review locked + +This section starts from the landed repo substrate as it exists on `feat/lift` today. + +The earlier sections answer "what Phase B is." This section records how Phase B landed without reopening Phase A or smuggling in Phase C. + +### 15.7.a Step 0, scope challenge + +Phase B must treat the shipped snapshot substrate as the baseline, not as fresh clay. + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase B decision | +|---|---|---| +| crate-private seam boundary | `src/lib.rs`, `src/repo/mod.rs` | keep `repo` crate-private, do not promote public API just to expose diffing | +| immutable snapshot access | `src/repo/snapshot.rs`, `RepoSnapshot::entry`, `RepoSnapshot::read_bytes` | diff only already-materialized snapshots, never reopen filesystem walking or option semantics | +| stable ordered inventory | `src/repo/inventory.rs`, `Inventory::iter`, `Inventory::get`, `fingerprint_entries` | build Phase B on the already-sorted inventory surface, not a new unordered map layer | +| stable repo-relative path identity | `src/kernel/path.rs`, `src/repo/inventory.rs` | key the diff on `RepoPath` only, one diff row per repo-relative path | +| stable content and canonical fingerprinting | `src/kernel/fingerprint.rs`, `src/repo/inventory.rs` | fingerprint diff output with canonical JSON, never with absolute roots or traversal state | +| landed schema/test harness pattern | `src/repo/schema.rs`, `tests/support/repo_support.rs`, `tests/repo_schema.rs` | extend the same embedded-schema plus generated-manifest pattern for diff fixtures | +| no-default-features build posture | `tests/compile_matrix.rs` | Phase B must preserve `cargo check -p substrate-lift --no-default-features` | +| downstream runtime boundary | `src/app/runtime.rs` | keep runtime bootstrap out of Phase B, repo still ends at immutable artifact plus pure diff | + +The landed scope for the Phase B implementation PR was: + +- add only pure path-based diffing over `RepoSnapshot`; +- add `src/repo/diff.rs`, `schemas/repo/diff_manifest.v1.json`, `fixtures/repo/diff/**`, `tests/repo_diff.rs`, and the minimal schema/test-harness extensions needed to support them; +- extend `src/repo/mod.rs` and `src/repo/schema.rs` only to expose the new Phase-B internal seam surface and embedded schema constants; +- keep `SnapshotSource`, `SnapshotOptions`, traversal behavior, root detection, ignore semantics, and snapshot diagnostics unchanged; +- do not add rename detection, blob-level patch hunks, diff-time filesystem reads, git revision materialization, runtime wiring, or pack-driven option translation. + +Complexity check: + +- Phase B is smaller than Phase A, but it still spans core seam code, schema, fixtures, and tests, so it should be treated as one focused repo slice rather than a "quick helper"; +- the minimum credible change still touches more than 8 files once fixtures and tests are counted, but that is acceptable because the implementation stays inside one seam and adds no new runtime integration; +- if a proposed change needs to modify `src/app/runtime.rs`, `src/pack/**`, or snapshot-construction behavior in `src/repo/snapshot.rs`, it is probably Phase C or later work and should be deferred. + +Search and distribution check: + +- `[Layer 1]` use the built-in sorted `BTreeMap` inventory order already landed in `Inventory`; do not pull in a diff crate or invent a second ordering primitive; +- `[Layer 1]` reuse the existing `jsonschema` + embedded-schema test pattern from `tests/repo_schema.rs`; do not introduce a second fixture-validation path; +- Phase B does not introduce a new binary, package, feature flag, or published artifact, so no CLI surface, release workflow, or install-path change belongs in this PR. + +Scope result: + +- scope accepted as-is, because the plan is already the reduced version and the remaining work is the complete implementation of that reduced scope. + +### 15.7.b Architecture review + +Phase B execution shape: + +```text +base: RepoSnapshot.inventory.iter() head: RepoSnapshot.inventory.iter() + | | + v v + peekable ordered stream peekable ordered stream + \ / + \ / + +---- merge-walk on RepoPath ----+ + | + +--> base path only -> Removed { before: Some, after: None } + +--> head path only -> Added { before: None, after: Some } + +--> same path, same blob -> omit + +--> same path, new blob -> Modified { before: Some, after: Some } + | + v + ordered Vec in path order + | + v + canonical diff fingerprint document: + { + version, + base_fingerprint, + head_fingerprint, + entries[path, kind, before_blob, after_blob] + } + | + v + RepoDiff { base_fingerprint, head_fingerprint, entries, fingerprint } +``` + +Architectural rules: + +- `build_diff(base, head)` compares `InventoryEntry` state only. It does not reopen the filesystem and does not need `BlobStore::read_bytes`. +- The algorithm should merge-walk the two already-sorted inventory iterators instead of first building a path union set and then sorting it again. That keeps the implementation explicit, deterministic, and linear in the number of inventory entries. +- `DiffEntry.before` and `DiffEntry.after` carry the already-materialized `InventoryEntry` values needed by downstream consumers. Blob bytes stay in the snapshots. +- There is at most one `DiffEntry` per `RepoPath`. +- The zero-diff case is first-class: identical snapshots produce `RepoDiff { entries: vec![] }` with a stable fingerprint. +- There is no meaningful new auth or external security surface here because Phase B is a crate-private pure transform. The risk is semantic drift, not permission bypass. The architecture therefore needs tests and invariants more than new runtime guards. + +Realistic production failure scenarios this plan must account for: + +- if the merge walk advances the wrong iterator on equal paths, a modified file can be emitted as `Added` plus `Removed` instead of `Modified`; +- if the fingerprint document is built from host-specific data or unstable entry order, identical semantic diffs hash differently across temp roots and test reruns; +- if the implementation consults `BlobStore` or the live filesystem during diffing, Phase B silently reintroduces non-determinism that Phase A was explicitly built to prevent. + +Module boundary for Phase B only: + +```text +src/repo/ + mod.rs + diff.rs + schema.rs + +tests/ + repo_diff.rs + repo_schema.rs + support/repo_support.rs + +fixtures/repo/ + diff/ +``` + +### 15.7.c Code quality review + +Phase B stayed boring on purpose. + +| Area | Code quality rule | Why | +|---|---|---| +| `src/repo/diff.rs` | keep one small data-model layer plus a few private helpers, not a trait hierarchy or backend abstraction | there is one backend and one diff mode, so extra abstraction is fake flexibility | +| diff algorithm | prefer a merge-walk over sorted iterators, not `HashMap` / `HashSet` plus a cleanup sort | the inventories are already ordered, so a second ordering phase is pure churn | +| fingerprinting | fingerprint a plain canonical document built from diff entries, not ad hoc string concatenation | matches landed seam style and keeps the hash input reviewable | +| `src/repo/schema.rs` | add diff schema constants in the same pattern as snapshot schema constants | keeps schema embedding DRY and discoverable | +| `tests/support/repo_support.rs` | add `diff_validator`, `manifest_from_diff`, and paired-snapshot helpers by extending the current helper module, not by creating a second support module | prevents fixture logic from splitting across parallel helper stacks | +| `tests/repo_schema.rs` | keep schema parity and manifest-shape assertions here | this file already owns repo-schema validation | +| `tests/repo_diff.rs` | keep behavior assertions here, not inside schema tests | separates "what the diff means" from "what the manifest looks like" | + +Hard quality rules: + +1. Do not change `src/repo/snapshot.rs` unless an actual Phase-A defect is discovered. Phase B should consume snapshots, not renegotiate them. +2. Do not duplicate `InventoryEntry` or `RepoSnapshot` fields into a second test-only manifest type when existing runtime types can already supply the values. +3. Do not add a provider trait, a generic diff backend, or a rename-similarity helper "for later." +4. Keep docs last. If prose reveals a missing semantic decision, fix the semantics in code/tests first. + +### 15.7.d Rules to lock now + +Lock these rules: + +1. `build_diff` is a pure function over `&RepoSnapshot` inputs. +2. `DiffKind` remains exactly `Added`, `Modified`, `Removed`. +3. Path comparison is by canonical repo-relative `RepoPath`, never absolute host path. +4. `Modified` is decided by `blob_fingerprint` inequality only. +5. Unchanged files are omitted even if file IDs or stats are inspected elsewhere. +6. Rename behavior is not inferred. A rename is one `Removed` plus one `Added`. +7. `DiffEntry.before` is present only for `Removed` and `Modified`. +8. `DiffEntry.after` is present only for `Added` and `Modified`. +9. `RepoDiff.fingerprint` excludes absolute roots, file IDs, stats, diagnostics, and blob bytes. +10. Phase B adds no new error surface beyond existing typed repo/schema failures needed to compile or validate fixtures. +11. The emitted `entries` vector is already in final order when constructed. No "sort at the end just to be safe" cleanup pass should be needed. +12. Schema/test helpers stay extensions of the landed repo test harness, not a second parallel fixture system. + +### 15.7.e Phase B implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| B1. diff seam surface + merge-walk algorithm | `src/repo/{mod.rs,diff.rs}` | real Phase-B diff types and pure builder land behind the existing crate-private seam | identical, add, remove, modify, and empty-diff cases all compile through one deterministic ordered codepath | +| B2. diff fingerprint helper | `src/repo/diff.rs` | canonical diff fingerprint is computed from the ordered diff document and stored on `RepoDiff` | equivalent snapshot pairs produce the same fingerprint across repeated calls and different temp roots | +| B3. diff schema embedding | `src/repo/schema.rs`, `schemas/repo/diff_manifest.v1.json` | embedded diff schema matches the on-disk manifest and follows the existing repo-schema pattern | diff schema constants compile and disk-vs-embedded validation passes | +| B4. diff fixture/test harness | `tests/support/repo_support.rs`, `fixtures/repo/diff/**` | generated diff manifests can be built and validated the same way snapshot manifests are | helper code can materialize paired snapshots and emit diff-manifest JSON without custom one-off logic | +| B5. integration tests | `tests/repo_diff.rs`, `tests/repo_schema.rs`, `tests/compile_matrix.rs` if needed | every required Phase-B path and invariant is covered | add/remove/modify/rename-as-two-events/empty-diff/fingerprint determinism all have deterministic assertions | +| B6. docs sweep | `lift_seam2_spec_reviewed.md`, `README.md`, `fixtures/repo/README.md`, `schemas/README.md` | prose matches the landed Phase-B contract and no file still describes diffing as hand-wavy future work | README surfaces describe repo diffing as pure over snapshots, not as git-aware history analysis | + +Implementation order: + +1. Land B1 and B2 together. The data model and the fingerprint semantics should freeze at the same time. +2. Land B3 immediately after the diff contract stops moving. +3. Land B4 once the schema filename, manifest fields, and helper names are stable. +4. Land B5 after B1 through B4 are stable. +5. Land B6 last, once schema filename and fixture names are frozen. + +### 15.7.f Test review, required coverage for Phase B + +CODE PATH COVERAGE +=========================== +[+] `src/repo/diff.rs` merge-walk + |- [REQUIRED] base exhausted first -> remaining head paths emit `Added` + |- [REQUIRED] head exhausted first -> remaining base paths emit `Removed` + |- [REQUIRED] same path with equal blob fingerprint is omitted + |- [REQUIRED] same path with different blob fingerprint emits exactly one `Modified` + |- [REQUIRED] path ordering follows `RepoPath` lexical order without an extra final sort + `- [REQUIRED] identical snapshots produce `entries.is_empty()` + +[+] `DiffEntry` shape invariants + |- [REQUIRED] `Added` has `before: None`, `after: Some` + |- [REQUIRED] `Removed` has `before: Some`, `after: None` + |- [REQUIRED] `Modified` has both `before` and `after` + `- [REQUIRED] rename-shaped change appears as `Removed` + `Added`, never a fourth kind + +[+] Fingerprinting + |- [REQUIRED] repeated `build_diff(base, head)` calls produce the same `RepoDiff.fingerprint` + |- [REQUIRED] changing entry order in fixture construction does not change the diff fingerprint + |- [REQUIRED] changing the diff entry set changes the diff fingerprint + |- [REQUIRED] equivalent trees under different absolute temp roots still diff to empty when snapshot fingerprints match + `- [REQUIRED] diffing two prebuilt snapshots after later live-tree mutation still yields the original result + +[+] Schema and manifest coverage + |- [REQUIRED] embedded `diff_manifest.v1.json` matches the on-disk schema + |- [REQUIRED] valid diff fixture manifests validate and deserialize + |- [REQUIRED] invalid diff fixture manifests fail validation deterministically + |- [REQUIRED] manifest ordering matches runtime ordering + `- [REQUIRED] manifest fingerprints match runtime fingerprints + +USER FLOW COVERAGE +=========================== +[+] Internal engine flow + |- [REQUIRED] materialize base snapshot -> materialize head snapshot -> build diff -> inspect ordered entries + |- [REQUIRED] modified path carries both `before` and `after` inventory entries through the handoff + |- [REQUIRED] empty diff still exposes stable base/head fingerprints plus a deterministic diff fingerprint + `- [REQUIRED] diff fixture generation reuses the same runtime codepath as the behavior tests + +--------------------------------- +COVERAGE GOAL: 18/18 required Phase-B paths covered before promotion + Code paths: 14/14 + Internal flows: 4/4 +QUALITY BAR: no smoke-only tests for acceptance paths +GAPS ALLOWED AT PROMOTION: 0 +--------------------------------- + +Required test files: + +- add `tests/repo_diff.rs` for diff assembly, ordering, before/after-shape invariants, and rename-as-add-remove semantics; +- extend `tests/repo_schema.rs` for diff-schema embedding, valid diff manifests, invalid diff manifests, and runtime-vs-manifest parity assertions; +- extend `tests/support/repo_support.rs` with the minimal helpers needed to materialize paired snapshots, generate diff-manifest JSON, and validate diff manifests; +- extend `tests/compile_matrix.rs` only if the new schema constants or diff module alter existing crate-level compile assertions. + +Required fixture inventory: + +```text +fixtures/repo/diff/ + valid/ + empty_diff.json + added_file.json + removed_file.json + modified_file.json + rename_as_add_remove.json + invalid/ + manifest_bad_repo_path.json + manifest_bad_kind.json + manifest_missing_diff_fingerprint.json + manifest_before_after_shape_invalid.json +``` + +Test-plan artifact to write during review/implementation: + +- write `~/.gstack/projects/$SLUG/{user}-{branch}-eng-review-test-plan-{timestamp}.md`; +- list the affected internal flow as `RepoSnapshot -> build_diff -> RepoDiff`; +- list the key interactions to verify as add/remove/modify/empty/rename-shaped changes plus schema parity; +- list the critical path as "materialize paired snapshots, build diff, validate manifest, rerun under a second temp root." + +Recommended validation commands: + +```bash +cargo fmt --all +cargo clippy -p substrate-lift --all-targets -- -D warnings +cargo test -p substrate-lift --test repo_diff -- --nocapture +cargo test -p substrate-lift --test repo_schema -- --nocapture +cargo test -p substrate-lift --test compile_matrix -- --nocapture +``` + +### 15.7.g Performance review + +Phase B has no database or network path, but it still has real performance decisions to lock: + +| Concern | Risk | Required decision | +|---|---|---| +| algorithmic shape | building a union set and sorting it later turns a simple ordered merge into avoidable extra allocation and CPU | use a merge-walk over the two ordered inventories | +| memory growth | cloning blob bytes or whole snapshots during diffing would scale with repository size, not with changed paths | clone only the `InventoryEntry` values needed for emitted `DiffEntry`s | +| fingerprint cost | fingerprinting the full snapshot again would repeat Phase-A work | fingerprint only the ordered diff document plus base/head snapshot fingerprints | +| hidden I/O | consulting `BlobStore::read_bytes` or host paths during diffing reintroduces live-tree coupling | Phase B stays metadata-only once the two snapshots already exist | + +Performance rules to lock now: + +- expected time complexity is `O(base_entries + head_entries + changed_entries_for_fingerprint)`; +- expected memory beyond the two snapshots is `O(changed_entries)`, not `O(all_paths)` plus blob bytes; +- do not add caches in Phase B. The data is already in memory, and a cache here would just be another invalidation problem; +- if an implementation reaches for `HashMap` / `HashSet` plus a cleanup sort, reject it unless a benchmark demonstrates a real regression in the ordered merge-walk. + +### 15.7.h Failure modes for Phase B + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| merge-walk advance logic | modified path is emitted as add+remove because the wrong iterator advances on equality | yes | yes, exact equal-path branch coverage | semantic drift in downstream change classification | +| base/head exhaustion | added or removed path is dropped because one tail of the iterator pair is never flushed | yes | yes, tail-handling tests | missing change never reaches downstream seams | +| modified detection | same-path content change is missed because comparison uses size or stale host reads instead of `blob_fingerprint` | yes | yes, compare the landed inventory fingerprints only | incorrect "unchanged" result | +| unchanged omission | unchanged paths leak into output and bloat every downstream consumer | yes | test-enforced invariant | noisy diff with unstable fanout | +| ordering | diff entries follow hash-map or traversal order instead of sorted `RepoPath` order | yes | test-enforced invariant | nondeterministic diff output and fingerprint drift | +| rename semantics | heuristic rename detection sneaks in through future convenience logic | yes | yes, hard rule to stay add/remove/modify only | backend-specific drift and unstable semantics | +| diff fingerprinting | absolute roots, file IDs, or output order leak into `RepoDiff.fingerprint` | yes | test-enforced invariant | same semantic diff hashes differently across runs | +| schema embedding | embedded diff schema drifts from the on-disk schema or fixture shape | yes | yes, schema parity tests must hard-fail | manifest validation mismatch in tests | + +Critical gap rule: + +- Phase B does not promote if any failure mode is both untested and capable of causing silent diff drift. + +### 15.7.i NOT in scope for the Phase B implementation PR + +- `SnapshotSource::GitRev { rev }` +- `SymlinkPolicy::Follow` +- any new filesystem walking or ignore semantics +- rename detection or similarity heuristics +- blob-level textual patches or hunk generation +- runtime, CLI, or pack integration work +- provider abstraction +- pack/profile schema changes to thread diff config into seam 2 + +### 15.7.j Worktree parallelization strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| B1 core diff model + merge-walk | `src/repo/` | - | +| B2 diff schema embedding | `src/repo/`, `schemas/repo/` | B1 | +| B3 diff fixture/harness helpers | `tests/support/`, `fixtures/repo/diff/` | B1, B2 | +| B4 behavior + schema tests | `tests/` | B1, B2, B3 | +| B5 docs sweep | spec + README surfaces | B2, B3 | + +Parallel lanes: + +- Lane A: B1 -> B2 (sequential core lane, shared `src/repo/` contract surfaces) +- Lane B: B3 -> B4 (test lane after the core contract and schema stabilize) +- Lane C: B5 (docs-only sidecar after schema filename and fixture names freeze) + +Execution order: + +1. Run Lane A first until the diff contract, helper names, and schema ID stop moving. +2. Once B2 lands, launch Lane B and Lane C in parallel worktrees. +3. Merge Lane B before promotion, because the tests are part of the gate. +4. Merge Lane C last, once code and fixture names are frozen. + +Conflict flags: + +- Lane A is strictly sequential because `src/repo/mod.rs`, `diff.rs`, and `schema.rs` define the seam surface. +- Lane B must not reopen snapshot-construction behavior in `src/repo/snapshot.rs`; if tests discover that need, route it back to Lane A and re-evaluate scope. +- Lane C is safe only if it stays docs-only. If prose uncovers unresolved semantics, fix the semantics first in Lane A. +- Do not split B1 into two parallel worktrees. `mod.rs` and `diff.rs` are one shared ownership surface in practice. + +### 15.7.k Phase B completion summary + +- Step 0: scope accepted as pure diff over landed snapshots +- Architecture review: ordered merge-walk plus canonical fingerprint pipeline written +- Code quality review: no new abstraction layers, no snapshot renegotiation, helper reuse locked +- Test review: 18 required Phase-B paths named, 0 gaps allowed at promotion +- Performance review: linear merge-walk, no blob rereads, no cache layer, no extra union-sort pass +- Failure modes: 0 silent-diff-drift gaps allowed +- NOT in scope: written +- What already exists: written +- TODOS.md updates: none required beyond the explicit deferrals already captured here +- Outside voice: skipped for this document rewrite, because this section is locking implementation structure rather than running a fresh multi-model review +- Parallelization: 3 lanes, 1 sequential core lane plus 2 sidecars +- Lake score: complete implementation path selected, no intentional shortcuts + +Phase-B promotion gate: + +1. `src/repo/diff.rs` exists and remains a pure function over `&RepoSnapshot`. +2. `build_diff(base, head)` emits only ordered add/remove/modify entries and omits unchanged files. +3. the implementation uses the landed ordered inventory surface, not a second unordered path-collection layer. +4. rename-shaped changes appear only as `Removed` + `Added`. +5. `schemas/repo/diff_manifest.v1.json` is embedded through `src/repo/schema.rs` and validated in tests. +6. all required Phase-B tests above are present and passing. +7. `cargo check -p substrate-lift --no-default-features` still passes. + +## 15.8 Phase C execution plan, eng-review locked + +This section starts from the landed seam-2 repo substrate as it exists after Phase B. + +The earlier sections answer "what Phase C is." This section locks how Phase C can expand materialization semantics without corrupting the already-landed snapshot and diff contracts. + +### 15.8.a Step 0, scope challenge + +Phase C is the first seam-2 slice that is allowed to make snapshot construction more expressive. + +That is precisely why the scope must stay narrow. + +What already exists and must be reused: + +| Sub-problem | Existing code | Phase C decision | +|---|---|---| +| crate-private seam boundary | `src/lib.rs`, `src/repo/mod.rs` | keep `repo` crate-private, do not promote public API just to expose `GitRev` or follow semantics | +| one request/response materialization contract | `src/repo/snapshot.rs`, `SnapshotRequest`, `SnapshotSource::Worktree`, `materialize_snapshot` | extend the existing request/response seam, do not add a second top-level repo entrypoint | +| explicit option surface | `src/repo/ignore.rs`, `SnapshotOptions`, `SymlinkPolicy::Skip`, caller glob compilation | extend the current option types directly for follow semantics and typed well-known excludes, not via parallel ad hoc flags | +| immutable snapshot artifact | `RepoSnapshot`, `Inventory`, `BlobStore`, `SnapshotStats`, `RepoDiagnostic` | both worktree and git-backed materialization must still end at the same immutable snapshot shape | +| source dispatch entrypoint | `materialize_snapshot` in `src/repo/snapshot.rs` currently has a single `Worktree` match arm | extend the current dispatch point, do not add a second top-level repo materializer | +| shared finalization pipeline | `materialize_worktree_snapshot`, `Inventory::from_entries`, `BlobStore::from_records`, and snapshot fingerprint assembly in `src/repo/snapshot.rs` | keep one final assembly path and feed it from source-specific candidate loaders rather than forking snapshot assembly | +| typed repo error surface | `src/repo/error.rs` already carries typed root, path, glob, source, option, and blob failures | add explicit GitRev and follow-resolution failures here, not stringly fallbacks hidden behind `RepoError::Io` | +| downstream diff transparency | `src/repo/diff.rs`, `RepoDiff`, `tests/repo_diff.rs` | Phase C must not change diff semantics; downstream diffing should consume Phase-C snapshots unchanged | +| schema/test harness pattern | `src/repo/schema.rs`, `tests/repo_schema.rs`, `tests/support/repo_support.rs` | extend the current snapshot-manifest embedding and helper pattern, do not create a second schema or fixture system | +| current manifest shape | `schemas/repo/snapshot_manifest.v1.json` plus `manifest_from_snapshot` in `tests/support/repo_support.rs` currently hard-code `source_kind = "worktree"` and no typed well-known exclude field | lock the minimal schema/helper expansion before broad test writing begins | +| no-default-features build posture | `tests/compile_matrix.rs` | Phase C must preserve `cargo check -p substrate-lift --no-default-features` | +| runtime boundary | `src/app/runtime.rs` | keep runtime bootstrap, CLI wiring, and pack-driven translation out of Phase C | + +The locked scope for the Phase C implementation PR is: + +- extend `SnapshotSource` with `GitRev { rev }`; +- extend `SymlinkPolicy` with `Follow`; +- add a typed, explicit, default-off well-known exclude policy for the canonical cache/build/vendor directories already named in this spec; +- add deterministic git-revision materialization below the seam, using exactly one runtime git backend path; +- update the snapshot schema, fixtures, tests, and helper surfaces to describe the new source and option variants honestly; +- add `RepoProvider` only if the implementation now has two genuinely distinct backends and a trait actually removes duplication instead of adding ceremony; +- do not change `RepoDiff`, rename semantics, runtime wiring, public API shape, or pack/profile schema. + +Complexity check: + +- Phase C is larger than Phase B because it touches materialization semantics, schema shape, and cross-source determinism, not just a pure transform; +- that extra scope is still justified because it remains inside `src/repo/**`, repo tests, schema fixtures, and docs, with no app/runtime integration; +- the likely touch set is still above the normal "challenge the plan" threshold, around 10 to 13 files once repo code, schema parity, helpers, fixtures, and new tests are counted; +- if a proposed change needs to modify `src/app/runtime.rs`, `src/pack/**`, or downstream consumer contracts, it is already Phase D or later work and should be deferred. + +Search and distribution check: + +- `[Layer 1]` reuse the existing `SnapshotRequest -> RepoSnapshot` contract and the landed schema/test-harness pattern; +- `[Layer 1]` reuse the current fingerprint, inventory, and blob-store contracts instead of introducing git-specific snapshot result types; +- choose one runtime git backend path, not both `git2` and `gix`, and do not shell out to `git` from runtime code; +- Phase C does not introduce a new binary, package, or published surface, so no release workflow or install-path change belongs here. + +Scope result: + +- scope accepted as-is, because this is the complete next slice of seam-2 materialization semantics without reopening downstream seams. + +Exact file-touch forecast: + +| File / surface | Required Phase-C change | Why this file belongs in scope | +|---|---|---| +| `src/repo/snapshot.rs` | extend `SnapshotSource`, widen dispatch, add GitRev materialization helpers, add bounded follow resolution, preserve one final assembly pipeline | this is the only real materialization engine today | +| `src/repo/ignore.rs` | add typed well-known exclude policy and compile it into the current ignore pipeline | excludes are already a first-class seam-2 concern here | +| `src/repo/error.rs` | add typed GitRev resolution and symlink-follow failures | current errors are too vague for Phase-C failure modes | +| `src/repo/mod.rs` | re-export only the new internal Phase-C seam types if needed | keeps the crate-private repo surface honest | +| `src/repo/schema.rs` | keep embedded schema constants in sync with the expanded manifest | schema parity is part of the gate | +| `schemas/repo/snapshot_manifest.v1.json` | represent `git_rev` and typed well-known excludes explicitly | tests need a concrete contract, not prose | +| `tests/support/repo_support.rs` | add request builders for `GitRev`, git-history fixture synthesis, and manifest generation for the expanded option/source surface | current helpers only know how to build worktree snapshots | +| `tests/repo_git_snapshot.rs` | new file | isolates GitRev materialization and cross-source parity coverage | +| `tests/repo_symlinks.rs` | new file | keeps follow semantics and failure modes out of the basic snapshot happy-path file | +| `tests/repo_ignore.rs` | extend | this is already the ignore-policy home | +| `tests/repo_purity.rs` | extend | purity is the place where GitRev and follow semantics can quietly rot | +| `tests/repo_schema.rs` | extend | manifest shape and runtime parity must stay coupled | +| `fixtures/repo/**` | add tree descriptions and manifest cases only, not raw git object stores | keeps fixtures reviewable and deterministic | + +The implementer should treat that file list as the allowed blast radius. If the work starts pushing outside it, stop and re-check scope. + +### 15.8.b Architecture review + +Phase C execution shape: + +```text +SnapshotRequest { root, source, options } + | + +--> compile ignore policy + | intrinsic `.git/**` + | + typed well-known excludes (optional, default off) + | + caller globs + | + +--> source dispatch + | + +--> Worktree + | walk repo root deterministically + | emit candidate entries from live tree + | + +--> GitRev { rev } + resolve rev against detected repo root exactly once + walk that revision's tree deterministically + emit the same candidate-entry shape + | + v +candidate normalization + | + +--> derive canonical repo-relative path + +--> apply non-UTF8 path policy + +--> apply ignore policy before blob load whenever the decision is path-only + +--> symlink policy + Skip -> diagnostic + omit + Follow -> resolve target within selected source, keep observed path, + reject escape, reject loop, reject dangling target, + reject directory target, materialize terminal regular-file bytes + | + v +materialize regular-file bytes once + | + +--> blob_fingerprint = sha256_bytes(bytes) + +--> file_id = FileId::from_identity("repo\0file\0v1\0") + | + v +Inventory + BlobStore + diagnostics + stats + | + v +one deterministic sort boundary by RepoPath + | + v +RepoSnapshot { root, source, inventory, blobs, diagnostics, stats, fingerprint } +``` + +Recommended implementation spine inside `src/repo/snapshot.rs`: + +```text +materialize_snapshot(request) + | + +--> compile_ignore_set(request.options) + +--> enumerate_source_entries(request) + | | + | +--> enumerate_worktree_entries(root) + | `--> enumerate_gitrev_entries(root, rev) + | + +--> normalize_candidate_entry(...) + +--> maybe_follow_symlink(...) + +--> maybe_prune_by_ignore(...) + +--> materialize_bytes_once(...) + `--> finalize_snapshot(...) +``` + +The names can differ, but the responsibility split should not. One source-enumeration layer, one normalization/follow layer, one final assembly layer. That is the smallest shape that keeps Worktree and GitRev from drifting while still avoiding fake abstraction. + +Architectural rules: + +- `SnapshotSource::GitRev { rev }` must resolve the requested revision against the detected repo root and either materialize that revision or return a typed error. There is no silent fallback to worktree semantics. +- Git-backed materialization must not be implemented by checking out a temp worktree and then reusing the worktree walker. That would reintroduce live-filesystem coupling, more I/O, and platform drift right where the seam is supposed to be deterministic. +- Worktree and GitRev are two ways to produce the same `RepoSnapshot` contract. If the selected tree content and options are equivalent, the resulting inventory order, blob fingerprints, and snapshot fingerprint must also be equivalent. +- `SymlinkPolicy::Follow` keeps the observed repo path as the inventory path. It reads bytes from the resolved in-repo terminal target, but downstream seams still see the path the caller requested materialized. +- `SymlinkPolicy::Follow` is intentionally bounded. Escapes outside the repo root, loops, dangling targets, and directory targets must all result in deterministic error or skip behavior with typed diagnostics. No silent subtree expansion. +- The well-known exclude policy is explicit and typed. It composes with intrinsic `.git/**` exclusion plus caller globs, and it must be visible in manifest/test surfaces instead of being a hidden default. +- `RepoProvider` is optional in Phase C. If the implementation still has one orchestration flow with two small source-specific helpers, keep it explicit. Add `provider.rs` only if there are now two real backends whose shared contract is stable enough to justify the trait. + +Module boundary for Phase C: + +```text +src/repo/ + mod.rs + ignore.rs + snapshot.rs + schema.rs + provider.rs # only if a second backend truly exists + +tests/ + repo_git_snapshot.rs + repo_symlinks.rs + repo_ignore.rs + repo_purity.rs + repo_schema.rs + repo_diff.rs # only if needed to prove diff transparency across sources + support/repo_support.rs + +fixtures/repo/ + trees/ + valid/ + invalid/ +``` + +Exact edit map to make the phase implementable: + +- `src/repo/snapshot.rs` + - extend `SnapshotSource` with `GitRev { rev: String }` + - widen `materialize_snapshot` dispatch instead of creating a second public function + - extract the current worktree-specific logic into a source-specific candidate enumerator + - add a bounded follow resolver that returns one terminal regular-file candidate or a typed failure/skip outcome + - keep `Inventory`, `BlobStore`, diagnostics sorting, and fingerprint finalization in one shared path +- `src/repo/ignore.rs` + - add a typed well-known exclude enum/set + - compile those entries into the same `CompiledIgnoreSet` used for caller globs + - keep `.git/**` as the only intrinsic exclusion +- `src/repo/error.rs` + - add explicit failures for invalid/missing revisions, revision object-kind mismatch if relevant, repo-boundary escape, dangling follow target, loop detection, and directory-target follow rejection + - do not hide those behind generic `Io` +- `tests/support/repo_support.rs` + - add `snapshot_request_with_source(...)` + - add temporary git-history synthesis helpers + - expand `manifest_from_snapshot(...)` so it can express both `worktree` and `git_rev` +- `schemas/repo/snapshot_manifest.v1.json` + - lock the exact JSON shape before fixture generation begins + - reject impossible source/option combinations at schema level where practical + +### 15.8.c Code quality review + +Phase C is exactly where fake flexibility and "just in case" abstraction will try to sneak in. + +Do not let it. + +| Area | Code quality rule | Why | +|---|---|---| +| `src/repo/snapshot.rs` | keep one orchestration flow with a few small source-specific helpers, not a nest of strategy objects | there is still one seam and one immutable output contract | +| git backend integration | choose one runtime backend path and hide it behind a narrow helper or adapter | mixing multiple git backends or CLI shellouts creates drift and review noise | +| symlink resolution | use an explicit resolver with a visited-set and clear bounds checks | this logic is security-sensitive and easy to make "clever" in the worst way | +| well-known excludes | compile or normalize them into the same ignore pipeline as caller globs, not a second late-stage filter pass | exclusion semantics should stay reviewable and deterministic | +| manifest/test helpers | extend `tests/support/repo_support.rs` and `tests/repo_schema.rs`, not a second repo-fixture framework | Phase A and B already established the test-harness pattern | +| provider abstraction | land `provider.rs` only if it removes real duplication between two backends | Phase C should not pay an abstraction tax for hypothetical future backends | + +Hard quality rules: + +1. Do not change `RepoDiff`, diff schema, or rename semantics in Phase C unless an actual defect is found. +2. Do not make `GitRev` resolution "helpful." Missing or invalid revisions fail. They do not degrade to worktree, HEAD, or some guessed ref. +3. Do not make runtime materialization shell out to `git`. If tests want to use the git CLI to synthesize temporary fixture history, that is a test-only helper choice, not a runtime contract. +4. Do not let follow semantics depend on platform quirks such as ambient cwd, symlink creation permissions, or host path casing. The selected source and repo root are the only authority. +5. Keep docs last. If prose reveals an unresolved semantic decision, fix the semantic hole in the plan and code/test shape first. + +### 15.8.d Rules to lock now + +Lock these rules: + +1. `SnapshotSource` remains explicit: `Worktree` or `GitRev { rev }`. +2. `GitRev { rev }` resolves against the detected repo root and either materializes that exact revision or returns a typed error. +3. Worktree and GitRev snapshots share the same `RepoSnapshot` contract and the same fingerprint rules. +4. `SymlinkPolicy::Skip` remains the default. +5. `SymlinkPolicy::Follow` keeps the observed path identity and materializes terminal in-repo regular-file content only. +6. Escaping symlinks, looping symlinks, dangling symlinks, and directory-target symlinks never trigger silent traversal. They deterministically error or skip with diagnostics. +7. The typed well-known exclude policy is explicit and default-off. +8. Well-known excludes compose as: intrinsic `.git/**` exclusion, then typed well-known excludes, then caller globs. +9. Phase C does not add `.gitignore`, `.ignore`, or global git-exclude interpretation. +10. Git-backed materialization must not consult live worktree contents once revision resolution is complete. +11. `RepoProvider` lands only if two real backends now exist and the trait meaningfully reduces duplication. +12. Downstream diff behavior remains path-based add/remove/modify only, independent of whether snapshots came from Worktree or GitRev. +13. `snapshot_manifest.v1.json` remains the schema file for this phase, but it expands minimally and explicitly: + - `source_kind: "worktree" | "git_rev"` + - `source_rev` is absent for `worktree` and required, non-empty, for `git_rev` + - `options.well_known_excludes` is required and defaults to `[]` +14. The typed well-known exclude surface should be a set of explicit enum values, not a boolean blob. Recommended canonical values: + - `rust_target` + - `node_modules` + - `python_hidden_venv` + - `python_venv` + - `python_pycache` + - `web_dist` + - `web_build` +15. The directory mapping above is literal. This phase does not introduce fuzzy matching, suffix heuristics, or broad "common junk" presets. + +Recommended manifest shape to lock before implementation: + +```json +{ + "version": 1, + "case": "gitrev-basic", + "source_kind": "git_rev", + "source_rev": "HEAD~1", + "options": { + "symlink_policy": "follow", + "well_known_excludes": ["rust_target", "web_dist"], + "exclude_globs": ["generated"], + "non_utf8_path_policy": "error", + "max_file_bytes": null, + "large_file_policy": "error" + } +} +``` + +This is the minimal honest expansion. It keeps the current top-level shape intact, avoids a schema version bump, and still makes impossible combinations testable. + +### 15.8.e Phase C implementation slices + +| Slice | Files / modules | Deliverable | Done when | +|---|---|---|---| +| C1. source/options seam surface | `src/repo/{mod.rs,ignore.rs,snapshot.rs,error.rs}` | `SnapshotSource::GitRev`, `SymlinkPolicy::Follow`, the typed well-known exclude surface, and typed Phase-C errors exist behind the current crate-private seam | the new enums/error variants compile without reopening downstream seams or hiding new failures behind generic `Io` | +| C2. git revision materializer | `src/repo/snapshot.rs` plus one private git helper or adapter | deterministic GitRev snapshot construction exists without temp checkout fallback | a known revision materializes through the same `RepoSnapshot` contract, invalid revs hard-fail, and no live-worktree reads occur after revision selection | +| C3. bounded symlink follow semantics | `src/repo/snapshot.rs` | follow mode can resolve in-repo terminal file targets safely | in-bounds file targets materialize under the observed path, while escape/loop/dangling/directory targets behave deterministically | +| C4. typed well-known excludes | `src/repo/ignore.rs`, `tests/repo_ignore.rs` | canonical cache/build/vendor directory exclusions are expressible as a typed policy | the policy is default-off, explicit, and identical across Worktree and GitRev | +| C5. schema + fixture harness updates | `src/repo/schema.rs`, `schemas/repo/snapshot_manifest.v1.json`, `tests/support/repo_support.rs`, `fixtures/repo/{valid,invalid}/**` | manifest/test surfaces describe the new source and option shapes honestly | embedded snapshot schema matches disk, helper-generated manifests round-trip, and impossible `source_kind` / `source_rev` shapes fail validation | +| C6. integration tests | `tests/repo_git_snapshot.rs`, `tests/repo_symlinks.rs`, `tests/repo_ignore.rs`, `tests/repo_purity.rs`, `tests/repo_schema.rs`, `tests/compile_matrix.rs`, `tests/repo_diff.rs` only if needed | every required Phase-C path and invariant is covered | all acceptance paths below have deterministic tests, including parity between equivalent Worktree and GitRev trees | +| C7. docs sweep | `lift_seam2_spec_reviewed.md`, `README.md`, `fixtures/repo/README.md`, `schemas/README.md` | prose matches the landed Phase-C contract and stops describing GitRev/follow/excludes as vague future work | doc surfaces describe explicit semantics, not implementation vibes | + +Implementation order: + +1. Land C1 first so the seam surface stops moving. +2. Land C2 next. GitRev is the biggest new semantic branch and should freeze before fixtures or docs harden around it. +3. Land C3 immediately after C2, because follow semantics and git-backed materialization both need the same deterministic candidate model. +4. Land C4 once the option surface is stable. +5. Land C5 after C1 through C4 stop moving. +6. Land C6 after C1 through C5 are stable. +7. Land C7 last. + +### 15.8.f Test review, required coverage for Phase C + +CODE PATH COVERAGE +=========================== +[+] `SnapshotSource::GitRev` + |- [REQUIRED] valid revision materializes a deterministic snapshot through the normal `RepoSnapshot` contract + |- [REQUIRED] invalid or missing revision fails with a typed repo error + |- [REQUIRED] the same committed tree under two different temp roots yields the same snapshot fingerprint + |- [REQUIRED] GitRev snapshot ignores later live-worktree mutation after the revision is resolved + `- [REQUIRED] equivalent tree content from Worktree and GitRev yields identical inventory ordering and snapshot fingerprint + +[+] `SymlinkPolicy::Follow` + |- [REQUIRED] `Skip` still omits symlinks exactly as in landed Phase A + |- [REQUIRED] in-repo symlink to a regular file materializes the target bytes under the observed symlink path + |- [REQUIRED] out-of-repo escape attempt errors or skips deterministically + |- [REQUIRED] symlink loop errors or skips deterministically + |- [REQUIRED] dangling symlink errors or skips deterministically + `- [REQUIRED] directory-target symlink never silently expands a subtree + +[+] Typed well-known excludes + |- [REQUIRED] default-off policy leaves `target/`, `node_modules/`, `.venv/`, `venv/`, `__pycache__/`, `dist/`, and `build/` visible unless explicitly selected + |- [REQUIRED] enabled typed policy excludes those canonical directories deterministically + |- [REQUIRED] caller globs still compose on top of the typed policy + `- [REQUIRED] Worktree and GitRev apply the same exclude semantics for the same logical tree + +[+] Schema and manifest coverage + |- [REQUIRED] embedded `snapshot_manifest.v1.json` matches the on-disk schema after the new source/options land + |- [REQUIRED] valid GitRev manifests validate and deserialize + |- [REQUIRED] valid follow-policy manifests validate and deserialize + |- [REQUIRED] invalid manifests fail deterministically for missing `rev`, bad exclude-policy values, or impossible source/option shapes + `- [REQUIRED] manifest ordering and fingerprints match runtime output + +USER FLOW COVERAGE +=========================== +[+] Internal engine flow + |- [REQUIRED] detect root -> materialize worktree snapshot with typed excludes -> inspect inventory/stats/fingerprint + |- [REQUIRED] detect root -> materialize GitRev snapshot -> inspect inventory/stats/fingerprint + |- [REQUIRED] materialize equivalent Worktree and GitRev trees -> build diff -> verify empty diff / unchanged downstream semantics + `- [REQUIRED] GitRev and follow-policy fixture generation reuse the same runtime codepaths as the behavior tests + +--------------------------------- +COVERAGE GOAL: 20/20 required Phase-C paths covered before promotion + Code paths: 16/16 + Internal flows: 4/4 +QUALITY BAR: no smoke-only tests for acceptance paths +GAPS ALLOWED AT PROMOTION: 0 +--------------------------------- + +Required test files: + +- add `tests/repo_git_snapshot.rs` for revision resolution, cross-root determinism, and worktree-vs-gitrev parity; +- add `tests/repo_symlinks.rs` for bounded follow behavior, including escape/loop/dangling/directory-target cases; +- extend `tests/repo_ignore.rs` for the typed well-known exclude policy and composition with caller globs; +- extend `tests/repo_purity.rs` to prove GitRev snapshots and followed symlink materialization remain immutable after later live-tree mutation; +- extend `tests/repo_schema.rs` for the expanded snapshot-manifest schema and manifest-shape validation; +- extend `tests/support/repo_support.rs` with the minimal helpers needed to synthesize temporary git history, generate manifests, and validate them; +- extend `tests/repo_diff.rs` only if needed to prove Phase-C snapshots stay transparent to downstream diff semantics; +- extend `tests/compile_matrix.rs` only as needed to preserve the no-default-features posture. + +Exact helper additions expected in `tests/support/repo_support.rs`: + +- `snapshot_request_with_source(root, source, options)` +- `init_git_fixture_tree(...)` +- `commit_all(repo_path, message)` +- `resolve_fixture_rev(...)` +- `manifest_from_snapshot_with_source(case, source, options, snapshot)` + +If the implementation needs more than that, challenge the helper design before adding it. The harness should stay small and boring. + +Required fixture strategy: + +- keep committed fixtures as tree descriptions and manifest JSON, not checked-in `.git/` directories; +- synthesize temporary git history in test helpers, so revision snapshots stay deterministic without polluting the repo with raw git object stores; +- keep the canonical directory examples for well-known excludes in tree fixtures that both Worktree and GitRev tests can reuse. + +Test-plan artifact to write during review/implementation: + +- write `~/.gstack/projects/$SLUG/{user}-{branch}-eng-review-test-plan-{timestamp}.md`; +- list the affected internal flow as `detect_root -> materialize_snapshot(source, options) -> RepoSnapshot`; +- list the critical path as "resolve revision, normalize candidate entries, apply ignore/follow policy, materialize bytes, verify cross-source determinism"; +- list the highest-risk cases as rev fallback, symlink escape, symlink loops, and hidden default excludes. + +Recommended validation commands: + +```bash +cargo fmt --all +cargo clippy -p substrate-lift --all-targets -- -D warnings +cargo test -p substrate-lift --test repo_git_snapshot -- --nocapture +cargo test -p substrate-lift --test repo_symlinks -- --nocapture +cargo test -p substrate-lift --test repo_ignore -- --nocapture +cargo test -p substrate-lift --test repo_purity -- --nocapture +cargo test -p substrate-lift --test repo_schema -- --nocapture +cargo test -p substrate-lift --test compile_matrix -- --nocapture +cargo test -p substrate-lift --test repo_diff -- --nocapture +``` + +### 15.8.g Performance review + +Phase C still has no database or network path, but it does have new ways to waste CPU and I/O if the implementation gets lazy. + +| Concern | Risk | Required decision | +|---|---|---| +| GitRev materialization shape | temp checkout or per-file CLI calls would add avoidable I/O, race risk, and platform drift | resolve the revision once and walk the selected tree directly through one runtime backend path | +| ignore pruning | applying typed well-known excludes after file bytes are already loaded wastes work | prune by path before blob load whenever the decision is path-only | +| symlink resolution | recursive follow without visited-state can loop forever or re-resolve the same targets repeatedly | use bounded, explicit resolution with visited-state and deterministic failure modes | +| cross-source determinism | leaking backend-specific metadata such as absolute roots, object IDs, or mode bits into fingerprints causes source-dependent drift | keep fingerprint input to the same canonical `(path, blob_fingerprint, size_bytes)` data shape | +| memory growth | followed symlinks can cause multiple observed paths to point at identical bytes | accept repeated path entries when semantics require them, but do not duplicate traversal state or add global caches in Phase C | + +Performance rules to lock now: + +- expected time complexity should stay one source walk plus one deterministic sort boundary, not source walk plus temp checkout plus cleanup walk; +- Worktree and GitRev should both short-circuit excluded paths before byte materialization whenever possible; +- do not add a global repo snapshot cache in Phase C; +- if the implementation reaches for shelling out to `git archive`, `git checkout`, or equivalent temp-tree materialization, reject it unless a later benchmark-and-hardening effort explicitly reopens the decision. + +### 15.8.h Failure modes for Phase C + +| Codepath | Failure mode | Test required? | Error handling required? | Consumer sees | +|---|---|---|---|---| +| GitRev resolution | invalid rev silently degrades to worktree, HEAD, or some guessed ref | yes | yes, typed rev-resolution failure only | explicit failure, never "helpful" fallback | +| GitRev materialization | backend consults live worktree files after revision resolution | yes | test-enforced invariant | stable historical snapshot, not host-state drift | +| cross-source fingerprinting | equivalent Worktree and GitRev trees hash differently because backend-specific data leaks in | yes | test-enforced invariant | source-dependent snapshot drift | +| follow semantics | out-of-repo symlink escape is materialized as if it were in-bounds | yes | yes, deterministic reject/skip with diagnostics | no host escape enters inventory | +| follow semantics | symlink loop hangs or produces duplicate unstable output | yes | yes, visited-state plus deterministic failure | explicit failure or omission, never infinite traversal | +| follow semantics | directory-target symlink silently expands a subtree and changes traversal semantics | yes | yes, deterministic reject/skip | bounded file-only semantics remain intact | +| well-known excludes | typed policy turns on implicitly and hides files the caller did not ask to hide | yes | yes, default-off invariant | explicit semantics, no surprising omissions | +| path normalization | git tree path bytes or followed targets are coerced into bogus `RepoPath` values | yes | yes, apply the same UTF-8 policy as the worktree path | typed failure or deterministic omission | +| provider abstraction | provider and non-provider paths drift semantically because two codepaths are maintained poorly | yes, if provider lands | yes, shared contract tests | same snapshot contract regardless of backend wrapper | + +Critical gap rule: + +- Phase C does not promote if any failure mode is both untested and capable of causing silent materialization drift or repo-boundary escape. + +### 15.8.i NOT in scope for the Phase C implementation PR + +- `.gitignore`, `.ignore`, or global git-exclude interpretation +- rename detection, similarity heuristics, or any other diff-contract changes +- blob-level textual patches or hunk generation +- runtime, CLI, or pack/profile integration work +- public API promotion of `repo` +- automatic caller-default selection of the typed well-known exclude policy +- directory-symlink subtree expansion +- git submodule recursion or gitlink-aware dependency modeling +- sparse include-path snapshots +- permission-bit, executable-bit, or line-ending modeling +- multiple runtime git backend crates in the same implementation + +### 15.8.j Worktree parallelization strategy + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| C1 seam surface + option/error types | `src/repo/{snapshot.rs,ignore.rs,error.rs,mod.rs}` | - | +| C2 GitRev materializer core | `src/repo/snapshot.rs` | C1 | +| C3 bounded symlink resolver | `src/repo/snapshot.rs` | C1, C2 | +| C4 typed well-known excludes | `src/repo/ignore.rs`, `tests/repo_ignore.rs` | C1 | +| C5 schema + fixture helper updates | `src/repo/schema.rs`, `schemas/repo/`, `tests/support/`, fixture manifests | C1, C2, C3, C4 | +| C6 behavior + parity tests | `tests/` | C2, C3, C4, C5 | +| C7 docs sweep | spec + README surfaces | C5 | + +Parallel lanes: + +- Lane A, `phase-c-core`: C1 -> C2 -> C3 (single-owner core lane for `src/repo/snapshot.rs` and `src/repo/error.rs`) +- Lane B, `phase-c-ignore-schema`: C4 -> C5 (schema/harness lane after Lane A freezes enum names and source semantics) +- Lane C, `phase-c-tests-docs`: C6 + C7 (sidecar lane once helper names, manifest fields, and fixture layout stop moving) + +Execution order: + +1. Run Lane A alone. This is the dangerous lane because `src/repo/snapshot.rs` is the single shared ownership surface for source dispatch, follow semantics, and final snapshot assembly. +2. Start Lane B only after Lane A has frozen: + - `SnapshotSource::GitRev { rev }` + - `SymlinkPolicy::Follow` + - the typed well-known exclude names + - the exact error variants used by GitRev and follow failures +3. Start Lane C only after Lane B has frozen helper names, manifest fields, and fixture locations. +4. Merge order is A -> B -> C. If C reveals missing semantic hooks in `src/repo/**`, route that back to A and reopen the gate. Do not paper over it in tests. + +Conflict flags: + +- `src/repo/snapshot.rs` is exclusive to Lane A. Do not split GitRev and follow work across two parallel branches. They will collide. +- `src/repo/ignore.rs` can move into Lane B only after Lane A locks the enum names it consumes. +- `tests/support/repo_support.rs` and `schemas/repo/snapshot_manifest.v1.json` are exclusive to Lane B. Lane C must consume them, not redefine them. +- `tests/repo_git_snapshot.rs`, `tests/repo_symlinks.rs`, and docs files are Lane-C territory. If those changes require altering source semantics, stop and send the change back upstream. +- If `RepoProvider` lands, it belongs inside Lane A, not as a fourth parallel lane. + +### 15.8.k Phase C completion summary + +- Step 0: scope accepted as expanded materialization semantics only, with no downstream seam churn +- Architecture review: explicit source dispatch, bounded follow semantics, and typed exclude layering written +- Code quality review: no runtime shellout fallback, no fake provider abstraction, no diff-contract churn locked +- Test review: 20 required Phase-C paths named, 0 gaps allowed at promotion +- Performance review: direct source materialization, early ignore pruning, no temp checkout fallback +- Failure modes: 0 silent-materialization-drift or repo-escape gaps allowed +- NOT in scope: written +- What already exists: written +- TODOS.md updates: none required beyond the explicit downstream deferrals already captured here +- Outside voice: skipped for this document expansion, because this section is locking the next implementation slice rather than rerunning the full multi-model review pipeline +- Parallelization: 3 lanes, 1 sequential core lane plus 2 sidecars +- Lake score: complete implementation path selected, no intentional shortcuts + +Phase-C promotion gate: + +1. `SnapshotSource::GitRev { rev }` exists and either materializes the requested revision or returns a typed error. +2. equivalent Worktree and GitRev trees produce the same ordered inventory and snapshot fingerprint for the same option set. +3. `SymlinkPolicy::Follow` never escapes repo bounds, never loops indefinitely, and never silently expands directory targets. +4. the typed well-known exclude policy is explicit, default-off, and deterministic across Worktree and GitRev. +5. the expanded `snapshot_manifest.v1.json` is embedded through `src/repo/schema.rs` and validated in tests. +6. all required Phase-C tests above are present and passing. +7. downstream diff behavior remains unchanged when fed Phase-C snapshots. +8. `cargo check -p substrate-lift --no-default-features` still passes. + +## 16. My recommended decision set + +These are the decisions I would lock now. + +1. `repo` stays `pub(crate)` for seam 2. +2. seam 2 phase A is **filesystem-first** and does not require pack schema changes. +3. `RepoSnapshot` is a **fully materialized immutable artifact**. +4. `FileId` is path-stable, not content-stable. +5. snapshot fingerprints are based only on sorted `(path, blob_fingerprint, size_bytes)`. +6. `.git/**` is the only intrinsic exclude in phase A. +7. `.gitignore` support is explicitly out of scope in phase A. +8. seam 2 has a canonical **A/B/C** phase map: + - A = worktree snapshot substrate + - B = pure diff over snapshots + - C = expanded materialization semantics +9. `SymlinkPolicy::Follow` and `SnapshotSource::GitRev` belong to Phase C, not Phase A. +10. `RepoProvider` is deferred until Phase C and only lands if a second backend now exists. +11. diff semantics are path-based add/remove/modify only, with **no rename detection**. +12. seam 2 introduces fixture schemas, not end-user runtime config schemas. +13. well-known cache/build/vendor excludes are deferred to Phase C as an explicit materialization policy expansion. +14. whole-snapshot size/file-count limits are deferred, but Phase A must expose stats so memory pressure is visible. +15. Phase A performance guidance is lightweight: one materialization pass, one deterministic sort boundary, no live re-read after snapshot creation. +16. Phase A should take `walkdir = "2"` instead of growing a handwritten recursive walker. +17. Phase B compares only already-materialized `InventoryEntry` state from two `RepoSnapshot`s. +18. Phase B ships `diff_manifest.v1.json`, `tests/repo_diff.rs`, and deterministic diff fingerprints before any GitRev work. +19. Phase C extends snapshot construction semantics only. It does not renegotiate `RepoDiff` or downstream repo consumers. +20. `SnapshotSource::GitRev { rev }` must resolve against the detected repo root and either materialize that exact revision or fail, never fallback to worktree semantics. +21. Runtime GitRev materialization should use one Rust git backend path, not shell out to `git`. Tests may use the git CLI to synthesize temporary fixture history if needed. +22. `SymlinkPolicy::Follow` keeps observed path identity but reads bytes from the resolved in-repo terminal regular-file target. +23. Escaping, looping, dangling, and directory-target symlinks stay deterministic error/skip outcomes, never silent traversal. +24. The typed well-known exclude policy is explicit and default-off. `.git/**` remains the only intrinsic exclude. +25. `RepoProvider` is optional in Phase C, not mandatory. Land it only if two real backends now exist and the trait removes duplication instead of adding ceremony. +26. Equivalent Worktree and GitRev trees should fingerprint identically when the selected options are the same. + +That gives seam 2 an honest progression: Phase A landed the immutable worktree snapshot substrate, Phase B landed pure snapshot diffing, and Phase C is now locked as the smallest complete expansion of materialization semantics. + +## GSTACK REVIEW REPORT + +| Review | Trigger | Why | Runs | Status | Findings | +|--------|---------|-----|------|--------|----------| +| CEO Review | `/plan-ceo-review` | Scope & strategy | 2 | ISSUES OPEN | mode: SELECTIVE_EXPANSION, 1 critical gaps | +| Codex Review | `/codex review` | Independent 2nd opinion | 0 | — | — | +| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 3 | CLEAR | 12 issues, 0 critical gaps | +| Design Review | `/plan-design-review` | UI/UX gaps | 0 | — | — | + +**UNRESOLVED:** 0 +**VERDICT:** ENG CLEARED — Phase A and Phase B are landed, and the next implementation target is the locked Phase-C expanded-materialization plan above. CEO review remains informational and has open issues. diff --git a/crates/lift/lift_seam3_spec_reviewed.md b/crates/lift/lift_seam3_spec_reviewed.md new file mode 100644 index 000000000..f1acc5ee0 --- /dev/null +++ b/crates/lift/lift_seam3_spec_reviewed.md @@ -0,0 +1,3109 @@ + +# substrate-lift seam 3 spec — language platform (reviewed against landed seam 0 + seam 1 + seam 2 + landed seam 3 Phases A + B) + +## 0. Ground truth from the landed crate + +This spec is intentionally anchored to the crate as it exists today, not to the earlier idealized seam sketch. + +Observed state in the landed crate: + +- `src/kernel/**` is real, tested, schema-backed, and publicly re-exported from `lib.rs`. +- `src/pack/**` is real code and now compiles profiles, topology packs, score models, query packs, rule packs, and recipe packs into crate-private compiled artifacts. +- `src/repo/**` is real, tested, schema-backed, and crate-private. `RepoSnapshot`, `BlobStore`, `Inventory`, `RepoDiff`, `SnapshotRequest`, and repo diagnostics are already landed. +- `src/lang/**` is now real, tested, schema-backed, and still crate-private: + - `src/lang/mod.rs` exports `adapter`, `cache`, `driver`, `error`, `model`, `registry`, and `schema`; + - `lang::LanguageId` is the expected re-export of `pack::LanguageId`; + - `ParseDriver::parse_snapshot(&self, ...)` already normalizes requests, computes `request_fingerprint`, + contains adapter panics, validates draft output, and emits deterministic `ParseSet` output; + - `ParseStats` already tracks `cache_hits` and `cache_misses` in addition to parse outcomes; + - `src/lang/cache.rs` is real and owns `ParseCache`, `ParseCacheKey`, `NoopParseCache`, + `InMemoryParseCache`, and `PLATFORM_CACHE_VERSION`; + - `src/lang/registry.rs` already exposes deterministic `built_in_registry()` bootstrap preparation, + and it is intentionally empty today because no concrete production adapters are landed yet; + - `lib.rs` still exposes `lang` only as `pub(crate) mod lang;`, so seam 3 is still not public API. +- `Cargo.toml` already contains feature flags for concrete language families: + - `config-lang` + - `rust-lang` + - `python-lang` + - `javascript-lang` + - `typescript-lang` +- `Cargo.toml` does **not** yet include any parser runtime crates such as tree-sitter language grammars, syn, serde_yaml-specific AST layers, etc. +- seam 1 already owns a crate-private `pack::LanguageId` enum with canonical values: + - `json` + - `toml` + - `yaml` + - `rust` + - `python` + - `javascript` + - `typescript` +- `CompiledAnalysisDefaults.languages`, `CompiledQueryPack.language`, and `CompiledRuleScope.languages` already use that `pack::LanguageId` type. +- seam 1 also already defines `QueryEngineKind::TreeSitter` in compiled query packs, but there is **no** query execution seam yet and no language runtime contract for query execution yet. +- `src/app/runtime.rs` currently stops at `ProfileBootstrap { bundle: CompiledPackSet }`; there is no lang-facing runtime orchestration yet. +- the compile-matrix test still asserts the crate builds with `--no-default-features`. +- the top-level `README.md` has already been updated to describe seam 3 Phase B as landed internal seam work. +- the real proof slice now exists in tests: + - `tests/lang_consumer.rs` + - `tests/lang_cache.rs` + - `tests/lang_registry.rs` + - `tests/support/lang_support.rs` + - a bounded TOML adapter proof that demonstrates deterministic parse output and a consumer path + deriving config-key inventory from `ParseSet` without rereading the filesystem. +- there is still **no** concrete production adapter family, no parser runtime crates wired into the crate, + no lang-facing runtime bootstrap object, and no query execution seam. + +That changes the planning posture. + +Seam 3 is no longer the first landing question. Phases A and B are already in the crate. The remaining +job for this document is to lock the next increment cleanly: + +> Phase C should add the narrow metadata and registry hooks that seam 4 concrete adapters and seam 7 +> query execution will need, without reopening the Phase-A/B boundary or smuggling concrete production +> parsers or query execution into seam 3. + +The seam-1 `LanguageId` consequence still holds: + +> seam 3 should **continue** to avoid a second competing `LanguageId` type in v1. + +The crate already has one internal language identifier contract in `pack`, and back-migrating seam 1 +just to move that enum is unnecessary churn. Seam 3 should keep consuming and re-exporting that +existing internal type under the `lang` module boundary. + +--- + +## 1. Mission + +Seam 3 owns the **language platform**. + +It is responsible for: + +- defining the adapter trait for language-specific parsers; +- defining deterministic adapter registration and lookup; +- defining the parse request and parse result contracts; +- defining the normalized parsed-unit contract used by later seams; +- defining normalized local symbol and local edge contracts; +- defining parse-time surface-marker contracts; +- validating adapter output into a canonical, deterministic representation; +- defining parse-cache keys and the cache trait; +- turning a `RepoSnapshot` into a deterministic `ParseSet` without rereading the live filesystem; +- surfacing typed language-platform errors and deterministic parse diagnostics; +- embedding fixture schemas for parse manifests. + +It is **not** responsible for: + +- concrete production parsing for Rust / Python / JSON / etc.; +- repo root detection or file walking; +- topology classification; +- repo-wide graph resolution; +- query execution; +- detector/fact execution; +- Lift score math; +- pack compilation; +- CLI rendering. + +A useful rule: + +> seam 3 ends at **validated, normalized per-file parse output**. +> It does not resolve repo-wide edges or emit app findings. + +--- + +## 2. Boundary with existing code + +### Existing seam-0 primitives seam 3 should reuse directly + +Use directly from `kernel`: + +- `RepoPath` +- `FileId` +- `SymbolId` +- `Fingerprint` +- `ByteSpan` +- `Locator` +- `Diagnostic` +- `DiagnosticCode` +- `Severity` +- `sha256_canonical_json` +- `sha256_bytes` + +Unlike seam 1 and seam 2, seam 3 **can** use kernel `Locator` and kernel `Diagnostic` directly, because parse work is always anchored to repo-relative files and optional byte spans. + +### Existing seam-2 contracts seam 3 should reuse directly + +Use directly from `repo`: + +- `RepoSnapshot` +- `Inventory` +- `InventoryEntry` +- `BlobStore` +- `BlobRecord` +- `SnapshotStats` + +Seam 3 must consume bytes **only** from `RepoSnapshot` / `BlobStore`. +It must not reopen files from disk. + +### Existing seam-1 contracts seam 3 should reuse carefully + +Seam 3 should **not** define a duplicate language-id enum. + +Instead, Phase A should do this: + +```rust +// src/lang/mod.rs +pub(crate) use crate::pack::LanguageId; +``` + +That yields these rules: + +- downstream seams should import `lang::LanguageId`, not `pack::LanguageId` directly; +- seam 3 may internally depend on `pack::LanguageId` in Phase A; +- seam 1 must not gain a dependency on `lang`; +- a later extraction of `LanguageId` into `kernel` or `lang` can happen behind the `lang::LanguageId` re-export without churning all downstream seams. + +### Dependency direction + +Seam 3 should depend only on: + +- `kernel` +- `repo` +- `pack::LanguageId` (and only that narrow type alias surface) +- `std` +- `serde` for fixture/manifest contracts +- `jsonschema` only in tests / schema validation helpers + +Seam 3 should **not** depend on: + +- `pack::compiler` +- `pack::compiled::*` other than the re-exported `LanguageId` +- `topo` +- `graph` +- `facts` +- `derive` +- `query` +- `patch` +- `app` +- `cli` +- `anyhow` +- parser runtime crates in Phase A + +--- + +## 3. Canonical phase map + +Because the current crate only has a placeholder `src/lang/mod.rs`, seam 3 should land in three explicit phases. + +This section is canonical. Later sections should reference these phases rather than inventing a competing rollout plan. + +### Phase A — platform foundation only + +Phase A lands: + +- `src/lang/**` real modules instead of the placeholder, but in a reduced first slice; +- adapter name and adapter descriptor contracts; +- the `LanguageAdapter` trait; +- the `LanguageRegistry` and deterministic registration rules; +- the `ParseRequest`, `ParseSet`, `ParsedUnit`, `FailedParse`, and `SkippedParse` contracts; +- normalized local symbol, local edge, and surface-marker contracts folded into `model.rs`; +- deterministic output validation, canonical ordering, and platform-owned symbol ID assignment; +- the parse-driver boundary from `RepoSnapshot` to `ParseSet`; +- explicit Phase-A handling for requested languages with no registered adapter; +- an embedded parse-manifest schema for fixtures; +- test-only fake adapters proving extensibility; +- one bounded real-adapter spike or one narrow real adapter plus one real consuming workflow, + specifically to stress the shared contracts before Phase A is considered complete; +- targeted unit/integration tests. + +Phase A does **not** land: + +- the full production adapter family for Rust / Python / JavaScript / TypeScript / config; +- tree-sitter integration; +- any cache contract or cache stats surface; +- on-disk persistent caching; +- query-engine execution hooks; +- public API promotion; +- adapter auto-registration from feature flags. + +### Phase B — cache and runtime-bootstrap preparation (landed) + +Phase B lands: + +- the cache contract and `cache.rs`; +- a no-op cache implementation plus an in-memory cache implementation; +- cache hit/miss accounting in `ParseStats`; +- adapter-version-aware cache invalidation plus a lang-platform cache format version; +- a Phase-B fixture schema/version update if cache counters remain serialized in `ParseSet`; +- a small registry-construction helper for built-in adapters as bootstrap preparation only. + +Phase B still does **not** land: + +- production parsing implementations; +- persistent cache files; +- query execution; +- graph resolution; +- app runtime orchestration of parse runs. + +### Phase C — platform integration hooks for later seams + +Phase C lands: + +- a narrow adapter-capability surface that real seam-4 adapters can report without widening `ParseSet`; +- `lang::QueryEngineKind` as a re-export of the existing pack-owned query-engine identifier, + so later seams do not import query-engine truth from two places; +- deterministic registry inspection helpers so seam 7 can ask what a registered adapter supports + before trying to run a query engine against it; +- deterministic feature-gated built-in registration rules, so seam 4 adapters have one canonical + bootstrap path when they start landing. + +Phase C should keep one explicit rule: + +> capability metadata lives on the adapter/registry boundary, not inside parsed-unit payloads. + +Phase C must still keep the rule that: + +> concrete parsers belong to seam 4, not seam 3. + +--- + +## 4. Exact module shape by phase + +### Phase A `src/lang/` + +```text +src/lang/ + mod.rs + error.rs + adapter.rs + registry.rs + driver.rs + model.rs + schema.rs +``` + +No production adapter modules live here in seam 3. +Those belong in seam 4. + +For the reduced Phase-A landing, keep the data model boring: + +- `LocalSymbolDraft`, `LocalSymbol`, `LocalEdgeDraft`, `LocalEdge`, `SurfaceMarkerDraft`, and + `SurfaceMarker` live in `model.rs` instead of being split into extra files; +- only split those types back out later if seam 4 proves the file is actually too dense. + +### Phase A `src/lang/mod.rs` + +`src/lang/mod.rs` should re-export only the stable internal seam surface: + +```rust +//! Internal language platform seam. + +#![allow(dead_code)] +#![allow(unused_imports)] + +pub(crate) mod adapter; +pub(crate) mod driver; +pub(crate) mod error; +pub(crate) mod model; +pub(crate) mod registry; +pub(crate) mod schema; + +pub(crate) use crate::pack::LanguageId; + +pub(crate) use adapter::{ + AdapterDescriptor, AdapterName, AdapterParseOutput, AdapterParseResult, LanguageAdapter, + ParseInput, +}; +pub(crate) use driver::ParseDriver; +pub(crate) use error::{LangError, LangResult}; +pub(crate) use model::{ + EdgeEndpoint, EdgeEndpointDraft, FailedParse, LocalEdge, LocalEdgeDraft, LocalEdgeKind, + LocalSymbol, LocalSymbolDraft, MissingRequestedLanguage, ParseRequest, ParseScope, ParseSet, + ParseStats, ParsedUnit, ReferenceTarget, ReferenceTargetDraft, SkippedParse, SkippedReason, + SurfaceMarker, SurfaceMarkerDraft, SurfaceMarkerKind, SymbolKind, SymbolVisibility, +}; +pub(crate) use registry::{LanguageRegistry, LanguageRegistryBuilder}; +pub(crate) use schema::{ + LANG_PARSE_MANIFEST_V1_SCHEMA_FILE, LANG_PARSE_MANIFEST_V1_SCHEMA_ID, + LANG_PARSE_MANIFEST_V1_SCHEMA_JSON, LANG_PARSE_MANIFEST_V1_SCHEMA_VERSION, +}; +``` + +### Phase B `src/lang/` + +Phase B added: + +```text +src/lang/ + cache.rs +``` + +Do **not** split `cache.rs` yet unless Phase-B implementation proves it is needed. + +The current Phase-A seam is still small enough that the explicit choice is: + +- keep `NoopParseCache` and `InMemoryParseCache` together in `cache.rs`; +- keep the built-in registry helper in `registry.rs`; +- keep `ParseDriver` responsible for cache lookup/store orchestration only, not cache internals. +- if `ParseStats` stays serialized in `ParseSet`, keep the schema/version work adjacent by adding + `schemas/lang/parse_manifest.v2.json` and extending `src/lang/schema.rs`, rather than hiding that + fallout in test cleanup. + +### Phase C `src/lang/` + +Phase C should add: + +```text +src/lang/ + capabilities.rs +``` + +Keep Phase C bounded: + +- `capabilities.rs` owns capability enums/structs only; +- `adapter.rs` owns the trait hook that returns those capabilities, with a default empty implementation; +- `registry.rs` owns deterministic inspection and built-in population helpers; +- do **not** add parser-runtime modules, query execution modules, or runtime bootstrap modules here. + +--- + +## 5. Exact Rust contract shape + +### 5.1 `error.rs` + +```rust +pub(crate) type LangResult = Result; + +#[derive(Debug, thiserror::Error, Clone, Eq, PartialEq)] +pub(crate) enum LangError { + #[error("duplicate adapter name")] + DuplicateAdapterName { name: String }, + + #[error("duplicate language adapter registration")] + DuplicateLanguageAdapter { + language: LanguageId, + existing: String, + duplicate: String, + }, + + #[error("invalid adapter name")] + InvalidAdapterName { input: String }, + + #[error("parse cache invariant failure")] + CacheInvariant { reason: String }, + + #[error("lang schema validation failure")] + SchemaViolation { + schema_id: &'static str, + reason: String, + }, +} +``` + +Rule: +- seam 3 exposes **typed** internal errors only; +- file-level parse failures and syntax failures are **not** `LangError`. + They must be represented as `FailedParse` records with diagnostics. + +### 5.2 `adapter.rs` + +```rust +use crate::kernel::{ByteSpan, Diagnostic, FileId, Fingerprint, RepoPath}; +use crate::lang::{LanguageId, LocalEdgeDraft, LocalSymbolDraft, SurfaceMarkerDraft}; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct AdapterName(String); +// Canonical form: +// - lowercase dot-separated segments +// - first char of each segment alphabetic +// - later chars alnum or underscore +// Example: "builtin.fake_config" + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct AdapterDescriptor { + pub name: AdapterName, + pub language: LanguageId, + pub version: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ParseInput<'a> { + pub path: &'a RepoPath, + pub file_id: &'a FileId, + pub blob_fingerprint: &'a Fingerprint, + pub bytes: &'a [u8], +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct AdapterParseOutput { + pub symbols: Vec, + pub edges: Vec, + pub surface_markers: Vec, + pub diagnostics: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum AdapterParseResult { + Parsed(AdapterParseOutput), + Failed { diagnostics: Vec }, +} + +pub(crate) trait LanguageAdapter: Send + Sync { + fn descriptor(&self) -> AdapterDescriptor; + fn recognizes(&self, input: &ParseInput<'_>) -> bool; + fn parse(&self, input: &ParseInput<'_>) -> AdapterParseResult; +} +``` + +Rules: + +- `recognizes()` must be pure and deterministic. +- in Phase A, the driver selects the single registered adapter for a requested `LanguageId` first, + then uses `recognizes()` only as a file-level inclusion filter inside that language, not as a + second routing system. +- `parse()` must be pure and deterministic. +- neither method may perform filesystem I/O, env access, wall-clock reads, or random generation. +- `parse()` may emit diagnostics but must not panic on bad input. + +### 5.3 `model.rs` + +```rust +use crate::kernel::{ByteSpan, SymbolId}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SymbolKind { + Module, + Namespace, + Package, + Function, + Method, + Class, + Struct, + Enum, + Trait, + Interface, + TypeAlias, + Field, + Constant, + Variable, + TestCase, + TestSuite, + ConfigKey, + Unknown, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SymbolVisibility { + Public, + Private, + Internal, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalSymbolDraft { + pub local_key: String, + pub kind: SymbolKind, + pub name: Option, + pub path: Vec, + pub span: ByteSpan, + pub visibility: SymbolVisibility, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalSymbol { + pub id: SymbolId, + pub kind: SymbolKind, + pub name: Option, + pub path: Vec, + pub span: ByteSpan, + pub visibility: SymbolVisibility, +} +``` + +Rules: + +- `local_key` must be unique within a single parsed file. +- `path` must be deterministic and already normalized by the adapter. +- `path` may be empty for anonymous symbols, but `local_key` may not be empty. +- final `LocalSymbol` must not retain `local_key`. + +```rust +use crate::kernel::{ByteSpan, RepoPath, SymbolId}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LocalEdgeKind { + Contains, + Import, + Export, + Call, + TypeRef, + Inherit, + Implement, + TestRef, + ConfigRef, + SchemaRef, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum EdgeEndpoint { + FileRoot, + Symbol(SymbolId), +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum ReferenceTargetDraft { + LocalSymbol { local_key: String }, + QualifiedName { parts: Vec }, + FilePath { path: RepoPath }, + ExternalPackage { package: String, symbol: Option }, + Opaque { value: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum ReferenceTarget { + LocalSymbol(SymbolId), + QualifiedName { parts: Vec }, + FilePath { path: RepoPath }, + ExternalPackage { package: String, symbol: Option }, + Opaque { value: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalEdgeDraft { + pub kind: LocalEdgeKind, + pub source: EdgeEndpointDraft, + pub target: ReferenceTargetDraft, + pub span: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum EdgeEndpointDraft { + FileRoot, + Symbol { local_key: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalEdge { + pub kind: LocalEdgeKind, + pub source: EdgeEndpoint, + pub target: ReferenceTarget, + pub span: Option, +} +``` + +Rules: + +- draft edge references must resolve against the draft symbol table for the file; +- unresolved draft local keys are a file-level adapter-output failure, not a global platform crash; +- cross-file or external targets remain unresolved here by design. + +```rust +use crate::kernel::{ByteSpan, SymbolId}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SurfaceMarkerKind { + PublicApi, + Test, + EntryPoint, + Export, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct SurfaceMarkerDraft { + pub kind: SurfaceMarkerKind, + pub symbol_local_key: Option, + pub span: Option, + pub label: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct SurfaceMarker { + pub kind: SurfaceMarkerKind, + pub symbol: Option, + pub span: Option, + pub label: Option, +} +``` + +Rule: +- a draft marker may reference either a symbol or a span or both; +- if `symbol_local_key` is present, it must resolve. + +```rust +use std::collections::BTreeSet; + +use crate::kernel::{Diagnostic, FileId, Fingerprint, RepoPath}; +use crate::lang::{AdapterName, LanguageId, LocalEdge, LocalSymbol, SurfaceMarker}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ParseScope { + Snapshot, + Paths(BTreeSet), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ParseRequest { + pub languages: BTreeSet, + pub scope: ParseScope, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct MissingRequestedLanguage { + pub language: LanguageId, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParsedUnit { + pub path: RepoPath, + pub file_id: FileId, + pub blob_fingerprint: Fingerprint, + pub language: LanguageId, + pub adapter: AdapterName, + pub adapter_version: String, + pub unit_fingerprint: Fingerprint, + pub symbols: Vec, + pub edges: Vec, + pub surface_markers: Vec, + pub diagnostics: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct FailedParse { + pub path: RepoPath, + pub file_id: FileId, + pub blob_fingerprint: Fingerprint, + pub language: LanguageId, + pub adapter: AdapterName, + pub adapter_version: String, + pub diagnostics: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SkippedReason { + NoMatchingAdapter, + PathNotInSnapshot, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct SkippedParse { + pub path: RepoPath, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_id: Option, + pub reason: SkippedReason, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParseStats { + pub considered_files: u64, + pub parsed_units: u64, + pub failed_units: u64, + pub skipped_no_adapter: u64, + pub skipped_missing_paths: u64, + pub missing_requested_languages: u64, + pub diagnostic_count: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParseSet { + pub snapshot_fingerprint: Fingerprint, + pub request: ParseRequest, + pub request_fingerprint: Fingerprint, + pub units: Vec, + pub failed: Vec, + pub skipped: Vec, + pub missing_languages: Vec, + pub diagnostics: Vec, + pub stats: ParseStats, +} +``` + +Rules: + +- `languages` in `ParseRequest` is sorted, unique, and empty means “all registered adapters”. +- `ParseScope::Paths` must be sorted and unique. +- `ParseScope::Paths(BTreeSet::new())` means a deterministic empty parse run with zero + `considered_files`, zero units, zero failures, zero skips, and no implicit fallback to + `ParseScope::Snapshot`. +- `ParseSet.units`, `failed`, and `skipped` must be sorted deterministically before serialization. +- `ParseSet.missing_languages` must be sorted by `language`. +- `ParseSet.diagnostics` is only for run-level diagnostics, for example malformed requests or + missing requested-language coverage; file-level diagnostics live only on `ParsedUnit` or + `FailedParse` and are not duplicated upward. +- `SkippedParse { reason: NoMatchingAdapter }` is materialized only for explicitly requested + paths. Snapshot-wide files that do not match any selected adapter are counted in + `ParseStats.skipped_no_adapter`, but are not emitted one-by-one into `ParseSet.skipped`. +- `request_fingerprint` must be computed from canonical JSON of `ParseRequest`. +- `unit_fingerprint` must be computed from canonical JSON of the final normalized `ParsedUnit` excluding the `unit_fingerprint` field itself. + +### 5.4 `registry.rs` + +```rust +use std::collections::BTreeMap; +use std::sync::Arc; + +use crate::lang::{AdapterDescriptor, AdapterName, LangError, LangResult, LanguageAdapter, LanguageId}; + +#[derive(Default)] +pub(crate) struct LanguageRegistryBuilder { + adapters: BTreeMap>, + languages: BTreeMap, +} + +pub(crate) struct LanguageRegistry { + adapters: BTreeMap>, + languages: BTreeMap, +} + +impl LanguageRegistryBuilder { + pub(crate) fn new() -> Self; + pub(crate) fn register(mut self, adapter: A) -> LangResult; + pub(crate) fn build(self) -> LangResult; +} + +impl LanguageRegistry { + pub(crate) fn descriptors(&self) -> Vec; + pub(crate) fn adapter_for_language(&self, language: LanguageId) -> Option<&Arc>; +} +``` + +Rules: + +- Phase A allows **at most one registered adapter per `LanguageId`**. +- duplicate adapter names are rejected. +- duplicate language registration is rejected. +- registry iteration order must be deterministic and independent of registration order. + +### 5.5 `driver.rs` + +```rust +use crate::lang::{LangResult, LanguageRegistry, ParseRequest, ParseSet}; +use crate::repo::RepoSnapshot; + +pub(crate) struct ParseDriver { + registry: LanguageRegistry, +} + +impl ParseDriver { + pub(crate) fn new(registry: LanguageRegistry) -> Self; + pub(crate) fn with_cache(registry: LanguageRegistry, cache: C) -> Self; + pub(crate) fn parse_snapshot( + &self, + snapshot: &RepoSnapshot, + request: &ParseRequest, + ) -> LangResult; +} +``` + +Rules: + +- `ParseDriver` is the only seam-3 entrypoint that may iterate the snapshot. +- `ParseDriver::parse_snapshot` must walk the selected snapshot inventory only once per request, + selecting the candidate adapter for each file during that pass and normalizing outputs + afterward, rather than rescanning the full inventory per language. +- it must never reread the live filesystem. +- it must never resolve repo-wide graph edges. +- it must treat file-level parse failures as data, not as a seam-wide error. +- it must contain adapter panics and convert them into deterministic `FailedParse` records with a + top-level file diagnostic, rather than allowing one hostile file to abort the whole run. +- it must keep diagnostic ownership non-overlapping: file-level diagnostics stay on units or failed + records, while run-level diagnostics stay on `ParseSet`. +- if `ParseRequest.languages` names a language with no registered adapter, the parse run must emit a + deterministic top-level diagnostic and a `MissingRequestedLanguage` record instead of silently + ignoring the request. +- Phase A must define and test hostile-input containment rules before the first real adapter spike + is accepted: panic containment, malformed-byte handling, and explicit output-budget behavior for + symbol/edge/marker/diagnostic counts per file. + +### 5.6 `cache.rs` + +Phase B should cache **normalized per-file outcomes**, not raw adapter drafts and not whole-run +`ParseSet` blobs. + +Exact contract shape: + +```rust +use crate::kernel::{FileId, Fingerprint}; +use crate::lang::{AdapterName, FailedParse, LanguageId, ParsedUnit}; + +pub(crate) const LANG_PARSE_CACHE_FORMAT_VERSION: u32 = 1; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct ParseCacheKey { + pub file_id: FileId, + pub blob_fingerprint: Fingerprint, + pub language: LanguageId, + pub adapter: AdapterName, + pub adapter_version: String, + pub platform_cache_version: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum CachedParseOutcome { + Parsed(ParsedUnit), + Failed(FailedParse), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CacheLookup { + Disabled, + Hit(CachedParseOutcome), + Miss, +} + +pub(crate) trait ParseCache: Send + Sync { + fn get(&self, key: &ParseCacheKey) -> LangResult; + fn put(&self, key: ParseCacheKey, value: CachedParseOutcome) -> LangResult<()>; +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct NoopParseCache; + +#[derive(Clone, Debug, Default)] +pub(crate) struct InMemoryParseCache { + // exact storage type intentionally not locked here, but it must be deterministic. +} +``` + +Rules: + +- `ParseCacheKey` must use `file_id` and `blob_fingerprint` together. + `file_id` is already path-sensitive in the repo seam, so Phase B does **not** need a duplicate + raw path string in the cache key. +- the key must also include `language`, `adapter`, and `adapter_version`, so adapter upgrades or + routing changes cannot return stale results. +- the key must include `platform_cache_version`, owned by `lang/cache.rs` and bumped whenever + normalization, ordering, fingerprinting, or other cache-visible lang-platform semantics change. + That prevents stale cached `ParsedUnit` values after seam-3 code changes even when adapter + versions do not move. +- cache values must store the **final normalized outcome**: + - `ParsedUnit` on success; + - `FailedParse` on deterministic file-level parse failure. +- cache values must **not** store: + - `SkippedParse` records; + - `MissingRequestedLanguage` records; + - whole-run `ParseSet` values. +- `NoopParseCache` must return `CacheLookup::Disabled` from `get()` and ignore `put()`. +- `InMemoryParseCache` must be deterministic for equivalent insertion sequences. A `BTreeMap` keyed + by `ParseCacheKey` is preferred over `HashMap`. +- cache failures remain seam-level `LangError::CacheInvariant`, because a broken cache is a platform + failure, not a file-level parse result. + +### 5.7 Phase-B `ParseDriver` cache behavior + +The driver should stay `&self` in Phase B. + +That is still the cleanest contract because: + +- callers do not need to learn a new mutability story just to opt into caching; +- `NoopParseCache` can remain the default behind `ParseDriver::new`; +- real cache implementations can use interior mutability without forcing the seam API to widen. + +Exact behavior on each candidate file: + +1. select the adapter exactly as Phase A already does; +2. build `ParseCacheKey` from the chosen adapter descriptor and the inventory entry; +3. query the cache; +4. on `Hit`, append the cached normalized outcome directly into the `ParseSet`; +5. on `Miss`, invoke the adapter, normalize the outcome, append it to `ParseSet`, then store it; +6. on `Disabled`, behave exactly like Phase A. + +Cache lookup happens **after** adapter selection. + +That means Phase B does **not** cache: + +- `recognizes()` routing decisions; +- snapshot-wide “no matching adapter” counts; +- missing-requested-language records. + +Those remain request-level or registry-level concerns. + +`recognizes()` panic containment therefore remains a deterministic but un-cached Phase-A behavior. +Phase B must keep that boundary explicit and add a regression test proving repeated recognize-stage +panic failures stay deterministic while `cache_hits` and `cache_misses` remain unchanged. + +### 5.8 `ParseStats` additions in Phase B + +Phase B should extend `ParseStats` with cache accounting: + +```rust +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParseStats { + pub considered_files: u64, + pub parsed_units: u64, + pub failed_units: u64, + pub skipped_no_adapter: u64, + pub skipped_missing_paths: u64, + pub missing_requested_languages: u64, + pub diagnostic_count: u64, + pub cache_hits: u64, + pub cache_misses: u64, +} +``` + +Rules: + +- `cache_hits` increments only when `CacheLookup::Hit(_)` is returned. +- `cache_misses` increments only when `CacheLookup::Miss` is returned. +- `NoopParseCache` must leave both counters at zero by returning `Disabled`, not synthetic misses. +- for an enabled cache, `cache_hits + cache_misses` must equal the number of candidate files that + actually reached parse-stage execution, which is `parsed_units + failed_units` for that run. +- because `ParseStats` is serialized inside `ParseSet`, Phase B must not mutate + `parse_manifest.v1.json` in place to add these fields. If cache counters remain serialized, add + `parse_manifest.v2.json`, companion constants in `src/lang/schema.rs`, and Phase-B fixtures/tests + atomically while keeping v1 fixtures valid. + +### 5.9 Built-in registry helper + +Phase B should add one tiny registry-construction helper in `registry.rs`: + +```rust +pub(crate) fn built_in_registry() -> LangResult; +``` + +Rules: + +- this helper is the **only** place that should know about feature-gated built-in adapter + registration order; +- it is allowed to return an empty registry today, because seam 4 still owns production adapter + landings; +- when seam 4 starts adding feature-gated built-ins, they must be registered here in deterministic + adapter-name order; +- `app/runtime.rs` and later seams should depend on this helper, not on concrete adapter types. +- this helper is runtime-bootstrap preparation, not proof that runtime parse orchestration exists. + `app/runtime.rs` must stay unchanged in Phase B, and an empty registry must not be treated as + successful runtime wiring. + +This gives the runtime one stable construction seam without forcing seam 3 to land production +adapters early. + +--- + +## 6. Determinism and identity rules + +These are non-negotiable. + +### 6.1 LanguageId ownership rule + +In Phase A: + +- `lang::LanguageId` is a re-export of `pack::LanguageId`; +- seam 3 must not define a duplicate enum with the same values; +- downstream seams should depend on `lang::LanguageId` spelling only. + +### 6.2 Adapter identity + +`AdapterName` canonical string pattern: + +```text +^[a-z][a-z0-9]*(\.[a-z][a-z0-9_]*)+$ +``` + +Examples: + +```text +builtin.fake_config +builtin.rust +builtin.typescript +``` + +Examples that must fail: + +```text +Builtin.rust +builtin/ +builtin-rust +builtin..rust +``` + +### 6.3 Parse request fingerprint + +`ParseRequest` fingerprint must be: + +```text +sha256_canonical_json(ParseRequest) +``` + +after request normalization: + +- sorted unique languages; +- sorted unique paths when scope is `Paths`. + +### 6.4 Symbol identity + +Adapters return `LocalSymbolDraft { local_key, ... }`, but `local_key` is **not** the final `SymbolId`. + +The platform computes final `SymbolId`s after canonical sorting using this identity lemma: + +```text +lang\0symbol\0v1\0\0\0\0\0 +``` + +Where: + +- `joined-symbol-path` is the symbol `path` joined with `\0`; +- `duplicate-ordinal` is the zero-based ordinal among otherwise identical symbols after deterministic sorting by: + 1. `kind` + 2. `path` + 3. `name` + 4. `span.start_byte` + 5. `span.end_byte` + 6. `local_key` + +Rule: +- `local_key` exists only so the platform can resolve draft-local references; +- final `SymbolId` generation remains platform-owned and deterministic. + +### 6.5 Unit fingerprint + +`unit_fingerprint` must be: + +```text +sha256_canonical_json(ParsedUnit_without_unit_fingerprint) +``` + +after canonical sorting of: + +- `symbols` +- `edges` +- `surface_markers` +- `diagnostics` + +### 6.6 Parse ordering + +`ParseSet` ordering must be deterministic: + +- `units` sorted by `path`, then `adapter`, then `file_id` +- `failed` sorted by `path`, then `adapter`, then `file_id` +- `skipped` sorted by `path`, then `reason` +- `missing_languages` sorted by `language` +- `diagnostics` sorted by kernel diagnostic ordering + +### 6.7 Requested language handling + +If `ParseRequest.languages` contains a language with no registered adapter: + +- seam 3 must not silently ignore it; +- the parse set must record a `MissingRequestedLanguage`; +- a deterministic top-level diagnostic must be emitted with code + `lang.parse.missing_registered_adapter`. + +### 6.8 Parse failure semantics + +If an adapter cannot parse a file or emits invalid draft output: + +- the file becomes a `FailedParse` record; +- diagnostics are preserved; +- the rest of the parse set continues; +- seam 3 does not return `LangError` for that file-level failure. + +--- + +## 7. Schema inventory + +Seam 3 should add one fixture-facing schema file in Phase A: + +```text +schemas/lang/parse_manifest.v1.json +``` + +Seam 3 should add one schema constants module: + +```text +src/lang/schema.rs +``` + +### `src/lang/schema.rs` + +```rust +#![allow(dead_code)] + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/lang/parse_manifest.v1.json"; + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_VERSION: u32 = 1; + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_FILE: &str = + "parse_manifest.v1.json"; + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/lang/parse_manifest.v1.json"); +``` + +Phase B note: + +- if cache counters remain serialized in `ParseSet.stats`, add + `schemas/lang/parse_manifest.v2.json` and matching constants alongside v1; +- do **not** rewrite `parse_manifest.v1.json` in place, because the current schema and fixtures are + strict and already represent the landed Phase-A contract. + +### `schemas/lang/parse_manifest.v1.json` + +This schema is for fixtures and manifest validation, not for a public API. + +Top-level exact shape: + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/lang/parse_manifest.v1.json", + "title": "Lift Language Parse Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "case", + "snapshot_fingerprint", + "request_fingerprint", + "request", + "units", + "failed", + "skipped", + "missing_languages", + "diagnostics", + "stats" + ], + "properties": { + "version": { "const": 1 }, + "case": { "type": "string", "minLength": 1 }, + "snapshot_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "request_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "request": { "$ref": "#/$defs/request" }, + "units": { + "type": "array", + "items": { "$ref": "#/$defs/parsed_unit" } + }, + "failed": { + "type": "array", + "items": { "$ref": "#/$defs/failed_parse" } + }, + "skipped": { + "type": "array", + "items": { "$ref": "#/$defs/skipped_parse" } + }, + "missing_languages": { + "type": "array", + "items": { "$ref": "#/$defs/missing_requested_language" } + }, + "diagnostics": { + "type": "array", + "items": { "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" } + }, + "stats": { "$ref": "#/$defs/stats" } + } +} +``` + +Required defs to include: + +- `language_id` +- `adapter_name` +- `symbol_kind` +- `symbol_visibility` +- `local_symbol` +- `edge_endpoint` +- `reference_target` +- `local_edge_kind` +- `local_edge` +- `surface_marker_kind` +- `surface_marker` +- `parsed_unit` +- `failed_parse` +- `skipped_reason` +- `skipped_parse` +- `missing_requested_language` +- `request` +- `scope` +- `stats` + +### Runtime-only invariants not fully expressible in schema + +These must be enforced in Rust validators: + +- all `ByteSpan`s must satisfy `end_byte >= start_byte` and `end_byte <= bytes.len()` when attached to a parsed file; +- `local_key` values must be unique within one file; +- draft references must resolve to existing local keys; +- `units`, `failed`, and `skipped` must already be sorted deterministically before serialization; +- `missing_languages` must already be sorted deterministically before serialization; +- `unit_fingerprint` must match the final normalized unit; +- `request_fingerprint` must match the normalized request; +- `AdapterName` must be valid and canonical; +- no final `LocalSymbol` may retain a draft-only `local_key`. + +--- + +## 8. Acceptance criteria + +Seam 3 is done only when all of these are true. + +### Contract completeness + +- `src/lang/mod.rs` is real and no longer a placeholder. +- every public serializable seam-3 contract used in fixtures has a matching schema entry in `schemas/lang/parse_manifest.v1.json`. +- every seam-3 type used by later seams is re-exported from `src/lang/mod.rs`. +- there is no duplicate `LanguageId` type introduced in `lang`. +- the accepted CEO gate is reflected in the actual seam definition of done: Phase A is not complete + until one bounded real-adapter spike or one narrow real adapter plus one real consuming workflow + has exercised the shared contracts. + +### Determinism + +- registry iteration order is deterministic and independent of registration order. +- the same `RepoSnapshot` + same `ParseRequest` + same registered adapters + same cache state yields + byte-identical parse manifests. +- when cache is enabled across repeated equivalent runs, the semantic payload remains identical and + `ParseStats` may differ only in `cache_hits` / `cache_misses`. +- `request_fingerprint` is stable for the same normalized request. +- `unit_fingerprint` is stable for the same normalized parse output. +- no public seam-3 serialization relies on `HashMap` iteration order. + +### Failure semantics + +- one file’s parse failure does not abort parsing the rest of the snapshot. +- invalid adapter output becomes a deterministic `FailedParse` with diagnostics, not a panic. +- adapter panics become deterministic `FailedParse` records with diagnostics, not process-wide + aborts. +- requested languages with no registered adapter become deterministic top-level records and + diagnostics, not silent no-ops. +- requested paths missing from the snapshot become deterministic `SkippedParse { reason: PathNotInSnapshot }` records. +- empty explicit path scopes return a deterministic empty `ParseSet`, not a whole-snapshot parse. +- snapshot-wide non-matching files are counted, but not exploded into one `SkippedParse` record per + file unless the caller explicitly requested those paths. + +### Boundary cleanliness + +- seam 3 remains `pub(crate)` only. +- seam 3 has zero dependencies on: + - `graph` + - `topo` + - `facts` + - `derive` + - `query` + - `patch` + - `app` + - `cli` + - parser runtime crates in Phase A +- seam 3 reads bytes only from `RepoSnapshot` / `BlobStore`. +- seam 3 performs no filesystem I/O, env access, time access, or randomness. + +### Extensibility proof + +- there is at least one test-only fake adapter proving that a new adapter can be implemented without changing app code. +- there is at least one test proving duplicate adapter name rejection. +- there is at least one test proving duplicate language registration rejection. +- there is at least one test proving registration order does not affect output. +- there is at least one bounded real-adapter spike or one narrow real adapter plus one real + consumer-path test proving the contracts survive contact with reality. + +### Schema parity + +- manifest fixtures validate against `schemas/lang/parse_manifest.v1.json`. +- Phase-B cache-aware fixtures, if added, validate against `schemas/lang/parse_manifest.v2.json` + while v1 fixtures continue to validate unchanged. +- schema constants in `src/lang/schema.rs` match the actual file on disk. +- parity tests fail if a serialized field changes without a schema update. + +### Feature-flag compatibility + +- `cargo check -p substrate-lift --no-default-features` continues to work after seam 3 lands. +- seam 3 does not require any concrete adapter feature to compile. + +### Value proof + +- one representative real repo is parsed through the bounded spike with stable output across repeat + runs on the same snapshot. +- one downstream consumer proves that the seam does not end at an orphan internal artifact. +- Phase A success metrics include at minimum: parsed files, failed files, skipped-no-adapter count, + missing-language count, and stable output diff behavior across repeat runs. + +--- + +## 9. Invariants + +These should go into the seam doc as a must-not-break section. + +1. `lang` remains crate-private in seam 3. +2. `lang::LanguageId` is a re-export, not a duplicate enum. +3. seam 3 consumes only immutable `RepoSnapshot` data. +4. seam 3 never rereads the live filesystem. +5. adapter registration order never changes parse results. +6. there is at most one registered adapter per `LanguageId` in Phase A. +7. adapter-local `local_key` values never leak into final serialized units. +8. final `SymbolId`s are platform-derived, not adapter-chosen. +9. file-level parse failures are data, not seam-wide errors. +10. requested languages with no registered adapter are surfaced explicitly, never dropped silently. +11. final parse outputs are sorted and deterministic. +12. seam 3 does not own graph resolution, topology, rule evaluation, score math, or query execution. +13. seam 3 does not pull parser-specific concrete types into shared contracts. +14. seam 3 does not create a second file-path or symbol-id contract outside `kernel`. + +--- + +## 10. Falsification questions + +These are the review questions that can invalidate the seam. + +1. Can seam 3 reopen a file from disk after receiving a `RepoSnapshot`? +2. Can registration order change which adapter wins for a file? +3. Can two adapters for the same `LanguageId` be silently registered at once? +4. Can a requested language with no registered adapter disappear silently instead of being reported explicitly? +5. Can a single syntax error abort the whole parse run? +6. Can an adapter choose final `SymbolId`s directly? +7. Can `local_key` leak into a final `ParsedUnit`? +8. Can `unit_fingerprint` change because collection insertion order changed? +9. Can `ParseRequest` with the same logical languages/paths produce different `request_fingerprint`s because inputs were unsorted? +10. Can seam 3 introduce a second competing `LanguageId` enum? +11. Can seam 3 take a dependency on `pack::compiler`, `graph`, `query`, or app code? +12. Can file-level adapter output contract violations cause a panic instead of a deterministic failed record? +13. Can a span exceed the underlying blob length without being rejected or turned into a failed parse? +14. Can a requested path that is not in the snapshot disappear silently instead of being represented explicitly? +15. Can seam 3 require a real parser runtime crate before any concrete adapters exist? + +If any answer is yes, seam 3 is not clean enough. + +--- + +## 11. Risks and mitigations + +### Risk 1: `LanguageId` ownership churn + +The crate already has `pack::LanguageId`. +Creating a second enum in `lang` would create avoidable drift. + +Mitigation: +- Phase A re-exports `pack::LanguageId` from `lang`. +- downstream code uses `lang::LanguageId` spelling. + +### Risk 2: overdesigning around tree-sitter before adapters exist + +The crate has `QueryEngineKind::TreeSitter` in seam 1, but there is no query execution seam yet. +If seam 3 tries to encode parser-engine specifics too early, it will likely overfit. + +Mitigation: +- Phase A keeps parser-engine details out of the shared platform contracts. +- Phase C adds capability metadata only when real adapters need it. + +### Risk 3: adapter output instability + +If adapters choose their own final IDs or emit unstable orderings, later seams inherit nondeterminism. + +Mitigation: +- adapters emit draft output; +- seam 3 validates, sorts, and canonicalizes into final `ParsedUnit`s. + +### Risk 4: cache invalidation bugs + +If cache keys omit adapter version or path-sensitive identity, parse results can go stale incorrectly. + +Mitigation: +- include `file_id`, `blob_fingerprint`, `adapter`, and `adapter_version` in cache keys. +- keep the entire cache contract in Phase B so Phase A does not promise behavior it does not need yet. + +### Risk 5: platform starts absorbing concrete adapter logic + +There is a strong temptation to sneak extension tables or parser specifics into `lang`. +That would collapse seam 3 into seam 4. + +Mitigation: +- seam 3 contains no production adapters. +- test-only fake adapters prove the platform without committing to real parser behavior. + +### Risk 6: too many fatal errors + +If parse failures bubble out as `LangError`, higher-level seams will have brittle behavior. + +Mitigation: +- reserve `LangError` for registry/cache/schema/platform failures only. +- represent file-level parse failures as `FailedParse` data. + +### Risk 7: hidden dependency on live repo state + +If adapters read the filesystem or environment directly, determinism breaks. + +Mitigation: +- `ParseInput` includes only repo-relative path, file id, blob fingerprint, and bytes. +- adapter trait docs should explicitly forbid live I/O. + +--- + +## 12. In scope / out of scope + +### In scope + +- real `src/lang/**` module tree +- adapter trait and registry +- parse request / parse set contracts +- draft symbol / edge / surface-marker contracts +- final normalized parsed-unit contracts +- deterministic validation and sorting +- explicit missing-requested-language handling +- embedded parse-manifest schema +- test-only fake adapters +- targeted tests for registry, parsing flow, schema parity, and determinism + +### Out of scope + +- the broad production JSON/TOML/YAML adapter family +- the broad production Rust adapter +- the broad production Python adapter +- the broad production JavaScript/TypeScript adapter family +- tree-sitter runtime integration +- parser-specific AST storage in shared contracts +- any cache contract or cache stats surface +- persistent on-disk cache +- cross-file symbol resolution +- repo-wide graph construction +- query execution +- detector/fact execution +- topology/path-class classification +- public API promotion of `lang` + +A deliberate rule: + +> seam 3 must be useful before the full adapter family lands. +> It earns that claim through one bounded real-adapter spike or one narrow real adapter plus one +> real consuming workflow, not through fake adapters alone. + +--- + +## 13. Specific implementation items + +### A. Replace the placeholder module + +Replace: + +```rust +//! Reserved for the future language platform seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; +``` + +with the real Phase-A module tree and re-exports. + +### B. Add schema directory and constants + +Add: + +```text +schemas/lang/parse_manifest.v1.json +src/lang/schema.rs +``` + +### C. Add tests + +Add at minimum: + +```text +tests/lang_registry.rs +tests/lang_parse.rs +tests/lang_schema.rs +``` + +### D. Add fixtures + +Add at minimum: + +```text +fixtures/lang/README.md +fixtures/lang/valid/ +fixtures/lang/invalid/ +``` + +Use a test-only fake adapter and repo-snapshot fixtures from seam 2. + +### E. Update top-level docs + +Update `README.md` so seam 3 is no longer described purely as an abstract future seam. + +### F. Keep feature flags clean + +Do **not** wire `config-lang`, `rust-lang`, `python-lang`, `javascript-lang`, or `typescript-lang` to production adapter modules in seam 3. +That belongs to seam 4. + +### G. Add one real proof slice + +Add one of these before Phase A is treated as complete: + +- a bounded real-adapter spike against one real language/config family plus a fixture-backed + representative repo, or +- one narrow real adapter plus one real consumer path proving `ParseSet` feeds a later seam. + +The goal is not broad language support. The goal is to force the contracts through one real value +path before they are treated as stable. + +--- + +## 14. Fixture and test plan + +### Required test-only adapter + +Define at least one deterministic fake adapter under `#[cfg(test)]`, for example: + +- `builtin.fake_config` +- language id: `json` +- recognizes only `*.fake.json` +- parse contract encoded by simple JSON fields or line-based conventions + +Its purpose is not realism. +Its purpose is proving: + +- registry registration works; +- `ParseDriver` delegates correctly; +- draft-to-final symbol resolution works; +- file-level failures are represented as data; +- deterministic ordering and hashing work. + +This fake adapter is necessary but not sufficient. It proves seam mechanics, not real-language +fitness. + +### Required real proof slice + +Add one bounded real-adapter spike or one narrow real adapter plus one real consumer-path test. + +Minimum proof expectations: + +- one representative real repo fixture or repo-derived snapshot input, +- one hostile-input case proving panic containment / deterministic failure conversion, +- one consumer-path assertion proving the result is not dead infrastructure, +- repeat-run stability on the same snapshot. + +### Minimum tests + +1. `lang_registry_rejects_duplicate_adapter_names` +2. `lang_registry_rejects_duplicate_language_registration` +3. `parse_driver_parses_matching_files_from_snapshot` +4. `parse_driver_skips_missing_requested_paths` +5. `parse_driver_reports_missing_requested_languages` +6. `parse_driver_skips_files_when_recognizes_returns_false_within_selected_language` +7. `parse_driver_uses_all_registered_adapters_when_request_languages_is_empty` +8. `parse_driver_turns_duplicate_local_keys_into_failed_parse` +9. `parse_driver_turns_unresolved_edge_local_keys_into_failed_parse` +10. `parse_driver_turns_unresolved_surface_marker_local_keys_into_failed_parse` +11. `parse_driver_turns_invalid_spans_into_failed_parse` +12. `symbol_id_generation_is_stable_for_equivalent_adapter_output` +13. `unit_fingerprint_matches_normalized_unit_and_is_stable` +14. `parse_set_run_level_diagnostics_do_not_duplicate_file_level_diagnostics` +15. `parse_manifest_fixture_validates_against_schema` +16. `parse_output_is_independent_of_registration_order` +17. `equivalent_parse_requests_normalize_to_the_same_request_fingerprint` +18. `parse_driver_does_not_read_live_filesystem_after_snapshot` +19. `crate_compiles_without_default_features_after_lang_seam_lands` +20. `parse_driver_returns_empty_result_for_empty_explicit_path_scope` +21. `parse_driver_converts_adapter_panic_into_failed_parse` +22. `parse_driver_does_not_emit_unbounded_no_matching_adapter_records_for_snapshot_scope` +23. `bounded_real_adapter_spike_proves_contract_fit_on_one_real_repo_or_snapshot` +24. `consumer_path_proves_parse_set_is_load_bearing` + +--- + +## 15. Locked review decisions + +These decisions are now locked for seam 3 based on the engineering review: + +1. Reduced Phase A scope: land only `mod.rs`, `error.rs`, `adapter.rs`, `registry.rs`, + `driver.rs`, `model.rs`, and `schema.rs`; fold symbol/edge/surface contracts into `model.rs`. +2. `lang` stays `pub(crate)` for seam 3. +3. seam 3 lands the platform first, but Phase A is not complete without one bounded real-adapter + spike or one narrow real adapter plus one real consumer path. +4. `lang::LanguageId` is a re-export of `pack::LanguageId` in Phase A. +5. seam 3 reuses kernel `Diagnostic` and `Locator` directly. +6. adapters emit **draft** output; seam 3 computes final symbol IDs and canonical order. +7. file-level parse failures are data, not `LangError`. +8. Phase A keeps a single-adapter-per-language registry; ambiguous adapter arbitration is out of + scope until a later seam proves it is needed. +9. requested languages with no registered adapter are handled explicitly in Phase A through + deterministic top-level diagnostics and `MissingRequestedLanguage` records. +10. Phase A fixtures include both normalized `request` and `request_fingerprint`. +11. the entire cache contract and cache stats surface move to Phase B. +12. seam 3 adds one fixture schema file: `schemas/lang/parse_manifest.v1.json`. +13. seam 3 proves extensibility with test-only fake adapters before any production parser is added. +14. `ParseSet.diagnostics` is run-level only; file-level diagnostics are not duplicated upward. +15. explanatory `detail` fields use `Option` consistently across non-happy-path records. +16. in Phase A, `recognizes()` is a file-level filter inside the selected language, not a second + adapter-routing mechanism. +17. request normalization gets explicit coverage: logically equivalent language/path sets must + produce the same normalized `ParseRequest` and the same `request_fingerprint`. +18. `ParseDriver::parse_snapshot` is specified as a single-pass inventory walk, not a per-language + rescan loop. +19. `ParseDriver::parse_snapshot` takes `&self` in Phase A; cache-bearing mutability is deferred + until Phase B introduces an explicit cache layer. +20. empty explicit path scopes produce deterministic empty results, not a whole-snapshot fallback. +21. snapshot-wide no-match files are counted, but not emitted as one skipped record per file unless + the caller explicitly requested those paths. +22. panic containment and hostile-input behavior are part of Phase-A proof, not hand-waved adapter + etiquette. + +These decisions keep seam 3 narrow, deterministic, and compatible with the crate that actually +exists today. + +## 16. Phase B detailed implementation plan + +This section is now a landed implementation record. +It is preserved because it explains why the cache/bootstrap surfaces look the way they do. +The live next-step plan starts in Section 19. + +Phase B exists because the crate has already proven the Phase-A seam shape in real code and tests. +The next bottleneck is repeat work, not missing architecture. `ParseDriver::parse_snapshot(...)` +already emits deterministic normalized output. Phase B should reuse that output safely, without +reopening Phase-A semantics or pretending runtime integration exists when `src/app/runtime.rs` still +stops at `ProfileBootstrap`. + +### Working summary + +Current repo facts that shape the plan: + +- `src/lang/cache.rs` does not exist. +- `src/lang/driver.rs` has no cache field, no cache constructor, and no cache-aware branch. +- `src/lang/model.rs` serializes `ParseStats` into `ParseSet`, and that struct has no cache counters. +- `schemas/lang/parse_manifest.v1.json` is strict with `additionalProperties: false`. +- `tests/lang_schema.rs` hard-checks the current manifest version and stats shape. +- `src/app/runtime.rs` has no language registry or parse bootstrap boundary yet. + +That means Phase B is not “add one cache file.” It is a bounded multi-surface change that touches +driver construction, serialized schema shape, tests, fixtures, and registry bootstrap preparation. + +### Step 0: Scope challenge + +Phase B should touch exactly these surfaces: + +| Surface | Current state | Phase-B change | +|---|---|---| +| `src/lang/cache.rs` | absent | add cache types, cache version constant, noop cache, in-memory cache | +| `src/lang/mod.rs` | no cache exports | re-export cache surface | +| `src/lang/driver.rs` | no cache plumbing | add default noop cache, `with_cache(...)`, hit/miss flow | +| `src/lang/model.rs` | no cache counters | add `cache_hits` / `cache_misses` and refresh rules | +| `src/lang/registry.rs` | builder + lookup only | add deterministic `built_in_registry()` helper | +| `src/lang/schema.rs` | v1 constants only | add v2 constants if serialized stats change | +| `schemas/lang/parse_manifest.v1.json` | landed Phase-A schema | leave untouched | +| `schemas/lang/parse_manifest.v2.json` | absent | add only if cache counters stay serialized | +| `tests/lang_*.rs`, `fixtures/lang/**` | Phase-A coverage only | add cache tests, v2 schema tests, and fixture coverage | + +What must not move in this phase: + +- `src/app/runtime.rs` +- concrete production adapters +- public API promotion of `lang` +- query, graph, facts, or app seam integration + +### Outside voice integration + +Focused Codex review on the current Phase-B draft surfaced six issues that must be designed out: + +- High: “byte-identical output” and “non-zero cache hits on second run” conflict while `ParseStats` + stays inside `ParseSet`. +- High: adapter version alone is not enough invalidation, because normalized output also depends on + seam-3 ordering/fingerprinting logic. +- High: adding cache counters is a schema/version change, not a local struct tweak. +- Medium: “runtime-readiness” is overstated if the only runtime-facing work is `built_in_registry()`. +- Medium: recognize-stage panic failures remain uncached unless explicitly called out as a non-goal. +- Medium: constructor/test/schema fallout is broader than the old seven-step sequence admitted. + +The revised plan below incorporates those fixes directly. + +### 1. Architecture review + +The right architecture stays narrow: + +- cache reusable normalized file outcomes inside `lang`; +- keep adapter selection and snapshot walking in `ParseDriver`; +- keep runtime work at bootstrap preparation only via `built_in_registry()`; +- keep schema/version fallout explicit instead of hiding it in fixture churn. + +Required ASCII dependency graph: + +```text + +-------------------+ + | kernel/** | + | ids, spans, diag | + +---------+---------+ + | + v + +-------------------+ +-----+-------------------------------+ + | repo/** |-->| lang/driver.rs | + | snapshot/blob | | - select adapter | + | inventory/stats | | - build ParseCacheKey | + +-------------------+ | - branch on Disabled/Hit/Miss | + +-----+-------------------+------------+ + | | + v v + +----------+--------+ +------+----------------+ + | src/lang/cache.rs | | src/lang/model.rs | + | cache contract | | ParseSet / ParseStats | + | cache format ver | | cache counters | + +----------+--------+ +------+----------------+ + | + v + +----------+-------------------------------+ + | registry.rs built_in_registry() | + | bootstrap preparation only | + +-------------------+---------------------+ + | + v + later seam, not Phase B runtime +``` + +Cache key rule: + +```text +file_id + blob_fingerprint + language + adapter + adapter_version + platform_cache_version +``` + +`platform_cache_version` is mandatory because cached `ParsedUnit` depends on seam-3 normalization +rules, not only adapter output. If `symbol_identity_lemma`, sort order, fingerprint composition, or +normalization semantics change, the cache must invalidate even when adapter binaries do not. + +Semantic-equality rule: + +- uncached run and cached run must match for all semantic payload fields: + `snapshot_fingerprint`, `request`, `request_fingerprint`, `units`, `failed`, `skipped`, + `missing_languages`, and `diagnostics`; +- `stats` may differ only in `cache_hits` and `cache_misses`. + +That removes the earlier contradiction between “byte-identical output” and “prove a cache hit.” + +### 2. Code quality review + +The implementation should stay explicit: + +- `ParseDriver::new(registry)` owns `NoopParseCache` internally and preserves Phase-A call sites. +- `ParseDriver::with_cache(registry, cache)` is the only opt-in constructor added in Phase B. +- `cache.rs` owns the cache trait, key, value, and in-memory implementation. Do not split it yet. +- `driver.rs` owns orchestration only. It should not absorb cache storage internals. +- `registry.rs` owns deterministic built-in adapter registration order only. It must not construct + or configure a runtime parse service. + +The main code-quality trap is hidden scope growth. Adding cache counters means either: + +1. bump the serialized manifest schema to v2, or +2. stop serializing cache counters. + +This plan chooses the explicit option: + +- keep cache counters serialized in `ParseStats`, +- keep `parse_manifest.v1.json` untouched as the landed Phase-A contract, +- add `parse_manifest.v2.json` plus new constants and v2 fixtures/tests in the same change. + +That is the whole game. No fake “local struct change” story. + +### 3. Test review + +This phase needs new tests, not just modified asserts. The affected codepaths are: + +```text +CODE PATH COVERAGE +=========================== +[+] ParseDriver constructors + ├── new() preserves Phase-A semantics with NoopParseCache + └── with_cache() enables cache flow without mutability leaks + +[+] Cache lookup flow + ├── Disabled -> parse/normalize path + ├── Hit -> append cached ParsedUnit + ├── Hit -> append cached FailedParse + └── Miss -> parse -> normalize -> cache put + +[+] Invalidation + ├── blob_fingerprint change + ├── adapter_version change + └── platform_cache_version change + +[+] Non-goals kept explicit + ├── no caching of missing-language records + ├── no caching of skipped/no-match bookkeeping + └── recognize-stage panic remains deterministic and uncached + +[+] Schema/version fallout + ├── v1 fixtures still validate unchanged + ├── v2 fixtures validate with cache counters + └── schema constants match disk for both versions in scope + +[+] Registry bootstrap preparation + └── built_in_registry() order is deterministic for active features +``` + +Required Phase-B tests: + +1. `parse_driver_with_noop_cache_matches_phase_a_semantics` +2. `in_memory_cache_hits_on_second_equivalent_run_except_stats` +3. `cache_miss_when_blob_fingerprint_changes` +4. `cache_miss_when_adapter_version_changes` +5. `cache_miss_when_platform_cache_version_changes` +6. `cache_reuses_failed_parse_outcomes` +7. `cache_does_not_cache_skipped_or_missing_language_records` +8. `recognize_panic_behavior_remains_deterministic_and_uncached` +9. `built_in_registry_is_deterministic_for_enabled_features` +10. `lang_manifest_v2_schema_matches_disk_and_validates_phase_b_fixtures` +11. `phase_b_cache_preserves_no_default_features_compile_matrix` + +Commands to lock before implementation: + +- `cargo test -p substrate-lift lang_registry -- --nocapture` +- `cargo test -p substrate-lift lang_parse -- --nocapture` +- `cargo test -p substrate-lift lang_consumer -- --nocapture` +- `cargo test -p substrate-lift lang_schema -- --nocapture` +- `cargo test -p substrate-lift lang_cache -- --nocapture` +- `cargo test -p substrate-lift compile_matrix -- --nocapture` +- `cargo check -p substrate-lift --no-default-features` + +Test plan artifact written to: + +- [spensermcconnell-feat-lift-phase-b-test-plan-20260418-102634.md](/Users/spensermcconnell/.gstack/projects/atomize-hq-substrate/spensermcconnell-feat-lift-phase-b-test-plan-20260418-102634.md) + +### 4. Performance review + +Phase B is a reuse optimization, so the performance story must stay honest: + +- it avoids repeat parse and normalization work after adapter selection; +- it does **not** avoid `recognizes()` routing work, because routing remains uncached in this seam; +- it does **not** introduce whole-run cache blobs, so `ParseSet` assembly stays request-local; +- it must not create large cache-unaware skip manifests for snapshot-wide no-match files. + +The primary performance risk is subtle, not dramatic: + +- if the plan compares full serialized `ParseSet` values across uncached and cached runs, the + cache-hit proof will fail because stats changed; +- if the plan mutates manifest v1 in place, schema tests will fail before performance tests even + matter; +- if the key omits `platform_cache_version`, repeat runs can be fast and wrong, which is worse than + slow and correct. + +### NOT in scope + +- persistent or on-disk cache files +- cross-process cache sharing +- caching routing decisions from `recognizes()` +- app runtime parse orchestration +- production adapter landings +- query-engine hooks +- graph/query/facts/app integration +- public API promotion of `lang` + +### What already exists + +- `src/lang/driver.rs` already owns deterministic snapshot walking and adapter panic containment +- `src/lang/model.rs` already owns `ParseSet`, `ParseStats`, sorting, and fingerprint helpers +- `tests/lang_parse.rs` already covers deterministic Phase-A parse flow and panic containment +- `tests/lang_registry.rs` already covers deterministic registry order and duplicate rejection +- `tests/lang_schema.rs` already proves strict schema parity for manifest v1 +- `src/pack/schema.rs` and `src/repo/schema.rs` already show the schema-constant pattern to copy + +### Failure modes specific to Phase B + +| Codepath | Failure mode | Rescued? | Phase-B handling | +|---|---|---|---| +| cache key design | stale normalized output reused after seam-3 normalization change | yes | include `platform_cache_version` in `ParseCacheKey` | +| cache key design | stale output reused after adapter upgrade | yes | include `adapter_version` in `ParseCacheKey` | +| schema/versioning | cache counters break strict manifest v1 fixtures | yes | add `parse_manifest.v2.json`, keep v1 untouched | +| acceptance tests | second run cannot be both identical and cache-hit proving | yes | compare semantic payload separately from stats | +| routing stage | repeated `recognizes()` panic never benefits from cache | partial | keep as explicit non-goal and add deterministic regression coverage | +| runtime framing | `built_in_registry()` mistaken for runtime integration | yes | document as bootstrap preparation only, keep `app/runtime.rs` untouched | +| disabled cache | `NoopParseCache` inflates miss counters | yes | `Disabled`, not synthetic `Miss` | + +### Worktree parallelization strategy + +Phase B is parallelizable, but only with explicit ownership and a frozen contract handoff. + +Lane ownership: + +- Lane A, primary: `src/lang/cache.rs`, `src/lang/driver.rs`, `src/lang/model.rs`, `src/lang/mod.rs`, `tests/lang_cache.rs` +- Lane B, schema/fixture side lane: `src/lang/schema.rs`, `schemas/lang/parse_manifest.v2.json`, `tests/lang_schema.rs`, `fixtures/lang/**` +- Lane C, bootstrap/doc side lane: `src/lang/registry.rs`, `tests/lang_registry.rs`, `README.md`, compile-matrix touchpoints if needed + +Gating rules: + +1. Lane A freezes the cache key shape, constructor names, and `ParseStats` field list first. +2. Lane B starts only after Lane A freezes whether cache counters stay serialized and confirms the + v2 schema decision. +3. Lane C can start once `built_in_registry()` signature is frozen. It must not edit driver/cache + internals. +4. Final integration happens in this order: Lane A merge, Lane B merge, Lane C merge, then full + test sweep. + +Why this split is safe: + +- Lane A owns behavioral semantics. +- Lane B owns schema/fixture fallout from those semantics. +- Lane C owns deterministic bootstrap preparation and docs. + +That keeps overlapping writes out of `driver.rs`, `model.rs`, and schema fixtures at the same time. + +### TODOS.md updates + +No new root `TODOS.md` update is proposed from this pass. + +Reason: + +- this crate directory does not currently maintain a local `TODOS.md`, +- and every deferred Phase-B item is better kept in this seam spec than in a disconnected todo list. + +### Completion summary + +```text + Scope challenge: 9 concrete surfaces identified, runtime overclaim removed + Outside voice: ran (codex only), 6 issues incorporated into the plan + Architecture Review: cache key, semantic equality, and runtime boundary corrected + Code Quality Review: constructor/test/schema fallout made explicit + Test Review: 11 required tests locked, commands listed, artifact written + Performance Review: routing-stage non-goal and invalidation risks documented + NOT in scope: written (8 items) + What already exists: written + Failure modes: 7 Phase-B-specific modes mapped + Parallelization: 3 lanes with gating and ownership + Lake Score: 6/6 recommendations favored complete over shortcut +``` + +## 17. Phase B locked decisions + +1. Cache normalized per-file outcomes, not raw adapter drafts. +2. Keep `ParseDriver::parse_snapshot(&self, ...)` in Phase B. +3. Use `NoopParseCache` as the default behind `ParseDriver::new(...)`. +4. Count cache hits and misses only when a cache is actually enabled. +5. Do not cache request-level bookkeeping like missing-language or no-match records. +6. Use `file_id + blob_fingerprint + language + adapter + adapter_version + platform_cache_version` as the cache key. +7. Keep all Phase-B cache implementation in `src/lang/cache.rs` unless the code proves it is too dense. +8. Add `built_in_registry()` now, even if it returns an empty registry until seam 4 lands adapters. +9. Treat `built_in_registry()` as bootstrap preparation only, not runtime parse orchestration. +10. Add `parse_manifest.v2.json` instead of mutating Phase-A manifest v1 in place. +11. Compare semantic parse payload separately from `ParseStats` when proving cache reuse. +12. Keep recognize-stage panic containment uncached in Phase B and prove it with a regression test. + +## 18. Phase B decision audit addendum + +| # | Phase | Decision | Classification | Principle | Rationale | Rejected | +|---|---|---|---|---|---|---| +| 13 | Phase B | Cache normalized `ParsedUnit` and `FailedParse` outcomes only | Mechanical | P5 | The driver already owns normalization and failure semantics, so caching raw drafts would create a second truth source. | Caching adapter draft output | +| 14 | Phase B | Keep `ParseDriver::parse_snapshot(&self, ...)` and inject cache via trait | Mechanical | P5 | Runtime callers should not take on fake mutability just to turn caching on. | Switching the driver API to `&mut self` | +| 15 | Phase B | Key the cache by `file_id`, `blob_fingerprint`, `language`, `adapter`, `adapter_version`, and `platform_cache_version` | Mechanical | P1 + P5 | This captures path-sensitive identity, adapter invalidation, and seam-3 normalization changes without dragging request-scope noise into the key. | Keying only on path, only on blob, or on adapter version alone | +| 16 | Phase B | Do not cache `SkippedParse`, missing-language, or whole-`ParseSet` results | Mechanical | P3 + P5 | Those are request-level bookkeeping artifacts, not reusable per-file parse outcomes. | Whole-run cache blobs or no-match negative caches | +| 17 | Phase B | Add `built_in_registry()` before real built-ins land, but treat it as bootstrap prep only | Mechanical | P3 + P5 | It gives later runtime code one stable seam without pretending Phase B wires runtime parse orchestration. | Wiring runtime code directly to concrete adapters or claiming runtime integration early | +| 18 | Phase B | Add `parse_manifest.v2.json` instead of mutating v1 in place | Mechanical | P1 + P5 | The landed v1 schema and fixtures are already strict, so cache-counter serialization needs an explicit version step. | Silent in-place schema mutation | +| 19 | Phase B | Prove cache reuse by semantic payload equality plus stats delta, not full byte-identical `ParseSet` equality | Mechanical | P5 | `cache_hits` and `cache_misses` must change on the second run, so full-value equality would assert against the point of the phase. | Requiring identical full serialized outputs across cached and uncached runs | +| 20 | Phase B | Keep recognize-stage panic failures deterministic and uncached in this seam | Mechanical | P3 + P5 | Caching routing failures would widen the seam more than the value justifies right now; the boundary must be explicit and tested instead. | Smuggling a second routing-failure cache into Phase B | + +## 19. Phase C detailed implementation plan + +Phase B solved the repeat-work problem inside `lang`. The next bottleneck is not raw performance. +It is handshake quality. + +Seam 4 needs one stable place to say what a real adapter actually provides. +Seam 7 needs one stable place to ask whether a compiled query pack can even run against that adapter. +Right now the crate has adapter identity (`name`, `language`, `version`) and normalized parse output, +but it does not have an explicit contract for adapter capabilities. + +That is the Phase C wedge. + +### Working summary + +Current repo facts that shape the plan: + +- `src/lang/cache.rs` is landed and already owns the Phase-B cache contract. +- `ParseStats` already includes `cache_hits` and `cache_misses`. +- `schemas/lang/parse_manifest.v2.json` is landed, so Phase C should avoid another manifest bump + unless it truly changes serialized parse payloads. +- `src/lang/registry.rs::built_in_registry()` exists, is deterministic, and intentionally returns an + empty registry today. +- `src/pack/compiled/query_pack.rs` already owns `QueryEngineKind::TreeSitter`. +- `src/app/runtime.rs` still bootstraps only `CompiledPackSet`; there is no lang/runtime join point yet. +- no concrete production adapters or parser runtime crates are registered in the crate today. +- `src/lang/mod.rs` currently re-exports cache, driver, error, model, registry, and schema surfaces, + but it does **not** yet expose a capability module or `QueryEngineKind`. +- `src/lang/adapter.rs` currently stops at `descriptor()`, `recognizes()`, and `parse()`. +- `tests/lang_registry.rs` currently proves deterministic descriptor ordering and duplicate rejection, + but it does **not** yet prove metadata lookup or built-in order contracts. + +That means Phase C is not "start landing real parsers." +It is "add the missing metadata handshake so real parsers can land cleanly in seam 4." + +It also means the old Phase-C draft carried one non-implementable expectation: + +- it asked for a test proving enabled built-in adapters register in canonical order, but the crate + does not have concrete built-ins yet. + +The corrected plan below keeps the future ordering contract, but makes the current seam fully +implementable now by freezing that order in code and keeping `built_in_registry()` behavior empty +until seam 4 has real adapters to plug into it. + +### Step 0: scope challenge + +Phase C should touch exactly these surfaces: + +| Surface | Current state | Phase-C change | +|---|---|---| +| `src/lang/capabilities.rs` | absent | add adapter-capability types and query-engine support metadata | +| `src/lang/adapter.rs` | descriptor + parse hooks only | add default `capabilities()` hook with empty/default behavior | +| `src/lang/registry.rs` | registration, lookup, deterministic descriptors, empty `built_in_registry()` | add capability inspection helpers, freeze canonical built-in language order, and keep built-in bootstrap empty until seam 4 | +| `src/lang/mod.rs` | re-exports `LanguageId` and Phase-B surface | re-export `QueryEngineKind` and Phase-C capability types | +| `tests/lang_registry.rs` | registry invariants + deterministic empty built-ins | extend for capability inspection and canonical built-in order scaffolding | +| `tests/lang_capabilities.rs` | absent | add dedicated capability-surface tests | +| `tests/lang_cache.rs` + `tests/lang_schema.rs` | Phase-B regression coverage only | add explicit "Phase C does not widen cache/schema" regression checks if missing | +| `README.md` + this spec | seam 3 described through Phase B | note that Phase C is the metadata handshake before seam-4 adapters | + +What must **not** move in this phase: + +- `src/lang/model.rs` +- `src/lang/cache.rs` +- `schemas/lang/parse_manifest.v1.json` +- `schemas/lang/parse_manifest.v2.json` +- `src/app/runtime.rs` +- query execution +- concrete production adapter code +- public API promotion of `lang` + +Why the scope is accepted as-is: + +- it is an 8-surface slice, which is large but still a boilable lake because the change is contract + plumbing, not a new runtime subsystem; +- every touched surface is directly in the blast radius of the missing adapter metadata handshake; +- the tempting shortcut, stuffing capability fields into `ParseSet` or waiting for seam 4, would + either widen schema churn now or force seam 4 to reopen seam 3 later. + +### Outside voice integration + +The remaining issues in the old Phase-C writeup were mechanical, not strategic: + +- it mixed a real current-state plan with one test that assumed future built-ins already existed; +- it skipped a code-quality pass, which left the exact `registry.rs` ownership split underspecified; +- it skipped worktree parallelization, which made the seam look more serial than it really is; +- it named capability helpers but did not pin their clone and ordering semantics tightly enough for + an implementer to move without rereading the whole document. + +The revised plan fixes those directly. No second premise-level review is needed to make this phase +implementable. The missing work was execution detail. + +### 1. Architecture review + +The clean architecture is: + +- keep parse payloads exactly where they are; +- add adapter metadata on the registry boundary, not inside `ParseSet`; +- let later seams join `ParsedUnit.adapter` + `ParsedUnit.adapter_version` back to registry metadata + when they need execution-time behavior; +- keep `built_in_registry()` as the single deterministic construction point for feature-gated built-ins, + while explicitly leaving it empty until seam 4 contributes real adapters. + +Required ASCII dependency graph: + +```text + seam 4 concrete adapter impl + | + v ++------------------------------+ +| lang/capabilities.rs | +| AdapterCapabilities | ++---------------+--------------+ + | + v ++---------------+--------------+ +| lang/adapter.rs | +| descriptor() | +| capabilities() default empty | +| recognizes() / parse() | ++---------------+--------------+ + | + v ++---------------+-----------------------+ +| lang/registry.rs | +| register() | +| descriptors() | +| capabilities_for_language() | +| supported_query_engines_for_language()| +| BUILT_IN_LANGUAGE_ORDER | +| built_in_registry() | ++---------------+-----------------------+ + | + +-------+--------+ + | | + v v ++-------+---------+ +---------------------------+ +| lang/driver.rs | | seam 7 query/match later | +| ParseSet flow | | joins language + engine | +| unchanged | | against registry metadata | ++-----------------+ +---------------------------+ +``` + +The important negative space: + +- Phase C does **not** add AST handles, parser sessions, or engine-specific runtime objects to + `ParsedUnit`. +- Phase C does **not** add query-pack execution helpers to `lang`. +- Phase C does **not** make cache keys depend on capabilities, because normalized parse payload does + not depend on registry metadata. +- Phase C does **not** pretend built-in adapters exist yet. The registry bootstrap path is frozen, + not populated. + +Realistic production failure scenarios and the rescue path: + +- If seam 4 later adds adapters through ad hoc constructors instead of `built_in_registry()`, later + boots will drift by feature-set and call site. Rescue: freeze one canonical order constant and one + bootstrap path now. +- If a later seam interprets empty `surface_markers` as "unsupported" instead of "none found," + downstream behavior becomes nondeterministic. Rescue: Phase C adds explicit capability flags. +- If `QueryEngineKind` forks inside `lang`, seam 7 will eventually compare two "tree_sitter" + truths and lose. Rescue: re-export the pack-owned enum instead of minting a second one. + +### 2. Code quality review + +Phase C needs one boring, explicit contract split: + +- `adapter.rs` owns trait hooks; +- `capabilities.rs` owns the capability data model; +- `registry.rs` owns lookup and bootstrap rules; +- `mod.rs` owns the seam-local re-export boundary; +- `driver.rs`, `cache.rs`, `model.rs`, and schemas stay untouched. + +That is the whole quality bar. The plan should not hide metadata in five places when one new module +and two small registry helpers are enough. + +Phase C should keep `AdapterDescriptor` identity-only: + +- `name` +- `language` +- `version` + +Do **not** smuggle capability flags into `AdapterDescriptor`. +Identity and capability are different concerns, and the cache key already depends on descriptor identity. + +Add a new Phase-C contract in `capabilities.rs`: + +```rust +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +use crate::lang::QueryEngineKind; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Default)] +pub(crate) struct AdapterCapabilities { + pub emits_local_edges: bool, + pub emits_surface_markers: bool, + pub query_engines: BTreeSet, +} +``` + +`mod.rs` should add the new module and re-export the existing pack-owned query engine truth: + +```rust +pub(crate) mod capabilities; + +pub(crate) use crate::pack::{LanguageId, QueryEngineKind}; +pub(crate) use capabilities::AdapterCapabilities; +``` + +And extend the adapter trait with a default hook: + +```rust +pub(crate) trait LanguageAdapter: Send + Sync { + fn descriptor(&self) -> AdapterDescriptor; + + fn capabilities(&self) -> AdapterCapabilities { + AdapterCapabilities::default() + } + + fn recognizes(&self, input: &ParseInput<'_>) -> bool; + fn parse(&self, input: &ParseInput<'_>) -> AdapterParseResult; +} +``` + +Rules: + +- default capabilities must be empty so existing fake adapters and Phase-B tests remain easy to write; +- `emits_local_edges` and `emits_surface_markers` describe structural support, not quality guarantees; +- `query_engines` states only engine-level compatibility in Phase C, because the compiled query-pack + contract already speaks in terms of `language + engine` and nothing narrower yet; +- if later work proves engine-level compatibility is insufficient, the next contract step belongs to + the pack/query seam, not to an ad hoc seam-3 escape hatch; +- `LanguageRegistry` should not grow a second stored capability map. It already stores adapters; the + correct move is to ask the adapter for capabilities on demand and clone the small result. + +`registry.rs` should gain the exact small lookup surface below: + +```rust +const BUILT_IN_LANGUAGE_ORDER: &[LanguageId] = &[ + LanguageId::Json, + LanguageId::Toml, + LanguageId::Yaml, + LanguageId::Rust, + LanguageId::Python, + LanguageId::Javascript, + LanguageId::Typescript, +]; + +impl LanguageRegistry { + pub(crate) fn capabilities_for_language( + &self, + language: LanguageId, + ) -> Option; + + pub(crate) fn supported_query_engines_for_language( + &self, + language: LanguageId, + ) -> BTreeSet; +} +``` + +`built_in_registry()` should route through one explicit bootstrap helper now, even though that helper +returns the builder unchanged until seam 4 adds real adapters. That keeps the seam implementable +today and prevents the future built-in path from splintering later. + +Phase C should avoid the same drift problem seam 3 already solved for `LanguageId`. + +Do this: + +- `pub(crate) use crate::pack::QueryEngineKind;` + +Do **not** do this: + +- create `lang::QueryEngineKind` as a second enum; +- copy string literals like `"tree_sitter"` into capability structs; +- or make seam 7 import engine identity from both `pack` and `lang`. + +That keeps the compatibility story boring: + +- compiled query packs still say `language + engine`; +- adapters still say `language + supported engines`; +- seam 7 can deterministically reject unsupported combinations without needing query execution in seam 3. + +### 3. Test review + +Phase C needs new tests because this is contract work, not implementation garnish. + +Required codepath diagram: + +```text +CODE PATH COVERAGE +=========================== +[+] src/lang/capabilities.rs + ├── [GAP] AdapterCapabilities::default() is all-false / empty + ├── [GAP] serde round-trip keeps query-engine ordering deterministic + └── [GAP] explicit override supports edge/marker/query-engine metadata + +[+] LanguageAdapter trait + ├── [GAP] default capabilities() is empty and backwards compatible + └── [GAP] fake adapter can override capabilities() without affecting parse path + +[+] LanguageRegistry inspection + ├── [GAP] capabilities lookup by language + ├── [GAP] supported query engines lookup by language + ├── [GAP] unknown language -> None / empty set + └── [GAP] deterministic descriptor order remains unchanged by capability metadata + +[+] built_in_registry bootstrap path + ├── [GAP] empty under current feature set remains deterministic + ├── [GAP] no-default-features remains empty and deterministic + └── [GAP] canonical built-in language order constant is frozen in code + +[+] Non-goals remain explicit + ├── [GAP] ParseSet schema unchanged + ├── [GAP] cache key unchanged + └── [GAP] no query execution hooks in lang +``` + +Required Phase-C tests: + +1. `default_adapter_capabilities_are_empty_and_backwards_compatible` +2. `adapter_capabilities_serde_round_trip_is_canonical` +3. `registry_exposes_capabilities_in_language_scoped_lookup` +4. `supported_query_engines_lookup_is_language_scoped` +5. `descriptor_order_is_unchanged_by_capability_metadata` +6. `built_in_registry_remains_deterministic_and_empty_before_seam4` +7. `canonical_built_in_language_order_is_frozen` +8. `phase_c_capabilities_do_not_change_cache_keys_or_parse_stats` +9. `phase_c_preserves_parse_manifest_v2_shape` +10. `phase_c_preserves_no_default_features_compile_matrix` + +Commands to lock before implementation: + +- `cargo test -p substrate-lift lang_registry -- --nocapture` +- `cargo test -p substrate-lift lang_parse -- --nocapture` +- `cargo test -p substrate-lift lang_capabilities -- --nocapture` +- `cargo test -p substrate-lift lang_cache -- --nocapture` +- `cargo test -p substrate-lift lang_schema -- --nocapture` +- `cargo test -p substrate-lift compile_matrix -- --nocapture` +- `cargo check -p substrate-lift --no-default-features` + +Test plan artifact written to: + +- [spensermcconnell-feat-lift-phase-c-test-plan-20260418-162143.md](/Users/spensermcconnell/.gstack/projects/atomize-hq-substrate/spensermcconnell-feat-lift-phase-c-test-plan-20260418-162143.md) + +Coverage judgment: + +- the old draft correctly identified the missing metadata seam, but it stopped short of proving the + current crate can land that seam without touching cache or manifest payloads; +- the corrected test matrix turns those non-goals into executable regressions; +- the canonical-order test is now implementable because it targets the frozen order contract itself, + not future concrete built-ins that do not exist yet. + +### 4. Performance review + +Phase C is almost entirely metadata, so the performance rule is simple: + +- registry inspection may allocate small sorted collections; +- parse throughput must stay unchanged; +- cache behavior must stay unchanged; +- manifest serialization must stay unchanged. + +If a Phase-C implementation makes `ParseDriver::parse_snapshot(...)` slower on the hot path, +it is probably doing the wrong work in the wrong seam. + +### NOT in scope + +- concrete parser implementations +- parser runtime crates +- AST/session handles in `ParsedUnit` +- query execution helpers +- query-pack schema expansion +- app/runtime integration +- schema v3 for parse manifests +- public API promotion of `lang` + +### What already exists + +- `ParsedUnit` already carries `adapter` and `adapter_version`, which is enough provenance to join + runtime metadata later without changing parse payload shape +- `tests/lang_registry.rs` already proves deterministic descriptor behavior +- `tests/lang_cache.rs` already proves the Phase-B cache key and stats behavior that Phase C must not break +- `src/pack/compiled/query_pack.rs` already defines the only real query-engine enum in the crate +- `built_in_registry()` already gives seam 4 one canonical registration choke point + +### Failure modes specific to Phase C + +| Codepath | Failure mode | Test? | Error handling? | User / operator sees | +|---|---|---|---|---| +| capability contract | adapter metadata gets stuffed into `ParsedUnit` and forces schema churn | yes | yes, regression tests hold schema steady | schema diff / fixture failure instead of silent drift | +| engine identity | seam 3 creates a second query-engine enum | yes | yes, compile/type mismatch reveals it | compile failure or review-visible duplicate enum | +| bootstrap path | seam-4 adapters later register through ad hoc constructors instead of `built_in_registry()` | partial | yes, one canonical helper is documented and frozen | deterministic bootstrap drift caught in review/tests | +| feature gating | built-in language order varies across builds | yes | yes, order constant is tested | unstable descriptor snapshots instead of silent randomization | +| downstream semantics | empty `surface_markers` or `edges` gets mistaken for "unsupported" | yes | partial, capability flags disambiguate support from absence | later seam sees explicit support flags | +| performance | Phase-C metadata leaks into parse hot path or cache key | yes | yes, cache/schema regression tests pin non-goals | cache misses or throughput drop caught by regression suite | + +No silent critical gap remains if the tests above land with the code. The one watch item is the +future seam-4 bootstrap path, because Phase C can freeze the choke point now but cannot prove real +feature-gated population until real built-ins exist. + +### Worktree parallelization strategy + +Phase C is parallelizable after one short contract freeze. + +Dependency table: + +| Step | Modules touched | Depends on | +|---|---|---| +| A. Capability contract foundation | `src/lang/`, `tests/` | — | +| B. Registry inspection + bootstrap order helpers | `src/lang/`, `tests/` | A | +| C. Regression + docs sweep | `tests/`, `schemas/`, crate docs | A | + +Parallel lanes: + +- Lane A: Step A → Step B + Contract foundation first, then registry helpers. Sequential because both touch `src/lang/adapter.rs`, + `src/lang/mod.rs`, and `src/lang/registry.rs`. +- Lane B: Step C + Regression and docs sweep after Step A freezes the new type names and method signatures. This lane + owns `tests/lang_cache.rs`, `tests/lang_schema.rs`, `README.md`, and this spec. + +Execution order: + +1. Land Step A first in the primary worktree. +2. Once `AdapterCapabilities`, `capabilities()`, and the `mod.rs` re-export names are frozen, launch + Lane A Step B and Lane B in parallel worktrees. +3. Merge Lane A first because it owns the source-of-truth registry semantics. +4. Merge Lane B second. +5. Run the full Phase-C command sweep only after both lanes are integrated. + +Conflict flags: + +- Lane A and Lane B both touch `tests/`, but not the same files if ownership stays clean. +- Do **not** split `src/lang/registry.rs` across worktrees. That file is the primary merge-conflict + risk in this phase. + +### TODOS.md updates + +No new root `TODOS.md` update is proposed from this pass. + +Reason: + +- this crate directory does not currently maintain a local `TODOS.md`; +- the one deferred item, real built-in adapter population, belongs to seam 4 execution planning, not + to a disconnected Phase-C todo entry. + +### Completion summary + +```text + Step 0: Scope Challenge — 8 concrete surfaces identified, future-built-in assumption removed + Architecture Review: 3 boundary issues found, all fixed in the plan + Code Quality Review: 4 contract-ownership issues fixed explicitly + Test Review: diagram produced, 10 required tests locked, artifact written + Performance Review: parse/cache/schema hot paths explicitly frozen + NOT in scope: written (8 items) + What already exists: written + TODOS.md updates: 0 + Failure modes: 6 Phase-C-specific modes mapped, 0 silent critical gaps + Outside voice: prior autoplan decisions incorporated, no new rerun required + Parallelization: 2 lanes, 1 early gate + 1 parallel side lane + Lake Score: 5/5 recommendations favored complete over shortcut +``` + +## 20. Phase C locked decisions + +1. Keep `AdapterDescriptor` identity-only in Phase C. +2. Add `AdapterCapabilities` in `src/lang/capabilities.rs` as a separate adapter/registry contract. +3. Add `LanguageAdapter::capabilities()` as a default method so existing fake adapters remain cheap. +4. Keep capability metadata off `ParseSet` and `ParsedUnit`. +5. Re-export the existing `pack::QueryEngineKind` from `lang`; do not mint a second enum. +6. Keep query-engine compatibility engine-level only in Phase C. +7. Freeze canonical built-in language order in `registry.rs`, but keep `built_in_registry()` empty + until seam 4 contributes real adapters. +8. Treat `built_in_registry()` as the only canonical feature-gated built-in adapter registration path. +9. Add registry inspection helpers that clone capability metadata on demand instead of storing a + second registry-side capability map. +10. Keep cache keys, `ParseStats`, and parse-manifest schemas unchanged in Phase C. + +## 21. Phase C decision audit addendum + +| # | Phase | Decision | Classification | Principle | Rationale | Rejected | +|---|---|---|---|---|---|---| +| 21 | Phase C | Keep `AdapterDescriptor` identity-only and add a separate `AdapterCapabilities` surface | Mechanical | P5 | Cache identity and runtime metadata change at different rates; combining them would blur invariants and risk accidental cache or schema coupling. | Stuffing capability flags into `AdapterDescriptor` | +| 22 | Phase C | Add `LanguageAdapter::capabilities()` as a default method | Mechanical | P3 + P5 | Existing fake adapters and Phase-B tests should not pay migration tax for metadata they do not need. | A new required trait method with no default | +| 23 | Phase C | Keep capability metadata registry-side, not serialized in `ParseSet` | Mechanical | P1 + P5 | Downstream seams can join runtime metadata by adapter identity, so schema churn would buy complexity without new user value. | Adding capability fields to `ParsedUnit` or `ParseSet` | +| 24 | Phase C | Re-export `pack::QueryEngineKind` from `lang` | Mechanical | P4 + P5 | The crate already has one query-engine identity source; duplicating it would recreate the exact drift problem seam 3 avoided for `LanguageId`. | Minting a second seam-3 query-engine enum | +| 25 | Phase C | Keep query compatibility engine-level only for now | Mechanical | P3 + P5 | The compiled query-pack contract already speaks in `language + engine`; introducing finer compatibility before seam 7 proves it is necessary would be speculative churn. | Pre-landing grammar/dialect/version matrices in seam 3 | +| 26 | Phase C | Freeze canonical built-in registration order in code, but keep `built_in_registry()` empty until seam 4 has real adapters | Mechanical | P3 + P5 | The order contract is real now, but fake built-ins would only add noise and maintenance burden. | Pretending built-ins exist already or deferring order freezing entirely | +| 27 | Phase C | Add registry inspection helpers that clone adapter capability metadata on demand | Mechanical | P4 + P5 | The registry already stores adapters; duplicating a second capability map would create unnecessary drift and update burden. | Storing parallel capability state inside `LanguageRegistry` | +| 28 | Phase C | Keep built-in adapter population behind one registry bootstrap helper | Mechanical | P5 | Seam 4 needs one choke point for feature-gated registration, not a spread of ad hoc constructors. | Registering built-ins directly from tests, docs, or future runtime code | +| 29 | Phase C | Do not change cache keys, `ParseStats`, or parse-manifest schemas in Phase C | Mechanical | P3 + P5 | Phase C is metadata plumbing, not parse-behavior change; touching cache or schema semantics would widen the blast radius for no gain. | Re-keying cache on capabilities or adding schema v3 | + +## GSTACK REVIEW REPORT + +| Review | Trigger | Why | Runs | Status | Findings | +|--------|---------|-----|------|--------|----------| +| CEO Review | `/plan-ceo-review` | Scope & strategy | 2 | OPEN | mode: SELECTIVE_EXPANSION, 1 critical gap | +| Codex Review | `/codex review` | Independent 2nd opinion | 0 | — | — | +| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 5 | CLEAR | 11 issues, 0 critical gaps | +| Design Review | `/plan-design-review` | UI/UX gaps | 0 | — | — | + +**UNRESOLVED:** 0 unresolved decisions across logged reviews. +**VERDICT:** ENG CLEARED — ready to implement. CEO review is optional and still has open scope-level concerns from the earlier autoplan pass. + +--- + +## AUTOPLAN PHASE 1 — CEO REVIEW (2026-04-17) + +### Working summary + +This plan is grounded in the real crate state. `src/lang/mod.rs` is still a placeholder, `src/repo/**` and `src/pack/**` are already real seams, `pack::LanguageId` already exists, and `app/runtime.rs` still stops at `ProfileBootstrap`. + +The strategic question is not whether a language seam is needed eventually. It is whether this seam should land first as pure platform contracts, or whether the first landing should prove one real end-to-end repo-intelligence outcome on one real language family. + +### 0A. Premise challenge + +| Premise | Assessment | Why it matters | +|---|---|---| +| Seam 3 should be the first real landing of `src/lang/**` | Valid | The current codebase has a real seam boundary to fill, and later seams need a parse artifact that is not a live-filesystem read. | +| Seam 3 should stay narrow and platform-only in Phase A | Medium confidence, challenged | This protects seam purity, but both outside voices converged that platform-only plus fake adapters risks shipping elegant plumbing with no proof of user value. | +| `lang::LanguageId` should re-export `pack::LanguageId` instead of creating a second enum | Valid | `pack::LanguageId` is already used by `CompiledAnalysisDefaults`, `CompiledQueryPack`, and `CompiledRuleScope`; duplicating it now would create churn without learning. | +| A fake adapter is enough to prove extensibility in Phase A | Rejected as sufficient proof | It proves internal coherence, not that the contracts survive one ugly real parser, one real repo, or one real consumer seam. | +| Single adapter per language is a safe Phase-A lock | Medium confidence, challenged | Reasonable as an initial simplifying rule, but not strong enough to lock as an architectural truth before one production adapter forces arbitration pressure. | +| Determinism is the dominant constraint | Valid, but incomplete | Determinism is core to this repo’s posture, but the plan does not yet connect that determinism to one user-visible win such as better replayability, lower false positives, or faster agent context capture. | + +### 0B. Existing code leverage + +| Sub-problem | Existing code | Reuse decision | +|---|---|---| +| Stable IDs, spans, fingerprints, diagnostics | `src/kernel/**` | Reuse directly. The seam should not mint parallel IDs, paths, span types, or diagnostic contracts. | +| Immutable repo bytes and deterministic inventory | `src/repo/**`, especially `RepoSnapshot`, `Inventory`, `BlobStore`, `SnapshotStats` | Reuse directly. This is the strongest part of the current wedge because it already enforces snapshot-based determinism. | +| Language identity contract | `src/pack/names.rs` `LanguageId` | Re-export through `lang`, do not duplicate. | +| Existing runtime ingress | `src/app/runtime.rs` `ProfileBootstrap` | Reuse as the example of how a seam should prove a real consumer instead of stopping at internal contracts. | +| Feature-flag posture | `tests/compile_matrix.rs` | Reuse and extend. `--no-default-features` is already guarded and should stay part of seam acceptance. | + +Existing leverage is good. The plan is not reinventing kernel or repo. The strategic gap is elsewhere: it is not reusing the repo’s existing pattern of “land the seam, then prove one consuming path” strongly enough. + +### 0C. Dream state mapping + +```text +CURRENT STATE THIS PLAN (as written) 12-MONTH IDEAL +placeholder lang seam deterministic parse contracts trusted repo intelligence +no parsed units fake-adapter validation only real adapters on real repos +real repo snapshots no production parser yet downstream seams consume ParseSet +real pack/runtime seams no user-visible wedge yet one secure/replayable analysis moat +``` + +### 0C-bis. Implementation alternatives + +APPROACH A: Platform-First Phase A + Summary: Land the seam exactly as written, with registry, parse contracts, fake adapters, schema parity, and no production adapter. + Effort: M + Risk: Medium + Pros: + - Cleanest seam boundary and lowest immediate blast radius + - Preserves deterministic contracts before parser-specific pressure arrives + - Reuses kernel/repo sharply + Cons: + - Ships no direct user-visible outcome + - Fake adapters can validate the shape and still miss the real contract breaks + - Risks six-month regret: beautiful substrate, no compounding leverage + Reuses: + - `kernel/**`, `repo/**`, `pack::LanguageId`, existing compile-matrix posture + +APPROACH B: Vertical Slice Wedge + Summary: Land the seam plus one production-grade adapter on one high-value family, likely Rust or config, and prove one downstream consumer consumes `ParseSet`. + Effort: L + Risk: Medium + Pros: + - Forces the contracts through real data immediately + - Produces a measurable user outcome instead of internal correctness only + - Validates whether single-adapter-per-language and current symbol contracts survive contact with reality + Cons: + - Larger first landing + - Increases scope and parser-specific complexity now + - Might require revising some Phase-A “locked” decisions quickly + Reuses: + - everything in Approach A plus current Rust-heavy repo reality and the existing runtime seam pattern + +APPROACH C: Adapter Spike Then Contract Freeze + Summary: Do a bounded spike for one real adapter first, use that evidence to tighten the contracts, then land the platform seam immediately afterward. + Effort: M + Risk: Low to Medium + Pros: + - Lowest risk of freezing the wrong contracts + - Still keeps the formal seam landing narrow + - Gives real evidence for marker taxonomy, symbol IDs, and arbitration rules + Cons: + - Slightly slower than jumping straight into the seam docs as implementation source of truth + - Requires discipline to keep the spike bounded and evidence-oriented + Reuses: + - existing repo snapshots, deterministic fixtures, current Rust/config repo surfaces + +**RECOMMENDATION:** Choose Approach C if the goal is to preserve seam quality without lying to yourself about product learning. Choose Approach B if the team explicitly wants a bolder wedge now. Approach A is the cleanest document and the weakest proof. + +### 0D. Mode-specific analysis + +Mode selected: `SELECTIVE_EXPANSION` + +Complexity check: +- The written Phase-A landing touches at least `src/lang/mod.rs`, `error.rs`, `adapter.rs`, `registry.rs`, `driver.rs`, `model.rs`, `schema.rs`, `schemas/lang/parse_manifest.v1.json`, `tests/lang_registry.rs`, `tests/lang_parse.rs`, `tests/lang_schema.rs`, `fixtures/lang/**`, and `README.md`. +- That is comfortably over the “8 files” smell threshold even before any production adapter or consumer is added. + +Minimum set of changes that still achieves the stated goal: +- real `src/lang/**` module tree +- `LanguageId` re-export +- registry and driver +- normalized parsed-unit contracts +- schema parity +- compile-matrix preservation + +Expansion scan: +- One real adapter spike +- One downstream consumer of `ParseSet` +- Hostile-input parse budget rules +- Parse coverage / failure-rate metrics +- Explicit moat framing in the plan + +Cherry-pick result under `/autoplan` rules: +- Auto-decided mechanical carry-forward: keep the seam internal-only for now, reuse `pack::LanguageId`, keep parse output deterministic, keep cache out of Phase A. +- Surfaced as **User Challenge**, not auto-decided: add one real adapter or one bounded real-adapter spike plus one real consuming workflow before treating the seam as strategically approved. + +### 0E. Temporal interrogation + +| Window | What the implementer needs resolved now | +|---|---| +| Hour 1 (human), ~5-10 min with CC | Is this a pure contract seam, or is one real adapter expected in the first landing? This changes every file count and acceptance gate. | +| Hour 2-3 (human), ~10-20 min with CC | What hostile-input guarantees must adapters honor? Time/space budget, panic containment, malformed UTF-8, and maximum blob/span expectations are not fully specified yet. | +| Hour 4-5 (human), ~10-20 min with CC | Which downstream consumer proves the seam is real? Without that, the implementer can finish the seam and still not know if the contracts are load-bearing. | +| Hour 6+ (human), ~15-30 min with CC | What metrics decide success on real repos: parse coverage, failure rate, runtime, and downstream utility? The current acceptance criteria mostly measure internal hygiene. | + +### 0F. Mode selection confirmation + +`SELECTIVE_EXPANSION` still fits. The baseline plan is useful, but the only scope addition worth fighting for is the one that proves the seam is not dead infrastructure. + +### CODEX SAYS (CEO — strategy challenge) + +- Critical: the document optimizes seam purity before proving a customer wedge. +- High: fake adapters are not enough to validate the contract. +- High: all acceptance criteria are internal-quality gates, not value gates. +- High: the plan excludes every real adapter and consumer, which makes “platform” sound bigger than what ships. +- High: the moat is implied, not stated. If the moat is secure, replayable, deterministic repo analysis inside Substrate constraints, the plan should say that plainly. + +### CLAUDE SUBAGENT (CEO — strategic independence) + +- Critical: the milestone is phrased as “language platform,” not as one real repo-intelligence outcome. +- High: several core premises are assumed rather than falsified with evidence. +- High: the six-month regret scenario is obvious: perfect substrate, still no shipped capability. +- High: viable alternatives were ruled out too early without a comparative decision record. +- High: competitive risk is understated because commodity parser stacks can ship “good enough” faster. + +### CEO DUAL VOICES — CONSENSUS TABLE + +```text +═══════════════════════════════════════════════════════════════ + Dimension Claude Codex Consensus + ──────────────────────────────────── ────── ───── ───────── + 1. Premises valid? mixed mixed DISAGREE on some premises, both flag gaps + 2. Right problem to solve? no no CONFIRMED + 3. Scope calibration correct? no no CONFIRMED + 4. Alternatives sufficiently explored? + no no CONFIRMED + 5. Competitive/market risks covered? + no no CONFIRMED + 6. 6-month trajectory sound? no no CONFIRMED +═══════════════════════════════════════════════════════════════ +``` + +Confirmed = 5/6. Disagreements = 1/6, but even the disagreement is about which premises survive, not about whether the current plan is strategically under-proved. + +### Section 1: Architecture review + +Examined: the seam boundary in `lift_seam3_spec_reviewed.md`, the existing module layout in `README.md`, the actual placeholder `src/lang/mod.rs`, and the pattern already used by `app/runtime.rs`. + +Finding: the architecture is internally coherent and reuse-heavy, but the seam currently stops one layer too early. The repo already has a healthier pattern in seam 1: a clean internal seam plus one real consumer path. This plan does not yet require that proof for seam 3. + +### Section 2: Error & Rescue Map + +The spec does a decent job distinguishing seam-wide `LangError` from file-level `FailedParse` data. That part is strong. + +The gap is operational rescue, not type-level rescue. There is no explicit contract yet for hostile parser behavior on malicious or pathological source files, which means parse failures are modeled but parse resource exhaustion is not. + +### Section 3: Security & Threat Model + +This seam parses untrusted repository content. That is the actual trust boundary, and the current plan only partially acknowledges it. + +Gap: the plan should explicitly state parser-hostile-input rules for future real adapters: no live I/O, no env/time/randomness, bounded memory posture, deterministic handling of malformed inputs, and no adapter panic leaking past the seam boundary. + +### Section 4: Data Flow & Interaction Edge Cases + +There is no UI scope here, so the relevant interaction surface is internal data flow: + +```text +RepoSnapshot ──▶ ParseDriver ──▶ Adapter parse draft ──▶ Validation/canonicalization ──▶ ParseSet + │ │ │ │ │ + │ │ │ │ ├── missing requested language + │ │ │ ├── invalid span/local key + │ │ ├── malformed bytes/input + │ ├── missing adapter + └── missing path / empty scope +``` + +The main gap is not a missing happy-path edge case. It is that no downstream consumer is named in the same data-flow slice, so the flow ends at an internal artifact instead of a proven use. + +### Section 5: Code Quality Review + +The document is precise and mostly explicit. The quality risk is not sloppy wording. It is false certainty. + +The most brittle pattern is the “Locked review decisions” block. Several items are worthy implementation defaults, but they are written with more finality than the evidence currently supports. + +### Section 6: Test Review + +The fixture and determinism matrix is thorough for contract correctness. It is not thorough for product proof. + +The missing tests are not more fake-adapter tests. The missing tests are one real-adapter stress case and one downstream-consumer test that proves `ParseSet` makes later seams materially easier. + +### Section 7: Performance Review + +No production parser lands in this phase, so there is limited concrete performance analysis to do. Still, the plan is missing one important invariant: what happens on very large blobs, symbol-heavy files, or adversarial nested structures once a real adapter appears. + +That should be called out now, because otherwise performance posture gets deferred until after the contracts are frozen. + +### Section 8: Observability & Debuggability Review + +The plan defines diagnostics, stats, and fingerprints, which is good seam-local observability. What it does not yet define is what success looks like on real repos. + +Add explicit run-level metrics expectations for later implementation: parse coverage by language, failed file count, skipped reasons, and stable output diffs between runs on the same snapshot. Those are the signals an operator will actually care about. + +### Section 9: Deployment & Rollout Review + +This seam is internal-only and not a separately distributed artifact, so there is no external rollout surface to map. That part is fine. + +The rollout gap is organizational: if the team implements this exactly as written, it can declare victory with no obligation to prove a downstream outcome. That is the rollout failure mode to prevent. + +### Section 10: Long-term trajectory review + +The long-term seam map in `README.md` is coherent. The risk is path dependency. + +If seam 3 lands as pure contracts plus fake adapters, seam 4 will inherit the pressure to preserve those contracts even if the first real adapters reveal they were wrong. That is how temporary scaffolding turns into permanent architecture. + +### Section 11: Design & UX review + +Skipped. No UI scope detected in the seam-3 plan. The only plan-file match for UI-like terms was a false-positive phrase in a contract comment, not a user surface. + +## NOT in scope + +- Full platform-first approval with no real adapter pressure. Deferred pending premise confirmation because both outside voices challenged it. +- Public API promotion of `lang`. Still correctly out of scope for this phase. +- Tree-sitter or parser-runtime integration. Still deferred, but the plan should say what evidence will trigger it. +- Multi-adapter arbitration. Keep out of Phase A unless the real-adapter spike immediately proves it is needed. + +## What already exists + +- `kernel/**` already owns the cross-seam primitives this seam needs. +- `repo/**` already gives the snapshot determinism story this seam should build on. +- `pack::LanguageId` already exists and should be reused. +- `app/runtime.rs` already shows a better precedent for “seam plus one consuming path.” + +## Dream state delta + +As written, this plan moves the repo from “no language seam” to “well-specified language contracts.” That is real progress. + +It does **not** yet move the repo all the way to “trusted repo intelligence that helps an agent or app do something materially better.” The delta to the 12-month ideal is therefore still missing one real adapter, one real consumer, and one explicit moat narrative. + +## Error & Rescue Registry + +| Method / Codepath | What can go wrong | Exception / Failure class | Rescued? | Rescue action | User / operator sees | +|---|---|---|---|---|---| +| `LanguageRegistryBuilder::register` | duplicate adapter name | `LangError::DuplicateAdapterName` | Y | reject registration deterministically | test failure / startup failure with typed error | +| `LanguageRegistryBuilder::register` | duplicate language registration | `LangError::DuplicateLanguageAdapter` | Y | reject registration deterministically | test failure / startup failure with typed error | +| `ParseDriver::parse_snapshot` | requested language has no registered adapter | run-level diagnostic + `MissingRequestedLanguage` | Y | continue parse, surface top-level record | operator sees explicit missing-language record | +| `ParseDriver::parse_snapshot` | requested path not in snapshot | `SkippedParse { PathNotInSnapshot }` | Y | continue parse, surface skipped record | operator sees skipped reason | +| adapter `parse` | syntax or semantic parse failure | `FailedParse` + diagnostics | Y | continue parse, preserve failure as data | operator sees file-level failure | +| adapter `parse` | pathological input causes excessive resource use | not yet specified | N | add explicit hostile-input/budget rules before implementation | currently ambiguous | + +## Failure Modes Registry + +| Codepath | Failure mode | Rescued? | Test? | User sees? | Logged? | +|---|---|---|---|---|---| +| parse request normalization | logically equivalent request fingerprints diverge | planned | planned | internal drift only | planned | +| adapter registration | two adapters claim one language | yes | planned | typed startup failure | planned | +| file parse | duplicate local keys or unresolved local refs | yes | planned | `FailedParse` record | planned | +| real adapter on hostile file | panic / blow-up / pathological runtime | no | no | ambiguous | no | +| seam-wide rollout | seam ships with no downstream consumer | no | no | silent strategic failure | no | + +Rows 4 and 5 are the important gaps. Row 4 is an implementation-critical gap. Row 5 is the strategy gap both outside voices converged on. + +## Completion Summary + +```text + +====================================================================+ + | MEGA PLAN REVIEW — COMPLETION SUMMARY | + +====================================================================+ + | Mode selected | SELECTIVE EXPANSION | + | System Audit | strong seam reuse, weak value proof | + | Step 0 | 3 approaches compared, 1 user challenge | + | Section 1 (Arch) | 1 major issue found | + | Section 2 (Errors) | 6 paths mapped, 1 GAP | + | Section 3 (Security)| 1 issue found, 0 immediate critical | + | Section 4 (Data/UX) | 1 edge-case gap, 1 missing consumer | + | Section 5 (Quality) | 1 issue found | + | Section 6 (Tests) | reviewed, 1 major gap | + | Section 7 (Perf) | 1 issue found | + | Section 8 (Observ) | 1 gap found | + | Section 9 (Deploy) | 1 rollout risk flagged | + | Section 10 (Future) | Reversibility: 3/5, debt items: 2 | + | Section 11 (Design) | SKIPPED (no UI scope) | + +--------------------------------------------------------------------+ + | NOT in scope | written (4 items) | + | What already exists | written | + | Dream state delta | written | + | Error/rescue registry| 6 methods, 1 CRITICAL-ISH GAP | + | Failure modes | 5 total, 2 critical strategic gaps | + | TODOS.md updates | 0 proposed yet | + | Scope proposals | 1 surfaced, pending user decision | + | CEO plan | existing prior artifact referenced | + | Outside voice | ran (codex + subagent) | + | Lake Score | 3/3 mechanical choices favored completeness | + | Diagrams produced | 2 (dream-state, internal data flow) | + | Stale diagrams found | 0 | + | Unresolved decisions | 1 (listed below) | + +====================================================================+ +``` + +## Unresolved decision + +### User Challenge 1 + +Both outside voices recommend changing the stated direction. + +- What the current plan says: land seam 3 as platform contracts first, with fake adapters and no production adapter or real consumer required in Phase A. +- What both models recommend: require either one bounded real-adapter spike or one real adapter plus one real consuming workflow before treating seam 3 as strategically approved. +- Why: this is the cleanest way to preserve determinism and seam quality without optimizing architecture before the repo proves value. +- What context we might be missing: you may intentionally want to de-risk the contract layer first because the internal seam map itself is the near-term goal. +- If we are wrong, the cost is: you take on more immediate scope and parser-specific churn than you wanted in the first landing. + +### Premise gate outcome + +User response: `A` + +Accepted direction: +- seam 3 may still land a clean internal platform surface, +- but it is no longer considered strategically complete without one bounded real-adapter spike or one real adapter plus one real consuming workflow. + +This resolves the CEO-phase user challenge in favor of proving one real value slice before the seam is treated as fully approved. + +## Decision Audit Trail + +| # | Phase | Decision | Classification | Principle | Rationale | Rejected | +|---|---|---|---|---|---|---| +| 1 | CEO | Use `SELECTIVE_EXPANSION` mode | Mechanical | P1 + P2 | The baseline seam is useful, but the only worthwhile expansion is the one that proves the seam is real. | HOLD SCOPE without surfacing the missing wedge | +| 2 | CEO | Treat UI scope as absent | Mechanical | P5 | The plan contains no real user interface surface, so a design phase would be noise. | Running design review from a false-positive keyword match | +| 3 | CEO | Keep `lang::LanguageId` as a re-export of `pack::LanguageId` | Mechanical | P4 + P5 | Existing compiled pack contracts already depend on this type, so duplication would create churn without learning. | Defining a second `LanguageId` enum in `lang` | +| 4 | CEO | Keep cache out of Phase A | Mechanical | P3 + P5 | The seam is already large; adding cache before a real adapter arrives grows complexity without proving value. | Shipping cache contracts in the first landing | +| 5 | CEO | Require user confirmation on adding one real adapter or adapter spike plus one consumer | User Challenge | P1 + P2 | Both outside voices converged that platform-only Phase A is strategically under-proved. | Auto-approving platform-only Phase A as sufficient proof | +| 6 | CEO | Accept gate option A: require one real adapter spike or one real adapter plus one consumer | User-approved challenge | P1 + P2 | The seam now has to prove value, not just internal correctness. | Keeping Phase A approved as fake-adapter-only infrastructure | +| 7 | Design | Skip design review | Mechanical | P5 | No UI scope exists in this seam plan, so a design pass would add noise instead of signal. | Running a UI review off a false-positive text match | +| 8 | Eng | Make the accepted proof slice part of the actual definition of done | Mechanical | P1 + P5 | The plan cannot have one review gate and a different implementation finish line. | Leaving the accepted CEO gate outside the seam acceptance criteria | +| 9 | Eng | Define empty explicit path scope as deterministic empty output | Mechanical | P5 | Empty inputs need explicit semantics or the first caller will accidentally trigger whole-repo work. | Implicit or ambiguous empty-scope behavior | +| 10 | Eng | Keep snapshot-scope no-match files in stats, not per-file skipped records | Mechanical | P3 + P5 | Prevents manifest-size blowups while preserving deterministic accounting. | Emitting one skipped record per non-matching file in snapshot scope | +| 11 | Eng | Use `ParseDriver::parse_snapshot(&self, ...)` in Phase A | Mechanical | P5 | There is no Phase-A cache/state yet, so mutability would only imply hidden coupling. | Smuggling future cache mutability into the first contract | +| 12 | Eng | Make panic containment and hostile-input behavior explicit Phase-A proof requirements | Mechanical | P1 + P5 | “Adapters must not panic” is etiquette, not a trust boundary. The seam has to own containment. | Leaving panic/resource-failure semantics implicit | + +### Phase-transition summary + +**Phase 1 complete.** Codex: 8 concerns. Claude subagent: 5 issues. Consensus: 5/6 confirmed, 1 disagreement surfaced as a premise-level caution. Passing to the premise gate before any later `/autoplan` phase continues. + +## AUTOPLAN PHASE 2 — DESIGN REVIEW + +Skipped. No UI scope was detected in this seam plan. The only earlier keyword hit was a false positive inside contract prose, not a user-visible screen, layout, or interaction surface. + +**Phase 2 complete.** Skipped, no UI scope. Passing to Phase 3. + +## AUTOPLAN PHASE 3 — ENG REVIEW (2026-04-17) + +### Step 0: Scope challenge + +What already exists: +- `src/kernel/**` already owns stable IDs, spans, diagnostics, paths, and fingerprints. +- `src/repo/**` already owns deterministic snapshot materialization, blob reads, stats, and schema-backed manifests. +- `src/pack/schema.rs` and `src/repo/schema.rs` already show the exact schema-constant pattern seam 3 should follow. +- `tests/compile_matrix.rs` already guards the `--no-default-features` posture. +- `README.md` already separates seam 3 “language platform” from seam 4 “concrete adapters”. + +Minimum complete slice after the accepted gate: +- the reduced `src/lang/**` module tree, +- deterministic parse contracts and driver, +- schema parity, +- fake-adapter seam proof, +- one bounded real-adapter spike or one narrow real adapter plus one real consumer path, +- explicit hostile-input containment semantics. + +Complexity check: +- This is still a >8-file seam even in the reduced cut. +- That is acceptable only because the seam already has strong reuse from `kernel`, `repo`, and the existing schema/test patterns. +- The dangerous shortcut would be pretending fake-adapter coverage is enough. That shortcut is no longer allowed after the accepted gate. + +### CLAUDE SUBAGENT (eng — independent review) + +- P1: contracts are being locked before one real adapter and one real consumer prove them. +- P1: “must not panic” is not a containment strategy for hostile input. +- P2: `ParseSet` can grow too large if skip semantics are materialized naively. +- P2: empty explicit path scope semantics were previously unspecified. +- P2: `lang` depending on `pack::LanguageId` is coupling pressure to watch. +- P3: `ParseDriver::parse_snapshot(&mut self, ...)` was unnecessary Phase-A mutability. + +### CODEX SAYS (eng — architecture challenge) + +- Critical: the accepted CEO gate was not yet reflected in the actual definition of done. +- High: failure isolation was aspirational, not enforceable. +- High: `pack::LanguageId` may be too coarse as a long-term runtime routing identity. +- Medium: `driver.rs` risked becoming a god-module with validation, normalization, and fingerprinting piled together. +- Medium: schema parity was at risk of ossifying the model before one real adapter validated it. + +### ENG DUAL VOICES — CONSENSUS TABLE + +```text +═══════════════════════════════════════════════════════════════ + Dimension Claude Codex Consensus + ──────────────────────────────────── ────── ───── ───────── + 1. Architecture sound? mixed mixed DISAGREE on details, both flag contract-proof gap + 2. Test coverage sufficient? no no CONFIRMED + 3. Performance risks addressed? no no CONFIRMED + 4. Security threats covered? no no CONFIRMED + 5. Error paths handled? no no CONFIRMED + 6. Deployment risk manageable? mixed mixed CONFIRMED +═══════════════════════════════════════════════════════════════ +``` + +Consensus confirmed: 5/6. The one mixed dimension is not “architecture is fine,” it is “the core shape is coherent, but only if the proof slice and containment rules become part of the source-of-truth plan.” + +### 1. Architecture review + +The seam boundary is still the right one. `kernel` owns primitives, `repo` owns bytes, `lang` should own per-file parsing contracts, and seam 4 should still own the broad adapter family. + +The architecture problem was not the seam split. It was the lack of a proof slice and the risk that `driver.rs` silently becomes the home for too many responsibilities. The accepted gate and the new containment rules fix the first issue. The second remains a watch item for implementation. + +Required ASCII architecture diagram: + +```text + +-------------------+ + | kernel/** | + | ids, spans, diag | + +---------+---------+ + | + v + +-------------------+ +----+----+ +-------------------+ + | repo/** |-->| lang |--->| later seams | + | snapshot/blob | | driver | | graph/query/app | + | inventory/stats | | registry | | consume ParseSet | + +-------------------+ | model | +-------------------+ + | schema | + +----+------+ + | + +-------------+-------------+ + | | + v v + fake adapter seam proof bounded real proof slice + (mechanics) (one real adapter spike or + one consumer path) +``` + +Coupling assessment: +- Good coupling: `lang` depends on `kernel` and `repo`. +- Acceptable temporary coupling: `lang::LanguageId` re-exporting `pack::LanguageId`. +- Bad future coupling to avoid: `lang` depending on `pack::compiler`, app code, or graph/query seams. + +### 2. Code quality review + +The spec is explicit, but the old version was explicit in the wrong place. It locked decisions before the proof slice existed. That is the kind of precision that causes rework instead of preventing it. + +Mechanical quality fixes now reflected in the plan: +- `ParseDriver::parse_snapshot` is `&self` in Phase A, not `&mut self`. +- empty explicit path scope is defined, not implied. +- snapshot-wide no-match files are counted, not emitted one-by-one as skipped records. +- proof-slice requirements are part of “done,” not just appended review text. + +The remaining code-quality risk is implementation density in `driver.rs`. If normalization, local-ref resolution, span checks, sorting, ID derivation, and fingerprinting all collapse into one long module, the first real adapter will force an immediate refactor. + +### 3. Test review + +This seam is library-only, so the critical flows are codepath flows, not UI flows. + +Required coverage diagram: + +```text +CODE PATH COVERAGE +=========================== +[+] LanguageRegistryBuilder + ├── duplicate adapter name rejected + ├── duplicate language rejected + └── deterministic descriptor order + +[+] ParseDriver::parse_snapshot + ├── ParseScope::Snapshot + ├── ParseScope::Paths(non-empty) + ├── ParseScope::Paths(empty) [GAP before eng review, now required] + ├── missing requested language + ├── missing requested path + ├── no-matching-adapter in snapshot scope (counter only) + └── no-matching-adapter for explicitly requested path (skipped record) + +[+] Adapter output normalization + ├── duplicate local keys -> FailedParse + ├── unresolved local refs -> FailedParse + ├── invalid spans -> FailedParse + ├── stable symbol ids + └── stable unit fingerprint + +[+] Hostile input containment + ├── adapter panic -> FailedParse [NEW REQUIRED] + ├── malformed bytes -> deterministic diagnostics [NEW REQUIRED] + └── pathological real-adapter spike input [NEW REQUIRED] + +[+] Real proof slice + ├── bounded real-adapter spike on one real repo/snapshot [NEW REQUIRED] + └── consumer-path test proving ParseSet is load-bearing [NEW REQUIRED] +``` + +Coverage judgment: +- The previous fake-adapter matrix was strong for seam mechanics. +- It was insufficient for production confidence. +- The new required tests are the ones that would matter at 2am Friday: empty scope, panic containment, skip-output size behavior, one real spike, one real consumer. + +Test plan artifact written to: +- [spensermcconnell-feat-lift-eng-review-test-plan-20260417-223430.md](/Users/spensermcconnell/.gstack/projects/atomize-hq-substrate/spensermcconnell-feat-lift-eng-review-test-plan-20260417-223430.md) + +### 4. Performance review + +No broad production parser lands yet, so most performance analysis is future-facing. The non-future-facing issue was output size, not runtime. + +The important correction is that `ParseSet.skipped` must not become a giant “we skipped everything” manifest on large snapshots. Counting snapshot-wide no-match files in stats and only materializing path-specific skip records is the right explicit Phase-A rule. + +The second performance watch item is the real proof slice. That spike needs repeat-run stability and at least a basic boundedness readout on one representative repo or repo-derived snapshot. + +### NOT in scope + +- the broad production adapter family +- tree-sitter/runtime integration +- public API promotion of `lang` +- cache contracts and cache stats +- graph/query/facts/app integration beyond one narrow consumer proof path + +### What already exists + +- schema constant pattern in `src/pack/schema.rs` and `src/repo/schema.rs` +- fixture/test folder conventions under `tests/**` and `fixtures/**` +- compile-matrix enforcement in `tests/compile_matrix.rs` +- repo snapshot purity and determinism posture already enforced below this seam + +### Failure modes + +| Codepath | Failure mode | Test? | Error handling? | User / operator sees | +|---|---|---|---|---| +| `ParseDriver::parse_snapshot` | empty explicit path set accidentally parses whole snapshot | now required | now specified | deterministic empty result | +| adapter parse | panic on hostile input | now required | now specified | `FailedParse` with diagnostics | +| snapshot-scope no-match files | output-size blow-up from per-file skipped records | now required | now specified | bounded manifest + stats | +| real proof slice | contracts fail on first real language/repo | now required | partially by proof gate | seam not accepted as complete | +| consumer path | `ParseSet` remains orphan internal artifact | now required | proof gate | seam not accepted as complete | + +### Worktree parallelization strategy + +Sequential implementation is still the safest default because most of the Phase-A work clusters under `src/lang/**`. + +There is only one safe side lane: +- Lane A: `src/lang/**` contracts, driver, and normalization +- Lane B: `schemas/lang/**`, `tests/lang_schema.rs`, fixture scaffolding, and README updates + +Execution order: +- Launch Lane B once the core model names stabilize. +- Keep Lane A authoritative because it owns the actual seam contracts. + +### TODOS.md updates + +No new root `TODOS.md` update is proposed from this pass. + +Reason: +- this repo does not currently maintain a root `TODOS.md`, +- and every material deferred item for seam 3 is already better tracked directly in this reviewed seam spec. + +### Completion summary + +```text + Step 0: Scope Challenge — accepted proof-slice requirement carried into the spec + Architecture Review: 2 major issues found, 1 fixed directly in the plan + Code Quality Review: 2 issues found, 2 fixed directly in the plan + Test Review: diagram produced, 5 critical gaps identified and converted into requirements + Performance Review: 1 issue found, fixed via explicit skip-output semantics + NOT in scope: written + What already exists: written + TODOS.md updates: 0 + Failure modes: 5 high-signal modes mapped + Outside voice: ran (codex + subagent) + Parallelization: 2 lanes, 1 primary + 1 side lane + Lake Score: 5/5 recommendations favored complete over shortcut +``` + +### Cross-phase themes + +**Theme: prove one real value slice** +- flagged in CEO phase and again in both engineering outside voices. +- high-confidence signal. This was the dominant cross-phase concern. + +**Theme: fake adapters are necessary but not sufficient** +- flagged in CEO phase and eng phase independently. +- high-confidence signal. The seam needs one real contract stress case. + +**Theme: determinism without containment is not enough** +- CEO phase surfaced the value side of determinism. +- Eng phase surfaced the hostile-input and panic-containment side. +- together they say the same thing: the seam must be trustworthy, not just tidy. + +### Phase-transition summary + +**Phase 3 complete.** Codex: 5 concerns. Claude subagent: 6 issues. Consensus: 5/6 confirmed, 1 mixed architecture dimension. The reviewed spec now includes the accepted proof-slice requirement, explicit empty-scope semantics, bounded no-match skip behavior, and Phase-A panic-containment expectations. diff --git a/crates/lift/profiles/README.md b/crates/lift/profiles/README.md new file mode 100644 index 000000000..47afd29b7 --- /dev/null +++ b/crates/lift/profiles/README.md @@ -0,0 +1,9 @@ +# Profiles + +This directory is part of the pack-authoring surface. Phase A landed the shipped profile compiler foundation, Phase B added topology selection, Phase C extends profiles to select advanced pack families for deterministic bundle resolution, and Phase D adds the narrow runtime bootstrap handoff over that compiled bundle. + +Profiles are declarative inputs consumed through the crate-private `pack` compiler seam. The compiler can build one standalone profile from builtin, file-backed, or inline sources, validate it against the embedded common/profile schemas, and produce deterministic compiled output plus typed diagnostics. + +The builtin profile name currently guaranteed is `builtin:generic/default`. That builtin is embedded in the compiler, and its topology slots now point at the builtin topology packs `builtin:generic/boundaries` and `builtin:generic/components`. Profiles may also name file- or builtin-backed score/query/rule/recipe packs. Resolving those refs now produces a crate-internal `CompiledPackSet`; that still does not imply repo walking, path classification, query execution, scoring, or rule/recipe execution. Phase D does add a narrow crate-internal activation boundary in `app::runtime`, where `bootstrap_profile(...)` wraps the resolved bundle as `ProfileBootstrap` for later runtime consumers. + +This directory still does not carry bundled profile files. Later seams may add bundled Substrate profiles, migration shims, and broader authoring UX here. Those remain deferred beyond the current compiler-plus-bootstrap boundary. The current runtime surface is bootstrap ingress only, not a public registry, dispatch layer, or broader app container. diff --git a/crates/lift/recipes/README.md b/crates/lift/recipes/README.md new file mode 100644 index 000000000..35bb109f9 --- /dev/null +++ b/crates/lift/recipes/README.md @@ -0,0 +1,5 @@ +# Recipes + +This directory is reserved for rewrite recipes consumed by the pack compiler and later patch-planning seams. + +Phase C now compiles recipe packs structurally inside the crate-private `pack` seam. Recipes remain declarative inputs only; transform execution and patch generation are still deferred to later seams. diff --git a/crates/lift/rules/README.md b/crates/lift/rules/README.md new file mode 100644 index 000000000..26f4be6c5 --- /dev/null +++ b/crates/lift/rules/README.md @@ -0,0 +1,5 @@ +# Rules + +This directory is reserved for detector and policy rule packs that compile into immutable runtime objects. + +Phase C now compiles rule packs structurally inside the crate-private `pack` seam. That means schema validation, typed query refs, deterministic fingerprints, and bundle closure support exist, but rule execution still belongs to later engine and app seams. diff --git a/crates/lift/schemas/README.md b/crates/lift/schemas/README.md new file mode 100644 index 000000000..08a95705d --- /dev/null +++ b/crates/lift/schemas/README.md @@ -0,0 +1,25 @@ +# Schemas + +This directory contains hand-authored JSON schemas for crate contracts. + +Current shipped schema sets are: + +- `schemas/kernel/primitives.v1.json` for shared kernel primitives +- `schemas/pack/common.v1.json` for pack-local shared definitions +- `schemas/pack/profile.v1.json` for profile documents +- `schemas/repo/snapshot_manifest.v1.json` for deterministic repo snapshot fixture manifests +- `schemas/repo/diff_manifest.v1.json` for deterministic repo pure-diff fixture manifests +- `schemas/pack/boundary_taxonomy.v1.json` for boundary taxonomy packs +- `schemas/pack/component_map.v1.json` for component map packs +- `schemas/pack/score_model.v1.json` for structural score-model packs +- `schemas/pack/query_pack.v1.json` for structural query packs +- `schemas/pack/rule_pack.v1.json` for structural rule packs +- `schemas/pack/recipe_pack.v1.json` for structural recipe packs + +The landed pack compiler embeds the pack schemas through `src/pack/schema.rs`, following the same embed-and-access pattern used by `src/kernel/schema.rs`. The internal repo seam uses that same pattern through `src/repo/schema.rs` for the snapshot and diff manifest contracts. Compilation and schema access use those embedded contracts and do not load schema files from disk at runtime. + +Phase A established two schema foundations: the common/profile pack contracts and the internal repo snapshot-manifest contract for filesystem-first immutable snapshot cases. The landed repo Phase B surface adds the internal diff-manifest contract for pure snapshot diffs, while the pack compiler Phase B surface added boundary taxonomy and component map topology schemas. Phase C extends the pack surface with score models, query packs, rule packs, and recipe packs, plus deterministic profile bundle resolution into a crate-internal `CompiledPackSet`. + +The repo schema is intentionally narrow: it describes worktree snapshot manifests plus pure diff manifests over already-materialized snapshots, with recorded options, file inventory metadata, aggregate fingerprints, and snapshot stats for deterministic fixture or golden use. It does not imply git-backed snapshot sources, rename detection, public serialization APIs, or runtime wiring have landed. + +The advanced schemas remain structural only in this seam. They do not imply score execution, tree-sitter query execution, rule runtime, recipe application, broader repo analysis, or app/runtime bootstrap. diff --git a/crates/lift/schemas/kernel/primitives.v1.json b/crates/lift/schemas/kernel/primitives.v1.json new file mode 100644 index 000000000..73813033c --- /dev/null +++ b/crates/lift/schemas/kernel/primitives.v1.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/kernel/primitives.v1.json", + "title": "Lift Kernel Primitives v1", + "$defs": { + "repo_path": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!.*\\\\)(?!.*//)(?!.*(?:^|/)\\.{1,2}(?:/|$))(?!.*/$).+$", + "description": "UTF-8 repo-root-relative POSIX path." + }, + "json_pointer": { + "type": "string", + "description": "RFC 6901 JSON Pointer. Empty string denotes root." + }, + "stable_id": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*:sha256:[0-9a-f]{64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "byte_span": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_byte", + "end_byte" + ], + "properties": { + "start_byte": { + "type": "integer", + "minimum": 0 + }, + "end_byte": { + "type": "integer", + "minimum": 0 + } + } + }, + "locator": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "$ref": "#/$defs/repo_path" + }, + "span": { + "$ref": "#/$defs/byte_span" + }, + "json_pointer": { + "$ref": "#/$defs/json_pointer" + } + } + }, + "diagnostic_code": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(\\.[a-z][a-z0-9_]*)+$" + }, + "severity": { + "type": "string", + "enum": [ + "error", + "warning", + "info" + ] + }, + "related_location": { + "type": "object", + "additionalProperties": false, + "required": [ + "locator", + "message" + ], + "properties": { + "locator": { + "$ref": "#/$defs/locator" + }, + "message": { + "type": "string", + "minLength": 1 + } + } + }, + "diagnostic": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "severity", + "message" + ], + "properties": { + "code": { + "$ref": "#/$defs/diagnostic_code" + }, + "severity": { + "$ref": "#/$defs/severity" + }, + "message": { + "type": "string", + "minLength": 1 + }, + "subject": { + "$ref": "#/$defs/locator" + }, + "related": { + "type": "array", + "items": { + "$ref": "#/$defs/related_location" + } + }, + "help": { + "type": "string", + "minLength": 1 + } + } + } + } +} diff --git a/crates/lift/schemas/lang/parse_manifest.v1.json b/crates/lift/schemas/lang/parse_manifest.v1.json new file mode 100644 index 000000000..6edac8e49 --- /dev/null +++ b/crates/lift/schemas/lang/parse_manifest.v1.json @@ -0,0 +1,631 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/lang/parse_manifest.v1.json", + "title": "Lift Language Parse Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "case", + "snapshot_fingerprint", + "request_fingerprint", + "request", + "units", + "failed", + "skipped", + "missing_languages", + "diagnostics", + "stats" + ], + "properties": { + "version": { + "const": 1 + }, + "case": { + "type": "string", + "minLength": 1 + }, + "snapshot_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "request_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "request": { + "$ref": "#/$defs/request" + }, + "units": { + "type": "array", + "items": { + "$ref": "#/$defs/parsed_unit" + } + }, + "failed": { + "type": "array", + "items": { + "$ref": "#/$defs/failed_parse" + } + }, + "skipped": { + "type": "array", + "items": { + "$ref": "#/$defs/skipped_parse" + } + }, + "missing_languages": { + "type": "array", + "items": { + "$ref": "#/$defs/missing_requested_language" + } + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" + } + }, + "stats": { + "$ref": "#/$defs/stats" + } + }, + "$defs": { + "language_id": { + "type": "string", + "enum": [ + "json", + "toml", + "yaml", + "rust", + "python", + "javascript", + "typescript" + ] + }, + "adapter_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(\\.[a-z][a-z0-9_]*)+$" + }, + "symbol_kind": { + "type": "string", + "enum": [ + "module", + "namespace", + "package", + "function", + "method", + "class", + "struct", + "enum", + "trait", + "interface", + "type_alias", + "field", + "constant", + "variable", + "test_case", + "test_suite", + "config_key", + "unknown" + ] + }, + "symbol_visibility": { + "type": "string", + "enum": [ + "public", + "private", + "internal", + "unknown" + ] + }, + "local_symbol": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "kind", + "name", + "path", + "span", + "visibility" + ], + "properties": { + "id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "kind": { + "$ref": "#/$defs/symbol_kind" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + }, + "span": { + "$ref": "../kernel/primitives.v1.json#/$defs/byte_span" + }, + "visibility": { + "$ref": "#/$defs/symbol_visibility" + } + } + }, + "edge_endpoint": { + "oneOf": [ + { + "type": "string", + "const": "FileRoot" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "Symbol" + ], + "properties": { + "Symbol": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + } + } + } + ] + }, + "reference_target": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "LocalSymbol" + ], + "properties": { + "LocalSymbol": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "QualifiedName" + ], + "properties": { + "QualifiedName": { + "type": "object", + "additionalProperties": false, + "required": [ + "parts" + ], + "properties": { + "parts": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "FilePath" + ], + "properties": { + "FilePath": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "ExternalPackage" + ], + "properties": { + "ExternalPackage": { + "type": "object", + "additionalProperties": false, + "required": [ + "package", + "symbol" + ], + "properties": { + "package": { + "type": "string" + }, + "symbol": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "Opaque" + ], + "properties": { + "Opaque": { + "type": "object", + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + } + } + } + ] + }, + "local_edge_kind": { + "type": "string", + "enum": [ + "contains", + "import", + "export", + "call", + "type_ref", + "inherit", + "implement", + "test_ref", + "config_ref", + "schema_ref", + "unknown" + ] + }, + "local_edge": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "source", + "target" + ], + "properties": { + "kind": { + "$ref": "#/$defs/local_edge_kind" + }, + "source": { + "$ref": "#/$defs/edge_endpoint" + }, + "target": { + "$ref": "#/$defs/reference_target" + }, + "span": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/byte_span" + }, + { + "type": "null" + } + ] + } + } + }, + "surface_marker_kind": { + "type": "string", + "enum": [ + "public_api", + "test", + "entry_point", + "export" + ] + }, + "surface_marker": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "symbol", + "span", + "label" + ], + "properties": { + "kind": { + "$ref": "#/$defs/surface_marker_kind" + }, + "symbol": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + { + "type": "null" + } + ] + }, + "span": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/byte_span" + }, + { + "type": "null" + } + ] + }, + "label": { + "type": [ + "string", + "null" + ] + } + } + }, + "parsed_unit": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "file_id", + "blob_fingerprint", + "language", + "adapter", + "adapter_version", + "unit_fingerprint", + "symbols", + "edges", + "surface_markers", + "diagnostics" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "language": { + "$ref": "#/$defs/language_id" + }, + "adapter": { + "$ref": "#/$defs/adapter_name" + }, + "adapter_version": { + "type": "string" + }, + "unit_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "symbols": { + "type": "array", + "items": { + "$ref": "#/$defs/local_symbol" + } + }, + "edges": { + "type": "array", + "items": { + "$ref": "#/$defs/local_edge" + } + }, + "surface_markers": { + "type": "array", + "items": { + "$ref": "#/$defs/surface_marker" + } + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" + } + } + } + }, + "failed_parse": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "file_id", + "blob_fingerprint", + "language", + "adapter", + "adapter_version", + "diagnostics" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "language": { + "$ref": "#/$defs/language_id" + }, + "adapter": { + "$ref": "#/$defs/adapter_name" + }, + "adapter_version": { + "type": "string" + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" + } + } + } + }, + "skipped_reason": { + "type": "string", + "enum": [ + "no_matching_adapter", + "path_not_in_snapshot" + ] + }, + "skipped_parse": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "reason" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "reason": { + "$ref": "#/$defs/skipped_reason" + }, + "detail": { + "type": "string" + } + } + }, + "missing_requested_language": { + "type": "object", + "additionalProperties": false, + "required": [ + "language" + ], + "properties": { + "language": { + "$ref": "#/$defs/language_id" + }, + "detail": { + "type": "string" + } + } + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": [ + "languages", + "scope" + ], + "properties": { + "languages": { + "type": "array", + "items": { + "$ref": "#/$defs/language_id" + }, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + } + } + }, + "scope": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "snapshot" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "paths" + ], + "properties": { + "kind": { + "const": "paths" + }, + "paths": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "uniqueItems": true + } + } + } + ] + }, + "stats": { + "type": "object", + "additionalProperties": false, + "required": [ + "considered_files", + "parsed_units", + "failed_units", + "skipped_no_adapter", + "skipped_missing_paths", + "missing_requested_languages", + "diagnostic_count" + ], + "properties": { + "considered_files": { + "type": "integer", + "minimum": 0 + }, + "parsed_units": { + "type": "integer", + "minimum": 0 + }, + "failed_units": { + "type": "integer", + "minimum": 0 + }, + "skipped_no_adapter": { + "type": "integer", + "minimum": 0 + }, + "skipped_missing_paths": { + "type": "integer", + "minimum": 0 + }, + "missing_requested_languages": { + "type": "integer", + "minimum": 0 + }, + "diagnostic_count": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/crates/lift/schemas/lang/parse_manifest.v2.json b/crates/lift/schemas/lang/parse_manifest.v2.json new file mode 100644 index 000000000..0d81c2aec --- /dev/null +++ b/crates/lift/schemas/lang/parse_manifest.v2.json @@ -0,0 +1,641 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/lang/parse_manifest.v2.json", + "title": "Lift Language Parse Manifest v2", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "case", + "snapshot_fingerprint", + "request_fingerprint", + "request", + "units", + "failed", + "skipped", + "missing_languages", + "diagnostics", + "stats" + ], + "properties": { + "version": { + "const": 2 + }, + "case": { + "type": "string", + "minLength": 1 + }, + "snapshot_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "request_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "request": { + "$ref": "#/$defs/request" + }, + "units": { + "type": "array", + "items": { + "$ref": "#/$defs/parsed_unit" + } + }, + "failed": { + "type": "array", + "items": { + "$ref": "#/$defs/failed_parse" + } + }, + "skipped": { + "type": "array", + "items": { + "$ref": "#/$defs/skipped_parse" + } + }, + "missing_languages": { + "type": "array", + "items": { + "$ref": "#/$defs/missing_requested_language" + } + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" + } + }, + "stats": { + "$ref": "#/$defs/stats" + } + }, + "$defs": { + "language_id": { + "type": "string", + "enum": [ + "json", + "toml", + "yaml", + "rust", + "python", + "javascript", + "typescript" + ] + }, + "adapter_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(\\.[a-z][a-z0-9_]*)+$" + }, + "symbol_kind": { + "type": "string", + "enum": [ + "module", + "namespace", + "package", + "function", + "method", + "class", + "struct", + "enum", + "trait", + "interface", + "type_alias", + "field", + "constant", + "variable", + "test_case", + "test_suite", + "config_key", + "unknown" + ] + }, + "symbol_visibility": { + "type": "string", + "enum": [ + "public", + "private", + "internal", + "unknown" + ] + }, + "local_symbol": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "kind", + "name", + "path", + "span", + "visibility" + ], + "properties": { + "id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "kind": { + "$ref": "#/$defs/symbol_kind" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + }, + "span": { + "$ref": "../kernel/primitives.v1.json#/$defs/byte_span" + }, + "visibility": { + "$ref": "#/$defs/symbol_visibility" + } + } + }, + "edge_endpoint": { + "oneOf": [ + { + "type": "string", + "const": "FileRoot" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "Symbol" + ], + "properties": { + "Symbol": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + } + } + } + ] + }, + "reference_target": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "LocalSymbol" + ], + "properties": { + "LocalSymbol": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "QualifiedName" + ], + "properties": { + "QualifiedName": { + "type": "object", + "additionalProperties": false, + "required": [ + "parts" + ], + "properties": { + "parts": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "FilePath" + ], + "properties": { + "FilePath": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "ExternalPackage" + ], + "properties": { + "ExternalPackage": { + "type": "object", + "additionalProperties": false, + "required": [ + "package", + "symbol" + ], + "properties": { + "package": { + "type": "string" + }, + "symbol": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "Opaque" + ], + "properties": { + "Opaque": { + "type": "object", + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "value": { + "type": "string" + } + } + } + } + } + ] + }, + "local_edge_kind": { + "type": "string", + "enum": [ + "contains", + "import", + "export", + "call", + "type_ref", + "inherit", + "implement", + "test_ref", + "config_ref", + "schema_ref", + "unknown" + ] + }, + "local_edge": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "source", + "target" + ], + "properties": { + "kind": { + "$ref": "#/$defs/local_edge_kind" + }, + "source": { + "$ref": "#/$defs/edge_endpoint" + }, + "target": { + "$ref": "#/$defs/reference_target" + }, + "span": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/byte_span" + }, + { + "type": "null" + } + ] + } + } + }, + "surface_marker_kind": { + "type": "string", + "enum": [ + "public_api", + "test", + "entry_point", + "export" + ] + }, + "surface_marker": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "symbol", + "span", + "label" + ], + "properties": { + "kind": { + "$ref": "#/$defs/surface_marker_kind" + }, + "symbol": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + { + "type": "null" + } + ] + }, + "span": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/byte_span" + }, + { + "type": "null" + } + ] + }, + "label": { + "type": [ + "string", + "null" + ] + } + } + }, + "parsed_unit": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "file_id", + "blob_fingerprint", + "language", + "adapter", + "adapter_version", + "unit_fingerprint", + "symbols", + "edges", + "surface_markers", + "diagnostics" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "language": { + "$ref": "#/$defs/language_id" + }, + "adapter": { + "$ref": "#/$defs/adapter_name" + }, + "adapter_version": { + "type": "string" + }, + "unit_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "symbols": { + "type": "array", + "items": { + "$ref": "#/$defs/local_symbol" + } + }, + "edges": { + "type": "array", + "items": { + "$ref": "#/$defs/local_edge" + } + }, + "surface_markers": { + "type": "array", + "items": { + "$ref": "#/$defs/surface_marker" + } + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" + } + } + } + }, + "failed_parse": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "file_id", + "blob_fingerprint", + "language", + "adapter", + "adapter_version", + "diagnostics" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "language": { + "$ref": "#/$defs/language_id" + }, + "adapter": { + "$ref": "#/$defs/adapter_name" + }, + "adapter_version": { + "type": "string" + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/diagnostic" + } + } + } + }, + "skipped_reason": { + "type": "string", + "enum": [ + "no_matching_adapter", + "path_not_in_snapshot" + ] + }, + "skipped_parse": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "reason" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "reason": { + "$ref": "#/$defs/skipped_reason" + }, + "detail": { + "type": "string" + } + } + }, + "missing_requested_language": { + "type": "object", + "additionalProperties": false, + "required": [ + "language" + ], + "properties": { + "language": { + "$ref": "#/$defs/language_id" + }, + "detail": { + "type": "string" + } + } + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": [ + "languages", + "scope" + ], + "properties": { + "languages": { + "type": "array", + "items": { + "$ref": "#/$defs/language_id" + }, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + } + } + }, + "scope": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "snapshot" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "paths" + ], + "properties": { + "kind": { + "const": "paths" + }, + "paths": { + "type": "array", + "items": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "uniqueItems": true + } + } + } + ] + }, + "stats": { + "type": "object", + "additionalProperties": false, + "required": [ + "considered_files", + "parsed_units", + "failed_units", + "skipped_no_adapter", + "skipped_missing_paths", + "missing_requested_languages", + "diagnostic_count", + "cache_hits", + "cache_misses" + ], + "properties": { + "considered_files": { + "type": "integer", + "minimum": 0 + }, + "parsed_units": { + "type": "integer", + "minimum": 0 + }, + "failed_units": { + "type": "integer", + "minimum": 0 + }, + "skipped_no_adapter": { + "type": "integer", + "minimum": 0 + }, + "skipped_missing_paths": { + "type": "integer", + "minimum": 0 + }, + "missing_requested_languages": { + "type": "integer", + "minimum": 0 + }, + "diagnostic_count": { + "type": "integer", + "minimum": 0 + }, + "cache_hits": { + "type": "integer", + "minimum": 0 + }, + "cache_misses": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/crates/lift/schemas/pack/boundary_taxonomy.v1.json b/crates/lift/schemas/pack/boundary_taxonomy.v1.json new file mode 100644 index 000000000..a845c9795 --- /dev/null +++ b/crates/lift/schemas/pack/boundary_taxonomy.v1.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/boundary_taxonomy.v1.json", + "title": "Substrate Lift Boundary Taxonomy Pack v1", + "type": "object", + "required": ["kind", "version", "id", "name", "counting", "boundaries"], + "properties": { + "kind": { + "const": "boundary_taxonomy" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "counting": { + "$ref": "#/$defs/counting" + }, + "boundaries": { + "type": "array", + "items": { + "$ref": "#/$defs/boundary" + } + } + }, + "$defs": { + "counting": { + "type": "object", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "enum": ["distinct_minus_one"] + } + }, + "additionalProperties": false + }, + "boundary": { + "type": "object", + "required": ["id", "label", "include"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "label": { + "type": "string", + "minLength": 1 + }, + "include": { + "$ref": "common.v1.json#/$defs/glob_list" + }, + "exclude": { + "$ref": "common.v1.json#/$defs/glob_list" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/common.v1.json b/crates/lift/schemas/pack/common.v1.json new file mode 100644 index 000000000..ae3e6daa9 --- /dev/null +++ b/crates/lift/schemas/pack/common.v1.json @@ -0,0 +1,117 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/common.v1.json", + "title": "Substrate Lift Pack Common Definitions v1", + "type": "object", + "$defs": { + "pack_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*(?:/[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*)*$" + }, + "pack_ref": { + "type": "string", + "pattern": "^(builtin:[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*(?:/[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*)*|file:(?:[a-z0-9_][a-z0-9._-]*)(?:/[a-z0-9_][a-z0-9._-]*)*)$" + }, + "pack_kind": { + "type": "string", + "enum": [ + "profile", + "boundary_taxonomy", + "component_map", + "score_model", + "rule_pack", + "query_pack", + "recipe_pack" + ] + }, + "language_id": { + "type": "string", + "enum": [ + "json", + "toml", + "yaml", + "rust", + "python", + "javascript", + "typescript" + ] + }, + "glob": { + "type": "string", + "minLength": 1 + }, + "glob_list": { + "type": "array", + "items": { + "$ref": "#/$defs/glob" + } + }, + "reserved_path_class": { + "type": "string", + "enum": [ + "test", + "docs", + "ci", + "migration", + "security", + "public_api", + "generated", + "vendor" + ] + }, + "severity": { + "type": "string", + "enum": ["error", "warning", "info"] + }, + "json_pointer": { + "type": "string" + }, + "query_ref": { + "type": "object", + "required": ["pack", "id"], + "properties": { + "pack": { + "$ref": "#/$defs/pack_ref" + }, + "id": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "expr": { + "type": "object", + "required": ["op"], + "properties": { + "op": { + "type": "string", + "enum": [ + "const_int", + "const_bool", + "field", + "exists", + "is_null", + "add", + "sub", + "mul", + "div", + "max", + "min", + "eq", + "ne", + "gt", + "ge", + "lt", + "le", + "and", + "or", + "not" + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/component_map.v1.json b/crates/lift/schemas/pack/component_map.v1.json new file mode 100644 index 000000000..eef2f94cb --- /dev/null +++ b/crates/lift/schemas/pack/component_map.v1.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/component_map.v1.json", + "title": "Substrate Lift Component Map Pack v1", + "type": "object", + "required": ["kind", "version", "id", "name", "counting", "components"], + "properties": { + "kind": { + "const": "component_map" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "counting": { + "$ref": "#/$defs/counting" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/$defs/component" + } + } + }, + "$defs": { + "counting": { + "type": "object", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "enum": ["distinct"] + } + }, + "additionalProperties": false + }, + "component": { + "type": "object", + "required": ["id", "label", "include"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "label": { + "type": "string", + "minLength": 1 + }, + "include": { + "$ref": "common.v1.json#/$defs/glob_list" + }, + "exclude": { + "$ref": "common.v1.json#/$defs/glob_list" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/profile.v1.json b/crates/lift/schemas/pack/profile.v1.json new file mode 100644 index 000000000..67f8b96a3 --- /dev/null +++ b/crates/lift/schemas/pack/profile.v1.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/profile.v1.json", + "title": "Substrate Lift Profile Pack v1", + "type": "object", + "required": ["kind", "version", "id", "name"], + "properties": { + "kind": { + "const": "profile" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "apps": { + "type": "object", + "properties": { + "enabled": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*$" + } + }, + "default": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*$" + } + }, + "additionalProperties": false + }, + "analysis": { + "type": "object", + "properties": { + "languages": { + "type": "array", + "items": { + "$ref": "common.v1.json#/$defs/language_id" + } + }, + "follow_symlinks": { + "type": "boolean" + }, + "max_scope_depth": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + }, + "additionalProperties": false + }, + "topology": { + "type": "object", + "properties": { + "boundary_taxonomy": { + "$ref": "common.v1.json#/$defs/pack_ref" + }, + "component_map": { + "$ref": "common.v1.json#/$defs/pack_ref" + } + }, + "additionalProperties": false + }, + "score": { + "type": "object", + "properties": { + "model": { + "$ref": "common.v1.json#/$defs/pack_ref" + } + }, + "additionalProperties": false + }, + "rules": { + "$ref": "#/$defs/include_section" + }, + "queries": { + "$ref": "#/$defs/include_section" + }, + "recipes": { + "$ref": "#/$defs/include_section" + } + }, + "$defs": { + "include_section": { + "type": "object", + "properties": { + "packs": { + "type": "array", + "items": { + "$ref": "common.v1.json#/$defs/pack_ref" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/query_pack.v1.json b/crates/lift/schemas/pack/query_pack.v1.json new file mode 100644 index 000000000..9ca74579e --- /dev/null +++ b/crates/lift/schemas/pack/query_pack.v1.json @@ -0,0 +1,85 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "title": "Substrate Lift Query Pack v1", + "type": "object", + "required": ["$schema", "kind", "version", "id", "name", "language", "engine", "queries"], + "properties": { + "$schema": { + "const": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json" + }, + "kind": { + "const": "query_pack" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "language": { + "$ref": "common.v1.json#/$defs/language_id" + }, + "engine": { + "$ref": "#/$defs/query_engine" + }, + "queries": { + "type": "array", + "items": { + "$ref": "#/$defs/query" + } + } + }, + "$defs": { + "query_engine": { + "type": "string", + "enum": ["tree_sitter"] + }, + "query": { + "type": "object", + "required": ["id", "pattern", "captures"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "summary": { + "type": "string" + }, + "pattern": { + "type": "string", + "minLength": 1 + }, + "captures": { + "type": "array", + "items": { + "$ref": "#/$defs/capture" + } + } + }, + "additionalProperties": false + }, + "capture": { + "type": "object", + "required": ["name", "required"], + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "required": { + "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/recipe_pack.v1.json b/crates/lift/schemas/pack/recipe_pack.v1.json new file mode 100644 index 000000000..567557678 --- /dev/null +++ b/crates/lift/schemas/pack/recipe_pack.v1.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json", + "title": "Substrate Lift Recipe Pack v1", + "type": "object", + "required": ["$schema", "kind", "version", "id", "name", "recipes"], + "properties": { + "$schema": { + "const": "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json" + }, + "kind": { + "const": "recipe_pack" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "recipes": { + "type": "array", + "items": { + "$ref": "#/$defs/recipe" + } + } + }, + "$defs": { + "recipe": { + "type": "object", + "required": ["id", "query", "transforms"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "summary": { + "type": "string" + }, + "query": { + "$ref": "common.v1.json#/$defs/query_ref" + }, + "transforms": { + "type": "array", + "items": { + "$ref": "#/$defs/transform" + } + } + }, + "additionalProperties": false + }, + "transform": { + "oneOf": [ + { + "$ref": "#/$defs/replace_capture_text_transform" + } + ] + }, + "replace_capture_text_transform": { + "type": "object", + "required": ["op", "capture", "text"], + "properties": { + "op": { + "const": "replace_capture_text" + }, + "capture": { + "type": "string", + "minLength": 1 + }, + "text": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/rule_pack.v1.json b/crates/lift/schemas/pack/rule_pack.v1.json new file mode 100644 index 000000000..e4a0b5772 --- /dev/null +++ b/crates/lift/schemas/pack/rule_pack.v1.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "title": "Substrate Lift Rule Pack v1", + "type": "object", + "required": ["$schema", "kind", "version", "id", "name", "rules"], + "properties": { + "$schema": { + "const": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json" + }, + "kind": { + "const": "rule_pack" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/rule" + } + } + }, + "$defs": { + "rule": { + "type": "object", + "required": ["id", "severity", "query", "emit"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "summary": { + "type": "string" + }, + "severity": { + "$ref": "common.v1.json#/$defs/severity" + }, + "scope": { + "$ref": "#/$defs/rule_scope" + }, + "query": { + "$ref": "common.v1.json#/$defs/query_ref" + }, + "emit": { + "type": "array", + "items": { + "$ref": "#/$defs/emit_action" + } + } + }, + "additionalProperties": false + }, + "rule_scope": { + "type": "object", + "properties": { + "languages": { + "type": "array", + "items": { + "$ref": "common.v1.json#/$defs/language_id" + } + }, + "path_classes": { + "type": "array", + "items": { + "$ref": "common.v1.json#/$defs/reserved_path_class" + } + } + }, + "additionalProperties": false + }, + "emit_action": { + "oneOf": [ + { + "$ref": "#/$defs/finding_emit_action" + } + ] + }, + "finding_emit_action": { + "type": "object", + "required": ["kind", "code", "message"], + "properties": { + "kind": { + "const": "finding" + }, + "code": { + "type": "string", + "minLength": 1 + }, + "message": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/pack/score_model.v1.json b/crates/lift/schemas/pack/score_model.v1.json new file mode 100644 index 000000000..e251fcdb4 --- /dev/null +++ b/crates/lift/schemas/pack/score_model.v1.json @@ -0,0 +1,138 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/pack/score_model.v1.json", + "title": "Substrate Lift Score Model Pack v1", + "type": "object", + "required": [ + "$schema", + "kind", + "version", + "id", + "name", + "vector_version", + "lift_score", + "estimated_slices", + "triggers", + "confidence", + "missing_input_rules" + ], + "properties": { + "$schema": { + "const": "https://schemas.substrate.dev/lift/pack/score_model.v1.json" + }, + "kind": { + "const": "score_model" + }, + "version": { + "const": 1 + }, + "id": { + "$ref": "common.v1.json#/$defs/pack_name" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "vector_version": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + }, + "lift_score": { + "$ref": "common.v1.json#/$defs/expr" + }, + "estimated_slices": { + "$ref": "common.v1.json#/$defs/expr" + }, + "triggers": { + "type": "array", + "items": { + "$ref": "#/$defs/trigger_rule" + } + }, + "confidence": { + "$ref": "#/$defs/confidence_model" + }, + "missing_input_rules": { + "type": "array", + "items": { + "$ref": "#/$defs/missing_input_rule" + } + } + }, + "$defs": { + "trigger_rule": { + "type": "object", + "required": ["id", "when"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "when": { + "$ref": "common.v1.json#/$defs/expr" + } + }, + "additionalProperties": false + }, + "confidence_model": { + "type": "object", + "required": ["default", "rules"], + "properties": { + "default": { + "type": "string", + "minLength": 1 + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/confidence_rule" + } + } + }, + "additionalProperties": false + }, + "confidence_rule": { + "type": "object", + "required": ["id", "when", "set", "causes"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "when": { + "$ref": "common.v1.json#/$defs/expr" + }, + "set": { + "type": "string", + "minLength": 1 + }, + "causes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "additionalProperties": false + }, + "missing_input_rule": { + "type": "object", + "required": ["field", "when"], + "properties": { + "field": { + "$ref": "common.v1.json#/$defs/json_pointer" + }, + "when": { + "$ref": "common.v1.json#/$defs/expr" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/crates/lift/schemas/repo/diff_manifest.v1.json b/crates/lift/schemas/repo/diff_manifest.v1.json new file mode 100644 index 000000000..2ad14b8cf --- /dev/null +++ b/crates/lift/schemas/repo/diff_manifest.v1.json @@ -0,0 +1,121 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/repo/diff_manifest.v1.json", + "title": "Lift Repo Diff Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "case", + "base_fingerprint", + "head_fingerprint", + "entries", + "diff_fingerprint" + ], + "properties": { + "version": { + "const": 1 + }, + "case": { + "type": "string", + "minLength": 1 + }, + "base_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "head_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/$defs/diff_entry" + } + }, + "diff_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + } + }, + "$defs": { + "diff_entry": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "kind", + "before_blob_fingerprint", + "after_blob_fingerprint" + ], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "kind": { + "type": "string", + "enum": ["added", "modified", "removed"] + }, + "before_blob_fingerprint": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + { + "type": "null" + } + ] + }, + "after_blob_fingerprint": { + "anyOf": [ + { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + { + "type": "null" + } + ] + } + }, + "oneOf": [ + { + "properties": { + "kind": { + "const": "added" + }, + "before_blob_fingerprint": { + "type": "null" + }, + "after_blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + } + } + }, + { + "properties": { + "kind": { + "const": "modified" + }, + "before_blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "after_blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + } + } + }, + { + "properties": { + "kind": { + "const": "removed" + }, + "before_blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "after_blob_fingerprint": { + "type": "null" + } + } + } + ] + } + } +} diff --git a/crates/lift/schemas/repo/snapshot_manifest.v1.json b/crates/lift/schemas/repo/snapshot_manifest.v1.json new file mode 100644 index 000000000..b9bf3a3a4 --- /dev/null +++ b/crates/lift/schemas/repo/snapshot_manifest.v1.json @@ -0,0 +1,186 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.substrate.dev/lift/repo/snapshot_manifest.v1.json", + "title": "Lift Repo Snapshot Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "case", + "source_kind", + "options", + "files", + "snapshot_fingerprint", + "stats" + ], + "properties": { + "version": { + "const": 1 + }, + "case": { + "type": "string", + "minLength": 1 + }, + "source_kind": { + "type": "string", + "enum": ["worktree", "git_rev"] + }, + "source_rev": { + "type": "string", + "minLength": 1 + }, + "options": { + "type": "object", + "additionalProperties": false, + "required": [ + "symlink_policy", + "well_known_excludes", + "exclude_globs", + "non_utf8_path_policy", + "max_file_bytes", + "large_file_policy" + ], + "properties": { + "symlink_policy": { + "type": "string", + "enum": ["skip", "follow"] + }, + "well_known_excludes": { + "type": "array", + "items": { + "$ref": "#/$defs/well_known_exclude" + }, + "uniqueItems": true + }, + "exclude_globs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "non_utf8_path_policy": { + "type": "string", + "enum": ["error", "skip"] + }, + "max_file_bytes": { + "type": ["integer", "null"], + "minimum": 0 + }, + "large_file_policy": { + "type": "string", + "enum": ["error", "skip"] + } + } + }, + "files": { + "type": "array", + "items": { + "$ref": "#/$defs/file_record" + } + }, + "snapshot_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "stats": { + "$ref": "#/$defs/stats" + } + }, + "allOf": [ + { + "if": { + "properties": { + "source_kind": { + "const": "git_rev" + } + } + }, + "then": { + "required": ["source_rev"] + }, + "else": { + "not": { + "required": ["source_rev"] + } + } + } + ], + "$defs": { + "well_known_exclude": { + "type": "string", + "enum": [ + "rust_target", + "node_modules", + "python_hidden_venv", + "python_venv", + "python_pycache", + "web_dist", + "web_build" + ] + }, + "file_record": { + "type": "object", + "additionalProperties": false, + "required": ["path", "file_id", "blob_fingerprint", "size_bytes"], + "properties": { + "path": { + "$ref": "../kernel/primitives.v1.json#/$defs/repo_path" + }, + "file_id": { + "$ref": "../kernel/primitives.v1.json#/$defs/stable_id" + }, + "blob_fingerprint": { + "$ref": "../kernel/primitives.v1.json#/$defs/fingerprint" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + } + } + }, + "stats": { + "type": "object", + "additionalProperties": false, + "required": [ + "file_count", + "total_bytes", + "skipped_by_ignore", + "skipped_symlinks", + "skipped_non_utf8_paths", + "skipped_large_files", + "skipped_unsupported_file_kinds" + ], + "properties": { + "file_count": { + "type": "integer", + "minimum": 0 + }, + "total_bytes": { + "type": "integer", + "minimum": 0 + }, + "skipped_by_ignore": { + "type": "integer", + "minimum": 0 + }, + "skipped_symlinks": { + "type": "integer", + "minimum": 0 + }, + "skipped_non_utf8_paths": { + "type": "integer", + "minimum": 0 + }, + "skipped_large_files": { + "type": "integer", + "minimum": 0 + }, + "skipped_unsupported_file_kinds": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/crates/lift/src/app/context/mod.rs b/crates/lift/src/app/context/mod.rs new file mode 100644 index 000000000..12ce06572 --- /dev/null +++ b/crates/lift/src/app/context/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future context app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/contract/mod.rs b/crates/lift/src/app/contract/mod.rs new file mode 100644 index 000000000..36768812b --- /dev/null +++ b/crates/lift/src/app/contract/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future contract app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/impact/mod.rs b/crates/lift/src/app/impact/mod.rs new file mode 100644 index 000000000..96c9966c3 --- /dev/null +++ b/crates/lift/src/app/impact/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future impact app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/index/mod.rs b/crates/lift/src/app/index/mod.rs new file mode 100644 index 000000000..7e9a4e3b2 --- /dev/null +++ b/crates/lift/src/app/index/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future index app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/mod.rs b/crates/lift/src/app/mod.rs new file mode 100644 index 000000000..f863c9747 --- /dev/null +++ b/crates/lift/src/app/mod.rs @@ -0,0 +1,11 @@ +//! App seams plus the Phase D bootstrap boundary for pack activation. + +pub(crate) mod context; +pub(crate) mod contract; +pub(crate) mod impact; +pub(crate) mod index; +pub(crate) mod policy; +pub(crate) mod query_app; +pub(crate) mod rewrite; +pub(crate) mod runtime; +pub(crate) mod score; diff --git a/crates/lift/src/app/policy/mod.rs b/crates/lift/src/app/policy/mod.rs new file mode 100644 index 000000000..922f75084 --- /dev/null +++ b/crates/lift/src/app/policy/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future policy app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/query_app/mod.rs b/crates/lift/src/app/query_app/mod.rs new file mode 100644 index 000000000..82871cade --- /dev/null +++ b/crates/lift/src/app/query_app/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future query app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/rewrite/mod.rs b/crates/lift/src/app/rewrite/mod.rs new file mode 100644 index 000000000..ce8e4807d --- /dev/null +++ b/crates/lift/src/app/rewrite/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future rewrite app seam. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/app/runtime.rs b/crates/lift/src/app/runtime.rs new file mode 100644 index 000000000..fb0ab5759 --- /dev/null +++ b/crates/lift/src/app/runtime.rs @@ -0,0 +1,24 @@ +//! Phase D runtime bootstrap boundary. + +use crate::pack::{CompiledPackSet, PackCompiler, PackResult, PackSource}; + +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub(crate) struct ProfileBootstrap { + pub bundle: CompiledPackSet, +} + +impl ProfileBootstrap { + #[allow(dead_code)] + pub(crate) fn from_pack_set(bundle: CompiledPackSet) -> Self { + Self { bundle } + } +} + +#[allow(dead_code)] +pub(crate) fn bootstrap_profile(source: PackSource) -> PackResult { + let compiler = PackCompiler::new(); + let profile = compiler.compile_profile(source)?; + let bundle = compiler.resolve_profile_pack_set(&profile)?; + Ok(ProfileBootstrap::from_pack_set(bundle)) +} diff --git a/crates/lift/src/app/score/compat_v1.rs b/crates/lift/src/app/score/compat_v1.rs new file mode 100644 index 000000000..d4b9e77fb --- /dev/null +++ b/crates/lift/src/app/score/compat_v1.rs @@ -0,0 +1,175 @@ +//! Legacy Work Lift v1 dotted-path compatibility helpers. +//! +//! This module intentionally quarantines dotted field path logic at the +//! score-app edge. The engine remains JSON-Pointer-native. + +#![allow(dead_code)] + +use crate::kernel::{JsonPointer, KernelError, KernelResult}; + +/// Converts a dotted Lift v1 field path into a canonical JSON Pointer. +pub(crate) fn dotted_to_json_pointer(input: &str) -> KernelResult { + if input.is_empty() { + return Err(invalid_pointer(input)); + } + + let mut pointer = JsonPointer::root(); + for segment in input.split('.') { + if segment.is_empty() || !is_v1_segment(segment) { + return Err(invalid_pointer(input)); + } + pointer = pointer.push_token(segment); + } + + Ok(pointer) +} + +/// Converts a canonical JSON Pointer into a dotted Lift v1 field path. +pub(crate) fn json_pointer_to_dotted(pointer: &JsonPointer) -> KernelResult { + let raw = pointer.as_str(); + if raw.is_empty() { + return Err(invalid_pointer(raw)); + } + + let mut segments = Vec::new(); + for token in raw[1..].split('/') { + if token.is_empty() { + return Err(invalid_pointer(raw)); + } + + let segment = unescape_token(token, raw)?; + if !is_v1_segment(&segment) { + return Err(invalid_pointer(raw)); + } + segments.push(segment); + } + + Ok(segments.join(".")) +} + +fn is_v1_segment(segment: &str) -> bool { + let mut chars = segment.chars(); + let Some(first) = chars.next() else { + return false; + }; + + if !(first.is_ascii_lowercase() || first == '_') { + return false; + } + + chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} + +fn unescape_token(token: &str, raw: &str) -> KernelResult { + let mut result = String::with_capacity(token.len()); + let mut chars = token.chars(); + + while let Some(ch) = chars.next() { + if ch != '~' { + result.push(ch); + continue; + } + + match chars.next() { + Some('0') => result.push('~'), + Some('1') => result.push('/'), + _ => return Err(invalid_pointer(raw)), + } + } + + Ok(result) +} + +fn invalid_pointer(input: &str) -> KernelError { + KernelError::InvalidJsonPointer { + input: input.to_owned(), + } +} + +#[cfg(all(test, feature = "compat-v1"))] +mod tests { + use super::{dotted_to_json_pointer, json_pointer_to_dotted}; + use crate::kernel::JsonPointer; + + #[test] + fn dotted_examples_convert_to_pointer_form() { + assert_eq!( + dotted_to_json_pointer("touch.crates_touched") + .expect("pointer") + .as_str(), + "/touch/crates_touched" + ); + assert_eq!( + dotted_to_json_pointer("contract.behavior_deltas") + .expect("pointer") + .as_str(), + "/contract/behavior_deltas" + ); + assert_eq!( + dotted_to_json_pointer("risk.unknowns_high") + .expect("pointer") + .as_str(), + "/risk/unknowns_high" + ); + } + + #[test] + fn dotted_paths_reject_invalid_segments() { + assert!(dotted_to_json_pointer("").is_err()); + assert!(dotted_to_json_pointer("touch..crates_touched").is_err()); + assert!(dotted_to_json_pointer(".touch").is_err()); + assert!(dotted_to_json_pointer("missing-inputs.value").is_err()); + assert!(dotted_to_json_pointer("0leading.segment").is_err()); + } + + #[test] + fn pointer_examples_convert_to_dotted_form() { + assert_eq!( + json_pointer_to_dotted(&JsonPointer::parse("/touch/crates_touched").expect("pointer")) + .expect("dotted"), + "touch.crates_touched" + ); + assert_eq!( + json_pointer_to_dotted( + &JsonPointer::parse("/contract/behavior_deltas").expect("pointer") + ) + .expect("dotted"), + "contract.behavior_deltas" + ); + assert_eq!( + json_pointer_to_dotted(&JsonPointer::parse("/risk/unknowns_high").expect("pointer")) + .expect("dotted"), + "risk.unknowns_high" + ); + } + + #[test] + fn pointers_reject_root_and_non_identifier_tokens() { + assert!(json_pointer_to_dotted(&JsonPointer::root()).is_err()); + assert!(json_pointer_to_dotted( + &JsonPointer::parse("/touch//crates_touched").expect("pointer") + ) + .is_err()); + assert!(JsonPointer::parse("/touch/~2bad").is_err()); + assert!( + json_pointer_to_dotted(&JsonPointer::parse("/touch/has~1slash").expect("pointer")) + .is_err() + ); + assert!( + json_pointer_to_dotted(&JsonPointer::parse("/touch/has~0tilde").expect("pointer")) + .is_err() + ); + } + + #[test] + fn examples_round_trip_cleanly() { + for dotted in [ + "touch.crates_touched", + "contract.behavior_deltas", + "risk.unknowns_high", + ] { + let pointer = dotted_to_json_pointer(dotted).expect("pointer"); + assert_eq!(json_pointer_to_dotted(&pointer).expect("dotted"), dotted); + } + } +} diff --git a/crates/lift/src/app/score/mod.rs b/crates/lift/src/app/score/mod.rs new file mode 100644 index 000000000..94e1e3584 --- /dev/null +++ b/crates/lift/src/app/score/mod.rs @@ -0,0 +1,7 @@ +//! Reserved for the future score app seam. + +#[cfg(feature = "compat-v1")] +pub(crate) mod compat_v1; + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/bin/lift.rs b/crates/lift/src/bin/lift.rs new file mode 100644 index 000000000..382ae6703 --- /dev/null +++ b/crates/lift/src/bin/lift.rs @@ -0,0 +1,17 @@ +#![allow(unused_crate_dependencies)] + +fn main() { + #[cfg(feature = "cli")] + { + if let Err(err) = substrate_lift::run_cli() { + eprintln!("{err}"); + std::process::exit(1); + } + } + + #[cfg(not(feature = "cli"))] + { + eprintln!("lift was built without the 'cli' feature"); + std::process::exit(1); + } +} diff --git a/crates/lift/src/cli/args.rs b/crates/lift/src/cli/args.rs new file mode 100644 index 000000000..c4c194574 --- /dev/null +++ b/crates/lift/src/cli/args.rs @@ -0,0 +1,65 @@ +use clap::{Args, Parser, Subcommand}; + +use crate::error::LiftError; + +#[derive(Debug, Parser)] +#[command( + name = "lift", + bin_name = "lift", + version, + about = "Deterministic code-intelligence engine CLI scaffold" +)] +pub(crate) struct Cli { + #[command(subcommand)] + pub(crate) command: Option, +} + +#[derive(Debug, Subcommand)] +pub(crate) enum Command { + Score(ScoreArgs), + Impact(ImpactArgs), + Policy(PolicyArgs), + Contract(ContractArgs), + Context(ContextArgs), + Index(IndexArgs), + Query(QueryArgs), + Rewrite(RewriteArgs), +} + +#[derive(Debug, Args)] +pub(crate) struct ScoreArgs {} + +#[derive(Debug, Args)] +pub(crate) struct ImpactArgs {} + +#[derive(Debug, Args)] +pub(crate) struct PolicyArgs {} + +#[derive(Debug, Args)] +pub(crate) struct ContractArgs {} + +#[derive(Debug, Args)] +pub(crate) struct ContextArgs {} + +#[derive(Debug, Args)] +pub(crate) struct IndexArgs {} + +#[derive(Debug, Args)] +pub(crate) struct QueryArgs {} + +#[derive(Debug, Args)] +pub(crate) struct RewriteArgs {} + +pub(crate) fn dispatch(cli: Cli) -> Result<(), LiftError> { + match cli.command { + Some(Command::Score(_)) => Err(LiftError::NotImplemented("lift score")), + Some(Command::Impact(_)) => Err(LiftError::NotImplemented("lift impact")), + Some(Command::Policy(_)) => Err(LiftError::NotImplemented("lift policy")), + Some(Command::Contract(_)) => Err(LiftError::NotImplemented("lift contract")), + Some(Command::Context(_)) => Err(LiftError::NotImplemented("lift context")), + Some(Command::Index(_)) => Err(LiftError::NotImplemented("lift index")), + Some(Command::Query(_)) => Err(LiftError::NotImplemented("lift query")), + Some(Command::Rewrite(_)) => Err(LiftError::NotImplemented("lift rewrite")), + None => Err(LiftError::NotImplemented("lift")), + } +} diff --git a/crates/lift/src/cli/mod.rs b/crates/lift/src/cli/mod.rs new file mode 100644 index 000000000..1c53f20f3 --- /dev/null +++ b/crates/lift/src/cli/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod args; +mod run; + +pub use run::run; diff --git a/crates/lift/src/cli/run.rs b/crates/lift/src/cli/run.rs new file mode 100644 index 000000000..8c695251f --- /dev/null +++ b/crates/lift/src/cli/run.rs @@ -0,0 +1,8 @@ +use clap::Parser; + +use crate::error::LiftError; + +pub fn run() -> Result<(), LiftError> { + let cli = crate::cli::args::Cli::parse(); + crate::cli::args::dispatch(cli) +} diff --git a/crates/lift/src/derive/mod.rs b/crates/lift/src/derive/mod.rs new file mode 100644 index 000000000..5667f07ee --- /dev/null +++ b/crates/lift/src/derive/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future derive seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/error.rs b/crates/lift/src/error.rs new file mode 100644 index 000000000..1df35a64e --- /dev/null +++ b/crates/lift/src/error.rs @@ -0,0 +1,16 @@ +#[derive(Debug)] +pub enum LiftError { + NotImplemented(&'static str), + Cli(String), +} + +impl std::fmt::Display for LiftError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotImplemented(message) => write!(f, "not implemented: {message}"), + Self::Cli(message) => write!(f, "{message}"), + } + } +} + +impl std::error::Error for LiftError {} diff --git a/crates/lift/src/export/mod.rs b/crates/lift/src/export/mod.rs new file mode 100644 index 000000000..aebde2a36 --- /dev/null +++ b/crates/lift/src/export/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future export seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/facts/mod.rs b/crates/lift/src/facts/mod.rs new file mode 100644 index 000000000..e6bcb8d5f --- /dev/null +++ b/crates/lift/src/facts/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future facts seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/graph/mod.rs b/crates/lift/src/graph/mod.rs new file mode 100644 index 000000000..1fb5898cc --- /dev/null +++ b/crates/lift/src/graph/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future graph seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/kernel/canonical_json.rs b/crates/lift/src/kernel/canonical_json.rs new file mode 100644 index 000000000..c71efed92 --- /dev/null +++ b/crates/lift/src/kernel/canonical_json.rs @@ -0,0 +1,19 @@ +//! Canonical JSON helpers. + +use serde::Serialize; + +use super::error::{KernelError, KernelResult}; + +/// Serializes a value to an RFC 8785 canonical JSON string. +pub fn canonical_json_string(value: &T) -> KernelResult { + serde_jcs::to_string(value).map_err(|error| KernelError::CanonicalJson { + reason: error.to_string(), + }) +} + +/// Serializes a value to RFC 8785 canonical JSON bytes. +pub fn canonical_json_bytes(value: &T) -> KernelResult> { + serde_jcs::to_vec(value).map_err(|error| KernelError::CanonicalJson { + reason: error.to_string(), + }) +} diff --git a/crates/lift/src/kernel/diagnostic.rs b/crates/lift/src/kernel/diagnostic.rs new file mode 100644 index 000000000..e671c3f99 --- /dev/null +++ b/crates/lift/src/kernel/diagnostic.rs @@ -0,0 +1,280 @@ +//! Machine-stable diagnostics. + +use std::cmp::Ordering; + +use serde::{Deserialize, Serialize}; + +use crate::kernel::{KernelError, KernelResult, Locator}; + +/// Diagnostic severity. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Severity { + /// Hard validation or contract failure. + Error, + /// Recoverable issue that should be addressed. + Warning, + /// Informational note. + Info, +} + +impl Severity { + fn rank(self) -> u8 { + match self { + Self::Error => 0, + Self::Warning => 1, + Self::Info => 2, + } + } +} + +/// Stable machine-readable diagnostic code. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct DiagnosticCode(String); + +impl DiagnosticCode { + /// Parses and validates a dot-namespaced diagnostic code. + pub fn parse(input: &str) -> KernelResult { + if is_valid_diagnostic_code(input) { + Ok(Self(input.to_owned())) + } else { + Err(KernelError::SchemaViolation { + reason: format!("invalid diagnostic code: {input}"), + }) + } + } + + /// Returns the code string. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for DiagnosticCode { + type Error = KernelError; + + fn try_from(value: String) -> KernelResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: DiagnosticCode) -> Self { + value.0 + } +} + +/// Secondary location attached to a diagnostic. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct RelatedLocation { + /// The related locator. + pub locator: Locator, + /// Human-readable relationship context. + pub message: String, +} + +/// Structured diagnostic record. +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct Diagnostic { + /// Stable diagnostic code. + pub code: DiagnosticCode, + /// Diagnostic severity. + pub severity: Severity, + /// Human-readable message. + pub message: String, + /// Primary subject of the diagnostic, when available. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + /// Additional related locations. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub related: Vec, + /// Optional fix or remediation hint. + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} + +impl Ord for Diagnostic { + fn cmp(&self, other: &Self) -> Ordering { + diagnostic_sort_key(self) + .cmp(&diagnostic_sort_key(other)) + .then_with(|| self.subject.cmp(&other.subject)) + .then_with(|| self.related.cmp(&other.related)) + .then_with(|| self.help.cmp(&other.help)) + } +} + +impl PartialOrd for Diagnostic { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +fn is_valid_diagnostic_code(input: &str) -> bool { + let mut segments = input.split('.'); + let Some(first) = segments.next() else { + return false; + }; + + if !is_valid_first_segment(first) { + return false; + } + + let mut saw_additional_segment = false; + for segment in segments { + saw_additional_segment = true; + if !is_valid_tail_segment(segment) { + return false; + } + } + + saw_additional_segment +} + +fn is_valid_first_segment(segment: &str) -> bool { + let mut chars = segment.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit()) +} + +fn is_valid_tail_segment(segment: &str) -> bool { + let mut chars = segment.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} + +type DiagnosticSortKey<'a> = ( + u8, + Option<&'a str>, + Option, + Option, + &'a str, + &'a str, + Option<&'a str>, +); + +fn diagnostic_sort_key(diagnostic: &Diagnostic) -> DiagnosticSortKey<'_> { + let subject_path = diagnostic + .subject + .as_ref() + .map(|locator| locator.path.as_str()); + let subject_start = diagnostic + .subject + .as_ref() + .and_then(|locator| locator.span.map(|span| span.start_byte)); + let subject_end = diagnostic + .subject + .as_ref() + .and_then(|locator| locator.span.map(|span| span.end_byte)); + let subject_pointer = diagnostic.subject.as_ref().and_then(|locator| { + locator + .json_pointer + .as_ref() + .map(|pointer| pointer.as_str()) + }); + + ( + diagnostic.severity.rank(), + subject_path, + subject_start, + subject_end, + diagnostic.code.as_str(), + diagnostic.message.as_str(), + subject_pointer, + ) +} + +#[cfg(test)] +mod tests { + use super::{Diagnostic, DiagnosticCode, RelatedLocation, Severity}; + use crate::kernel::{ByteSpan, JsonPointer, Locator, RepoPath}; + + #[test] + fn parses_valid_diagnostic_codes() { + let code = + DiagnosticCode::parse("kernel.repo_path.invalid_absolute").expect("code should parse"); + assert_eq!(code.as_str(), "kernel.repo_path.invalid_absolute"); + } + + #[test] + fn rejects_invalid_diagnostic_codes() { + let invalid = [ + "", + "kernel", + "Kernel.repo_path.invalid", + "kernel.repo- path.invalid", + "kernel._segment.invalid", + "kernel.repo_path.Invalid", + "kernel.repo_path.", + ]; + + for candidate in invalid { + assert!( + DiagnosticCode::parse(candidate).is_err(), + "{candidate} should be rejected" + ); + } + } + + #[test] + fn diagnostics_sort_by_contract_order() { + let code = + DiagnosticCode::parse("kernel.repo_path.invalid_absolute").expect("code should parse"); + let path = RepoPath::parse("crates/lift/src/kernel/path.rs").expect("path"); + + let info = Diagnostic { + code: code.clone(), + severity: Severity::Info, + message: "info".to_owned(), + subject: None, + related: Vec::new(), + help: None, + }; + let warning = Diagnostic { + code: code.clone(), + severity: Severity::Warning, + message: "warning".to_owned(), + subject: Some(Locator { + path: path.clone(), + span: Some(ByteSpan::new(7, 8).expect("span")), + json_pointer: None, + }), + related: Vec::new(), + help: None, + }; + let error = Diagnostic { + code, + severity: Severity::Error, + message: "error".to_owned(), + subject: Some(Locator { + path, + span: Some(ByteSpan::new(1, 2).expect("span")), + json_pointer: Some(JsonPointer::parse("/touch/crates_touched").expect("pointer")), + }), + related: vec![RelatedLocation { + locator: Locator { + path: RepoPath::parse("README.md").expect("path"), + span: None, + json_pointer: None, + }, + message: "context".to_owned(), + }], + help: Some("fix it".to_owned()), + }; + + let mut diagnostics = [info, warning, error.clone()]; + diagnostics.sort(); + + assert_eq!(diagnostics[0], error); + assert_eq!(diagnostics[1].severity, Severity::Warning); + assert_eq!(diagnostics[2].severity, Severity::Info); + } + + #[test] + fn serde_round_trip_uses_validation() { + let parsed: DiagnosticCode = + serde_json::from_str("\"kernel.repo_path.invalid_absolute\"").expect("code"); + assert_eq!(parsed.as_str(), "kernel.repo_path.invalid_absolute"); + assert!(serde_json::from_str::("\"kernel\"").is_err()); + } +} diff --git a/crates/lift/src/kernel/error.rs b/crates/lift/src/kernel/error.rs new file mode 100644 index 000000000..1fe37284c --- /dev/null +++ b/crates/lift/src/kernel/error.rs @@ -0,0 +1,59 @@ +//! Typed kernel errors. + +use thiserror::Error; + +/// Result type used by kernel primitives. +pub type KernelResult = Result; + +/// Typed failures emitted by kernel contracts. +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub enum KernelError { + /// The provided logical repository path is invalid. + #[error("invalid repo path: {reason}")] + InvalidRepoPath { + /// The original user-provided input. + input: String, + /// The validation failure reason. + reason: String, + }, + /// The provided JSON Pointer is invalid. + #[error("invalid JSON pointer")] + InvalidJsonPointer { + /// The original user-provided input. + input: String, + }, + /// The provided stable identifier is invalid. + #[error("invalid stable id")] + InvalidStableId { + /// The original user-provided input. + input: String, + /// The expected identifier kind when a typed wrapper rejects the value. + expected_kind: Option<&'static str>, + }, + /// The provided fingerprint is invalid. + #[error("invalid fingerprint")] + InvalidFingerprint { + /// The original user-provided input. + input: String, + }, + /// The provided byte span is invalid. + #[error("invalid byte span: start {start_byte} > end {end_byte}")] + InvalidByteSpan { + /// The requested start byte. + start_byte: u64, + /// The requested end byte. + end_byte: u64, + }, + /// Canonical JSON serialization failed. + #[error("canonical JSON failure: {reason}")] + CanonicalJson { + /// Human-readable failure context. + reason: String, + }, + /// Schema validation failed. + #[error("schema validation failure: {reason}")] + SchemaViolation { + /// Human-readable failure context. + reason: String, + }, +} diff --git a/crates/lift/src/kernel/fingerprint.rs b/crates/lift/src/kernel/fingerprint.rs new file mode 100644 index 000000000..8ba8f6da3 --- /dev/null +++ b/crates/lift/src/kernel/fingerprint.rs @@ -0,0 +1,110 @@ +//! Fingerprint primitives. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::canonical_json::canonical_json_bytes; +use super::error::{KernelError, KernelResult}; + +const PREFIX: &str = "sha256:"; +const DIGEST_LEN: usize = 64; + +/// A content fingerprint with the wire format `sha256:<64-lower-hex>`. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct Fingerprint(String); + +impl Fingerprint { + /// Parses and validates a fingerprint. + pub fn parse(input: &str) -> KernelResult { + let digest = input + .strip_prefix(PREFIX) + .ok_or_else(|| KernelError::InvalidFingerprint { + input: input.to_owned(), + })?; + + if digest.len() != DIGEST_LEN || !digest.bytes().all(is_lower_hex) { + return Err(KernelError::InvalidFingerprint { + input: input.to_owned(), + }); + } + + Ok(Self(input.to_owned())) + } + + /// Returns the fingerprint as a string slice. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for Fingerprint { + type Error = KernelError; + + fn try_from(value: String) -> KernelResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: Fingerprint) -> Self { + value.0 + } +} + +/// Computes a SHA-256 fingerprint from raw bytes. +pub fn sha256_bytes(bytes: &[u8]) -> Fingerprint { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let digest = lower_hex(&hasher.finalize()); + Fingerprint(format!("{PREFIX}{digest}")) +} + +/// Computes a SHA-256 fingerprint from RFC 8785 canonical JSON bytes. +pub fn sha256_canonical_json(value: &T) -> KernelResult { + Ok(sha256_bytes(&canonical_json_bytes(value)?)) +} + +fn is_lower_hex(byte: u8) -> bool { + byte.is_ascii_digit() || matches!(byte, b'a'..=b'f') +} + +fn lower_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(nibble_to_hex(byte >> 4)); + out.push(nibble_to_hex(byte & 0x0f)); + } + out +} + +fn nibble_to_hex(nibble: u8) -> char { + match nibble { + 0..=9 => (b'0' + nibble) as char, + 10..=15 => (b'a' + (nibble - 10)) as char, + _ => unreachable!("nibble must be in 0..=15"), + } +} + +#[cfg(test)] +mod tests { + use super::Fingerprint; + + #[test] + fn parses_valid_fingerprint() { + let fingerprint = Fingerprint::parse( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ) + .expect("fingerprint"); + assert_eq!( + fingerprint.as_str(), + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + } + + #[test] + fn rejects_invalid_fingerprint() { + assert!(Fingerprint::parse("md5:abcd").is_err()); + assert!(serde_json::from_str::("\"sha256:nothex\"").is_err()); + } +} diff --git a/crates/lift/src/kernel/id.rs b/crates/lift/src/kernel/id.rs new file mode 100644 index 000000000..54815d805 --- /dev/null +++ b/crates/lift/src/kernel/id.rs @@ -0,0 +1,257 @@ +//! Stable identifier primitives. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::error::{KernelError, KernelResult}; + +const DIGEST_LEN: usize = 64; +const KIND_SEPARATOR: &str = ":sha256:"; +const IDENTITY_PREFIX: &str = "lift-kernel\0v1\0"; + +/// A stable identifier with the wire format `:sha256:<64-lower-hex>`. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct StableId(String); + +impl StableId { + /// Parses and validates a stable identifier. + pub fn parse(input: &str) -> KernelResult { + validate_stable_id(input, None)?; + Ok(Self(input.to_owned())) + } + + /// Builds a deterministic stable identifier from a kind and identity lemma. + pub fn from_identity(kind: &'static str, identity_lemma: &str) -> Self { + let mut hasher = Sha256::new(); + hasher.update(IDENTITY_PREFIX.as_bytes()); + hasher.update(kind.as_bytes()); + hasher.update([0]); + hasher.update(identity_lemma.as_bytes()); + + let digest = lower_hex(&hasher.finalize()); + Self(format!("{kind}{KIND_SEPARATOR}{digest}")) + } + + /// Returns the kind prefix of the stable identifier. + pub fn kind(&self) -> &str { + self.0 + .split_once(KIND_SEPARATOR) + .map_or("", |(kind, _)| kind) + } + + /// Returns the stable identifier as a string slice. + pub fn as_str(&self) -> &str { + &self.0 + } + + fn parse_typed(input: &str, expected_kind: &'static str) -> KernelResult { + validate_stable_id(input, Some(expected_kind))?; + Ok(Self(input.to_owned())) + } +} + +impl TryFrom for StableId { + type Error = KernelError; + + fn try_from(value: String) -> KernelResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: StableId) -> Self { + value.0 + } +} + +macro_rules! typed_stable_id { + ($name:ident, $kind:literal, $doc:literal) => { + #[doc = $doc] + #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] + #[serde(try_from = "String", into = "String")] + pub struct $name(StableId); + + impl $name { + /// The stable identifier kind for this typed wrapper. + pub const KIND: &'static str = $kind; + + /// Parses and validates a typed stable identifier. + pub fn parse(input: &str) -> KernelResult { + Ok(Self(StableId::parse_typed(input, Self::KIND)?)) + } + + /// Builds a deterministic typed stable identifier from an identity lemma. + pub fn from_identity(identity_lemma: &str) -> Self { + Self(StableId::from_identity(Self::KIND, identity_lemma)) + } + + /// Returns the identifier as a string slice. + pub fn as_str(&self) -> &str { + self.0.as_str() + } + } + + impl TryFrom for $name { + type Error = KernelError; + + fn try_from(value: String) -> KernelResult { + Self::parse(&value) + } + } + + impl From<$name> for String { + fn from(value: $name) -> Self { + value.0 .0 + } + } + + impl From<$name> for StableId { + fn from(value: $name) -> Self { + value.0 + } + } + }; +} + +typed_stable_id!(FileId, "file", "Typed stable identifier for file entities."); +typed_stable_id!( + SymbolId, + "symbol", + "Typed stable identifier for symbol entities." +); +typed_stable_id!( + ComponentId, + "component", + "Typed stable identifier for component entities." +); +typed_stable_id!( + BoundaryId, + "boundary", + "Typed stable identifier for boundary entities." +); +typed_stable_id!(FactId, "fact", "Typed stable identifier for fact entities."); +typed_stable_id!( + EvidenceId, + "evidence", + "Typed stable identifier for evidence entities." +); +typed_stable_id!(RuleId, "rule", "Typed stable identifier for rule entities."); +typed_stable_id!( + QueryId, + "query", + "Typed stable identifier for query entities." +); +typed_stable_id!( + RecipeId, + "recipe", + "Typed stable identifier for recipe entities." +); +typed_stable_id!( + MatchId, + "match", + "Typed stable identifier for match entities." +); + +fn validate_stable_id(input: &str, expected_kind: Option<&'static str>) -> KernelResult<()> { + let (kind, digest) = + input + .split_once(KIND_SEPARATOR) + .ok_or_else(|| KernelError::InvalidStableId { + input: input.to_owned(), + expected_kind, + })?; + + if !valid_kind(kind) || digest.len() != DIGEST_LEN || !digest.bytes().all(is_lower_hex) { + return Err(KernelError::InvalidStableId { + input: input.to_owned(), + expected_kind, + }); + } + + if let Some(expected_kind) = expected_kind { + if kind != expected_kind { + return Err(KernelError::InvalidStableId { + input: input.to_owned(), + expected_kind: Some(expected_kind), + }); + } + } + + Ok(()) +} + +fn valid_kind(kind: &str) -> bool { + let mut chars = kind.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} + +fn is_lower_hex(byte: u8) -> bool { + byte.is_ascii_digit() || matches!(byte, b'a'..=b'f') +} + +fn lower_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(nibble_to_hex(byte >> 4)); + out.push(nibble_to_hex(byte & 0x0f)); + } + out +} + +fn nibble_to_hex(nibble: u8) -> char { + match nibble { + 0..=9 => (b'0' + nibble) as char, + 10..=15 => (b'a' + (nibble - 10)) as char, + _ => unreachable!("nibble must be in 0..=15"), + } +} + +#[cfg(test)] +mod tests { + use super::{FileId, QueryId, StableId}; + + #[test] + fn stable_id_from_identity_is_deterministic() { + let left = StableId::from_identity("file", "src/lib.rs"); + let right = StableId::from_identity("file", "src/lib.rs"); + + assert_eq!(left, right); + assert_eq!( + left.as_str(), + "file:sha256:13ec57d807bcff8c7ee2ffc89b3adfb999e5b183d93474f425ebbe2ce371c416" + ); + } + + #[test] + fn typed_ids_reject_wrong_kinds() { + let err = QueryId::parse( + "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ) + .expect_err("query ids must reject file ids"); + + match err { + crate::kernel::KernelError::InvalidStableId { + expected_kind: Some(kind), + .. + } => assert_eq!(kind, QueryId::KIND), + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn serde_round_trip_uses_validation() { + let stable: StableId = serde_json::from_str( + "\"file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"", + ) + .expect("stable id should parse"); + assert_eq!(stable.kind(), "file"); + assert!(serde_json::from_str::("\"bad\"").is_err()); + + let file = FileId::from_identity("src/lib.rs"); + let round_trip: FileId = + serde_json::from_str(&serde_json::to_string(&file).expect("serialize")).expect("parse"); + assert_eq!(round_trip.as_str(), file.as_str()); + } +} diff --git a/crates/lift/src/kernel/json_pointer.rs b/crates/lift/src/kernel/json_pointer.rs new file mode 100644 index 000000000..7281540d7 --- /dev/null +++ b/crates/lift/src/kernel/json_pointer.rs @@ -0,0 +1,144 @@ +//! RFC 6901 JSON Pointer handling. + +use serde::{Deserialize, Serialize}; + +use crate::kernel::error::{KernelError, KernelResult}; + +/// Canonical JSON Pointer field path. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct JsonPointer(String); + +/// Canonical field-path alias used throughout the engine. +pub type FieldPath = JsonPointer; + +impl JsonPointer { + /// Parses and validates a canonical JSON Pointer. + pub fn parse(input: &str) -> KernelResult { + validate_pointer(input)?; + Ok(Self(input.to_owned())) + } + + /// Returns the root JSON Pointer. + pub fn root() -> Self { + Self(String::new()) + } + + /// Returns the canonical string form. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Returns a new pointer with one escaped token appended. + pub fn push_token(&self, token: &str) -> Self { + let escaped = escape_token(token); + if self.0.is_empty() { + Self(format!("/{escaped}")) + } else { + Self(format!("{}/{escaped}", self.0)) + } + } +} + +impl TryFrom for JsonPointer { + type Error = KernelError; + + fn try_from(value: String) -> KernelResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: JsonPointer) -> Self { + value.0 + } +} + +fn validate_pointer(input: &str) -> KernelResult<()> { + if input.is_empty() { + return Ok(()); + } + if !input.starts_with('/') { + return Err(invalid_json_pointer(input)); + } + + let mut chars = input.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '~' { + match chars.next() { + Some('0' | '1') => {} + _ => return Err(invalid_json_pointer(input)), + } + } + } + + Ok(()) +} + +fn escape_token(token: &str) -> String { + token.replace('~', "~0").replace('/', "~1") +} + +fn invalid_json_pointer(input: &str) -> KernelError { + KernelError::InvalidJsonPointer { + input: input.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::JsonPointer; + use crate::kernel::error::KernelError; + + #[test] + fn parses_root_and_nested_pointers() { + assert_eq!( + JsonPointer::parse("").expect("root should parse"), + JsonPointer::root() + ); + assert_eq!( + JsonPointer::parse("/touch/crates_touched") + .expect("nested pointer should parse") + .as_str(), + "/touch/crates_touched" + ); + assert_eq!( + JsonPointer::parse("/a//b") + .expect("empty token pointer should parse") + .as_str(), + "/a//b" + ); + } + + #[test] + fn rejects_invalid_pointers() { + let cases = ["field", "~0", "/~", "/~2", "/foo~bar"]; + + for input in cases { + let error = JsonPointer::parse(input).expect_err("pointer should fail"); + assert_eq!( + error, + KernelError::InvalidJsonPointer { + input: input.to_owned(), + } + ); + } + } + + #[test] + fn push_token_escapes_reserved_characters() { + let pointer = JsonPointer::root().push_token("a~/b"); + assert_eq!(pointer.as_str(), "/a~0~1b"); + + let nested = pointer.push_token("tail"); + assert_eq!(nested.as_str(), "/a~0~1b/tail"); + } + + #[test] + fn serde_round_trip_uses_parser() { + let parsed: JsonPointer = + serde_json::from_str("\"/touch/crates_touched\"").expect("pointer"); + assert_eq!(parsed.as_str(), "/touch/crates_touched"); + assert!(serde_json::from_str::("\"touch.crates_touched\"").is_err()); + } +} diff --git a/crates/lift/src/kernel/locator.rs b/crates/lift/src/kernel/locator.rs new file mode 100644 index 000000000..0a976e1c3 --- /dev/null +++ b/crates/lift/src/kernel/locator.rs @@ -0,0 +1,41 @@ +//! Source and document locators. + +use serde::{Deserialize, Serialize}; + +use crate::kernel::{ByteSpan, JsonPointer, RepoPath}; + +/// A deterministic location inside a repository file or structured document. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct Locator { + /// The repo-relative logical path. + pub path: RepoPath, + /// Optional byte range within the file. + #[serde(skip_serializing_if = "Option::is_none")] + pub span: Option, + /// Optional JSON Pointer into a structured payload. + #[serde(skip_serializing_if = "Option::is_none")] + pub json_pointer: Option, +} + +#[cfg(test)] +mod tests { + use super::Locator; + use crate::kernel::{ByteSpan, JsonPointer, RepoPath}; + + #[test] + fn locator_is_ordered_by_path_then_span_then_pointer() { + let path = RepoPath::parse("crates/lift/src/kernel/path.rs").expect("path should parse"); + let earlier = Locator { + path: path.clone(), + span: Some(ByteSpan::new(0, 4).expect("span should parse")), + json_pointer: None, + }; + let later = Locator { + path, + span: Some(ByteSpan::new(4, 8).expect("span should parse")), + json_pointer: Some(JsonPointer::parse("/touch/crates_touched").expect("pointer")), + }; + + assert!(earlier < later); + } +} diff --git a/crates/lift/src/kernel/mod.rs b/crates/lift/src/kernel/mod.rs new file mode 100644 index 000000000..bebebb476 --- /dev/null +++ b/crates/lift/src/kernel/mod.rs @@ -0,0 +1,39 @@ +//! Kernel contracts shared across the lift engine. +//! +//! This module owns deterministic, portable primitives that remain valid even +//! if higher-level apps disappear. Later seams can depend on these contracts, +//! but the kernel does not depend on them. + +#![forbid(unsafe_code)] +#![deny(missing_docs)] + +pub mod canonical_json; +pub mod diagnostic; +pub mod error; +pub mod fingerprint; +pub mod id; +pub mod json_pointer; +pub mod locator; +pub mod path; +pub mod schema; +pub mod span; + +pub use canonical_json::{canonical_json_bytes, canonical_json_string}; +pub use diagnostic::{Diagnostic, DiagnosticCode, RelatedLocation, Severity}; +pub use error::{KernelError, KernelResult}; +pub use fingerprint::{sha256_bytes, sha256_canonical_json, Fingerprint}; +pub use id::{ + BoundaryId, ComponentId, EvidenceId, FactId, FileId, MatchId, QueryId, RecipeId, RuleId, + StableId, SymbolId, +}; +pub use json_pointer::{FieldPath, JsonPointer}; +pub use locator::Locator; +pub use path::RepoPath; +pub use schema::{ + PRIMITIVES_V1_SCHEMA_FILE, PRIMITIVES_V1_SCHEMA_ID, PRIMITIVES_V1_SCHEMA_JSON, + PRIMITIVES_V1_SCHEMA_VERSION, +}; +pub use span::ByteSpan; + +/// Byte offset within a file-like payload. +pub type ByteOffset = u64; diff --git a/crates/lift/src/kernel/path.rs b/crates/lift/src/kernel/path.rs new file mode 100644 index 000000000..69bb1a6a3 --- /dev/null +++ b/crates/lift/src/kernel/path.rs @@ -0,0 +1,166 @@ +//! Deterministic logical repository paths. + +use serde::{Deserialize, Serialize}; + +use crate::kernel::error::{KernelError, KernelResult}; + +/// Canonical repo-root-relative logical path. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct RepoPath(String); + +impl RepoPath { + /// Parses and validates a repo-root-relative logical path. + pub fn parse(input: &str) -> KernelResult { + validate_repo_path(input)?; + Ok(Self(input.to_owned())) + } + + /// Returns the normalized string form. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Joins this path with an already-normalized child path. + pub fn join(&self, child: &RepoPath) -> KernelResult { + Self::parse(&format!("{}/{}", self.0, child.0)) + } + + /// Returns the logical parent path when one exists. + pub fn parent(&self) -> Option { + let (parent, _) = self.0.rsplit_once('/')?; + Some(Self(parent.to_owned())) + } +} + +impl TryFrom for RepoPath { + type Error = KernelError; + + fn try_from(value: String) -> KernelResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: RepoPath) -> Self { + value.0 + } +} + +fn validate_repo_path(input: &str) -> KernelResult<()> { + if input.is_empty() { + return Err(invalid_repo_path(input, "path must not be empty")); + } + if input.starts_with('/') { + return Err(invalid_repo_path(input, "path must be relative")); + } + if input.ends_with('/') { + return Err(invalid_repo_path( + input, + "path must not have a trailing slash", + )); + } + if input.contains('\\') { + return Err(invalid_repo_path( + input, + "path must use forward slashes only", + )); + } + + for segment in input.split('/') { + if segment.is_empty() { + return Err(invalid_repo_path( + input, + "path must not contain empty segments", + )); + } + if segment == "." { + return Err(invalid_repo_path( + input, + "path must not contain '.' segments", + )); + } + if segment == ".." { + return Err(invalid_repo_path( + input, + "path must not contain '..' segments", + )); + } + } + + Ok(()) +} + +fn invalid_repo_path(input: &str, reason: &str) -> KernelError { + KernelError::InvalidRepoPath { + input: input.to_owned(), + reason: reason.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::RepoPath; + use crate::kernel::error::KernelError; + + #[test] + fn parses_valid_paths() { + let path = RepoPath::parse("crates/lift/src/kernel/path.rs").expect("path should parse"); + assert_eq!(path.as_str(), "crates/lift/src/kernel/path.rs"); + } + + #[test] + fn rejects_invalid_paths() { + let cases = [ + ("", "path must not be empty"), + ("/src/lib.rs", "path must be relative"), + ("src/lib.rs/", "path must not have a trailing slash"), + ("src//lib.rs", "path must not contain empty segments"), + ("src/./lib.rs", "path must not contain '.' segments"), + ("src/../lib.rs", "path must not contain '..' segments"), + (r"src\lib.rs", "path must use forward slashes only"), + ]; + + for (input, reason) in cases { + let error = RepoPath::parse(input).expect_err("path should fail"); + assert_eq!( + error, + KernelError::InvalidRepoPath { + input: input.to_owned(), + reason: reason.to_owned(), + } + ); + } + } + + #[test] + fn join_preserves_logical_normalization() { + let base = RepoPath::parse("crates/lift").expect("base should parse"); + let child = RepoPath::parse("src/kernel/path.rs").expect("child should parse"); + + let joined = base.join(&child).expect("join should succeed"); + assert_eq!(joined.as_str(), "crates/lift/src/kernel/path.rs"); + } + + #[test] + fn parent_returns_none_for_single_segment_paths() { + let path = RepoPath::parse("Cargo.toml").expect("path should parse"); + assert_eq!(path.parent(), None); + } + + #[test] + fn parent_returns_parent_for_nested_paths() { + let path = RepoPath::parse("crates/lift/src/kernel").expect("path should parse"); + assert_eq!( + path.parent().expect("parent should exist").as_str(), + "crates/lift/src" + ); + } + + #[test] + fn serde_round_trip_uses_parser() { + let parsed: RepoPath = serde_json::from_str("\"src/lib.rs\"").expect("path"); + assert_eq!(parsed.as_str(), "src/lib.rs"); + assert!(serde_json::from_str::("\"src//lib.rs\"").is_err()); + } +} diff --git a/crates/lift/src/kernel/schema.rs b/crates/lift/src/kernel/schema.rs new file mode 100644 index 000000000..9629023da --- /dev/null +++ b/crates/lift/src/kernel/schema.rs @@ -0,0 +1,14 @@ +//! Embedded kernel schema metadata. + +/// The schema identifier for seam-0 kernel primitives. +pub const PRIMITIVES_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/kernel/primitives.v1.json"; + +/// The current seam-0 kernel schema version. +pub const PRIMITIVES_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-0 kernel primitives schema. +pub const PRIMITIVES_V1_SCHEMA_FILE: &str = "primitives.v1.json"; + +/// The embedded seam-0 kernel primitives schema source. +pub const PRIMITIVES_V1_SCHEMA_JSON: &str = include_str!("../../schemas/kernel/primitives.v1.json"); diff --git a/crates/lift/src/kernel/span.rs b/crates/lift/src/kernel/span.rs new file mode 100644 index 000000000..95f1f7a59 --- /dev/null +++ b/crates/lift/src/kernel/span.rs @@ -0,0 +1,90 @@ +//! Byte-span primitives. + +use serde::{Deserialize, Serialize}; + +use crate::kernel::error::{KernelError, KernelResult}; + +/// Half-open byte interval `[start_byte, end_byte)`. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "ByteSpanRepr")] +pub struct ByteSpan { + /// Inclusive start byte offset. + pub start_byte: u64, + /// Exclusive end byte offset. + pub end_byte: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct ByteSpanRepr { + start_byte: u64, + end_byte: u64, +} + +impl ByteSpan { + /// Creates a validated half-open byte span. + pub fn new(start_byte: u64, end_byte: u64) -> KernelResult { + if start_byte > end_byte { + return Err(KernelError::InvalidByteSpan { + start_byte, + end_byte, + }); + } + + Ok(Self { + start_byte, + end_byte, + }) + } + + /// Returns the byte length of the span. + pub fn len(&self) -> u64 { + self.end_byte - self.start_byte + } + + /// Returns true when the span covers zero bytes. + pub fn is_empty(&self) -> bool { + self.start_byte == self.end_byte + } +} + +impl TryFrom for ByteSpan { + type Error = KernelError; + + fn try_from(value: ByteSpanRepr) -> KernelResult { + Self::new(value.start_byte, value.end_byte) + } +} + +#[cfg(test)] +mod tests { + use super::ByteSpan; + use crate::kernel::error::KernelError; + + #[test] + fn accepts_empty_and_non_empty_spans() { + let empty = ByteSpan::new(4, 4).expect("empty span should be valid"); + assert!(empty.is_empty()); + assert_eq!(empty.len(), 0); + + let span = ByteSpan::new(4, 9).expect("span should be valid"); + assert!(!span.is_empty()); + assert_eq!(span.len(), 5); + } + + #[test] + fn rejects_reversed_spans() { + let error = ByteSpan::new(9, 4).expect_err("span should fail"); + assert_eq!( + error, + KernelError::InvalidByteSpan { + start_byte: 9, + end_byte: 4, + } + ); + } + + #[test] + fn serde_rejects_reversed_spans() { + assert!(serde_json::from_str::(r#"{"start_byte":9,"end_byte":4}"#).is_err()); + } +} diff --git a/crates/lift/src/lang/adapter.rs b/crates/lift/src/lang/adapter.rs new file mode 100644 index 000000000..684728b83 --- /dev/null +++ b/crates/lift/src/lang/adapter.rs @@ -0,0 +1,112 @@ +use serde::{Deserialize, Serialize}; + +use crate::kernel::{Diagnostic, FileId, Fingerprint, RepoPath}; +use crate::lang::{ + capabilities::AdapterCapabilities, LangError, LangResult, LanguageId, LocalEdgeDraft, + LocalSymbolDraft, SurfaceMarkerDraft, +}; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct AdapterName(String); + +impl AdapterName { + pub(crate) fn parse(input: &str) -> LangResult { + if is_valid_adapter_name(input) { + Ok(Self(input.to_owned())) + } else { + Err(LangError::InvalidAdapterName { + input: input.to_owned(), + }) + } + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for AdapterName { + type Error = LangError; + + fn try_from(value: String) -> LangResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: AdapterName) -> Self { + value.0 + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct AdapterDescriptor { + pub name: AdapterName, + pub language: LanguageId, + pub version: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ParseInput<'a> { + pub path: &'a RepoPath, + pub file_id: &'a FileId, + pub blob_fingerprint: &'a Fingerprint, + pub bytes: &'a [u8], +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct AdapterParseOutput { + pub symbols: Vec, + pub edges: Vec, + pub surface_markers: Vec, + pub diagnostics: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum AdapterParseResult { + Parsed(AdapterParseOutput), + Failed { diagnostics: Vec }, +} + +pub(crate) trait LanguageAdapter: Send + Sync { + fn descriptor(&self) -> AdapterDescriptor; + fn capabilities(&self) -> AdapterCapabilities { + AdapterCapabilities::default() + } + fn recognizes(&self, input: &ParseInput<'_>) -> bool; + fn parse(&self, input: &ParseInput<'_>) -> AdapterParseResult; +} + +fn is_valid_adapter_name(input: &str) -> bool { + let mut segments = input.split('.'); + let Some(first) = segments.next() else { + return false; + }; + + if !is_valid_head_segment(first) { + return false; + } + + let mut saw_tail_segment = false; + for segment in segments { + saw_tail_segment = true; + if !is_valid_tail_segment(segment) { + return false; + } + } + + saw_tail_segment +} + +fn is_valid_head_segment(segment: &str) -> bool { + let mut chars = segment.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit()) +} + +fn is_valid_tail_segment(segment: &str) -> bool { + let mut chars = segment.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} diff --git a/crates/lift/src/lang/cache.rs b/crates/lift/src/lang/cache.rs new file mode 100644 index 000000000..738eea496 --- /dev/null +++ b/crates/lift/src/lang/cache.rs @@ -0,0 +1,107 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use crate::kernel::{FileId, Fingerprint}; +use crate::lang::{ + AdapterDescriptor, AdapterName, FailedParse, LangError, LangResult, LanguageId, ParseInput, + ParsedUnit, +}; + +pub(crate) const PLATFORM_CACHE_VERSION: &str = "phase_b_v1"; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub(crate) struct ParseCacheKey { + pub file_id: FileId, + pub blob_fingerprint: Fingerprint, + pub language: LanguageId, + pub adapter: AdapterName, + pub adapter_version: String, + pub platform_cache_version: String, +} + +impl ParseCacheKey { + pub(crate) fn for_input(input: &ParseInput<'_>, descriptor: &AdapterDescriptor) -> Self { + Self::with_platform_cache_version(input, descriptor, PLATFORM_CACHE_VERSION) + } + + pub(crate) fn with_platform_cache_version( + input: &ParseInput<'_>, + descriptor: &AdapterDescriptor, + platform_cache_version: &str, + ) -> Self { + Self { + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name.clone(), + adapter_version: descriptor.version.clone(), + platform_cache_version: platform_cache_version.to_owned(), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CachedParseOutcome { + Parsed(ParsedUnit), + Failed(FailedParse), +} + +pub(crate) enum ParseCacheLookup { + Disabled, + Hit(Box), + Miss, +} + +pub(crate) trait ParseCache: Send + Sync { + fn get(&self, key: &ParseCacheKey) -> LangResult; + fn put(&self, key: ParseCacheKey, value: CachedParseOutcome) -> LangResult<()>; +} + +#[derive(Default)] +pub(crate) struct NoopParseCache; + +impl ParseCache for NoopParseCache { + fn get(&self, _key: &ParseCacheKey) -> LangResult { + Ok(ParseCacheLookup::Disabled) + } + + fn put(&self, _key: ParseCacheKey, _value: CachedParseOutcome) -> LangResult<()> { + Ok(()) + } +} + +#[derive(Clone, Default)] +pub(crate) struct InMemoryParseCache { + entries: Arc>>, +} + +impl InMemoryParseCache { + pub(crate) fn len(&self) -> LangResult { + Ok(self.lock_entries()?.len()) + } + + fn lock_entries( + &self, + ) -> LangResult>> { + self.entries + .lock() + .map_err(|error| LangError::CacheInvariant { + reason: format!("in-memory parse cache lock poisoned: {error}"), + }) + } +} + +impl ParseCache for InMemoryParseCache { + fn get(&self, key: &ParseCacheKey) -> LangResult { + let entries = self.lock_entries()?; + Ok(match entries.get(key).cloned() { + Some(value) => ParseCacheLookup::Hit(Box::new(value)), + None => ParseCacheLookup::Miss, + }) + } + + fn put(&self, key: ParseCacheKey, value: CachedParseOutcome) -> LangResult<()> { + self.lock_entries()?.insert(key, value); + Ok(()) + } +} diff --git a/crates/lift/src/lang/capabilities.rs b/crates/lift/src/lang/capabilities.rs new file mode 100644 index 000000000..5fda1dc8c --- /dev/null +++ b/crates/lift/src/lang/capabilities.rs @@ -0,0 +1,55 @@ +use std::collections::BTreeSet; +use std::fmt; + +use serde::de::{self, Visitor}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::pack::QueryEngineKind; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Default)] +#[serde(default)] +pub(crate) struct AdapterCapabilities { + pub emits_local_edges: bool, + pub emits_surface_markers: bool, + pub query_engines: BTreeSet, +} + +// Keep this manual serde in sync with crate::pack::QueryEngineKind variants. +// Adding a new engine does not automatically update these string mappings. +impl Serialize for QueryEngineKind { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for QueryEngineKind { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(QueryEngineKindVisitor) + } +} + +struct QueryEngineKindVisitor; + +impl Visitor<'_> for QueryEngineKindVisitor { + type Value = QueryEngineKind; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a supported query engine kind") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + match value { + "tree_sitter" => Ok(QueryEngineKind::TreeSitter), + _ => Err(E::unknown_variant(value, &["tree_sitter"])), + } + } +} diff --git a/crates/lift/src/lang/driver.rs b/crates/lift/src/lang/driver.rs new file mode 100644 index 000000000..a74c98f4f --- /dev/null +++ b/crates/lift/src/lang/driver.rs @@ -0,0 +1,733 @@ +use std::collections::BTreeMap; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::Arc; + +use crate::kernel::{Diagnostic, DiagnosticCode, Locator, RepoPath, Severity, SymbolId}; +use crate::lang::{ + compare_symbol_drafts, sort_local_edges, sort_local_symbols, sort_surface_markers, + symbol_identity_lemma, AdapterDescriptor, AdapterParseOutput, AdapterParseResult, + CachedParseOutcome, EdgeEndpoint, EdgeEndpointDraft, FailedParse, LangError, LangResult, + LanguageAdapter, LanguageId, LanguageRegistry, LocalEdge, LocalSymbol, LocalSymbolDraft, + MissingRequestedLanguage, NoopParseCache, ParseCache, ParseCacheKey, ParseCacheLookup, + ParseInput, ParseRequest, ParseScope, ParseSet, ParseStats, ParsedUnit, ReferenceTarget, + ReferenceTargetDraft, SkippedParse, SkippedReason, SurfaceMarker, +}; +use crate::repo::{InventoryEntry, RepoSnapshot}; + +pub(crate) struct ParseDriver { + registry: LanguageRegistry, + cache: Arc, +} + +impl ParseDriver { + pub(crate) fn new(registry: LanguageRegistry) -> Self { + Self::with_cache(registry, NoopParseCache) + } + + pub(crate) fn with_cache(registry: LanguageRegistry, cache: C) -> Self + where + C: ParseCache + 'static, + { + Self { + registry, + cache: Arc::new(cache), + } + } + + pub(crate) fn parse_snapshot( + &self, + snapshot: &RepoSnapshot, + request: &ParseRequest, + ) -> LangResult { + let normalized_request = request.normalized(); + let request_fingerprint = + normalized_request + .fingerprint() + .map_err(|error| LangError::SchemaViolation { + schema_id: "lang.parse.request", + reason: error.to_string(), + })?; + + let mut parse_set = ParseSet { + snapshot_fingerprint: snapshot.fingerprint.clone(), + request: normalized_request.clone(), + request_fingerprint, + units: Vec::new(), + failed: Vec::new(), + skipped: Vec::new(), + missing_languages: Vec::new(), + diagnostics: Vec::new(), + stats: ParseStats::default(), + }; + + let selected_languages = self.selected_languages(&normalized_request, &mut parse_set); + match &normalized_request.scope { + ParseScope::Snapshot => { + for entry in snapshot.inventory.iter() { + self.process_entry( + snapshot, + entry, + &selected_languages, + false, + &mut parse_set, + )?; + } + } + ParseScope::Paths(paths) => { + for path in paths { + let Some(entry) = snapshot.entry(path) else { + parse_set.stats.skipped_missing_paths += 1; + parse_set.skipped.push(SkippedParse { + path: path.clone(), + file_id: None, + reason: SkippedReason::PathNotInSnapshot, + detail: Some("requested path not present in snapshot".to_owned()), + }); + continue; + }; + self.process_entry(snapshot, entry, &selected_languages, true, &mut parse_set)?; + } + } + } + + parse_set.sort_all(); + parse_set.refresh_stats(); + Ok(parse_set) + } + + fn selected_languages( + &self, + request: &ParseRequest, + parse_set: &mut ParseSet, + ) -> Vec { + if request.languages.is_empty() { + return self + .registry + .descriptors() + .into_iter() + .map(|descriptor| descriptor.language) + .collect(); + } + + let mut selected = Vec::new(); + for language in &request.languages { + if self.registry.adapter_for_language(*language).is_some() { + selected.push(*language); + continue; + } + + parse_set.missing_languages.push(MissingRequestedLanguage { + language: *language, + detail: Some(format!( + "no registered adapter is available for language '{}'", + language.as_str() + )), + }); + parse_set.diagnostics.push(error_diagnostic( + "lang.parse.missing_registered_adapter", + format!( + "requested language '{}' has no registered adapter", + language.as_str() + ), + None, + )); + } + + selected + } + + fn process_entry( + &self, + snapshot: &RepoSnapshot, + entry: &InventoryEntry, + selected_languages: &[LanguageId], + explicit_path: bool, + parse_set: &mut ParseSet, + ) -> LangResult<()> { + parse_set.stats.considered_files += 1; + let bytes = + snapshot + .read_bytes(&entry.path) + .map_err(|error| LangError::CacheInvariant { + reason: error.to_string(), + })?; + let input = ParseInput { + path: &entry.path, + file_id: &entry.file_id, + blob_fingerprint: &entry.blob_fingerprint, + bytes, + }; + + let selection = self.select_candidate(&input, selected_languages); + match selection { + CandidateSelection::NoMatch => { + parse_set.stats.skipped_no_adapter += 1; + if explicit_path { + parse_set.skipped.push(SkippedParse { + path: entry.path.clone(), + file_id: Some(entry.file_id.clone()), + reason: SkippedReason::NoMatchingAdapter, + detail: Some("no selected adapter recognized file".to_owned()), + }); + } + } + CandidateSelection::Failed(failed) => parse_set.failed.push(failed), + CandidateSelection::Adapter { + adapter, + descriptor, + } => { + let cache_key = ParseCacheKey::for_input(&input, &descriptor); + match self.cache.get(&cache_key)? { + ParseCacheLookup::Disabled => { + let outcome = self.parse_and_normalize(&adapter, &descriptor, &input)?; + apply_cached_outcome(parse_set, outcome); + } + ParseCacheLookup::Hit(outcome) => { + parse_set.stats.cache_hits += 1; + apply_cached_outcome(parse_set, *outcome); + } + ParseCacheLookup::Miss => { + parse_set.stats.cache_misses += 1; + let outcome = self.parse_and_normalize(&adapter, &descriptor, &input)?; + self.cache.put(cache_key, outcome.clone())?; + apply_cached_outcome(parse_set, outcome); + } + } + } + } + + Ok(()) + } + + fn select_candidate( + &self, + input: &ParseInput<'_>, + selected_languages: &[LanguageId], + ) -> CandidateSelection { + for language in selected_languages { + let Some(adapter) = self.registry.adapter_for_language(*language).cloned() else { + continue; + }; + let descriptor = adapter.descriptor(); + match catch_unwind(AssertUnwindSafe(|| adapter.recognizes(input))) { + Ok(true) => { + return CandidateSelection::Adapter { + adapter, + descriptor, + }; + } + Ok(false) => {} + Err(payload) => { + return CandidateSelection::Failed(FailedParse { + path: input.path.clone(), + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name, + adapter_version: descriptor.version, + diagnostics: vec![error_diagnostic( + "lang.parse.adapter_panic", + format!( + "adapter panicked during recognizes(): {}", + panic_message(payload) + ), + Some(Locator { + path: input.path.clone(), + span: None, + json_pointer: None, + }), + )], + }); + } + } + } + + CandidateSelection::NoMatch + } + + fn invoke_parse( + &self, + adapter: &Arc, + descriptor: &AdapterDescriptor, + input: &ParseInput<'_>, + ) -> ParseAttempt { + match catch_unwind(AssertUnwindSafe(|| adapter.parse(input))) { + Ok(AdapterParseResult::Parsed(output)) => ParseAttempt::Parsed(output), + Ok(AdapterParseResult::Failed { diagnostics }) => { + ParseAttempt::Failed(normalize_failed_parse(descriptor, input, diagnostics)) + } + Err(payload) => ParseAttempt::Failed(FailedParse { + path: input.path.clone(), + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name.clone(), + adapter_version: descriptor.version.clone(), + diagnostics: vec![error_diagnostic( + "lang.parse.adapter_panic", + format!( + "adapter panicked during parse(): {}", + panic_message(payload) + ), + Some(Locator { + path: input.path.clone(), + span: None, + json_pointer: None, + }), + )], + }), + } + } + + fn parse_and_normalize( + &self, + adapter: &Arc, + descriptor: &AdapterDescriptor, + input: &ParseInput<'_>, + ) -> LangResult { + match self.invoke_parse(adapter, descriptor, input) { + ParseAttempt::Failed(failed) => Ok(CachedParseOutcome::Failed(failed)), + ParseAttempt::Parsed(output) => { + match normalize_parsed_output(descriptor, input, output)? { + NormalizedFile::Parsed(unit) => Ok(CachedParseOutcome::Parsed(unit)), + NormalizedFile::Failed(failed) => Ok(CachedParseOutcome::Failed(failed)), + } + } + } + } +} + +fn apply_cached_outcome(parse_set: &mut ParseSet, outcome: CachedParseOutcome) { + match outcome { + CachedParseOutcome::Parsed(unit) => parse_set.units.push(unit), + CachedParseOutcome::Failed(failed) => parse_set.failed.push(failed), + } +} + +enum CandidateSelection { + NoMatch, + Failed(FailedParse), + Adapter { + adapter: Arc, + descriptor: AdapterDescriptor, + }, +} + +enum ParseAttempt { + Failed(FailedParse), + Parsed(AdapterParseOutput), +} + +enum NormalizedFile { + Parsed(ParsedUnit), + Failed(FailedParse), +} + +fn normalize_parsed_output( + descriptor: &AdapterDescriptor, + input: &ParseInput<'_>, + output: AdapterParseOutput, +) -> LangResult { + let file_len = input.bytes.len() as u64; + let (mut diagnostics, mut issues) = + sanitize_diagnostics(input.path, file_len, output.diagnostics); + + let mut by_local_key = BTreeMap::::new(); + let mut symbol_drafts = output.symbols; + symbol_drafts.sort_by(compare_symbol_drafts); + for draft in &symbol_drafts { + if draft.local_key.is_empty() { + issues.push(error_diagnostic( + "lang.parse.invalid_local_key", + "symbol local_key must not be empty".to_owned(), + Some(Locator { + path: input.path.clone(), + span: Some(draft.span), + json_pointer: None, + }), + )); + } + if draft.span.end_byte > file_len { + issues.push(invalid_span_diagnostic( + input.path, + Some(draft.span), + format!( + "symbol span [{}..{}) exceeds file length {}", + draft.span.start_byte, draft.span.end_byte, file_len + ), + )); + } + if by_local_key + .insert(draft.local_key.clone(), draft.clone()) + .is_some() + { + issues.push(error_diagnostic( + "lang.parse.duplicate_local_key", + format!("duplicate symbol local_key '{}'", draft.local_key), + Some(Locator { + path: input.path.clone(), + span: Some(draft.span), + json_pointer: None, + }), + )); + } + } + + if !issues.is_empty() { + diagnostics.extend(issues); + diagnostics.sort(); + return Ok(NormalizedFile::Failed(FailedParse { + path: input.path.clone(), + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name.clone(), + adapter_version: descriptor.version.clone(), + diagnostics, + })); + } + + let mut symbols = Vec::with_capacity(symbol_drafts.len()); + let mut symbol_ids = BTreeMap::::new(); + let mut previous_identity: Option<( + crate::lang::SymbolKind, + Vec, + Option, + crate::kernel::ByteSpan, + )> = None; + let mut duplicate_ordinal = 0usize; + for draft in &symbol_drafts { + let current_identity = ( + draft.kind, + draft.path.clone(), + draft.name.clone(), + draft.span, + ); + if previous_identity.as_ref() == Some(¤t_identity) { + duplicate_ordinal += 1; + } else { + duplicate_ordinal = 0; + previous_identity = Some(current_identity); + } + + let id = SymbolId::from_identity(&symbol_identity_lemma( + descriptor.language, + input.path, + draft, + duplicate_ordinal, + )); + symbol_ids.insert(draft.local_key.clone(), id.clone()); + symbols.push(LocalSymbol { + id, + kind: draft.kind, + name: draft.name.clone(), + path: draft.path.clone(), + span: draft.span, + visibility: draft.visibility, + }); + } + + let mut edges = Vec::with_capacity(output.edges.len()); + for edge in output.edges { + if let Some(span) = edge.span { + if span.end_byte > file_len { + issues.push(invalid_span_diagnostic( + input.path, + Some(span), + format!( + "edge span [{}..{}) exceeds file length {}", + span.start_byte, span.end_byte, file_len + ), + )); + continue; + } + } + + let source = match resolve_edge_endpoint(&symbol_ids, &edge.source) { + Ok(source) => source, + Err(local_key) => { + issues.push(error_diagnostic( + "lang.parse.unresolved_local_ref", + format!("edge source references missing local_key '{}'", local_key), + Some(Locator { + path: input.path.clone(), + span: edge.span, + json_pointer: None, + }), + )); + continue; + } + }; + let target = match resolve_reference_target(&symbol_ids, &edge.target) { + Ok(target) => target, + Err(local_key) => { + issues.push(error_diagnostic( + "lang.parse.unresolved_local_ref", + format!("edge target references missing local_key '{}'", local_key), + Some(Locator { + path: input.path.clone(), + span: edge.span, + json_pointer: None, + }), + )); + continue; + } + }; + + edges.push(LocalEdge { + kind: edge.kind, + source, + target, + span: edge.span, + }); + } + + let mut surface_markers = Vec::with_capacity(output.surface_markers.len()); + for marker in output.surface_markers { + if let Some(span) = marker.span { + if span.end_byte > file_len { + issues.push(invalid_span_diagnostic( + input.path, + Some(span), + format!( + "surface marker span [{}..{}) exceeds file length {}", + span.start_byte, span.end_byte, file_len + ), + )); + continue; + } + } + + let symbol = match marker.symbol_local_key { + Some(local_key) => match symbol_ids.get(&local_key).cloned() { + Some(symbol) => Some(symbol), + None => { + issues.push(error_diagnostic( + "lang.parse.unresolved_marker_ref", + format!( + "surface marker references missing local_key '{}'", + local_key + ), + Some(Locator { + path: input.path.clone(), + span: marker.span, + json_pointer: None, + }), + )); + continue; + } + }, + None => None, + }; + + surface_markers.push(SurfaceMarker { + kind: marker.kind, + symbol, + span: marker.span, + label: marker.label, + }); + } + + if !issues.is_empty() { + diagnostics.extend(issues); + diagnostics.sort(); + return Ok(NormalizedFile::Failed(FailedParse { + path: input.path.clone(), + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name.clone(), + adapter_version: descriptor.version.clone(), + diagnostics, + })); + } + + sort_local_symbols(&mut symbols); + sort_local_edges(&mut edges); + sort_surface_markers(&mut surface_markers); + diagnostics.sort(); + + let mut unit = ParsedUnit { + path: input.path.clone(), + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name.clone(), + adapter_version: descriptor.version.clone(), + unit_fingerprint: crate::kernel::sha256_bytes(b""), + symbols, + edges, + surface_markers, + diagnostics, + }; + unit.unit_fingerprint = unit + .fingerprint() + .map_err(|error| LangError::SchemaViolation { + schema_id: "lang.parse.unit", + reason: error.to_string(), + })?; + + Ok(NormalizedFile::Parsed(unit)) +} + +fn normalize_failed_parse( + descriptor: &AdapterDescriptor, + input: &ParseInput<'_>, + diagnostics: Vec, +) -> FailedParse { + let file_len = input.bytes.len() as u64; + let (mut valid, mut issues) = sanitize_diagnostics(input.path, file_len, diagnostics); + if valid.is_empty() && issues.is_empty() { + valid.push(error_diagnostic( + "lang.parse.failed", + "adapter reported parse failure".to_owned(), + Some(Locator { + path: input.path.clone(), + span: None, + json_pointer: None, + }), + )); + } + valid.append(&mut issues); + valid.sort(); + + FailedParse { + path: input.path.clone(), + file_id: input.file_id.clone(), + blob_fingerprint: input.blob_fingerprint.clone(), + language: descriptor.language, + adapter: descriptor.name.clone(), + adapter_version: descriptor.version.clone(), + diagnostics: valid, + } +} + +fn sanitize_diagnostics( + path: &RepoPath, + file_len: u64, + diagnostics: Vec, +) -> (Vec, Vec) { + let mut valid = Vec::new(); + let mut issues = Vec::new(); + + for diagnostic in diagnostics { + if locator_span_invalid(diagnostic.subject.as_ref(), path, file_len) { + issues.push(invalid_span_diagnostic( + path, + diagnostic.subject.as_ref().and_then(|locator| locator.span), + "diagnostic subject span exceeds file length".to_owned(), + )); + continue; + } + if diagnostic + .related + .iter() + .any(|related| locator_span_invalid(Some(&related.locator), path, file_len)) + { + let span = diagnostic + .related + .iter() + .find_map(|related| (related.locator.path == *path).then_some(related.locator.span)) + .flatten(); + issues.push(invalid_span_diagnostic( + path, + span, + "diagnostic related span exceeds file length".to_owned(), + )); + continue; + } + valid.push(diagnostic); + } + + valid.sort(); + issues.sort(); + (valid, issues) +} + +fn locator_span_invalid(locator: Option<&Locator>, path: &RepoPath, file_len: u64) -> bool { + locator + .filter(|locator| locator.path == *path) + .and_then(|locator| locator.span) + .is_some_and(|span| span.end_byte > file_len) +} + +fn resolve_edge_endpoint( + symbol_ids: &BTreeMap, + source: &EdgeEndpointDraft, +) -> Result { + match source { + EdgeEndpointDraft::FileRoot => Ok(EdgeEndpoint::FileRoot), + EdgeEndpointDraft::Symbol { local_key } => symbol_ids + .get(local_key) + .cloned() + .map(EdgeEndpoint::Symbol) + .ok_or_else(|| local_key.clone()), + } +} + +fn resolve_reference_target( + symbol_ids: &BTreeMap, + target: &ReferenceTargetDraft, +) -> Result { + match target { + ReferenceTargetDraft::LocalSymbol { local_key } => symbol_ids + .get(local_key) + .cloned() + .map(ReferenceTarget::LocalSymbol) + .ok_or_else(|| local_key.clone()), + ReferenceTargetDraft::QualifiedName { parts } => Ok(ReferenceTarget::QualifiedName { + parts: parts.clone(), + }), + ReferenceTargetDraft::FilePath { path } => { + Ok(ReferenceTarget::FilePath { path: path.clone() }) + } + ReferenceTargetDraft::ExternalPackage { package, symbol } => { + Ok(ReferenceTarget::ExternalPackage { + package: package.clone(), + symbol: symbol.clone(), + }) + } + ReferenceTargetDraft::Opaque { value } => Ok(ReferenceTarget::Opaque { + value: value.clone(), + }), + } +} + +fn error_diagnostic(code: &str, message: String, subject: Option) -> Diagnostic { + Diagnostic { + code: DiagnosticCode::parse(code).expect("lang diagnostic code should be valid"), + severity: Severity::Error, + message, + subject, + related: Vec::new(), + help: None, + } +} + +fn invalid_span_diagnostic( + path: &RepoPath, + span: Option, + message: String, +) -> Diagnostic { + error_diagnostic( + "lang.parse.invalid_span", + message, + Some(Locator { + path: path.clone(), + span, + json_pointer: None, + }), + ) +} + +fn panic_message(payload: Box) -> String { + let payload = match payload.downcast::() { + Ok(message) => return *message, + Err(payload) => payload, + }; + let payload = match payload.downcast::<&'static str>() { + Ok(message) => return (*message).to_owned(), + Err(payload) => payload, + }; + format!("non-string panic payload ({:?})", payload.type_id()) +} diff --git a/crates/lift/src/lang/error.rs b/crates/lift/src/lang/error.rs new file mode 100644 index 000000000..0a37a4eeb --- /dev/null +++ b/crates/lift/src/lang/error.rs @@ -0,0 +1,30 @@ +use thiserror::Error; + +use crate::lang::LanguageId; + +pub(crate) type LangResult = Result; + +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub(crate) enum LangError { + #[error("duplicate adapter name")] + DuplicateAdapterName { name: String }, + + #[error("duplicate language adapter registration")] + DuplicateLanguageAdapter { + language: LanguageId, + existing: String, + duplicate: String, + }, + + #[error("invalid adapter name")] + InvalidAdapterName { input: String }, + + #[error("parse cache invariant failure")] + CacheInvariant { reason: String }, + + #[error("lang schema validation failure")] + SchemaViolation { + schema_id: &'static str, + reason: String, + }, +} diff --git a/crates/lift/src/lang/mod.rs b/crates/lift/src/lang/mod.rs new file mode 100644 index 000000000..a8b990b68 --- /dev/null +++ b/crates/lift/src/lang/mod.rs @@ -0,0 +1,42 @@ +//! Internal language platform seam. + +#![allow(dead_code)] +#![allow(unused_imports)] + +pub(crate) mod adapter; +pub(crate) mod cache; +pub(crate) mod capabilities; +pub(crate) mod driver; +pub(crate) mod error; +pub(crate) mod model; +pub(crate) mod registry; +pub(crate) mod schema; + +pub(crate) use crate::pack::{LanguageId, QueryEngineKind}; + +pub(crate) use adapter::{ + AdapterDescriptor, AdapterName, AdapterParseOutput, AdapterParseResult, LanguageAdapter, + ParseInput, +}; +pub(crate) use cache::{ + CachedParseOutcome, InMemoryParseCache, NoopParseCache, ParseCache, ParseCacheKey, + ParseCacheLookup, PLATFORM_CACHE_VERSION, +}; +pub(crate) use capabilities::AdapterCapabilities; +pub(crate) use driver::ParseDriver; +pub(crate) use error::{LangError, LangResult}; +pub(crate) use model::{ + compare_symbol_drafts, sort_local_edges, sort_local_symbols, sort_surface_markers, + symbol_identity_lemma, EdgeEndpoint, EdgeEndpointDraft, FailedParse, LocalEdge, LocalEdgeDraft, + LocalEdgeKind, LocalSymbol, LocalSymbolDraft, MissingRequestedLanguage, ParseRequest, + ParseScope, ParseSet, ParseStats, ParsedUnit, ReferenceTarget, ReferenceTargetDraft, + SkippedParse, SkippedReason, SurfaceMarker, SurfaceMarkerDraft, SurfaceMarkerKind, SymbolKind, + SymbolVisibility, +}; +pub(crate) use registry::{LanguageRegistry, LanguageRegistryBuilder}; +pub(crate) use schema::{ + LANG_PARSE_MANIFEST_V1_SCHEMA_FILE, LANG_PARSE_MANIFEST_V1_SCHEMA_ID, + LANG_PARSE_MANIFEST_V1_SCHEMA_JSON, LANG_PARSE_MANIFEST_V1_SCHEMA_VERSION, + LANG_PARSE_MANIFEST_V2_SCHEMA_FILE, LANG_PARSE_MANIFEST_V2_SCHEMA_ID, + LANG_PARSE_MANIFEST_V2_SCHEMA_JSON, LANG_PARSE_MANIFEST_V2_SCHEMA_VERSION, +}; diff --git a/crates/lift/src/lang/model.rs b/crates/lift/src/lang/model.rs new file mode 100644 index 000000000..5d6e681d6 --- /dev/null +++ b/crates/lift/src/lang/model.rs @@ -0,0 +1,516 @@ +use std::cmp::Ordering; +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +use crate::kernel::{ + sha256_canonical_json, ByteSpan, Diagnostic, FileId, Fingerprint, RepoPath, SymbolId, +}; +use crate::lang::{AdapterName, LanguageId}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SymbolKind { + Module, + Namespace, + Package, + Function, + Method, + Class, + Struct, + Enum, + Trait, + Interface, + TypeAlias, + Field, + Constant, + Variable, + TestCase, + TestSuite, + ConfigKey, + Unknown, +} + +impl SymbolKind { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Module => "module", + Self::Namespace => "namespace", + Self::Package => "package", + Self::Function => "function", + Self::Method => "method", + Self::Class => "class", + Self::Struct => "struct", + Self::Enum => "enum", + Self::Trait => "trait", + Self::Interface => "interface", + Self::TypeAlias => "type_alias", + Self::Field => "field", + Self::Constant => "constant", + Self::Variable => "variable", + Self::TestCase => "test_case", + Self::TestSuite => "test_suite", + Self::ConfigKey => "config_key", + Self::Unknown => "unknown", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SymbolVisibility { + Public, + Private, + Internal, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalSymbolDraft { + pub local_key: String, + pub kind: SymbolKind, + pub name: Option, + pub path: Vec, + pub span: ByteSpan, + pub visibility: SymbolVisibility, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalSymbol { + pub id: SymbolId, + pub kind: SymbolKind, + pub name: Option, + pub path: Vec, + pub span: ByteSpan, + pub visibility: SymbolVisibility, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LocalEdgeKind { + Contains, + Import, + Export, + Call, + TypeRef, + Inherit, + Implement, + TestRef, + ConfigRef, + SchemaRef, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum EdgeEndpoint { + FileRoot, + Symbol(SymbolId), +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum ReferenceTargetDraft { + LocalSymbol { + local_key: String, + }, + QualifiedName { + parts: Vec, + }, + FilePath { + path: RepoPath, + }, + ExternalPackage { + package: String, + symbol: Option, + }, + Opaque { + value: String, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum ReferenceTarget { + LocalSymbol(SymbolId), + QualifiedName { + parts: Vec, + }, + FilePath { + path: RepoPath, + }, + ExternalPackage { + package: String, + symbol: Option, + }, + Opaque { + value: String, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalEdgeDraft { + pub kind: LocalEdgeKind, + pub source: EdgeEndpointDraft, + pub target: ReferenceTargetDraft, + pub span: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum EdgeEndpointDraft { + FileRoot, + Symbol { local_key: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct LocalEdge { + pub kind: LocalEdgeKind, + pub source: EdgeEndpoint, + pub target: ReferenceTarget, + pub span: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SurfaceMarkerKind { + PublicApi, + Test, + EntryPoint, + Export, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct SurfaceMarkerDraft { + pub kind: SurfaceMarkerKind, + pub symbol_local_key: Option, + pub span: Option, + pub label: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct SurfaceMarker { + pub kind: SurfaceMarkerKind, + pub symbol: Option, + pub span: Option, + pub label: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "paths", rename_all = "snake_case")] +pub(crate) enum ParseScope { + Snapshot, + Paths(BTreeSet), +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParseRequest { + pub languages: BTreeSet, + pub scope: ParseScope, +} + +impl ParseRequest { + pub(crate) fn normalized(&self) -> Self { + Self { + languages: self.languages.clone(), + scope: match &self.scope { + ParseScope::Snapshot => ParseScope::Snapshot, + ParseScope::Paths(paths) => ParseScope::Paths(paths.clone()), + }, + } + } + + pub(crate) fn fingerprint(&self) -> crate::kernel::KernelResult { + sha256_canonical_json(&self.normalized()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct MissingRequestedLanguage { + pub language: LanguageId, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParsedUnit { + pub path: RepoPath, + pub file_id: FileId, + pub blob_fingerprint: Fingerprint, + pub language: LanguageId, + pub adapter: AdapterName, + pub adapter_version: String, + pub unit_fingerprint: Fingerprint, + pub symbols: Vec, + pub edges: Vec, + pub surface_markers: Vec, + pub diagnostics: Vec, +} + +impl ParsedUnit { + pub(crate) fn fingerprint(&self) -> crate::kernel::KernelResult { + #[derive(Serialize)] + struct FingerprintDoc<'a> { + path: &'a RepoPath, + file_id: &'a FileId, + blob_fingerprint: &'a Fingerprint, + language: LanguageId, + adapter: &'a AdapterName, + adapter_version: &'a str, + symbols: &'a [LocalSymbol], + edges: &'a [LocalEdge], + surface_markers: &'a [SurfaceMarker], + diagnostics: &'a [Diagnostic], + } + + sha256_canonical_json(&FingerprintDoc { + path: &self.path, + file_id: &self.file_id, + blob_fingerprint: &self.blob_fingerprint, + language: self.language, + adapter: &self.adapter, + adapter_version: &self.adapter_version, + symbols: &self.symbols, + edges: &self.edges, + surface_markers: &self.surface_markers, + diagnostics: &self.diagnostics, + }) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct FailedParse { + pub path: RepoPath, + pub file_id: FileId, + pub blob_fingerprint: Fingerprint, + pub language: LanguageId, + pub adapter: AdapterName, + pub adapter_version: String, + pub diagnostics: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum SkippedReason { + NoMatchingAdapter, + PathNotInSnapshot, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct SkippedParse { + pub path: RepoPath, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_id: Option, + pub reason: SkippedReason, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParseStats { + pub considered_files: u64, + pub parsed_units: u64, + pub failed_units: u64, + pub skipped_no_adapter: u64, + pub skipped_missing_paths: u64, + pub missing_requested_languages: u64, + pub diagnostic_count: u64, + #[serde(default)] + pub cache_hits: u64, + #[serde(default)] + pub cache_misses: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ParseSet { + pub snapshot_fingerprint: Fingerprint, + pub request: ParseRequest, + pub request_fingerprint: Fingerprint, + pub units: Vec, + pub failed: Vec, + pub skipped: Vec, + pub missing_languages: Vec, + pub diagnostics: Vec, + pub stats: ParseStats, +} + +impl ParseSet { + pub(crate) fn sort_all(&mut self) { + self.units.sort_by(compare_parsed_units); + self.failed.sort_by(compare_failed_parses); + self.skipped.sort_by(compare_skipped_parses); + self.missing_languages.sort_by(compare_missing_languages); + self.diagnostics.sort(); + } + + pub(crate) fn refresh_stats(&mut self) { + self.stats.parsed_units = self.units.len() as u64; + self.stats.failed_units = self.failed.len() as u64; + self.stats.missing_requested_languages = self.missing_languages.len() as u64; + self.stats.diagnostic_count = self.diagnostics.len() as u64 + + self + .units + .iter() + .map(|unit| unit.diagnostics.len() as u64) + .sum::() + + self + .failed + .iter() + .map(|unit| unit.diagnostics.len() as u64) + .sum::(); + } +} + +pub(crate) fn sort_local_symbols(symbols: &mut [LocalSymbol]) { + symbols.sort_by(compare_local_symbols); +} + +pub(crate) fn sort_local_edges(edges: &mut [LocalEdge]) { + edges.sort_by(compare_local_edges); +} + +pub(crate) fn sort_surface_markers(markers: &mut [SurfaceMarker]) { + markers.sort_by(compare_surface_markers); +} + +pub(crate) fn symbol_identity_lemma( + language: LanguageId, + path: &RepoPath, + draft: &LocalSymbolDraft, + duplicate_ordinal: usize, +) -> String { + format!( + "lang\0symbol\0v1\0{}\0{}\0{}\0{}\0{}", + language.as_str(), + path.as_str(), + draft.kind.as_str(), + draft.path.join("\0"), + duplicate_ordinal + ) +} + +pub(crate) fn compare_symbol_drafts(left: &LocalSymbolDraft, right: &LocalSymbolDraft) -> Ordering { + left.kind + .cmp(&right.kind) + .then_with(|| left.path.cmp(&right.path)) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.span.cmp(&right.span)) + .then_with(|| left.local_key.cmp(&right.local_key)) +} + +pub(crate) fn compare_parsed_units(left: &ParsedUnit, right: &ParsedUnit) -> Ordering { + left.path + .cmp(&right.path) + .then_with(|| left.adapter.cmp(&right.adapter)) + .then_with(|| left.file_id.as_str().cmp(right.file_id.as_str())) +} + +pub(crate) fn compare_failed_parses(left: &FailedParse, right: &FailedParse) -> Ordering { + left.path + .cmp(&right.path) + .then_with(|| left.adapter.cmp(&right.adapter)) + .then_with(|| left.file_id.as_str().cmp(right.file_id.as_str())) +} + +pub(crate) fn compare_skipped_parses(left: &SkippedParse, right: &SkippedParse) -> Ordering { + left.path + .cmp(&right.path) + .then_with(|| left.reason.cmp(&right.reason)) + .then_with(|| { + left.file_id + .as_ref() + .map(|id| id.as_str()) + .cmp(&right.file_id.as_ref().map(|id| id.as_str())) + }) + .then_with(|| left.detail.cmp(&right.detail)) +} + +pub(crate) fn compare_missing_languages( + left: &MissingRequestedLanguage, + right: &MissingRequestedLanguage, +) -> Ordering { + left.language + .cmp(&right.language) + .then_with(|| left.detail.cmp(&right.detail)) +} + +fn compare_local_symbols(left: &LocalSymbol, right: &LocalSymbol) -> Ordering { + left.kind + .cmp(&right.kind) + .then_with(|| left.path.cmp(&right.path)) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.span.cmp(&right.span)) + .then_with(|| left.id.as_str().cmp(right.id.as_str())) +} + +fn compare_local_edges(left: &LocalEdge, right: &LocalEdge) -> Ordering { + left.kind + .cmp(&right.kind) + .then_with(|| compare_edge_endpoints(&left.source, &right.source)) + .then_with(|| compare_reference_targets(&left.target, &right.target)) + .then_with(|| left.span.cmp(&right.span)) +} + +fn compare_surface_markers(left: &SurfaceMarker, right: &SurfaceMarker) -> Ordering { + left.kind + .cmp(&right.kind) + .then_with(|| { + left.symbol + .as_ref() + .map(|id| id.as_str()) + .cmp(&right.symbol.as_ref().map(|id| id.as_str())) + }) + .then_with(|| left.span.cmp(&right.span)) + .then_with(|| left.label.cmp(&right.label)) +} + +fn compare_edge_endpoints(left: &EdgeEndpoint, right: &EdgeEndpoint) -> Ordering { + match (left, right) { + (EdgeEndpoint::FileRoot, EdgeEndpoint::FileRoot) => Ordering::Equal, + (EdgeEndpoint::FileRoot, EdgeEndpoint::Symbol(_)) => Ordering::Less, + (EdgeEndpoint::Symbol(_), EdgeEndpoint::FileRoot) => Ordering::Greater, + (EdgeEndpoint::Symbol(left), EdgeEndpoint::Symbol(right)) => { + left.as_str().cmp(right.as_str()) + } + } +} + +fn compare_reference_targets(left: &ReferenceTarget, right: &ReferenceTarget) -> Ordering { + reference_target_rank(left) + .cmp(&reference_target_rank(right)) + .then_with(|| match (left, right) { + (ReferenceTarget::LocalSymbol(left), ReferenceTarget::LocalSymbol(right)) => { + left.as_str().cmp(right.as_str()) + } + ( + ReferenceTarget::QualifiedName { parts: left }, + ReferenceTarget::QualifiedName { parts: right }, + ) => left.cmp(right), + ( + ReferenceTarget::FilePath { path: left }, + ReferenceTarget::FilePath { path: right }, + ) => left.cmp(right), + ( + ReferenceTarget::ExternalPackage { + package: left_package, + symbol: left_symbol, + }, + ReferenceTarget::ExternalPackage { + package: right_package, + symbol: right_symbol, + }, + ) => left_package + .cmp(right_package) + .then_with(|| left_symbol.cmp(right_symbol)), + (ReferenceTarget::Opaque { value: left }, ReferenceTarget::Opaque { value: right }) => { + left.cmp(right) + } + _ => Ordering::Equal, + }) +} + +fn reference_target_rank(target: &ReferenceTarget) -> u8 { + match target { + ReferenceTarget::LocalSymbol(_) => 0, + ReferenceTarget::QualifiedName { .. } => 1, + ReferenceTarget::FilePath { .. } => 2, + ReferenceTarget::ExternalPackage { .. } => 3, + ReferenceTarget::Opaque { .. } => 4, + } +} diff --git a/crates/lift/src/lang/registry.rs b/crates/lift/src/lang/registry.rs new file mode 100644 index 000000000..602830be3 --- /dev/null +++ b/crates/lift/src/lang/registry.rs @@ -0,0 +1,109 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use crate::lang::{ + AdapterCapabilities, AdapterDescriptor, AdapterName, LangError, LangResult, LanguageAdapter, + LanguageId, QueryEngineKind, +}; + +pub(crate) const BUILT_IN_LANGUAGE_ORDER: &[LanguageId] = &[ + LanguageId::Json, + LanguageId::Toml, + LanguageId::Yaml, + LanguageId::Rust, + LanguageId::Python, + LanguageId::Javascript, + LanguageId::Typescript, +]; + +#[derive(Default)] +pub(crate) struct LanguageRegistryBuilder { + adapters: BTreeMap>, + languages: BTreeMap, +} + +pub(crate) struct LanguageRegistry { + adapters: BTreeMap>, + languages: BTreeMap, +} + +pub(crate) fn built_in_registry() -> LangResult { + bootstrap_built_in_registry(LanguageRegistryBuilder::new())?.build() +} + +impl LanguageRegistryBuilder { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn register(mut self, adapter: A) -> LangResult { + let descriptor = adapter.descriptor(); + let name = descriptor.name.clone(); + let language = descriptor.language; + + if self.adapters.contains_key(&name) { + return Err(LangError::DuplicateAdapterName { + name: name.as_str().to_owned(), + }); + } + + if let Some(existing) = self.languages.get(&language) { + return Err(LangError::DuplicateLanguageAdapter { + language, + existing: existing.as_str().to_owned(), + duplicate: name.as_str().to_owned(), + }); + } + + self.languages.insert(language, name.clone()); + self.adapters.insert(name, Arc::new(adapter)); + Ok(self) + } + + pub(crate) fn build(self) -> LangResult { + Ok(LanguageRegistry { + adapters: self.adapters, + languages: self.languages, + }) + } +} + +impl LanguageRegistry { + pub(crate) fn descriptors(&self) -> Vec { + self.adapters + .values() + .map(|adapter| adapter.descriptor()) + .collect() + } + + pub(crate) fn adapter_for_language( + &self, + language: LanguageId, + ) -> Option<&Arc> { + let name = self.languages.get(&language)?; + self.adapters.get(name) + } + + pub(crate) fn capabilities_for_language( + &self, + language: LanguageId, + ) -> Option { + self.adapter_for_language(language) + .map(|adapter| adapter.capabilities()) + } + + pub(crate) fn supported_query_engines_for_language( + &self, + language: LanguageId, + ) -> BTreeSet { + self.capabilities_for_language(language) + .map(|capabilities| capabilities.query_engines) + .unwrap_or_default() + } +} + +fn bootstrap_built_in_registry( + builder: LanguageRegistryBuilder, +) -> LangResult { + Ok(builder) +} diff --git a/crates/lift/src/lang/schema.rs b/crates/lift/src/lang/schema.rs new file mode 100644 index 000000000..6b1620e1a --- /dev/null +++ b/crates/lift/src/lang/schema.rs @@ -0,0 +1,19 @@ +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/lang/parse_manifest.v1.json"; + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_VERSION: u32 = 1; + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_FILE: &str = "parse_manifest.v1.json"; + +pub(crate) const LANG_PARSE_MANIFEST_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/lang/parse_manifest.v1.json"); + +pub(crate) const LANG_PARSE_MANIFEST_V2_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/lang/parse_manifest.v2.json"; + +pub(crate) const LANG_PARSE_MANIFEST_V2_SCHEMA_VERSION: u32 = 2; + +pub(crate) const LANG_PARSE_MANIFEST_V2_SCHEMA_FILE: &str = "parse_manifest.v2.json"; + +pub(crate) const LANG_PARSE_MANIFEST_V2_SCHEMA_JSON: &str = + include_str!("../../schemas/lang/parse_manifest.v2.json"); diff --git a/crates/lift/src/lib.rs b/crates/lift/src/lib.rs new file mode 100644 index 000000000..500858344 --- /dev/null +++ b/crates/lift/src/lib.rs @@ -0,0 +1,43 @@ +#![forbid(unsafe_code)] + +extern crate self as substrate_lift; + +#[cfg(test)] +use assert_cmd as _; +#[cfg(test)] +use jsonschema as _; +#[cfg(test)] +use predicates as _; +#[cfg(test)] +use serde as _; +#[cfg(test)] +use serde_jcs as _; +use serde_json as _; +#[cfg(test)] +use sha2 as _; +#[cfg(test)] +use thiserror as _; + +pub mod error; + +#[cfg(feature = "cli")] +pub use cli::run as run_cli; + +#[cfg(feature = "cli")] +pub(crate) mod cli; + +pub(crate) mod app; +pub(crate) mod derive; +pub(crate) mod export; +pub(crate) mod facts; +pub(crate) mod graph; +pub mod kernel; +pub(crate) mod lang; +pub(crate) mod pack; +pub(crate) mod patch; +pub(crate) mod query; +pub(crate) mod repo; +pub(crate) mod topo; + +#[cfg(test)] +mod runtime_bootstrap_tests; diff --git a/crates/lift/src/pack/builtin.rs b/crates/lift/src/pack/builtin.rs new file mode 100644 index 000000000..d71538393 --- /dev/null +++ b/crates/lift/src/pack/builtin.rs @@ -0,0 +1,165 @@ +//! Embedded built-in packs. + +use crate::pack::source::{PackFormat, PackSource}; + +const GENERIC_DEFAULT_PROFILE: &str = r#"kind = "profile" +version = 1 +id = "generic/default" +name = "Generic default profile" +description = "Default deterministic profile for generic repositories" + +[apps] +enabled = ["score"] +default = "score" + +[topology] +boundary_taxonomy = "builtin:generic/boundaries" +component_map = "builtin:generic/components" + +[analysis] +languages = ["json", "toml", "yaml", "rust", "python", "javascript", "typescript"] +follow_symlinks = false +max_scope_depth = 2 +"#; + +const GENERIC_BOUNDARIES: &str = r#"{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "generic/boundaries", + "name": "Generic repository boundaries", + "description": "Coarse boundary taxonomy for generic repositories.", + "counting": { + "mode": "distinct_minus_one" + }, + "boundaries": [ + { + "id": "application", + "label": "Application surface", + "include": ["apps/**", "services/**", "bin/**", "cmd/**"] + }, + { + "id": "library", + "label": "Shared library surface", + "include": ["crates/**", "packages/**", "libs/**", "src/**"] + }, + { + "id": "interface", + "label": "Interface and contract surface", + "include": ["api/**", "proto/**", "schemas/**", "contracts/**"] + }, + { + "id": "tests", + "label": "Test surface", + "include": ["tests/**", "test/**", "e2e/**", "integration-tests/**"] + }, + { + "id": "operations", + "label": "Operations surface", + "include": ["infra/**", "ops/**", "deploy/**", ".github/**", "scripts/**"] + }, + { + "id": "docs", + "label": "Documentation surface", + "include": ["docs/**"] + } + ] +}"#; + +const GENERIC_COMPONENTS: &str = r#"{ + "kind": "component_map", + "version": 1, + "id": "generic/components", + "name": "Generic repository components", + "description": "Generic component map for common repository layouts.", + "counting": { + "mode": "distinct" + }, + "components": [ + { + "id": "frontend", + "label": "Frontend", + "include": ["web/**", "ui/**", "frontend/**", "apps/web/**", "packages/ui/**"], + "tags": ["ui", "client"] + }, + { + "id": "backend", + "label": "Backend", + "include": ["api/**", "backend/**", "server/**", "services/**", "apps/api/**"], + "tags": ["service", "server"] + }, + { + "id": "shared", + "label": "Shared libraries", + "include": ["crates/**", "packages/shared/**", "packages/config/**", "libs/**", "src/**"], + "tags": ["shared", "library"] + }, + { + "id": "tooling", + "label": "Tooling", + "include": ["scripts/**", "tools/**", ".github/**", "infra/**"], + "tags": ["tooling", "ops"] + }, + { + "id": "tests", + "label": "Tests", + "include": ["tests/**", "test/**", "e2e/**", "integration-tests/**"], + "tags": ["test"] + } + ] +}"#; + +/// Returns a built-in profile source by logical pack name. +pub(crate) fn profile_source(logical_name: &str) -> Option { + match logical_name { + "generic/default" => Some(PackSource::Builtin { + logical_name: "generic/default", + format: PackFormat::Toml, + bytes: GENERIC_DEFAULT_PROFILE.as_bytes(), + }), + _ => None, + } +} + +/// Returns a built-in boundary taxonomy source by logical pack name. +pub(crate) fn boundary_taxonomy_source(logical_name: &str) -> Option { + match logical_name { + "generic/boundaries" => Some(PackSource::Builtin { + logical_name: "generic/boundaries", + format: PackFormat::Json, + bytes: GENERIC_BOUNDARIES.as_bytes(), + }), + _ => None, + } +} + +/// Returns a built-in component map source by logical pack name. +pub(crate) fn component_map_source(logical_name: &str) -> Option { + match logical_name { + "generic/components" => Some(PackSource::Builtin { + logical_name: "generic/components", + format: PackFormat::Json, + bytes: GENERIC_COMPONENTS.as_bytes(), + }), + _ => None, + } +} + +/// Returns a built-in score model source by logical pack name. +pub(crate) fn score_model_source(_logical_name: &str) -> Option { + None +} + +/// Returns a built-in query-pack source by logical pack name. +pub(crate) fn query_pack_source(_logical_name: &str) -> Option { + None +} + +/// Returns a built-in rule-pack source by logical pack name. +pub(crate) fn rule_pack_source(_logical_name: &str) -> Option { + None +} + +/// Returns a built-in recipe-pack source by logical pack name. +pub(crate) fn recipe_pack_source(_logical_name: &str) -> Option { + None +} diff --git a/crates/lift/src/pack/compiled/header.rs b/crates/lift/src/pack/compiled/header.rs new file mode 100644 index 000000000..25a62f62b --- /dev/null +++ b/crates/lift/src/pack/compiled/header.rs @@ -0,0 +1,20 @@ +//! Compiled pack header metadata. + +use crate::kernel::Fingerprint; +use crate::pack::names::PackName; +use crate::pack::raw::PackKind; +use crate::pack::source::PackOrigin; + +/// Stable header emitted for every compiled pack. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledPackHeader { + pub kind: PackKind, + pub id: PackName, + pub version: u32, + pub name: String, + pub description: Option, + pub schema_id: &'static str, + pub origin: PackOrigin, + pub source_fingerprint: Fingerprint, + pub semantic_fingerprint: Fingerprint, +} diff --git a/crates/lift/src/pack/compiled/mod.rs b/crates/lift/src/pack/compiled/mod.rs new file mode 100644 index 000000000..b20ff8da4 --- /dev/null +++ b/crates/lift/src/pack/compiled/mod.rs @@ -0,0 +1,58 @@ +//! Compiled pack output types. + +use std::collections::BTreeMap; + +use crate::kernel::Fingerprint; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::names::PackName; + +mod header; +mod profile; +mod query_pack; +mod recipe_pack; +mod rule_pack; +mod score_model; +mod topology; + +pub(crate) use header::CompiledPackHeader; +pub(crate) use profile::{ + CompiledAnalysisDefaults, CompiledPathClasses, CompiledProfile, CompiledProfileApps, + CompiledProfileIncludes, CompiledProfileScore, CompiledProfileTopology, +}; +pub(crate) use query_pack::{ + CompiledQueryCapture, CompiledQueryDef, CompiledQueryPack, QueryEngineKind, +}; +pub(crate) use recipe_pack::{CompiledRecipeDef, CompiledRecipePack, CompiledRecipeTransform}; +pub(crate) use rule_pack::{ + CompiledRuleDef, CompiledRuleEmit, CompiledRulePack, CompiledRuleScope, ReservedPathClass, +}; +pub(crate) use score_model::{ + CompiledConfidenceModel, CompiledConfidenceRule, CompiledMissingInputRule, CompiledScoreModel, + CompiledTriggerRule, +}; +pub(crate) use topology::{ + BoundaryCountingMode, CompiledBoundary, CompiledBoundaryTaxonomy, CompiledComponent, + CompiledComponentMap, CompiledGlobMatcher, ComponentCountingMode, ResolvedProfileTopology, +}; + +/// Fully resolved pack bundle for one compiled profile. +/// +/// Phase C bundle shape: +/// profile +/// + optional topology packs +/// + optional score model +/// + query packs +/// + rule packs +/// + recipe packs +#[derive(Clone, Debug)] +pub(crate) struct CompiledPackSet { + pub profile: CompiledProfile, + pub boundary_taxonomy: Option, + pub component_map: Option, + pub score_model: Option, + pub rule_packs: BTreeMap, + pub query_packs: BTreeMap, + pub recipe_packs: BTreeMap, + pub diagnostics: Vec, + pub semantic_fingerprint: Fingerprint, +} diff --git a/crates/lift/src/pack/compiled/profile.rs b/crates/lift/src/pack/compiled/profile.rs new file mode 100644 index 000000000..eb0ecad74 --- /dev/null +++ b/crates/lift/src/pack/compiled/profile.rs @@ -0,0 +1,63 @@ +//! Compiled profile output types. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use crate::pack::compiled::CompiledPackHeader; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::names::{AppName, LanguageId}; +use crate::pack::refs::PackRef; + +/// Compiled seam-1 profile. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledProfile { + pub header: CompiledPackHeader, + pub source_file_base_dir: Option, + pub apps: CompiledProfileApps, + pub analysis: CompiledAnalysisDefaults, + pub topology: CompiledProfileTopology, + pub score: CompiledProfileScore, + pub includes: CompiledProfileIncludes, + pub diagnostics: Vec, +} + +/// Compiled application defaults. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledProfileApps { + pub enabled: BTreeSet, + pub default: AppName, +} + +/// Compiled analysis defaults. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledAnalysisDefaults { + pub languages: BTreeSet, + pub follow_symlinks: bool, + pub max_scope_depth: u8, +} + +/// Deferred topology references selected by a profile. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct CompiledProfileTopology { + pub boundary_taxonomy: Option, + pub component_map: Option, + pub classes: CompiledPathClasses, +} + +/// Deferred score model reference selected by a profile. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct CompiledProfileScore { + pub model: Option, +} + +/// Deferred rule/query/recipe include lists selected by a profile. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct CompiledProfileIncludes { + pub rule_packs: BTreeSet, + pub query_packs: BTreeSet, + pub recipe_packs: BTreeSet, +} + +/// Placeholder for later topology class compilation. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct CompiledPathClasses; diff --git a/crates/lift/src/pack/compiled/query_pack.rs b/crates/lift/src/pack/compiled/query_pack.rs new file mode 100644 index 000000000..403560d8e --- /dev/null +++ b/crates/lift/src/pack/compiled/query_pack.rs @@ -0,0 +1,53 @@ +//! Compiled query-pack output types. + +use std::collections::BTreeMap; + +use crate::kernel::QueryId; +use crate::pack::compiled::CompiledPackHeader; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::names::LanguageId; + +/// Supported compiled query engines for seam-1 query packs. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum QueryEngineKind { + TreeSitter, +} + +impl QueryEngineKind { + /// Returns the canonical engine identifier. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::TreeSitter => "tree_sitter", + } + } +} + +/// Compiled query pack keyed by deterministic query ids. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledQueryPack { + pub header: CompiledPackHeader, + pub language: LanguageId, + pub engine: QueryEngineKind, + pub queries: BTreeMap, + pub diagnostics: Vec, +} + +/// Compiled query definition. +/// +/// Query ids are derived from the identity lemma +/// `pack\0query_pack\0\0query\0`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledQueryDef { + pub local_id: String, + pub id: QueryId, + pub summary: Option, + pub pattern: String, + pub captures: Vec, +} + +/// Compiled query capture metadata preserved for later engine compilation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledQueryCapture { + pub name: String, + pub required: bool, +} diff --git a/crates/lift/src/pack/compiled/recipe_pack.rs b/crates/lift/src/pack/compiled/recipe_pack.rs new file mode 100644 index 000000000..47d5fa5bb --- /dev/null +++ b/crates/lift/src/pack/compiled/recipe_pack.rs @@ -0,0 +1,35 @@ +//! Compiled recipe-pack contracts. + +use std::collections::BTreeMap; + +use crate::kernel::RecipeId; +use crate::pack::compiled::CompiledPackHeader; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::CompiledQueryRef; + +/// Compiled recipe pack keyed by deterministic recipe ids. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledRecipePack { + pub header: CompiledPackHeader, + pub recipes: BTreeMap, + pub diagnostics: Vec, +} + +/// Compiled recipe definition. +/// +/// Recipe ids are derived from the identity lemma +/// `pack\0recipe_pack\0\0recipe\0`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledRecipeDef { + pub local_id: String, + pub id: RecipeId, + pub summary: Option, + pub query: CompiledQueryRef, + pub transforms: Vec, +} + +/// Compiled recipe transforms preserved in source order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CompiledRecipeTransform { + ReplaceCaptureText { capture: String, text: String }, +} diff --git a/crates/lift/src/pack/compiled/rule_pack.rs b/crates/lift/src/pack/compiled/rule_pack.rs new file mode 100644 index 000000000..2e9a26ba1 --- /dev/null +++ b/crates/lift/src/pack/compiled/rule_pack.rs @@ -0,0 +1,58 @@ +//! Compiled rule-pack contracts. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::kernel::{RuleId, Severity}; +use crate::pack::compiled::CompiledPackHeader; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::names::LanguageId; +use crate::pack::CompiledQueryRef; + +/// Reserved path-class vocabulary for compiled rule scopes. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum ReservedPathClass { + Test, + Docs, + Ci, + Migration, + Security, + PublicApi, + Generated, + Vendor, +} + +/// Compiled rule pack keyed by deterministic rule ids. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledRulePack { + pub header: CompiledPackHeader, + pub rules: BTreeMap, + pub diagnostics: Vec, +} + +/// Compiled rule definition. +/// +/// Rule ids are derived from the identity lemma +/// `pack\0rule_pack\0\0rule\0`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledRuleDef { + pub local_id: String, + pub id: RuleId, + pub summary: Option, + pub severity: Severity, + pub scope: Option, + pub query: CompiledQueryRef, + pub emit: Vec, +} + +/// Optional compiled scope for a rule definition. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct CompiledRuleScope { + pub languages: BTreeSet, + pub path_classes: BTreeSet, +} + +/// Compiled emit actions preserved in source order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CompiledRuleEmit { + Finding { code: String, message: String }, +} diff --git a/crates/lift/src/pack/compiled/score_model.rs b/crates/lift/src/pack/compiled/score_model.rs new file mode 100644 index 000000000..0fd4c2d97 --- /dev/null +++ b/crates/lift/src/pack/compiled/score_model.rs @@ -0,0 +1,51 @@ +//! Compiled score-model contracts. + +use std::collections::BTreeSet; + +use crate::kernel::JsonPointer; +use crate::pack::compiled::CompiledPackHeader; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::CompiledExpr; + +/// Compiled seam-1 score model. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledScoreModel { + pub header: CompiledPackHeader, + pub vector_version: u32, + pub lift_score: CompiledExpr, + pub estimated_slices: CompiledExpr, + pub triggers: Vec, + pub confidence: CompiledConfidenceModel, + pub missing_input_rules: Vec, + pub diagnostics: Vec, +} + +/// Compiled trigger rule, preserving source order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledTriggerRule { + pub id: String, + pub when: CompiledExpr, +} + +/// Compiled confidence model. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledConfidenceModel { + pub default_level: String, + pub rules: Vec, +} + +/// Compiled confidence rule, preserving source order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledConfidenceRule { + pub id: String, + pub when: CompiledExpr, + pub set: String, + pub causes: BTreeSet, +} + +/// Compiled missing-input rule, preserving source order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledMissingInputRule { + pub field: JsonPointer, + pub when: CompiledExpr, +} diff --git a/crates/lift/src/pack/compiled/topology.rs b/crates/lift/src/pack/compiled/topology.rs new file mode 100644 index 000000000..b59bde5c8 --- /dev/null +++ b/crates/lift/src/pack/compiled/topology.rs @@ -0,0 +1,81 @@ +//! Compiled topology contracts for boundary taxonomies and component maps. + +use std::collections::{BTreeMap, BTreeSet}; + +use globset::GlobSet; + +use crate::kernel::Fingerprint; +use crate::pack::compiled::CompiledPackHeader; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::{BoundaryId, ComponentId}; + +/// Compiled counting mode for a boundary taxonomy. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum BoundaryCountingMode { + DistinctMinusOne, +} + +/// Compiled counting mode for a component map. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum ComponentCountingMode { + Distinct, +} + +/// Compiled boundary taxonomy keyed by deterministic boundary ids. +#[derive(Clone, Debug)] +pub(crate) struct CompiledBoundaryTaxonomy { + pub header: CompiledPackHeader, + pub counting_mode: BoundaryCountingMode, + pub boundaries: BTreeMap, + pub diagnostics: Vec, +} + +/// Compiled boundary entry. +/// +/// Boundary ids are derived from the identity lemma +/// `pack\0boundary_taxonomy\0\0boundary\0`. +#[derive(Clone, Debug)] +pub(crate) struct CompiledBoundary { + pub local_id: String, + pub id: BoundaryId, + pub label: String, + pub include_patterns: Vec, + pub exclude_patterns: Vec, + pub include_matcher: GlobSet, + pub exclude_matcher: GlobSet, +} + +/// Compiled component map keyed by deterministic component ids. +#[derive(Clone, Debug)] +pub(crate) struct CompiledComponentMap { + pub header: CompiledPackHeader, + pub counting_mode: ComponentCountingMode, + pub components: BTreeMap, + pub diagnostics: Vec, +} + +/// Compiled component entry. +/// +/// Component ids are derived from the identity lemma +/// `pack\0component_map\0\0component\0`. +#[derive(Clone, Debug)] +pub(crate) struct CompiledComponent { + pub local_id: String, + pub id: ComponentId, + pub label: String, + pub include_patterns: Vec, + pub exclude_patterns: Vec, + pub tags: BTreeSet, + pub include_matcher: GlobSet, + pub exclude_matcher: GlobSet, +} + +/// Topology packs resolved for a compiled profile. +#[derive(Clone, Debug)] +pub(crate) struct ResolvedProfileTopology { + pub boundary_taxonomy: Option, + pub component_map: Option, + pub semantic_fingerprint: Fingerprint, +} + +pub(crate) type CompiledGlobMatcher = GlobSet; diff --git a/crates/lift/src/pack/compiler.rs b/crates/lift/src/pack/compiler.rs new file mode 100644 index 000000000..cd5576fc6 --- /dev/null +++ b/crates/lift/src/pack/compiler.rs @@ -0,0 +1,3978 @@ +//! Seam-1 pack compiler spine. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use globset::{Glob, GlobSet, GlobSetBuilder}; +use jsonschema::error::ValidationErrorKind; +use jsonschema::{Retrieve, Uri, ValidationError, Validator}; +use serde::Serialize; +use serde_json::{Map, Number, Value}; + +use crate::kernel::{ + sha256_bytes, sha256_canonical_json, Fingerprint, JsonPointer, QueryId, RecipeId, RuleId, +}; +use crate::pack::builtin; +use crate::pack::compiled::{ + BoundaryCountingMode, CompiledAnalysisDefaults, CompiledBoundary, CompiledBoundaryTaxonomy, + CompiledComponent, CompiledComponentMap, CompiledConfidenceModel, CompiledConfidenceRule, + CompiledMissingInputRule, CompiledPackHeader, CompiledPackSet, CompiledPathClasses, + CompiledProfile, CompiledProfileApps, CompiledProfileIncludes, CompiledProfileScore, + CompiledProfileTopology, CompiledQueryCapture, CompiledQueryDef, CompiledQueryPack, + CompiledRecipeDef, CompiledRecipePack, CompiledRecipeTransform, CompiledRuleDef, + CompiledRuleEmit, CompiledRulePack, CompiledRuleScope, CompiledScoreModel, CompiledTriggerRule, + ComponentCountingMode, QueryEngineKind, ReservedPathClass, ResolvedProfileTopology, +}; +use crate::pack::diagnostics::{PackDiagnostic, PackLocation}; +use crate::pack::error::{PackError, PackResult}; +use crate::pack::expr::{compile_expr, compile_query_ref, CompiledQueryRef}; +use crate::pack::names::{AppName, LanguageId, PackName}; +use crate::pack::raw::{ + PackKind, RawBoundaryEntry, RawBoundaryTaxonomy, RawBoundaryTaxonomyCountingMode, + RawComponentEntry, RawComponentMap, RawComponentMapCountingMode, RawIncludeSection, RawProfile, + RawProfileAnalysis, RawProfileApps, RawProfileScore, RawProfileTopology, RawQueryPack, + RawRecipePack, RawRecipeTransform, RawReservedPathClass, RawRuleEmit, RawRulePack, + RawScoreModel, +}; +use crate::pack::refs::PackRef; +use crate::pack::schema::{ + PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_JSON, + PACK_COMMON_V1_SCHEMA_ID, PACK_COMMON_V1_SCHEMA_JSON, PACK_COMPONENT_MAP_V1_SCHEMA_ID, + PACK_COMPONENT_MAP_V1_SCHEMA_JSON, PACK_PROFILE_V1_SCHEMA_ID, PACK_QUERY_PACK_V1_SCHEMA_ID, + PACK_QUERY_PACK_V1_SCHEMA_JSON, PACK_RECIPE_PACK_V1_SCHEMA_ID, PACK_RECIPE_PACK_V1_SCHEMA_JSON, + PACK_RULE_PACK_V1_SCHEMA_ID, PACK_RULE_PACK_V1_SCHEMA_JSON, PACK_SCORE_MODEL_V1_SCHEMA_ID, + PACK_SCORE_MODEL_V1_SCHEMA_JSON, +}; +use crate::pack::source::{PackFormat, PackOrigin, PackSource}; +use crate::pack::{BoundaryId, ComponentId}; + +const DEFAULT_APP: &str = "score"; +const DEFAULT_LANGUAGES: &[&str] = &[ + "javascript", + "json", + "python", + "rust", + "toml", + "typescript", + "yaml", +]; +const DEFAULT_FOLLOW_SYMLINKS: bool = false; +const DEFAULT_MAX_SCOPE_DEPTH: u8 = 2; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TopologySchemaKind { + BoundaryTaxonomy, + ComponentMap, +} + +impl TopologySchemaKind { + fn invalid_root_code(self) -> &'static str { + match self { + Self::BoundaryTaxonomy => "pack.boundary_taxonomy.invalid_root", + Self::ComponentMap => "pack.component_map.invalid_root", + } + } + + fn validator(self) -> &'static Validator { + match self { + Self::BoundaryTaxonomy => boundary_taxonomy_schema_validator(), + Self::ComponentMap => component_map_schema_validator(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AdvancedSchemaKind { + ScoreModel, + QueryPack, + RulePack, + RecipePack, +} + +impl AdvancedSchemaKind { + fn invalid_root_code(self) -> &'static str { + match self { + Self::ScoreModel => "pack.score_model.invalid_root", + Self::QueryPack => "pack.query_pack.invalid_root", + Self::RulePack => "pack.rule_pack.invalid_root", + Self::RecipePack => "pack.recipe_pack.invalid_root", + } + } + + fn validator(self) -> &'static Validator { + match self { + Self::ScoreModel => score_model_schema_validator(), + Self::QueryPack => query_pack_schema_validator(), + Self::RulePack => rule_pack_schema_validator(), + Self::RecipePack => recipe_pack_schema_validator(), + } + } + + fn schema_id(self) -> &'static str { + match self { + Self::ScoreModel => PACK_SCORE_MODEL_V1_SCHEMA_ID, + Self::QueryPack => PACK_QUERY_PACK_V1_SCHEMA_ID, + Self::RulePack => PACK_RULE_PACK_V1_SCHEMA_ID, + Self::RecipePack => PACK_RECIPE_PACK_V1_SCHEMA_ID, + } + } +} + +#[derive(Clone, Debug, Default)] +struct EmbeddedSchemaRetriever; + +impl Retrieve for EmbeddedSchemaRetriever { + fn retrieve( + &self, + uri: &Uri, + ) -> Result> { + match uri.as_str() { + PACK_COMMON_V1_SCHEMA_ID | "common.v1.json" => { + serde_json::from_str(PACK_COMMON_V1_SCHEMA_JSON).map_err(|error| { + format!("failed to parse embedded common schema: {error}").into() + }) + } + other => Err(format!("schema not found: {other}").into()), + } + } +} + +/// Profile-only compiler entrypoint for Phase A. +#[derive(Clone, Debug, Default)] +pub(crate) struct PackCompiler; + +impl PackCompiler { + /// Creates a new pack compiler. + pub(crate) fn new() -> Self { + Self + } + + /// Compiles one standalone profile from builtin, file, or inline sources. + pub(crate) fn compile_profile(&self, source: PackSource) -> PackResult { + let loaded = self.load_source(source)?; + let LoadedSource { + origin, + format, + bytes, + file_base_dir, + } = loaded; + if format != PackFormat::Toml { + return Err(PackError::UnsupportedFormat { + origin: origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&bytes); + let json_value = parse_profile_toml(&origin, &bytes)?; + let mut diagnostics = validate_profile_document(&origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: origin.display(), + schema_id: PACK_PROFILE_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw_profile: RawProfile = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: origin.display(), + schema_id: PACK_PROFILE_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.profile.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )], + })?; + + let normalized = normalize_profile(&raw_profile); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.profile.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )], + })?; + + compile_normalized_profile( + origin, + file_base_dir, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + /// Compiles one standalone boundary taxonomy from builtin, file, or inline JSON sources. + pub(crate) fn compile_boundary_taxonomy( + &self, + source: PackSource, + ) -> PackResult { + let loaded = self.load_source(source)?; + if loaded.format != PackFormat::Json { + return Err(PackError::UnsupportedFormat { + origin: loaded.origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&loaded.bytes); + let json_value = parse_json_document(&loaded.origin, &loaded.bytes, "boundary_taxonomy")?; + let mut diagnostics = validate_boundary_taxonomy_document(&loaded.origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw: RawBoundaryTaxonomy = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.boundary_taxonomy.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + + let normalized = normalize_boundary_taxonomy(&raw); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: loaded.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.boundary_taxonomy.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + + compile_normalized_boundary_taxonomy( + loaded.origin, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + /// Compiles one standalone component map from builtin, file, or inline JSON sources. + pub(crate) fn compile_component_map( + &self, + source: PackSource, + ) -> PackResult { + let loaded = self.load_source(source)?; + if loaded.format != PackFormat::Json { + return Err(PackError::UnsupportedFormat { + origin: loaded.origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&loaded.bytes); + let json_value = parse_json_document(&loaded.origin, &loaded.bytes, "component_map")?; + let mut diagnostics = validate_component_map_document(&loaded.origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_COMPONENT_MAP_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw: RawComponentMap = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_COMPONENT_MAP_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.component_map.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + + let normalized = normalize_component_map(&raw); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: loaded.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.component_map.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + + compile_normalized_component_map( + loaded.origin, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + /// Compiles one standalone score model from builtin, file, or inline JSON sources. + pub(crate) fn compile_score_model(&self, source: PackSource) -> PackResult { + self.compile_score_model_loaded(source) + .map(|loaded| loaded.pack) + } + + /// Compiles one standalone query pack from builtin, file, or inline JSON sources. + pub(crate) fn compile_query_pack(&self, source: PackSource) -> PackResult { + self.compile_query_pack_loaded(source) + .map(|loaded| loaded.pack) + } + + /// Compiles one standalone rule pack from builtin, file, or inline JSON sources. + pub(crate) fn compile_rule_pack(&self, source: PackSource) -> PackResult { + self.compile_rule_pack_loaded(source) + .map(|loaded| loaded.pack) + } + + /// Compiles one standalone recipe pack from builtin, file, or inline JSON sources. + pub(crate) fn compile_recipe_pack(&self, source: PackSource) -> PackResult { + self.compile_recipe_pack_loaded(source) + .map(|loaded| loaded.pack) + } + + /// Resolves the profile topology refs into compiled topology artifacts. + pub(crate) fn resolve_profile_topology( + &self, + profile: &CompiledProfile, + ) -> PackResult { + let boundary_taxonomy = profile + .topology + .boundary_taxonomy + .as_ref() + .map(|reference| resolve_boundary_taxonomy(self, profile, reference)) + .transpose()?; + let component_map = profile + .topology + .component_map + .as_ref() + .map(|reference| resolve_component_map(self, profile, reference)) + .transpose()?; + + let fingerprint_input = [ + boundary_taxonomy + .as_ref() + .map(|compiled| compiled.header.semantic_fingerprint.as_str().to_owned()), + component_map + .as_ref() + .map(|compiled| compiled.header.semantic_fingerprint.as_str().to_owned()), + ]; + let semantic_fingerprint = + sha256_canonical_json(&fingerprint_input).map_err(|error| PackError::ParseFailure { + origin: profile.header.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.profile_topology.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: profile.header.origin.clone(), + path: None, + }), + )], + })?; + + Ok(ResolvedProfileTopology { + boundary_taxonomy, + component_map, + semantic_fingerprint, + }) + } + + /// Resolves all selected advanced packs for a compiled profile into one bundle. + pub(crate) fn resolve_profile_pack_set( + &self, + profile: &CompiledProfile, + ) -> PackResult { + let topology = self.resolve_profile_topology(profile)?; + let mut memo = BTreeMap::new(); + let mut seen_pack_ids = BTreeMap::new(); + register_bundle_pack_id( + &profile.header, + &resolved_pack_origin_key(&profile.header.origin)?, + &mut seen_pack_ids, + )?; + if let Some(boundary_taxonomy) = topology.boundary_taxonomy.as_ref() { + register_bundle_pack_id( + &boundary_taxonomy.header, + &resolved_pack_origin_key(&boundary_taxonomy.header.origin)?, + &mut seen_pack_ids, + )?; + } + if let Some(component_map) = topology.component_map.as_ref() { + register_bundle_pack_id( + &component_map.header, + &resolved_pack_origin_key(&component_map.header.origin)?, + &mut seen_pack_ids, + )?; + } + + let score_model = profile + .score + .model + .as_ref() + .map(|reference| { + self.resolve_bundle_score_model( + profile.header.id.as_str(), + profile.source_file_base_dir.as_ref(), + reference, + &mut memo, + &mut seen_pack_ids, + ) + }) + .transpose()? + .map(|loaded| loaded.pack); + + let mut query_packs = BTreeMap::new(); + let mut rule_packs = BTreeMap::new(); + let mut recipe_packs = BTreeMap::new(); + + for reference in &profile.includes.query_packs { + let loaded = self.resolve_bundle_query_pack( + profile.header.id.as_str(), + profile.source_file_base_dir.as_ref(), + reference, + &mut memo, + &mut seen_pack_ids, + )?; + query_packs.insert(loaded.pack.header.id.clone(), loaded.pack); + } + + for reference in &profile.includes.rule_packs { + let loaded = self.resolve_bundle_rule_pack( + profile.header.id.as_str(), + profile.source_file_base_dir.as_ref(), + reference, + &mut memo, + &mut seen_pack_ids, + )?; + self.resolve_rule_pack_queries( + &loaded, + &mut query_packs, + &mut memo, + &mut seen_pack_ids, + )?; + rule_packs.insert(loaded.pack.header.id.clone(), loaded.pack); + } + + for reference in &profile.includes.recipe_packs { + let loaded = self.resolve_bundle_recipe_pack( + profile.header.id.as_str(), + profile.source_file_base_dir.as_ref(), + reference, + &mut memo, + &mut seen_pack_ids, + )?; + self.resolve_recipe_pack_queries( + &loaded, + &mut query_packs, + &mut memo, + &mut seen_pack_ids, + )?; + recipe_packs.insert(loaded.pack.header.id.clone(), loaded.pack); + } + + let semantic_fingerprint = compile_pack_set_fingerprint( + profile, + &topology, + score_model.as_ref(), + &rule_packs, + &query_packs, + &recipe_packs, + )?; + + Ok(CompiledPackSet { + profile: profile.clone(), + boundary_taxonomy: topology.boundary_taxonomy, + component_map: topology.component_map, + score_model, + rule_packs, + query_packs, + recipe_packs, + diagnostics: Vec::new(), + semantic_fingerprint, + }) + } + + fn compile_score_model_loaded( + &self, + source: PackSource, + ) -> PackResult { + let loaded = self.load_source(source)?; + if loaded.format != PackFormat::Json { + return Err(PackError::UnsupportedFormat { + origin: loaded.origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&loaded.bytes); + let json_value = parse_json_document(&loaded.origin, &loaded.bytes, "score_model")?; + let mut diagnostics = validate_score_model_document(&loaded.origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_SCORE_MODEL_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw: RawScoreModel = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_SCORE_MODEL_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.score_model.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + let normalized = normalize_score_model(&raw); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: loaded.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.score_model.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + compile_normalized_score_model( + loaded.origin, + loaded.file_base_dir, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + fn compile_query_pack_loaded(&self, source: PackSource) -> PackResult { + let loaded = self.load_source(source)?; + if loaded.format != PackFormat::Json { + return Err(PackError::UnsupportedFormat { + origin: loaded.origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&loaded.bytes); + let json_value = parse_json_document(&loaded.origin, &loaded.bytes, "query_pack")?; + let mut diagnostics = validate_query_pack_document(&loaded.origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_QUERY_PACK_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw: RawQueryPack = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_QUERY_PACK_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.query_pack.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + let normalized = normalize_query_pack(&raw); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: loaded.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.query_pack.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + compile_normalized_query_pack( + loaded.origin, + loaded.file_base_dir, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + fn compile_rule_pack_loaded(&self, source: PackSource) -> PackResult { + let loaded = self.load_source(source)?; + if loaded.format != PackFormat::Json { + return Err(PackError::UnsupportedFormat { + origin: loaded.origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&loaded.bytes); + let json_value = parse_json_document(&loaded.origin, &loaded.bytes, "rule_pack")?; + let mut diagnostics = validate_rule_pack_document(&loaded.origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_RULE_PACK_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw: RawRulePack = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_RULE_PACK_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.rule_pack.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + let normalized = normalize_rule_pack(&raw); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: loaded.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.rule_pack.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + compile_normalized_rule_pack( + loaded.origin, + loaded.file_base_dir, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + fn compile_recipe_pack_loaded( + &self, + source: PackSource, + ) -> PackResult { + let loaded = self.load_source(source)?; + if loaded.format != PackFormat::Json { + return Err(PackError::UnsupportedFormat { + origin: loaded.origin.display(), + }); + } + + let source_fingerprint = sha256_bytes(&loaded.bytes); + let json_value = parse_json_document(&loaded.origin, &loaded.bytes, "recipe_pack")?; + let mut diagnostics = validate_recipe_pack_document(&loaded.origin, &json_value); + if !diagnostics.is_empty() { + diagnostics.sort(); + return Err(PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_RECIPE_PACK_V1_SCHEMA_ID, + diagnostics, + }); + } + + let raw: RawRecipePack = + serde_json::from_value(json_value).map_err(|error| PackError::SchemaViolation { + origin: loaded.origin.display(), + schema_id: PACK_RECIPE_PACK_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.recipe_pack.deserialize_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + let normalized = normalize_recipe_pack(&raw); + let semantic_fingerprint = + sha256_canonical_json(&normalized).map_err(|error| PackError::ParseFailure { + origin: loaded.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.recipe_pack.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: loaded.origin.clone(), + path: None, + }), + )], + })?; + compile_normalized_recipe_pack( + loaded.origin, + loaded.file_base_dir, + source_fingerprint, + semantic_fingerprint, + normalized, + ) + } + + fn resolve_bundle_score_model( + &self, + referring_pack: &str, + referring_base_dir: Option<&PathBuf>, + reference: &PackRef, + memo: &mut BTreeMap, + seen_pack_ids: &mut BTreeMap, + ) -> PackResult { + let (source, key) = resolve_bundle_ref_source( + self, + referring_pack, + referring_base_dir, + reference, + PackKind::ScoreModel, + )?; + if let Some(existing) = memo.get(&key) { + return match existing { + ResolvedBundlePack::ScoreModel(loaded) => Ok(loaded.clone()), + other => Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::ScoreModel, + actual: resolved_bundle_pack_kind(other), + }), + }; + } + + let loaded = self.compile_score_model_loaded(source)?; + register_bundle_pack_id(&loaded.pack.header, &key, seen_pack_ids)?; + memo.insert(key, ResolvedBundlePack::ScoreModel(loaded.clone())); + Ok(loaded) + } + + fn resolve_bundle_query_pack( + &self, + referring_pack: &str, + referring_base_dir: Option<&PathBuf>, + reference: &PackRef, + memo: &mut BTreeMap, + seen_pack_ids: &mut BTreeMap, + ) -> PackResult { + let (source, key) = resolve_bundle_ref_source( + self, + referring_pack, + referring_base_dir, + reference, + PackKind::QueryPack, + )?; + if let Some(existing) = memo.get(&key) { + return match existing { + ResolvedBundlePack::QueryPack(loaded) => Ok(loaded.clone()), + other => Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::QueryPack, + actual: resolved_bundle_pack_kind(other), + }), + }; + } + + let loaded = self.compile_query_pack_loaded(source)?; + register_bundle_pack_id(&loaded.pack.header, &key, seen_pack_ids)?; + memo.insert(key, ResolvedBundlePack::QueryPack(loaded.clone())); + Ok(loaded) + } + + fn resolve_bundle_rule_pack( + &self, + referring_pack: &str, + referring_base_dir: Option<&PathBuf>, + reference: &PackRef, + memo: &mut BTreeMap, + seen_pack_ids: &mut BTreeMap, + ) -> PackResult { + let (source, key) = resolve_bundle_ref_source( + self, + referring_pack, + referring_base_dir, + reference, + PackKind::RulePack, + )?; + if let Some(existing) = memo.get(&key) { + return match existing { + ResolvedBundlePack::RulePack(loaded) => Ok(loaded.clone()), + other => Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::RulePack, + actual: resolved_bundle_pack_kind(other), + }), + }; + } + + let loaded = self.compile_rule_pack_loaded(source)?; + register_bundle_pack_id(&loaded.pack.header, &key, seen_pack_ids)?; + memo.insert(key, ResolvedBundlePack::RulePack(loaded.clone())); + Ok(loaded) + } + + fn resolve_bundle_recipe_pack( + &self, + referring_pack: &str, + referring_base_dir: Option<&PathBuf>, + reference: &PackRef, + memo: &mut BTreeMap, + seen_pack_ids: &mut BTreeMap, + ) -> PackResult { + let (source, key) = resolve_bundle_ref_source( + self, + referring_pack, + referring_base_dir, + reference, + PackKind::RecipePack, + )?; + if let Some(existing) = memo.get(&key) { + return match existing { + ResolvedBundlePack::RecipePack(loaded) => Ok(loaded.clone()), + other => Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::RecipePack, + actual: resolved_bundle_pack_kind(other), + }), + }; + } + + let loaded = self.compile_recipe_pack_loaded(source)?; + register_bundle_pack_id(&loaded.pack.header, &key, seen_pack_ids)?; + memo.insert(key, ResolvedBundlePack::RecipePack(loaded.clone())); + Ok(loaded) + } + + fn resolve_rule_pack_queries( + &self, + loaded: &LoadedCompiledRulePack, + query_packs: &mut BTreeMap, + memo: &mut BTreeMap, + seen_pack_ids: &mut BTreeMap, + ) -> PackResult<()> { + for rule in loaded.pack.rules.values() { + let query_pack = self.resolve_bundle_query_pack( + loaded.pack.header.id.as_str(), + loaded.source_file_base_dir.as_ref(), + &rule.query.pack, + memo, + seen_pack_ids, + )?; + ensure_query_ref_exists(&loaded.pack.header.id, &rule.query, &query_pack.pack)?; + query_packs + .entry(query_pack.pack.header.id.clone()) + .or_insert(query_pack.pack); + } + Ok(()) + } + + fn resolve_recipe_pack_queries( + &self, + loaded: &LoadedCompiledRecipePack, + query_packs: &mut BTreeMap, + memo: &mut BTreeMap, + seen_pack_ids: &mut BTreeMap, + ) -> PackResult<()> { + for recipe in loaded.pack.recipes.values() { + let query_pack = self.resolve_bundle_query_pack( + loaded.pack.header.id.as_str(), + loaded.source_file_base_dir.as_ref(), + &recipe.query.pack, + memo, + seen_pack_ids, + )?; + ensure_query_ref_exists(&loaded.pack.header.id, &recipe.query, &query_pack.pack)?; + query_packs + .entry(query_pack.pack.header.id.clone()) + .or_insert(query_pack.pack); + } + Ok(()) + } + + fn detect_source_pack_kind(&self, source: &PackSource) -> PackResult> { + let loaded = self.load_source(source.clone())?; + let value = match loaded.format { + PackFormat::Json => { + parse_json_document(&loaded.origin, &loaded.bytes, "pack_kind_probe")? + } + PackFormat::Toml => parse_profile_toml(&loaded.origin, &loaded.bytes)?, + }; + Ok(value + .as_object() + .and_then(|object| object.get("kind")) + .and_then(Value::as_str) + .and_then(parse_pack_kind_string)) + } + + fn load_source(&self, source: PackSource) -> PackResult { + match source { + PackSource::Builtin { + logical_name, + format, + bytes, + } => Ok(LoadedSource { + origin: PackOrigin::Builtin { + logical_name: logical_name.to_owned(), + }, + format, + bytes: bytes.to_vec(), + file_base_dir: None, + }), + PackSource::File { path, format_hint } => { + let bytes = fs::read(&path).map_err(|error| PackError::Io { + origin: path.display().to_string(), + reason: error.to_string(), + })?; + let resolved_path = absolutize_path(&path)?; + let display_path = path.display().to_string(); + let format = match format_hint.or_else(|| infer_file_format(&path)) { + Some(format) => format, + None => { + return Err(PackError::UnsupportedFormat { + origin: display_path, + }); + } + }; + Ok(LoadedSource { + origin: PackOrigin::File { display_path }, + format, + bytes, + file_base_dir: resolved_path.parent().map(Path::to_path_buf), + }) + } + PackSource::Inline { + logical_name, + format, + bytes, + } => Ok(LoadedSource { + origin: PackOrigin::Inline { logical_name }, + format, + bytes, + file_base_dir: None, + }), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct LoadedSource { + origin: PackOrigin, + format: PackFormat, + bytes: Vec, + file_base_dir: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +enum ResolvedPackSourceKey { + Builtin(PackName), + File(PathBuf), + Inline(String), +} + +#[derive(Clone, Debug)] +struct LoadedCompiledScoreModel { + pack: CompiledScoreModel, + source_file_base_dir: Option, +} + +#[derive(Clone, Debug)] +struct LoadedCompiledQueryPack { + pack: CompiledQueryPack, + source_file_base_dir: Option, +} + +#[derive(Clone, Debug)] +struct LoadedCompiledRulePack { + pack: CompiledRulePack, + source_file_base_dir: Option, +} + +#[derive(Clone, Debug)] +struct LoadedCompiledRecipePack { + pack: CompiledRecipePack, + source_file_base_dir: Option, +} + +#[derive(Clone, Debug)] +enum ResolvedBundlePack { + ScoreModel(LoadedCompiledScoreModel), + QueryPack(LoadedCompiledQueryPack), + RulePack(LoadedCompiledRulePack), + RecipePack(LoadedCompiledRecipePack), +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedProfileDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + apps: NormalizedProfileApps, + analysis: NormalizedProfileAnalysis, + topology: NormalizedProfileTopology, + score: NormalizedProfileScore, + rules: NormalizedIncludeSection, + queries: NormalizedIncludeSection, + recipes: NormalizedIncludeSection, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedProfileApps { + enabled: Vec, + default: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedProfileAnalysis { + languages: Vec, + follow_symlinks: bool, + max_scope_depth: u8, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +struct NormalizedProfileTopology { + #[serde(skip_serializing_if = "Option::is_none")] + boundary_taxonomy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + component_map: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +struct NormalizedProfileScore { + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +struct NormalizedIncludeSection { + packs: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedBoundaryTaxonomyDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + counting: NormalizedBoundaryTaxonomyCounting, + boundaries: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedBoundaryTaxonomyCounting { + mode: RawBoundaryTaxonomyCountingMode, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedBoundaryEntry { + id: String, + label: String, + include: Vec, + exclude: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedComponentMapDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + counting: NormalizedComponentMapCounting, + components: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedComponentMapCounting { + mode: RawComponentMapCountingMode, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedComponentEntry { + id: String, + label: String, + include: Vec, + exclude: Vec, + tags: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedScoreModelDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + vector_version: u32, + lift_score: Value, + estimated_slices: Value, + triggers: Vec, + confidence: NormalizedScoreConfidenceModel, + missing_input_rules: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedScoreTriggerRule { + id: String, + when: Value, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedScoreConfidenceModel { + default: String, + rules: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedScoreConfidenceRule { + id: String, + when: Value, + set: String, + causes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedScoreMissingInputRule { + field: String, + when: Value, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedQueryPackDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + language: String, + engine: String, + queries: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedQueryDef { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + pattern: String, + captures: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedQueryCapture { + name: String, + required: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedRulePackDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + rules: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedRuleDef { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + severity: crate::kernel::Severity, + #[serde(skip_serializing_if = "Option::is_none")] + scope: Option, + query: Value, + emit: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedRuleScope { + #[serde(skip_serializing_if = "Vec::is_empty")] + languages: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + path_classes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedRecipePackDocument { + kind: PackKind, + version: u32, + id: String, + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + recipes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct NormalizedRecipeDef { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + query: Value, + transforms: Vec, +} + +type ArrayItemValidator = fn(&PackOrigin, &str, &str, &mut Vec); + +fn infer_file_format(path: &Path) -> Option { + match path.extension().and_then(|extension| extension.to_str()) { + Some("toml") => Some(PackFormat::Toml), + Some("json") => Some(PackFormat::Json), + _ => None, + } +} + +fn parse_profile_toml(origin: &PackOrigin, bytes: &[u8]) -> PackResult { + let text = std::str::from_utf8(bytes).map_err(|error| PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.profile.invalid_utf8", + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )], + })?; + let toml_value: toml::Value = + toml::from_str(text).map_err(|error| PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.profile.invalid_toml", + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )], + })?; + toml_to_json(origin, JsonPointer::root(), &toml_value) +} + +fn parse_json_document(origin: &PackOrigin, bytes: &[u8], pack_code: &str) -> PackResult { + let text = std::str::from_utf8(bytes).map_err(|error| PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + &format!("pack.{pack_code}.invalid_utf8"), + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )], + })?; + + serde_json::from_str(text).map_err(|error| PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + &format!("pack.{pack_code}.invalid_json"), + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )], + }) +} + +fn toml_to_json( + origin: &PackOrigin, + pointer: JsonPointer, + value: &toml::Value, +) -> PackResult { + match value { + toml::Value::String(string) => Ok(Value::String(string.clone())), + toml::Value::Integer(number) => Ok(Value::Number(Number::from(*number))), + toml::Value::Boolean(boolean) => Ok(Value::Bool(*boolean)), + toml::Value::Float(number) => { + Number::from_f64(*number) + .map(Value::Number) + .ok_or_else(|| PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.profile.unsupported_toml_value", + "non-finite TOML floats are unsupported", + Some(PackLocation { + origin: origin.clone(), + path: Some(pointer), + }), + )], + }) + } + toml::Value::Datetime(_) => Err(PackError::ParseFailure { + origin: origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.profile.unsupported_toml_value", + "TOML datetime values are unsupported in Phase A profiles", + Some(PackLocation { + origin: origin.clone(), + path: Some(pointer), + }), + )], + }), + toml::Value::Array(items) => { + let mut out = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + out.push(toml_to_json( + origin, + pointer.push_token(&index.to_string()), + item, + )?); + } + Ok(Value::Array(out)) + } + toml::Value::Table(table) => { + let mut out = Map::new(); + for (key, item) in table { + out.insert( + key.clone(), + toml_to_json(origin, pointer.push_token(key), item)?, + ); + } + Ok(Value::Object(out)) + } + } +} + +fn boundary_taxonomy_schema_validator() -> &'static Validator { + static VALIDATOR: OnceLock = OnceLock::new(); + VALIDATOR + .get_or_init(|| compile_topology_schema_validator(PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_JSON)) +} + +fn component_map_schema_validator() -> &'static Validator { + static VALIDATOR: OnceLock = OnceLock::new(); + VALIDATOR.get_or_init(|| compile_topology_schema_validator(PACK_COMPONENT_MAP_V1_SCHEMA_JSON)) +} + +fn score_model_schema_validator() -> &'static Validator { + static VALIDATOR: OnceLock = OnceLock::new(); + VALIDATOR.get_or_init(|| compile_embedded_schema_validator(PACK_SCORE_MODEL_V1_SCHEMA_JSON)) +} + +fn query_pack_schema_validator() -> &'static Validator { + static VALIDATOR: OnceLock = OnceLock::new(); + VALIDATOR.get_or_init(|| compile_embedded_schema_validator(PACK_QUERY_PACK_V1_SCHEMA_JSON)) +} + +fn rule_pack_schema_validator() -> &'static Validator { + static VALIDATOR: OnceLock = OnceLock::new(); + VALIDATOR.get_or_init(|| compile_embedded_schema_validator(PACK_RULE_PACK_V1_SCHEMA_JSON)) +} + +fn recipe_pack_schema_validator() -> &'static Validator { + static VALIDATOR: OnceLock = OnceLock::new(); + VALIDATOR.get_or_init(|| compile_embedded_schema_validator(PACK_RECIPE_PACK_V1_SCHEMA_JSON)) +} + +fn compile_topology_schema_validator(schema_json: &str) -> Validator { + compile_embedded_schema_validator(schema_json) +} + +fn compile_embedded_schema_validator(schema_json: &str) -> Validator { + let root_schema: Value = + serde_json::from_str(schema_json).expect("embedded topology schema JSON should parse"); + jsonschema::draft202012::options() + .with_retriever(EmbeddedSchemaRetriever) + .build(&root_schema) + .expect("embedded topology schema should compile") +} + +fn validate_topology_document( + origin: &PackOrigin, + value: &Value, + schema_kind: TopologySchemaKind, +) -> Vec { + let mut diagnostics = Vec::new(); + for error in schema_kind.validator().iter_errors(value) { + diagnostics.extend(topology_schema_error_to_diagnostics( + origin, + schema_kind, + &error, + )); + } + diagnostics +} + +fn topology_schema_error_to_diagnostics( + origin: &PackOrigin, + schema_kind: TopologySchemaKind, + error: &ValidationError<'_>, +) -> Vec { + match error.kind() { + ValidationErrorKind::AdditionalProperties { unexpected } => { + let mut fields = sorted_unique_strings(unexpected.clone()); + fields + .drain(..) + .map(|field| { + let path = append_pointer_segment(error.instance_path().as_str(), &field); + PackDiagnostic::error( + "pack.schema.unknown_field", + format!("unexpected field `{field}`"), + Some(location(origin, &path)), + ) + }) + .collect() + } + ValidationErrorKind::Required { property } => { + let Some(property) = property.as_str() else { + return vec![PackDiagnostic::error( + "pack.schema.missing_required_field", + error.to_string(), + pointer_subject(origin, error.instance_path().as_str()), + )]; + }; + let pointer = append_pointer_segment(error.instance_path().as_str(), property); + vec![PackDiagnostic::error( + missing_field_code(schema_kind, error.instance_path().as_str(), property), + format!("required field `{property}` is missing"), + Some(location(origin, &pointer)), + )] + } + _ => vec![PackDiagnostic::error( + topology_schema_error_code(schema_kind, error), + error.to_string(), + topology_schema_error_subject(origin, schema_kind, error), + )], + } +} + +fn topology_schema_error_code( + schema_kind: TopologySchemaKind, + error: &ValidationError<'_>, +) -> &'static str { + let path = error.instance_path().as_str(); + match error.kind() { + ValidationErrorKind::Type { .. } if path.is_empty() => schema_kind.invalid_root_code(), + ValidationErrorKind::Type { .. } => "pack.schema.invalid_type", + ValidationErrorKind::Constant { .. } if path == "/kind" => "pack.schema.invalid_kind", + ValidationErrorKind::Constant { .. } if path == "/version" => "pack.schema.invalid_version", + ValidationErrorKind::Enum { .. } if path.ends_with("/counting/mode") => { + "pack.schema.invalid_counting_mode" + } + ValidationErrorKind::Pattern { .. } if path == "/id" => "pack.schema.invalid_pack_name", + ValidationErrorKind::MinLength { .. } => min_length_code(path), + _ => "pack.schema.invalid_value", + } +} + +fn topology_schema_error_subject( + origin: &PackOrigin, + _schema_kind: TopologySchemaKind, + error: &ValidationError<'_>, +) -> Option { + let path = error.instance_path().as_str(); + if path.is_empty() && matches!(error.kind(), ValidationErrorKind::Type { .. }) { + Some(PackLocation { + origin: origin.clone(), + path: None, + }) + } else { + pointer_subject(origin, path) + } +} + +fn pointer_subject(origin: &PackOrigin, path: &str) -> Option { + if path.is_empty() { + None + } else { + Some(location(origin, path)) + } +} + +fn append_pointer_segment(base: &str, segment: &str) -> String { + if base.is_empty() { + format!("/{}", escape_json_pointer_segment(segment)) + } else { + format!("{base}/{}", escape_json_pointer_segment(segment)) + } +} + +fn escape_json_pointer_segment(segment: &str) -> String { + segment.replace('~', "~0").replace('/', "~1") +} + +fn missing_field_code( + schema_kind: TopologySchemaKind, + parent_path: &str, + property: &str, +) -> &'static str { + match (schema_kind, parent_path, property) { + (_, "", "kind") => "pack.schema.missing_kind", + (_, "", "version") => "pack.schema.missing_version", + (_, "", "id") => "pack.schema.missing_id", + (_, "", "name") => "pack.schema.missing_name", + (TopologySchemaKind::BoundaryTaxonomy, "", "counting") => "pack.schema.missing_counting", + (TopologySchemaKind::BoundaryTaxonomy, "", "boundaries") => { + "pack.schema.missing_boundaries" + } + (TopologySchemaKind::ComponentMap, "", "counting") => "pack.schema.missing_counting", + (TopologySchemaKind::ComponentMap, "", "components") => "pack.schema.missing_components", + (_, "/counting", "mode") => "pack.schema.missing_counting_mode", + (TopologySchemaKind::BoundaryTaxonomy, path, "id") + if is_topology_entry_path(path, "boundaries") => + { + "pack.schema.missing_boundary_id" + } + (TopologySchemaKind::BoundaryTaxonomy, path, "label") + if is_topology_entry_path(path, "boundaries") => + { + "pack.schema.missing_boundary_label" + } + (TopologySchemaKind::BoundaryTaxonomy, path, "include") + if is_topology_entry_path(path, "boundaries") => + { + "pack.schema.missing_boundary_include" + } + (TopologySchemaKind::ComponentMap, path, "id") + if is_topology_entry_path(path, "components") => + { + "pack.schema.missing_component_id" + } + (TopologySchemaKind::ComponentMap, path, "label") + if is_topology_entry_path(path, "components") => + { + "pack.schema.missing_component_label" + } + (TopologySchemaKind::ComponentMap, path, "include") + if is_topology_entry_path(path, "components") => + { + "pack.schema.missing_component_include" + } + _ => "pack.schema.missing_required_field", + } +} + +fn is_topology_entry_path(path: &str, collection: &str) -> bool { + path.strip_prefix(&format!("/{collection}/")) + .map(|suffix| suffix.parse::().is_ok()) + .unwrap_or(false) +} + +fn min_length_code(path: &str) -> &'static str { + if path == "/name" || path.ends_with("/id") || path.ends_with("/label") { + "pack.schema.invalid_name" + } else if path.contains("/include/") || path.contains("/exclude/") { + "pack.schema.invalid_glob" + } else { + "pack.schema.invalid_value" + } +} + +fn validate_profile_document(origin: &PackOrigin, value: &Value) -> Vec { + let mut diagnostics = Vec::new(); + let Some(root) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.profile.invalid_root", + "profile document must be an object", + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )); + return diagnostics; + }; + + require_string( + root.get("kind"), + origin, + "/kind", + "pack.schema.missing_kind", + &mut diagnostics, + ); + require_u64( + root.get("version"), + origin, + "/version", + "pack.schema.missing_version", + &mut diagnostics, + ); + require_string( + root.get("id"), + origin, + "/id", + "pack.schema.missing_id", + &mut diagnostics, + ); + require_string( + root.get("name"), + origin, + "/name", + "pack.schema.missing_name", + &mut diagnostics, + ); + + if let Some(kind) = root.get("kind").and_then(Value::as_str) { + if kind != "profile" { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_kind", + format!("expected profile kind, found {kind}"), + Some(location(origin, "/kind")), + )); + } + } + if let Some(version) = root.get("version").and_then(Value::as_u64) { + if version != 1 { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_version", + format!("expected profile version 1, found {version}"), + Some(location(origin, "/version")), + )); + } + } + if let Some(id) = root.get("id").and_then(Value::as_str) { + if PackName::parse(id).is_err() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_pack_name", + format!("invalid profile id: {id}"), + Some(location(origin, "/id")), + )); + } + } + if let Some(name) = root.get("name").and_then(Value::as_str) { + if name.is_empty() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_name", + "name must be a non-empty string", + Some(location(origin, "/name")), + )); + } + } + + validate_optional_string( + root.get("description"), + origin, + "/description", + &mut diagnostics, + ); + validate_apps(root.get("apps"), origin, &mut diagnostics); + validate_analysis(root.get("analysis"), origin, &mut diagnostics); + validate_optional_ref_object( + root.get("topology"), + origin, + "boundary_taxonomy", + "component_map", + "/topology", + &mut diagnostics, + ); + validate_single_optional_ref( + root.get("score"), + origin, + "model", + "/score", + &mut diagnostics, + ); + validate_include_section(root.get("rules"), origin, "/rules", &mut diagnostics); + validate_include_section(root.get("queries"), origin, "/queries", &mut diagnostics); + validate_include_section(root.get("recipes"), origin, "/recipes", &mut diagnostics); + + diagnostics +} + +fn validate_boundary_taxonomy_document(origin: &PackOrigin, value: &Value) -> Vec { + validate_topology_document(origin, value, TopologySchemaKind::BoundaryTaxonomy) +} + +fn validate_component_map_document(origin: &PackOrigin, value: &Value) -> Vec { + validate_topology_document(origin, value, TopologySchemaKind::ComponentMap) +} + +fn validate_score_model_document(origin: &PackOrigin, value: &Value) -> Vec { + validate_advanced_document(origin, value, AdvancedSchemaKind::ScoreModel) +} + +fn validate_query_pack_document(origin: &PackOrigin, value: &Value) -> Vec { + validate_advanced_document(origin, value, AdvancedSchemaKind::QueryPack) +} + +fn validate_rule_pack_document(origin: &PackOrigin, value: &Value) -> Vec { + validate_advanced_document(origin, value, AdvancedSchemaKind::RulePack) +} + +fn validate_recipe_pack_document(origin: &PackOrigin, value: &Value) -> Vec { + validate_advanced_document(origin, value, AdvancedSchemaKind::RecipePack) +} + +fn validate_advanced_document( + origin: &PackOrigin, + value: &Value, + schema_kind: AdvancedSchemaKind, +) -> Vec { + let mut diagnostics = Vec::new(); + for error in schema_kind.validator().iter_errors(value) { + diagnostics.extend(advanced_schema_error_to_diagnostics( + origin, + schema_kind, + &error, + )); + } + diagnostics +} + +fn advanced_schema_error_to_diagnostics( + origin: &PackOrigin, + schema_kind: AdvancedSchemaKind, + error: &ValidationError<'_>, +) -> Vec { + match error.kind() { + ValidationErrorKind::AdditionalProperties { unexpected } => { + let mut fields = sorted_unique_strings(unexpected.clone()); + fields + .drain(..) + .map(|field| { + let path = append_pointer_segment(error.instance_path().as_str(), &field); + PackDiagnostic::error( + "pack.schema.unknown_field", + format!("unexpected field `{field}`"), + Some(location(origin, &path)), + ) + }) + .collect() + } + ValidationErrorKind::Required { property } => { + let Some(property) = property.as_str() else { + return vec![PackDiagnostic::error( + "pack.schema.missing_required_field", + error.to_string(), + pointer_subject(origin, error.instance_path().as_str()), + )]; + }; + let pointer = append_pointer_segment(error.instance_path().as_str(), property); + vec![PackDiagnostic::error( + "pack.schema.missing_required_field", + format!("required field `{property}` is missing"), + Some(location(origin, &pointer)), + )] + } + ValidationErrorKind::Type { .. } if error.instance_path().as_str().is_empty() => { + vec![PackDiagnostic::error( + schema_kind.invalid_root_code(), + error.to_string(), + Some(PackLocation { + origin: origin.clone(), + path: None, + }), + )] + } + ValidationErrorKind::Constant { .. } if error.instance_path().as_str() == "/kind" => { + vec![PackDiagnostic::error( + "pack.schema.invalid_kind", + error.to_string(), + Some(location(origin, "/kind")), + )] + } + ValidationErrorKind::Constant { .. } if error.instance_path().as_str() == "/version" => { + vec![PackDiagnostic::error( + "pack.schema.invalid_version", + error.to_string(), + Some(location(origin, "/version")), + )] + } + ValidationErrorKind::Pattern { .. } if error.instance_path().as_str() == "/id" => { + vec![PackDiagnostic::error( + "pack.schema.invalid_pack_name", + error.to_string(), + Some(location(origin, "/id")), + )] + } + _ => vec![PackDiagnostic::error( + "pack.schema.invalid_value", + error.to_string(), + pointer_subject(origin, error.instance_path().as_str()), + )], + } +} + +fn require_string( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + code: &str, + diagnostics: &mut Vec, +) { + match value { + Some(Value::String(_)) => {} + Some(_) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected string value", + Some(location(origin, path)), + )), + None => diagnostics.push(PackDiagnostic::error( + code, + "required string field is missing", + Some(location(origin, path)), + )), + } +} + +fn require_u64( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + code: &str, + diagnostics: &mut Vec, +) { + match value { + Some(Value::Number(number)) if number.as_u64().is_some() => {} + Some(_) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected integer value", + Some(location(origin, path)), + )), + None => diagnostics.push(PackDiagnostic::error( + code, + "required integer field is missing", + Some(location(origin, path)), + )), + } +} + +fn require_object( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + code: &str, + diagnostics: &mut Vec, +) { + match value { + Some(Value::Object(_)) => {} + Some(_) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected object value", + Some(location(origin, path)), + )), + None => diagnostics.push(PackDiagnostic::error( + code, + "required object field is missing", + Some(location(origin, path)), + )), + } +} + +fn require_array( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + code: &str, + diagnostics: &mut Vec, +) { + match value { + Some(Value::Array(_)) => {} + Some(_) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected array value", + Some(location(origin, path)), + )), + None => diagnostics.push(PackDiagnostic::error( + code, + "required array field is missing", + Some(location(origin, path)), + )), + } +} + +fn validate_optional_string( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + diagnostics: &mut Vec, +) { + if let Some(value) = value { + if !value.is_string() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected string value", + Some(location(origin, path)), + )); + } + } +} + +fn validate_non_empty_string( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + diagnostics: &mut Vec, +) { + if let Some(value) = value { + match value.as_str() { + Some(text) if !text.is_empty() => {} + Some(_) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_name", + "value must be a non-empty string", + Some(location(origin, path)), + )), + None => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected string value", + Some(location(origin, path)), + )), + } + } +} + +fn validate_apps( + value: Option<&Value>, + origin: &PackOrigin, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(object) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "apps must be an object", + Some(location(origin, "/apps")), + )); + return; + }; + if let Some(enabled) = object.get("enabled") { + validate_string_array( + enabled, + origin, + "/apps/enabled", + diagnostics, + Some(validate_app_name), + ); + } + if let Some(default) = object.get("default") { + if let Some(default) = default.as_str() { + validate_app_name(origin, "/apps/default", default, diagnostics); + } else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "apps.default must be a string", + Some(location(origin, "/apps/default")), + )); + } + } +} + +fn validate_analysis( + value: Option<&Value>, + origin: &PackOrigin, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(object) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "analysis must be an object", + Some(location(origin, "/analysis")), + )); + return; + }; + if let Some(languages) = object.get("languages") { + validate_string_array( + languages, + origin, + "/analysis/languages", + diagnostics, + Some(validate_language_id), + ); + } + if let Some(follow_symlinks) = object.get("follow_symlinks") { + if !follow_symlinks.is_boolean() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "analysis.follow_symlinks must be a boolean", + Some(location(origin, "/analysis/follow_symlinks")), + )); + } + } + if let Some(max_scope_depth) = object.get("max_scope_depth") { + if max_scope_depth.as_u64().is_none() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "analysis.max_scope_depth must be an integer", + Some(location(origin, "/analysis/max_scope_depth")), + )); + } + } +} + +fn validate_optional_ref_object( + value: Option<&Value>, + origin: &PackOrigin, + first: &str, + second: &str, + base_path: &str, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(object) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "section must be an object", + Some(location(origin, base_path)), + )); + return; + }; + for field in [first, second] { + if let Some(item) = object.get(field) { + if !item.is_string() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + format!("{field} must be a string reference"), + Some(location(origin, &format!("{base_path}/{field}"))), + )); + } + } + } +} + +fn validate_single_optional_ref( + value: Option<&Value>, + origin: &PackOrigin, + field: &str, + base_path: &str, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(object) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "section must be an object", + Some(location(origin, base_path)), + )); + return; + }; + if let Some(item) = object.get(field) { + if !item.is_string() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + format!("{field} must be a string reference"), + Some(location(origin, &format!("{base_path}/{field}"))), + )); + } + } +} + +fn validate_include_section( + value: Option<&Value>, + origin: &PackOrigin, + base_path: &str, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(object) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "section must be an object", + Some(location(origin, base_path)), + )); + return; + }; + if let Some(packs) = object.get("packs") { + validate_string_array( + packs, + origin, + &format!("{base_path}/packs"), + diagnostics, + Some(validate_pack_ref_string), + ); + } +} + +fn validate_string_array( + value: &Value, + origin: &PackOrigin, + path: &str, + diagnostics: &mut Vec, + validate_item: Option, +) { + let Some(items) = value.as_array() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "expected string array", + Some(location(origin, path)), + )); + return; + }; + for (index, item) in items.iter().enumerate() { + if let Some(item) = item.as_str() { + if let Some(validate_item) = validate_item { + validate_item(origin, &format!("{path}/{index}"), item, diagnostics); + } + } else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "array entries must be strings", + Some(location(origin, &format!("{path}/{index}"))), + )); + } + } +} + +fn location(origin: &PackOrigin, path: &str) -> PackLocation { + PackLocation { + origin: origin.clone(), + path: Some(JsonPointer::parse(path).expect("pointer should be valid")), + } +} + +fn validate_app_name( + origin: &PackOrigin, + path: &str, + value: &str, + diagnostics: &mut Vec, +) { + if AppName::parse(value).is_err() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_app_name", + format!("invalid app name: {value}"), + Some(location(origin, path)), + )); + } +} + +fn validate_language_id( + origin: &PackOrigin, + path: &str, + value: &str, + diagnostics: &mut Vec, +) { + if LanguageId::parse(value).is_err() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_language_id", + format!("invalid language id: {value}"), + Some(location(origin, path)), + )); + } +} + +fn validate_pack_ref_string( + origin: &PackOrigin, + path: &str, + value: &str, + diagnostics: &mut Vec, +) { + if PackRef::parse(value).is_err() { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_pack_ref", + format!("invalid pack reference: {value}"), + Some(location(origin, path)), + )); + } +} + +fn validate_counting_mode( + value: Option<&Value>, + origin: &PackOrigin, + path: &str, + expected_mode: &str, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(object) = value.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "counting must be an object", + Some(location(origin, path)), + )); + return; + }; + + match object.get("mode") { + Some(Value::String(mode)) if mode == expected_mode => {} + Some(Value::String(mode)) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_counting_mode", + format!("expected counting.mode {expected_mode}, found {mode}"), + Some(location(origin, &format!("{path}/mode"))), + )), + Some(_) => diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "counting.mode must be a string", + Some(location(origin, &format!("{path}/mode"))), + )), + None => diagnostics.push(PackDiagnostic::error( + "pack.schema.missing_counting_mode", + "counting.mode is required", + Some(location(origin, &format!("{path}/mode"))), + )), + } +} + +fn validate_boundary_entries( + value: Option<&Value>, + origin: &PackOrigin, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(items) = value.as_array() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "boundaries must be an array", + Some(location(origin, "/boundaries")), + )); + return; + }; + + for (index, item) in items.iter().enumerate() { + let base = format!("/boundaries/{index}"); + let Some(object) = item.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "boundary entries must be objects", + Some(location(origin, &base)), + )); + continue; + }; + + require_string( + object.get("id"), + origin, + &format!("{base}/id"), + "pack.schema.missing_boundary_id", + diagnostics, + ); + require_string( + object.get("label"), + origin, + &format!("{base}/label"), + "pack.schema.missing_boundary_label", + diagnostics, + ); + require_array( + object.get("include"), + origin, + &format!("{base}/include"), + "pack.schema.missing_boundary_include", + diagnostics, + ); + + validate_non_empty_string(object.get("id"), origin, &format!("{base}/id"), diagnostics); + validate_non_empty_string( + object.get("label"), + origin, + &format!("{base}/label"), + diagnostics, + ); + if let Some(include) = object.get("include") { + validate_string_array( + include, + origin, + &format!("{base}/include"), + diagnostics, + None, + ); + validate_non_empty_string_array( + include, + origin, + &format!("{base}/include"), + diagnostics, + ); + } + if let Some(exclude) = object.get("exclude") { + validate_string_array( + exclude, + origin, + &format!("{base}/exclude"), + diagnostics, + None, + ); + validate_non_empty_string_array( + exclude, + origin, + &format!("{base}/exclude"), + diagnostics, + ); + } + } +} + +fn validate_component_entries( + value: Option<&Value>, + origin: &PackOrigin, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + let Some(items) = value.as_array() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "components must be an array", + Some(location(origin, "/components")), + )); + return; + }; + + for (index, item) in items.iter().enumerate() { + let base = format!("/components/{index}"); + let Some(object) = item.as_object() else { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_type", + "component entries must be objects", + Some(location(origin, &base)), + )); + continue; + }; + + require_string( + object.get("id"), + origin, + &format!("{base}/id"), + "pack.schema.missing_component_id", + diagnostics, + ); + require_string( + object.get("label"), + origin, + &format!("{base}/label"), + "pack.schema.missing_component_label", + diagnostics, + ); + require_array( + object.get("include"), + origin, + &format!("{base}/include"), + "pack.schema.missing_component_include", + diagnostics, + ); + + validate_non_empty_string(object.get("id"), origin, &format!("{base}/id"), diagnostics); + validate_non_empty_string( + object.get("label"), + origin, + &format!("{base}/label"), + diagnostics, + ); + if let Some(include) = object.get("include") { + validate_string_array( + include, + origin, + &format!("{base}/include"), + diagnostics, + None, + ); + validate_non_empty_string_array( + include, + origin, + &format!("{base}/include"), + diagnostics, + ); + } + if let Some(exclude) = object.get("exclude") { + validate_string_array( + exclude, + origin, + &format!("{base}/exclude"), + diagnostics, + None, + ); + validate_non_empty_string_array( + exclude, + origin, + &format!("{base}/exclude"), + diagnostics, + ); + } + if let Some(tags) = object.get("tags") { + validate_string_array(tags, origin, &format!("{base}/tags"), diagnostics, None); + } + } +} + +fn validate_non_empty_string_array( + value: &Value, + origin: &PackOrigin, + path: &str, + diagnostics: &mut Vec, +) { + let Some(items) = value.as_array() else { + return; + }; + + for (index, item) in items.iter().enumerate() { + if matches!(item.as_str(), Some(text) if text.is_empty()) { + diagnostics.push(PackDiagnostic::error( + "pack.schema.invalid_glob", + "glob entries must be non-empty strings", + Some(location(origin, &format!("{path}/{index}"))), + )); + } + } +} + +fn normalize_profile(raw: &RawProfile) -> NormalizedProfileDocument { + let apps = normalize_apps(raw.apps.as_ref()); + let analysis = normalize_analysis(raw.analysis.as_ref()); + let topology = normalize_topology(raw.topology.as_ref()); + let score = normalize_score(raw.score.as_ref()); + + NormalizedProfileDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + apps, + analysis, + topology, + score, + rules: normalize_include_section(raw.rules.as_ref()), + queries: normalize_include_section(raw.queries.as_ref()), + recipes: normalize_include_section(raw.recipes.as_ref()), + } +} + +fn normalize_apps(raw: Option<&RawProfileApps>) -> NormalizedProfileApps { + let mut enabled = sorted_unique_strings( + raw.and_then(|apps| apps.enabled.clone()) + .unwrap_or_else(|| vec![DEFAULT_APP.to_owned()]), + ); + let default = raw + .and_then(|apps| apps.default.clone()) + .unwrap_or_else(|| DEFAULT_APP.to_owned()); + if !enabled.iter().any(|app| app == &default) { + enabled.push(default.clone()); + enabled.sort(); + } + NormalizedProfileApps { enabled, default } +} + +fn normalize_analysis(raw: Option<&RawProfileAnalysis>) -> NormalizedProfileAnalysis { + NormalizedProfileAnalysis { + languages: sorted_unique_strings( + raw.and_then(|analysis| analysis.languages.clone()) + .unwrap_or_else(|| { + DEFAULT_LANGUAGES + .iter() + .map(|language| (*language).to_owned()) + .collect() + }), + ), + follow_symlinks: raw + .and_then(|analysis| analysis.follow_symlinks) + .unwrap_or(DEFAULT_FOLLOW_SYMLINKS), + max_scope_depth: raw + .and_then(|analysis| analysis.max_scope_depth) + .unwrap_or(DEFAULT_MAX_SCOPE_DEPTH), + } +} + +fn normalize_topology(raw: Option<&RawProfileTopology>) -> NormalizedProfileTopology { + NormalizedProfileTopology { + boundary_taxonomy: raw.and_then(|topology| topology.boundary_taxonomy.clone()), + component_map: raw.and_then(|topology| topology.component_map.clone()), + } +} + +fn normalize_score(raw: Option<&RawProfileScore>) -> NormalizedProfileScore { + NormalizedProfileScore { + model: raw.and_then(|score| score.model.clone()), + } +} + +fn normalize_include_section(raw: Option<&RawIncludeSection>) -> NormalizedIncludeSection { + NormalizedIncludeSection { + packs: sorted_unique_strings(raw.map(|section| section.packs.clone()).unwrap_or_default()), + } +} + +fn normalize_boundary_taxonomy(raw: &RawBoundaryTaxonomy) -> NormalizedBoundaryTaxonomyDocument { + let mut boundaries = raw + .boundaries + .iter() + .map(normalize_boundary_entry) + .collect::>(); + boundaries.sort_by(|left, right| { + left.id + .cmp(&right.id) + .then_with(|| left.label.cmp(&right.label)) + .then_with(|| left.include.cmp(&right.include)) + .then_with(|| left.exclude.cmp(&right.exclude)) + }); + + NormalizedBoundaryTaxonomyDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + counting: NormalizedBoundaryTaxonomyCounting { + mode: raw.counting.mode, + }, + boundaries, + } +} + +fn normalize_boundary_entry(raw: &RawBoundaryEntry) -> NormalizedBoundaryEntry { + NormalizedBoundaryEntry { + id: raw.id.clone(), + label: raw.label.clone(), + include: sorted_unique_strings(raw.include.clone()), + exclude: sorted_unique_strings(raw.exclude.clone()), + } +} + +fn normalize_component_map(raw: &RawComponentMap) -> NormalizedComponentMapDocument { + let mut components = raw + .components + .iter() + .map(normalize_component_entry) + .collect::>(); + components.sort_by(|left, right| { + left.id + .cmp(&right.id) + .then_with(|| left.label.cmp(&right.label)) + .then_with(|| left.include.cmp(&right.include)) + .then_with(|| left.exclude.cmp(&right.exclude)) + .then_with(|| left.tags.cmp(&right.tags)) + }); + + NormalizedComponentMapDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + counting: NormalizedComponentMapCounting { + mode: raw.counting.mode, + }, + components, + } +} + +fn normalize_component_entry(raw: &RawComponentEntry) -> NormalizedComponentEntry { + NormalizedComponentEntry { + id: raw.id.clone(), + label: raw.label.clone(), + include: sorted_unique_strings(raw.include.clone()), + exclude: sorted_unique_strings(raw.exclude.clone()), + tags: sorted_unique_strings(raw.tags.clone()), + } +} + +fn normalize_score_model(raw: &RawScoreModel) -> NormalizedScoreModelDocument { + NormalizedScoreModelDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + vector_version: raw.vector_version, + lift_score: raw.lift_score.clone(), + estimated_slices: raw.estimated_slices.clone(), + triggers: raw + .triggers + .iter() + .map(|trigger| NormalizedScoreTriggerRule { + id: trigger.id.clone(), + when: trigger.when.clone(), + }) + .collect(), + confidence: NormalizedScoreConfidenceModel { + default: raw.confidence.default_level.clone(), + rules: raw + .confidence + .rules + .iter() + .map(|rule| NormalizedScoreConfidenceRule { + id: rule.id.clone(), + when: rule.when.clone(), + set: rule.set.clone(), + causes: sorted_unique_strings(rule.causes.clone()), + }) + .collect(), + }, + missing_input_rules: raw + .missing_input_rules + .iter() + .map(|rule| NormalizedScoreMissingInputRule { + field: rule.field.clone(), + when: rule.when.clone(), + }) + .collect(), + } +} + +fn normalize_query_pack(raw: &RawQueryPack) -> NormalizedQueryPackDocument { + NormalizedQueryPackDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + language: raw.language.clone(), + engine: raw.engine.clone(), + queries: raw + .queries + .iter() + .map(|query| NormalizedQueryDef { + id: query.id.clone(), + summary: query.summary.clone(), + pattern: query.pattern.clone(), + captures: query + .captures + .iter() + .map(|capture| NormalizedQueryCapture { + name: capture.name.clone(), + required: capture.required, + }) + .collect(), + }) + .collect(), + } +} + +fn normalize_rule_pack(raw: &RawRulePack) -> NormalizedRulePackDocument { + NormalizedRulePackDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + rules: raw + .rules + .iter() + .map(|rule| NormalizedRuleDef { + id: rule.id.clone(), + summary: rule.summary.clone(), + severity: rule.severity, + scope: rule.scope.as_ref().map(|scope| NormalizedRuleScope { + languages: scope.languages.clone().unwrap_or_default(), + path_classes: scope.path_classes.clone().unwrap_or_default(), + }), + query: serde_json::json!({ + "pack": rule.query.pack.clone(), + "id": rule.query.id.clone(), + }), + emit: rule.emit.clone(), + }) + .collect(), + } +} + +fn normalize_recipe_pack(raw: &RawRecipePack) -> NormalizedRecipePackDocument { + NormalizedRecipePackDocument { + kind: raw.kind, + version: raw.version, + id: raw.id.clone(), + name: raw.name.clone(), + description: raw.description.clone(), + recipes: raw + .recipes + .iter() + .map(|recipe| NormalizedRecipeDef { + id: recipe.id.clone(), + summary: recipe.summary.clone(), + query: serde_json::json!({ + "pack": recipe.query.pack.clone(), + "id": recipe.query.id.clone(), + }), + transforms: recipe.transforms.clone(), + }) + .collect(), + } +} + +fn sorted_unique_strings(mut values: Vec) -> Vec { + values.sort(); + values.dedup(); + values +} + +fn parse_pack_kind_string(input: &str) -> Option { + match input { + "profile" => Some(PackKind::Profile), + "boundary_taxonomy" => Some(PackKind::BoundaryTaxonomy), + "component_map" => Some(PackKind::ComponentMap), + "score_model" => Some(PackKind::ScoreModel), + "rule_pack" => Some(PackKind::RulePack), + "query_pack" => Some(PackKind::QueryPack), + "recipe_pack" => Some(PackKind::RecipePack), + _ => None, + } +} + +fn compile_normalized_profile( + origin: PackOrigin, + source_file_base_dir: Option, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedProfileDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let enabled_apps = normalized + .apps + .enabled + .iter() + .map(|app| AppName::parse(app)) + .collect::>>()?; + let default_app = AppName::parse(&normalized.apps.default)?; + let languages = normalized + .analysis + .languages + .iter() + .map(|language| LanguageId::parse(language)) + .collect::>>()?; + + let topology = CompiledProfileTopology { + boundary_taxonomy: compile_optional_ref( + &origin, + normalized.topology.boundary_taxonomy.as_deref(), + "/topology/boundary_taxonomy", + )?, + component_map: compile_optional_ref( + &origin, + normalized.topology.component_map.as_deref(), + "/topology/component_map", + )?, + classes: CompiledPathClasses, + }; + let score = CompiledProfileScore { + model: compile_optional_ref(&origin, normalized.score.model.as_deref(), "/score/model")?, + }; + let includes = CompiledProfileIncludes { + rule_packs: compile_ref_set(&origin, &normalized.rules.packs, "/rules/packs")?, + query_packs: compile_ref_set(&origin, &normalized.queries.packs, "/queries/packs")?, + recipe_packs: compile_ref_set(&origin, &normalized.recipes.packs, "/recipes/packs")?, + }; + + Ok(CompiledProfile { + header: CompiledPackHeader { + kind: PackKind::Profile, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_PROFILE_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + source_file_base_dir, + apps: CompiledProfileApps { + enabled: enabled_apps, + default: default_app, + }, + analysis: CompiledAnalysisDefaults { + languages, + follow_symlinks: normalized.analysis.follow_symlinks, + max_scope_depth: normalized.analysis.max_scope_depth, + }, + topology, + score, + includes, + diagnostics: Vec::new(), + }) +} + +fn absolutize_path(path: &Path) -> PackResult { + if path.is_absolute() { + return Ok(path.to_path_buf()); + } + + let cwd = std::env::current_dir().map_err(|error| PackError::Io { + origin: path.display().to_string(), + reason: error.to_string(), + })?; + Ok(cwd.join(path)) +} + +fn compile_normalized_boundary_taxonomy( + origin: PackOrigin, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedBoundaryTaxonomyDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let pack_id = normalized.id; + let mut boundaries = BTreeMap::new(); + let mut local_ids = BTreeSet::new(); + + for boundary in normalized.boundaries { + if !local_ids.insert(boundary.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::BoundaryTaxonomy, + pack_id: pack_id.clone(), + entry_kind: "boundary", + entry_id: boundary.id, + }); + } + + let boundary_id = compile_boundary_id(&pack_id, &boundary.id); + let compiled = CompiledBoundary { + local_id: boundary.id, + id: boundary_id.clone(), + label: boundary.label, + include_patterns: boundary.include.clone(), + exclude_patterns: boundary.exclude.clone(), + include_matcher: compile_glob_set(&boundary.include)?, + exclude_matcher: compile_glob_set(&boundary.exclude)?, + }; + boundaries.insert(boundary_id, compiled); + } + + Ok(CompiledBoundaryTaxonomy { + header: CompiledPackHeader { + kind: PackKind::BoundaryTaxonomy, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + counting_mode: match normalized.counting.mode { + RawBoundaryTaxonomyCountingMode::DistinctMinusOne => { + BoundaryCountingMode::DistinctMinusOne + } + }, + boundaries, + diagnostics: Vec::new(), + }) +} + +fn compile_normalized_component_map( + origin: PackOrigin, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedComponentMapDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let pack_id = normalized.id; + let mut components = BTreeMap::new(); + let mut local_ids = BTreeSet::new(); + + for component in normalized.components { + if !local_ids.insert(component.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::ComponentMap, + pack_id: pack_id.clone(), + entry_kind: "component", + entry_id: component.id, + }); + } + + let component_id = compile_component_id(&pack_id, &component.id); + let compiled = CompiledComponent { + local_id: component.id, + id: component_id.clone(), + label: component.label, + include_patterns: component.include.clone(), + exclude_patterns: component.exclude.clone(), + tags: component.tags.into_iter().collect(), + include_matcher: compile_glob_set(&component.include)?, + exclude_matcher: compile_glob_set(&component.exclude)?, + }; + components.insert(component_id, compiled); + } + + Ok(CompiledComponentMap { + header: CompiledPackHeader { + kind: PackKind::ComponentMap, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_COMPONENT_MAP_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + counting_mode: match normalized.counting.mode { + RawComponentMapCountingMode::Distinct => ComponentCountingMode::Distinct, + }, + components, + diagnostics: Vec::new(), + }) +} + +fn compile_normalized_score_model( + origin: PackOrigin, + source_file_base_dir: Option, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedScoreModelDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let pack_id = normalized.id; + let lift_score = compile_expr( + &pack_id, + &JsonPointer::parse("/lift_score").expect("pointer"), + &normalized.lift_score, + )?; + let estimated_slices = compile_expr( + &pack_id, + &JsonPointer::parse("/estimated_slices").expect("pointer"), + &normalized.estimated_slices, + )?; + + let mut trigger_ids = BTreeSet::new(); + let mut triggers = Vec::with_capacity(normalized.triggers.len()); + for (index, trigger) in normalized.triggers.into_iter().enumerate() { + if !trigger_ids.insert(trigger.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::ScoreModel, + pack_id: pack_id.clone(), + entry_kind: "trigger", + entry_id: trigger.id, + }); + } + triggers.push(CompiledTriggerRule { + id: trigger.id, + when: compile_expr( + &pack_id, + &JsonPointer::parse(&format!("/triggers/{index}/when")).expect("pointer"), + &trigger.when, + )?, + }); + } + + let mut confidence_rule_ids = BTreeSet::new(); + let mut confidence_rules = Vec::with_capacity(normalized.confidence.rules.len()); + for (index, rule) in normalized.confidence.rules.into_iter().enumerate() { + if !confidence_rule_ids.insert(rule.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::ScoreModel, + pack_id: pack_id.clone(), + entry_kind: "confidence_rule", + entry_id: rule.id, + }); + } + confidence_rules.push(CompiledConfidenceRule { + id: rule.id, + when: compile_expr( + &pack_id, + &JsonPointer::parse(&format!("/confidence/rules/{index}/when")).expect("pointer"), + &rule.when, + )?, + set: rule.set, + causes: rule.causes.into_iter().collect(), + }); + } + + let mut missing_fields = BTreeSet::new(); + let mut missing_input_rules = Vec::with_capacity(normalized.missing_input_rules.len()); + for (index, rule) in normalized.missing_input_rules.into_iter().enumerate() { + if !missing_fields.insert(rule.field.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::ScoreModel, + pack_id: pack_id.clone(), + entry_kind: "missing_input_field", + entry_id: rule.field, + }); + } + let field = + JsonPointer::parse(&rule.field).map_err(|error| PackError::ExpressionCompile { + pack_id: pack_id.clone(), + path: JsonPointer::parse(&format!("/missing_input_rules/{index}/field")) + .expect("pointer"), + reason: error.to_string(), + })?; + missing_input_rules.push(CompiledMissingInputRule { + field, + when: compile_expr( + &pack_id, + &JsonPointer::parse(&format!("/missing_input_rules/{index}/when")) + .expect("pointer"), + &rule.when, + )?, + }); + } + + Ok(LoadedCompiledScoreModel { + pack: CompiledScoreModel { + header: CompiledPackHeader { + kind: PackKind::ScoreModel, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_SCORE_MODEL_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + vector_version: normalized.vector_version, + lift_score, + estimated_slices, + triggers, + confidence: CompiledConfidenceModel { + default_level: normalized.confidence.default, + rules: confidence_rules, + }, + missing_input_rules, + diagnostics: Vec::new(), + }, + source_file_base_dir, + }) +} + +fn compile_normalized_query_pack( + origin: PackOrigin, + source_file_base_dir: Option, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedQueryPackDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let pack_id = normalized.id; + let language = LanguageId::parse(&normalized.language)?; + let engine = match normalized.engine.as_str() { + "tree_sitter" => QueryEngineKind::TreeSitter, + other => { + return Err(PackError::SchemaViolation { + origin: origin.display(), + schema_id: PACK_QUERY_PACK_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.query_pack.unsupported_engine", + format!("unsupported query engine `{other}`"), + Some(location(&origin, "/engine")), + )], + }); + } + }; + + let mut local_ids = BTreeSet::new(); + let mut queries = BTreeMap::new(); + for query in normalized.queries { + if !local_ids.insert(query.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::QueryPack, + pack_id: pack_id.clone(), + entry_kind: "query", + entry_id: query.id, + }); + } + let query_id = compile_query_id(&pack_id, &query.id); + queries.insert( + query_id.clone(), + CompiledQueryDef { + local_id: query.id, + id: query_id, + summary: query.summary, + pattern: query.pattern, + captures: query + .captures + .into_iter() + .map(|capture| CompiledQueryCapture { + name: capture.name, + required: capture.required, + }) + .collect(), + }, + ); + } + + Ok(LoadedCompiledQueryPack { + pack: CompiledQueryPack { + header: CompiledPackHeader { + kind: PackKind::QueryPack, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_QUERY_PACK_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + language, + engine, + queries, + diagnostics: Vec::new(), + }, + source_file_base_dir, + }) +} + +fn compile_normalized_rule_pack( + origin: PackOrigin, + source_file_base_dir: Option, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedRulePackDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let pack_id = normalized.id; + let mut local_ids = BTreeSet::new(); + let mut rules = BTreeMap::new(); + + for (index, rule) in normalized.rules.into_iter().enumerate() { + if !local_ids.insert(rule.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::RulePack, + pack_id: pack_id.clone(), + entry_kind: "rule", + entry_id: rule.id, + }); + } + let rule_id = compile_rule_id(&pack_id, &rule.id); + let query = compile_query_ref( + &pack_id, + &JsonPointer::parse(&format!("/rules/{index}/query")).expect("pointer"), + &rule.query, + )?; + rules.insert( + rule_id.clone(), + CompiledRuleDef { + local_id: rule.id, + id: rule_id, + summary: rule.summary, + severity: rule.severity, + scope: rule.scope.map(|scope| CompiledRuleScope { + languages: scope.languages.into_iter().collect(), + path_classes: scope + .path_classes + .into_iter() + .map(map_reserved_path_class) + .collect(), + }), + query, + emit: rule + .emit + .into_iter() + .map(|emit| match emit { + RawRuleEmit::Finding { code, message } => { + CompiledRuleEmit::Finding { code, message } + } + }) + .collect(), + }, + ); + } + + Ok(LoadedCompiledRulePack { + pack: CompiledRulePack { + header: CompiledPackHeader { + kind: PackKind::RulePack, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_RULE_PACK_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + rules, + diagnostics: Vec::new(), + }, + source_file_base_dir, + }) +} + +fn compile_normalized_recipe_pack( + origin: PackOrigin, + source_file_base_dir: Option, + source_fingerprint: Fingerprint, + semantic_fingerprint: Fingerprint, + normalized: NormalizedRecipePackDocument, +) -> PackResult { + let id = PackName::parse(&normalized.id)?; + let pack_id = normalized.id; + let mut local_ids = BTreeSet::new(); + let mut recipes = BTreeMap::new(); + + for (index, recipe) in normalized.recipes.into_iter().enumerate() { + if !local_ids.insert(recipe.id.clone()) { + return Err(PackError::DuplicateEntryId { + pack_kind: PackKind::RecipePack, + pack_id: pack_id.clone(), + entry_kind: "recipe", + entry_id: recipe.id, + }); + } + let recipe_id = compile_recipe_id(&pack_id, &recipe.id); + let query = compile_query_ref( + &pack_id, + &JsonPointer::parse(&format!("/recipes/{index}/query")).expect("pointer"), + &recipe.query, + )?; + recipes.insert( + recipe_id.clone(), + CompiledRecipeDef { + local_id: recipe.id, + id: recipe_id, + summary: recipe.summary, + query, + transforms: recipe + .transforms + .into_iter() + .map(|transform| match transform { + RawRecipeTransform::ReplaceCaptureText { capture, text } => { + CompiledRecipeTransform::ReplaceCaptureText { capture, text } + } + }) + .collect(), + }, + ); + } + + Ok(LoadedCompiledRecipePack { + pack: CompiledRecipePack { + header: CompiledPackHeader { + kind: PackKind::RecipePack, + id, + version: normalized.version, + name: normalized.name, + description: normalized.description, + schema_id: PACK_RECIPE_PACK_V1_SCHEMA_ID, + origin, + source_fingerprint, + semantic_fingerprint, + }, + recipes, + diagnostics: Vec::new(), + }, + source_file_base_dir, + }) +} + +fn compile_query_id(pack_id: &str, local_id: &str) -> QueryId { + QueryId::from_identity(&format!("pack\0query_pack\0{pack_id}\0query\0{local_id}")) +} + +fn compile_rule_id(pack_id: &str, local_id: &str) -> RuleId { + RuleId::from_identity(&format!("pack\0rule_pack\0{pack_id}\0rule\0{local_id}")) +} + +fn compile_recipe_id(pack_id: &str, local_id: &str) -> RecipeId { + RecipeId::from_identity(&format!("pack\0recipe_pack\0{pack_id}\0recipe\0{local_id}")) +} + +fn map_reserved_path_class(value: RawReservedPathClass) -> ReservedPathClass { + match value { + RawReservedPathClass::Test => ReservedPathClass::Test, + RawReservedPathClass::Docs => ReservedPathClass::Docs, + RawReservedPathClass::Ci => ReservedPathClass::Ci, + RawReservedPathClass::Migration => ReservedPathClass::Migration, + RawReservedPathClass::Security => ReservedPathClass::Security, + RawReservedPathClass::PublicApi => ReservedPathClass::PublicApi, + RawReservedPathClass::Generated => ReservedPathClass::Generated, + RawReservedPathClass::Vendor => ReservedPathClass::Vendor, + } +} + +fn compile_glob_set(patterns: &[String]) -> PackResult { + let mut builder = GlobSetBuilder::new(); + for pattern in patterns { + let glob = Glob::new(pattern).map_err(|error| PackError::GlobCompile { + pattern: pattern.clone(), + reason: error.to_string(), + })?; + builder.add(glob); + } + + builder.build().map_err(|error| PackError::GlobCompile { + pattern: patterns.join(","), + reason: error.to_string(), + }) +} + +fn compile_boundary_id(pack_id: &str, local_id: &str) -> BoundaryId { + BoundaryId::from_identity(&format!( + "pack\0boundary_taxonomy\0{pack_id}\0boundary\0{local_id}" + )) +} + +fn compile_component_id(pack_id: &str, local_id: &str) -> ComponentId { + ComponentId::from_identity(&format!( + "pack\0component_map\0{pack_id}\0component\0{local_id}" + )) +} + +fn resolve_boundary_taxonomy( + compiler: &PackCompiler, + profile: &CompiledProfile, + reference: &PackRef, +) -> PackResult { + match reference { + PackRef::Builtin(name) => { + if let Some(source) = builtin::boundary_taxonomy_source(name.as_str()) { + compiler.compile_boundary_taxonomy(source) + } else if builtin::component_map_source(name.as_str()).is_some() { + Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::BoundaryTaxonomy, + actual: PackKind::ComponentMap, + }) + } else { + Err(unknown_pack_reference(profile, reference)) + } + } + PackRef::File(path) => { + let source = resolve_file_source(profile, path.as_str())?; + match compiler.compile_boundary_taxonomy(source) { + Ok(compiled) => Ok(compiled), + Err(error) if is_invalid_kind_schema_violation(&error) => { + let alternate_source = resolve_file_source(profile, path.as_str())?; + match compiler.compile_component_map(alternate_source) { + Ok(_) => Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::BoundaryTaxonomy, + actual: PackKind::ComponentMap, + }), + Err(_) => Err(error), + } + } + Err(error) => Err(error), + } + } + } +} + +fn resolve_component_map( + compiler: &PackCompiler, + profile: &CompiledProfile, + reference: &PackRef, +) -> PackResult { + match reference { + PackRef::Builtin(name) => { + if let Some(source) = builtin::component_map_source(name.as_str()) { + compiler.compile_component_map(source) + } else if builtin::boundary_taxonomy_source(name.as_str()).is_some() { + Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::ComponentMap, + actual: PackKind::BoundaryTaxonomy, + }) + } else { + Err(unknown_pack_reference(profile, reference)) + } + } + PackRef::File(path) => { + let source = resolve_file_source(profile, path.as_str())?; + match compiler.compile_component_map(source) { + Ok(compiled) => Ok(compiled), + Err(error) if is_invalid_kind_schema_violation(&error) => { + let alternate_source = resolve_file_source(profile, path.as_str())?; + match compiler.compile_boundary_taxonomy(alternate_source) { + Ok(_) => Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected: PackKind::ComponentMap, + actual: PackKind::BoundaryTaxonomy, + }), + Err(_) => Err(error), + } + } + Err(error) => Err(error), + } + } + } +} + +/// Phase B file refs resolve lexically from the compiled profile's captured +/// parent directory, never from a display string or the ambient cwd. +fn resolve_file_source(profile: &CompiledProfile, relative_path: &str) -> PackResult { + resolve_file_source_from_base( + profile.header.id.as_str(), + profile.source_file_base_dir.as_ref(), + relative_path, + Some(PackFormat::Json), + ) +} + +fn unknown_pack_reference(profile: &CompiledProfile, reference: &PackRef) -> PackError { + PackError::UnknownPackReference { + referring_pack: profile.header.id.as_str().to_owned(), + reference: reference.as_str(), + } +} + +fn unknown_file_reference(profile: &CompiledProfile, relative_path: &str) -> PackError { + unknown_file_reference_by_pack(profile.header.id.as_str(), relative_path) +} + +fn unknown_file_reference_by_pack(referring_pack: &str, relative_path: &str) -> PackError { + PackError::UnknownPackReference { + referring_pack: referring_pack.to_owned(), + reference: format!("file:{relative_path}"), + } +} + +fn resolve_file_source_from_base( + referring_pack: &str, + base_dir: Option<&PathBuf>, + relative_path: &str, + format_hint: Option, +) -> PackResult { + let Some(base) = base_dir else { + return Err(unknown_file_reference_by_pack( + referring_pack, + relative_path, + )); + }; + + let path = base.join(relative_path); + match path.try_exists() { + Ok(true) => {} + Ok(false) => { + if let Some(reason) = blocked_file_reference_reason(base, &path) { + return Err(PackError::Io { + origin: path.display().to_string(), + reason, + }); + } + return Err(unknown_file_reference_by_pack( + referring_pack, + relative_path, + )); + } + Err(error) => { + return Err(PackError::Io { + origin: path.display().to_string(), + reason: error.to_string(), + }); + } + } + + Ok(PackSource::File { path, format_hint }) +} + +fn blocked_file_reference_reason(base: &Path, path: &Path) -> Option { + let parent = path.parent()?; + let relative_parent = parent.strip_prefix(base).ok()?; + let mut probe = base.to_path_buf(); + + for component in relative_parent.components() { + probe.push(component.as_os_str()); + match fs::metadata(&probe) { + Ok(metadata) => { + if !metadata.is_dir() { + return Some(format!( + "path component is not a directory: {}", + probe.display() + )); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + Err(error) => return Some(error.to_string()), + } + } + + None +} + +fn resolve_bundle_ref_source( + compiler: &PackCompiler, + referring_pack: &str, + referring_base_dir: Option<&PathBuf>, + reference: &PackRef, + expected: PackKind, +) -> PackResult<(PackSource, ResolvedPackSourceKey)> { + match reference { + PackRef::Builtin(name) => { + if let Some(source) = builtin_source_for_kind(expected, name.as_str()) { + Ok((source, ResolvedPackSourceKey::Builtin(name.clone()))) + } else if let Some(actual) = builtin_pack_kind(name.as_str()) { + Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected, + actual, + }) + } else { + Err(PackError::UnknownPackReference { + referring_pack: referring_pack.to_owned(), + reference: reference.as_str(), + }) + } + } + PackRef::File(path) => { + let source = resolve_file_source_from_base( + referring_pack, + referring_base_dir, + path.as_str(), + None, + )?; + if let Some(actual) = compiler.detect_source_pack_kind(&source)? { + if actual != expected { + return Err(PackError::RefKindMismatch { + reference: reference.as_str(), + expected, + actual, + }); + } + } + let key = bundle_source_key(&source)?; + Ok((source, key)) + } + } +} + +fn bundle_source_key(source: &PackSource) -> PackResult { + match source { + PackSource::Builtin { logical_name, .. } => Ok(ResolvedPackSourceKey::Builtin( + PackName::parse(logical_name)?, + )), + PackSource::File { path, .. } => Ok(ResolvedPackSourceKey::File(absolutize_path(path)?)), + PackSource::Inline { .. } => Err(PackError::InvalidPackRef { + input: "inline bundle sources are unsupported".to_owned(), + }), + } +} + +fn builtin_source_for_kind(expected: PackKind, logical_name: &str) -> Option { + match expected { + PackKind::Profile => builtin::profile_source(logical_name), + PackKind::BoundaryTaxonomy => builtin::boundary_taxonomy_source(logical_name), + PackKind::ComponentMap => builtin::component_map_source(logical_name), + PackKind::ScoreModel => builtin::score_model_source(logical_name), + PackKind::RulePack => builtin::rule_pack_source(logical_name), + PackKind::QueryPack => builtin::query_pack_source(logical_name), + PackKind::RecipePack => builtin::recipe_pack_source(logical_name), + } +} + +fn builtin_pack_kind(logical_name: &str) -> Option { + [ + ( + PackKind::Profile, + builtin::profile_source(logical_name).is_some(), + ), + ( + PackKind::BoundaryTaxonomy, + builtin::boundary_taxonomy_source(logical_name).is_some(), + ), + ( + PackKind::ComponentMap, + builtin::component_map_source(logical_name).is_some(), + ), + ( + PackKind::ScoreModel, + builtin::score_model_source(logical_name).is_some(), + ), + ( + PackKind::RulePack, + builtin::rule_pack_source(logical_name).is_some(), + ), + ( + PackKind::QueryPack, + builtin::query_pack_source(logical_name).is_some(), + ), + ( + PackKind::RecipePack, + builtin::recipe_pack_source(logical_name).is_some(), + ), + ] + .into_iter() + .find_map(|(kind, present)| present.then_some(kind)) +} + +fn resolved_bundle_pack_kind(value: &ResolvedBundlePack) -> PackKind { + match value { + ResolvedBundlePack::ScoreModel(_) => PackKind::ScoreModel, + ResolvedBundlePack::QueryPack(_) => PackKind::QueryPack, + ResolvedBundlePack::RulePack(_) => PackKind::RulePack, + ResolvedBundlePack::RecipePack(_) => PackKind::RecipePack, + } +} + +fn register_bundle_pack_id( + header: &CompiledPackHeader, + key: &ResolvedPackSourceKey, + seen_pack_ids: &mut BTreeMap, +) -> PackResult<()> { + match seen_pack_ids.get(&header.id) { + Some(existing) if existing != key => Err(PackError::DuplicatePackId { + kind: header.kind, + id: header.id.as_str().to_owned(), + }), + Some(_) => Ok(()), + None => { + seen_pack_ids.insert(header.id.clone(), key.clone()); + Ok(()) + } + } +} + +fn resolved_pack_origin_key(origin: &PackOrigin) -> PackResult { + match origin { + PackOrigin::Builtin { logical_name } => Ok(ResolvedPackSourceKey::Builtin( + PackName::parse(logical_name)?, + )), + PackOrigin::File { display_path } => Ok(ResolvedPackSourceKey::File(absolutize_path( + Path::new(display_path), + )?)), + PackOrigin::Inline { logical_name } => { + Ok(ResolvedPackSourceKey::Inline(logical_name.clone())) + } + } +} + +fn ensure_query_ref_exists( + referring_pack: &PackName, + query_ref: &CompiledQueryRef, + query_pack: &CompiledQueryPack, +) -> PackResult<()> { + let query_id = compile_query_id(query_pack.header.id.as_str(), &query_ref.id); + if query_pack.queries.contains_key(&query_id) { + Ok(()) + } else { + Err(PackError::UnknownPackReference { + referring_pack: referring_pack.as_str().to_owned(), + reference: format!("{}#{}", query_ref.pack.as_str(), query_ref.id), + }) + } +} + +fn compile_pack_set_fingerprint( + profile: &CompiledProfile, + topology: &ResolvedProfileTopology, + score_model: Option<&CompiledScoreModel>, + rule_packs: &BTreeMap, + query_packs: &BTreeMap, + recipe_packs: &BTreeMap, +) -> PackResult { + #[derive(Serialize)] + struct BundleFingerprintInput { + profile: String, + #[serde(skip_serializing_if = "Option::is_none")] + boundary_taxonomy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + component_map: Option, + #[serde(skip_serializing_if = "Option::is_none")] + score_model: Option, + rule_packs: BTreeMap, + query_packs: BTreeMap, + recipe_packs: BTreeMap, + } + + let input = BundleFingerprintInput { + profile: profile.header.semantic_fingerprint.as_str().to_owned(), + boundary_taxonomy: topology + .boundary_taxonomy + .as_ref() + .map(|pack| pack.header.semantic_fingerprint.as_str().to_owned()), + component_map: topology + .component_map + .as_ref() + .map(|pack| pack.header.semantic_fingerprint.as_str().to_owned()), + score_model: score_model.map(|pack| pack.header.semantic_fingerprint.as_str().to_owned()), + rule_packs: rule_packs + .iter() + .map(|(name, pack)| { + ( + name.as_str().to_owned(), + pack.header.semantic_fingerprint.as_str().to_owned(), + ) + }) + .collect(), + query_packs: query_packs + .iter() + .map(|(name, pack)| { + ( + name.as_str().to_owned(), + pack.header.semantic_fingerprint.as_str().to_owned(), + ) + }) + .collect(), + recipe_packs: recipe_packs + .iter() + .map(|(name, pack)| { + ( + name.as_str().to_owned(), + pack.header.semantic_fingerprint.as_str().to_owned(), + ) + }) + .collect(), + }; + + sha256_canonical_json(&input).map_err(|error| PackError::ParseFailure { + origin: profile.header.origin.display(), + diagnostics: vec![PackDiagnostic::error( + "pack.bundle.canonicalization_failed", + error.to_string(), + Some(PackLocation { + origin: profile.header.origin.clone(), + path: None, + }), + )], + }) +} + +fn is_invalid_kind_schema_violation(error: &PackError) -> bool { + matches!( + error, + PackError::SchemaViolation { diagnostics, .. } + if diagnostics.iter().any(|diagnostic| diagnostic.code.as_str() == "pack.schema.invalid_kind") + ) +} + +fn compile_optional_ref( + origin: &PackOrigin, + reference: Option<&str>, + pointer: &str, +) -> PackResult> { + reference + .map(|reference| compile_ref(origin, reference, pointer)) + .transpose() +} + +fn compile_ref_set( + origin: &PackOrigin, + references: &[String], + pointer: &str, +) -> PackResult> { + let mut out = BTreeSet::new(); + for (index, reference) in references.iter().enumerate() { + out.insert(compile_ref( + origin, + reference, + &format!("{pointer}/{index}"), + )?); + } + Ok(out) +} + +fn compile_ref(origin: &PackOrigin, reference: &str, pointer: &str) -> PackResult { + let parsed = PackRef::parse(reference).map_err(|_| PackError::SchemaViolation { + origin: origin.display(), + schema_id: PACK_PROFILE_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.refs.invalid_reference", + format!("invalid pack reference: {reference}"), + Some(location(origin, pointer)), + )], + })?; + + if !origin.is_file_backed() && matches!(parsed, PackRef::File(_)) { + return Err(PackError::SchemaViolation { + origin: origin.display(), + schema_id: PACK_PROFILE_V1_SCHEMA_ID, + diagnostics: vec![PackDiagnostic::error( + "pack.refs.file_requires_file_origin", + "builtin and inline profile sources may only use builtin references in Phase A", + Some(location(origin, pointer)), + )], + }); + } + + Ok(parsed) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + use crate::pack::builtin; + use crate::pack::compiler::PackCompiler; + use crate::pack::error::PackError; + use crate::pack::source::{PackFormat, PackSource}; + + #[test] + fn compiles_builtin_file_and_inline_profiles() { + let compiler = PackCompiler::new(); + let builtin = compiler + .compile_profile(builtin::profile_source("generic/default").expect("builtin source")) + .expect("builtin profile"); + let inline = compiler + .compile_profile(PackSource::Inline { + logical_name: "inline".to_owned(), + format: PackFormat::Toml, + bytes: br#" +kind = "profile" +version = 1 +id = "generic/default" +name = "Generic default profile" +description = "Default deterministic profile for generic repositories" + +[topology] +boundary_taxonomy = "builtin:generic/boundaries" +component_map = "builtin:generic/components" + +[analysis] +max_scope_depth = 2 +follow_symlinks = false +languages = ["typescript", "javascript", "python", "rust", "yaml", "toml", "json"] + +[apps] +default = "score" +enabled = ["score"] +"# + .to_vec(), + }) + .expect("inline profile"); + let path = unique_temp_file("phase-a-profile.toml"); + std::fs::write(&path, builtin_source_bytes()).expect("write temp profile"); + let file = compiler + .compile_profile(PackSource::File { + path: path.clone(), + format_hint: None, + }) + .expect("file profile"); + std::fs::remove_file(path).expect("remove temp profile"); + + assert_eq!( + builtin.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_eq!( + builtin.header.semantic_fingerprint, + file.header.semantic_fingerprint + ); + assert_ne!( + builtin.header.source_fingerprint, + inline.header.source_fingerprint + ); + } + + #[test] + fn rejects_file_refs_from_builtin_and_inline_sources() { + let compiler = PackCompiler::new(); + let error = compiler + .compile_profile(PackSource::Inline { + logical_name: "inline".to_owned(), + format: PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "generic/default" +name = "Invalid profile" + +[rules] +packs = ["file:rules/security.v1.json"] +"# + .to_vec(), + }) + .expect_err("inline file refs should fail"); + + assert!(matches!(error, PackError::SchemaViolation { .. })); + } + + #[test] + fn rejects_unsupported_format_and_missing_files() { + let compiler = PackCompiler::new(); + let unsupported = compiler + .compile_profile(PackSource::Inline { + logical_name: "inline".to_owned(), + format: PackFormat::Json, + bytes: br#"{}"#.to_vec(), + }) + .expect_err("json profiles should fail"); + assert!(matches!(unsupported, PackError::UnsupportedFormat { .. })); + + let missing = compiler + .compile_profile(PackSource::File { + path: PathBuf::from("definitely-missing-profile.toml"), + format_hint: None, + }) + .expect_err("missing profile should fail"); + assert!(matches!(missing, PackError::Io { .. })); + } + + #[test] + fn semantic_fingerprint_ignores_toml_key_order() { + let compiler = PackCompiler::new(); + let first = compiler + .compile_profile(PackSource::Inline { + logical_name: "first".to_owned(), + format: PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "generic/default" +name = "Generic default profile" + +[apps] +enabled = ["score"] +default = "score" + +[analysis] +languages = ["json", "toml"] +follow_symlinks = false +max_scope_depth = 2 +"# + .to_vec(), + }) + .expect("first profile"); + let second = compiler + .compile_profile(PackSource::Inline { + logical_name: "second".to_owned(), + format: PackFormat::Toml, + bytes: br#"name = "Generic default profile" +id = "generic/default" +version = 1 +kind = "profile" + +[analysis] +max_scope_depth = 2 +follow_symlinks = false +languages = ["toml", "json"] + +[apps] +default = "score" +enabled = ["score"] +"# + .to_vec(), + }) + .expect("second profile"); + + assert_eq!( + first.header.semantic_fingerprint, + second.header.semantic_fingerprint + ); + } + + fn builtin_source_bytes() -> &'static [u8] { + match builtin::profile_source("generic/default").expect("builtin") { + PackSource::Builtin { bytes, .. } => bytes, + _ => unreachable!("builtin registry should return builtin source"), + } + } + + fn unique_temp_file(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + std::env::temp_dir().join(format!("{nanos}-{name}")) + } +} diff --git a/crates/lift/src/pack/diagnostics.rs b/crates/lift/src/pack/diagnostics.rs new file mode 100644 index 000000000..5216efc2c --- /dev/null +++ b/crates/lift/src/pack/diagnostics.rs @@ -0,0 +1,145 @@ +//! Machine-readable pack diagnostics. + +use std::cmp::Ordering; + +use serde::{Deserialize, Serialize}; + +use crate::kernel::{DiagnosticCode, JsonPointer, Severity}; +use crate::pack::source::PackOrigin; + +/// Pack-local location that does not require repo-relative paths. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct PackLocation { + /// Pack source identity. + pub origin: PackOrigin, + /// Optional JSON pointer into the structured pack document. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// Additional context location attached to a diagnostic. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct PackRelatedLocation { + /// Related pack-local location. + pub location: PackLocation, + /// Human-readable relationship context. + pub message: String, +} + +/// Structured pack compiler diagnostic. +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub(crate) struct PackDiagnostic { + /// Stable diagnostic code. + pub code: DiagnosticCode, + /// Diagnostic severity. + pub severity: Severity, + /// Human-readable message. + pub message: String, + /// Primary subject of the diagnostic. + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + /// Additional related locations. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub related: Vec, + /// Optional remediation hint. + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} + +impl PackDiagnostic { + pub(crate) fn error( + code: &str, + message: impl Into, + subject: Option, + ) -> Self { + Self { + code: DiagnosticCode::parse(code).expect("pack diagnostic code should be valid"), + severity: Severity::Error, + message: message.into(), + subject, + related: Vec::new(), + help: None, + } + } +} + +impl Ord for PackDiagnostic { + fn cmp(&self, other: &Self) -> Ordering { + diagnostic_sort_key(self) + .cmp(&diagnostic_sort_key(other)) + .then_with(|| self.subject.cmp(&other.subject)) + .then_with(|| self.related.cmp(&other.related)) + .then_with(|| self.help.cmp(&other.help)) + } +} + +impl PartialOrd for PackDiagnostic { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +type DiagnosticSortKey<'a> = ( + u8, + Option<&'a PackOrigin>, + Option<&'a str>, + &'a str, + &'a str, +); + +fn diagnostic_sort_key(diagnostic: &PackDiagnostic) -> DiagnosticSortKey<'_> { + ( + severity_rank(diagnostic.severity), + diagnostic.subject.as_ref().map(|subject| &subject.origin), + diagnostic + .subject + .as_ref() + .and_then(|subject| subject.path.as_ref().map(|path| path.as_str())), + diagnostic.code.as_str(), + diagnostic.message.as_str(), + ) +} + +fn severity_rank(severity: Severity) -> u8 { + match severity { + Severity::Error => 0, + Severity::Warning => 1, + Severity::Info => 2, + } +} + +#[cfg(test)] +mod tests { + use super::{PackDiagnostic, PackLocation}; + use crate::kernel::{JsonPointer, Severity}; + use crate::pack::source::PackOrigin; + + #[test] + fn diagnostics_sort_deterministically() { + let earlier = PackDiagnostic { + code: crate::kernel::DiagnosticCode::parse("pack.schema.invalid_version") + .expect("code"), + severity: Severity::Error, + message: "first".to_owned(), + subject: Some(PackLocation { + origin: PackOrigin::Inline { + logical_name: "a".to_owned(), + }, + path: Some(JsonPointer::parse("/version").expect("pointer")), + }), + related: Vec::new(), + help: None, + }; + let later = PackDiagnostic { + code: crate::kernel::DiagnosticCode::parse("pack.schema.invalid_version") + .expect("code"), + severity: Severity::Warning, + message: "second".to_owned(), + subject: None, + related: Vec::new(), + help: None, + }; + + assert!(earlier < later); + } +} diff --git a/crates/lift/src/pack/error.rs b/crates/lift/src/pack/error.rs new file mode 100644 index 000000000..f8ac986dc --- /dev/null +++ b/crates/lift/src/pack/error.rs @@ -0,0 +1,74 @@ +//! Typed pack compiler errors. + +use thiserror::Error; + +use crate::kernel::JsonPointer; +use crate::pack::diagnostics::PackDiagnostic; +use crate::pack::raw::PackKind; + +/// Result type used by pack compiler contracts. +pub(crate) type PackResult = Result; + +/// Typed failures emitted by the pack compiler seam. +#[allow(dead_code)] +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub(crate) enum PackError { + #[error("pack I/O failure: {reason}")] + Io { origin: String, reason: String }, + + #[error("unsupported pack format")] + UnsupportedFormat { origin: String }, + + #[error("pack parse failure")] + ParseFailure { + origin: String, + diagnostics: Vec, + }, + + #[error("schema validation failure")] + SchemaViolation { + origin: String, + schema_id: &'static str, + diagnostics: Vec, + }, + + #[error("invalid pack name")] + InvalidPackName { input: String }, + + #[error("invalid pack reference")] + InvalidPackRef { input: String }, + + #[error("duplicate pack id")] + DuplicatePackId { kind: PackKind, id: String }, + + #[error("duplicate entry id")] + DuplicateEntryId { + pack_kind: PackKind, + pack_id: String, + entry_kind: &'static str, + entry_id: String, + }, + + #[error("unknown pack reference")] + UnknownPackReference { + referring_pack: String, + reference: String, + }, + + #[error("pack reference kind mismatch")] + RefKindMismatch { + reference: String, + expected: PackKind, + actual: PackKind, + }, + + #[error("glob compile failure")] + GlobCompile { pattern: String, reason: String }, + + #[error("expression compile failure")] + ExpressionCompile { + pack_id: String, + path: JsonPointer, + reason: String, + }, +} diff --git a/crates/lift/src/pack/expr.rs b/crates/lift/src/pack/expr.rs new file mode 100644 index 000000000..c38d53aa9 --- /dev/null +++ b/crates/lift/src/pack/expr.rs @@ -0,0 +1,308 @@ +//! Shared expression and query-reference compilation. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::kernel::JsonPointer; +use crate::pack::error::{PackError, PackResult}; +use crate::pack::refs::PackRef; + +/// Structural query reference used by rule and recipe packs. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct CompiledQueryRef { + pub pack: PackRef, + pub id: String, +} + +/// Structural expression tree for score-model compilation. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum CompiledExpr { + ConstInt(i64), + ConstBool(bool), + Field(JsonPointer), + Exists(JsonPointer), + IsNull(JsonPointer), + Add(Vec), + Sub(Vec), + Mul(Vec), + Div(Vec), + Max(Vec), + Min(Vec), + Eq(Box, Box), + Ne(Box, Box), + Gt(Box, Box), + Ge(Box, Box), + Lt(Box, Box), + Le(Box, Box), + And(Vec), + Or(Vec), + Not(Box), +} + +/// Compiles one expression subtree from JSON. +pub(crate) fn compile_expr( + pack_id: &str, + path: &JsonPointer, + value: &Value, +) -> PackResult { + let object = value + .as_object() + .ok_or_else(|| expression_error(pack_id, path, "expression must be an object"))?; + let op = object + .get("op") + .and_then(Value::as_str) + .ok_or_else(|| expression_error(pack_id, path, "expression op must be a string"))?; + + match op { + "const_int" => compile_const_int(pack_id, path, object.get("value")), + "const_bool" => compile_const_bool(pack_id, path, object.get("value")), + "field" => compile_pointer_op(pack_id, path, object.get("path"), CompiledExpr::Field), + "exists" => compile_pointer_op(pack_id, path, object.get("path"), CompiledExpr::Exists), + "is_null" => compile_pointer_op(pack_id, path, object.get("path"), CompiledExpr::IsNull), + "add" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Add), + "sub" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Sub), + "mul" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Mul), + "div" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Div), + "max" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Max), + "min" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Min), + "and" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::And), + "or" => compile_variadic_expr(pack_id, path, object.get("args"), CompiledExpr::Or), + "eq" => compile_binary_expr( + pack_id, + path, + object.get("lhs"), + object.get("rhs"), + CompiledExpr::Eq, + ), + "ne" => compile_binary_expr( + pack_id, + path, + object.get("lhs"), + object.get("rhs"), + CompiledExpr::Ne, + ), + "gt" => compile_binary_expr( + pack_id, + path, + object.get("lhs"), + object.get("rhs"), + CompiledExpr::Gt, + ), + "ge" => compile_binary_expr( + pack_id, + path, + object.get("lhs"), + object.get("rhs"), + CompiledExpr::Ge, + ), + "lt" => compile_binary_expr( + pack_id, + path, + object.get("lhs"), + object.get("rhs"), + CompiledExpr::Lt, + ), + "le" => compile_binary_expr( + pack_id, + path, + object.get("lhs"), + object.get("rhs"), + CompiledExpr::Le, + ), + "not" => compile_unary_expr(pack_id, path, object.get("arg"), CompiledExpr::Not), + other => Err(expression_error( + pack_id, + path, + format!("unsupported expression operator `{other}`"), + )), + } +} + +/// Compiles one structural query reference object from JSON. +pub(crate) fn compile_query_ref( + pack_id: &str, + path: &JsonPointer, + value: &Value, +) -> PackResult { + let object = value + .as_object() + .ok_or_else(|| expression_error(pack_id, path, "query ref must be an object"))?; + let pack = object + .get("pack") + .and_then(Value::as_str) + .ok_or_else(|| expression_error(pack_id, path, "query ref pack must be a string"))?; + let id = object + .get("id") + .and_then(Value::as_str) + .ok_or_else(|| expression_error(pack_id, path, "query ref id must be a string"))?; + if id.is_empty() { + return Err(expression_error( + pack_id, + path, + "query ref id must be non-empty", + )); + } + + Ok(CompiledQueryRef { + pack: PackRef::parse(pack)?, + id: id.to_owned(), + }) +} + +fn compile_const_int( + pack_id: &str, + path: &JsonPointer, + value: Option<&Value>, +) -> PackResult { + let number = value + .and_then(Value::as_i64) + .ok_or_else(|| expression_error(pack_id, path, "const_int requires integer value"))?; + Ok(CompiledExpr::ConstInt(number)) +} + +fn compile_const_bool( + pack_id: &str, + path: &JsonPointer, + value: Option<&Value>, +) -> PackResult { + let boolean = value + .and_then(Value::as_bool) + .ok_or_else(|| expression_error(pack_id, path, "const_bool requires boolean value"))?; + Ok(CompiledExpr::ConstBool(boolean)) +} + +fn compile_pointer_op( + pack_id: &str, + path: &JsonPointer, + value: Option<&Value>, + constructor: fn(JsonPointer) -> CompiledExpr, +) -> PackResult { + let pointer = value.and_then(Value::as_str).ok_or_else(|| { + expression_error(pack_id, path, "pointer expression requires string path") + })?; + let pointer = JsonPointer::parse(pointer).map_err(|error| { + expression_error(pack_id, path, format!("invalid JSON Pointer: {error}")) + })?; + Ok(constructor(pointer)) +} + +fn compile_variadic_expr( + pack_id: &str, + path: &JsonPointer, + value: Option<&Value>, + constructor: fn(Vec) -> CompiledExpr, +) -> PackResult { + let items = value.and_then(Value::as_array).ok_or_else(|| { + expression_error(pack_id, path, "variadic expression requires args array") + })?; + if items.is_empty() { + return Err(expression_error( + pack_id, + path, + "variadic expression requires at least one arg", + )); + } + + let mut compiled = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + compiled.push(compile_expr( + pack_id, + &path.push_token("args").push_token(&index.to_string()), + item, + )?); + } + Ok(constructor(compiled)) +} + +fn compile_binary_expr( + pack_id: &str, + path: &JsonPointer, + lhs: Option<&Value>, + rhs: Option<&Value>, + constructor: fn(Box, Box) -> CompiledExpr, +) -> PackResult { + let lhs = compile_expr( + pack_id, + &path.push_token("lhs"), + lhs.ok_or_else(|| expression_error(pack_id, path, "binary expression requires lhs"))?, + )?; + let rhs = compile_expr( + pack_id, + &path.push_token("rhs"), + rhs.ok_or_else(|| expression_error(pack_id, path, "binary expression requires rhs"))?, + )?; + Ok(constructor(Box::new(lhs), Box::new(rhs))) +} + +fn compile_unary_expr( + pack_id: &str, + path: &JsonPointer, + arg: Option<&Value>, + constructor: fn(Box) -> CompiledExpr, +) -> PackResult { + let arg = compile_expr( + pack_id, + &path.push_token("arg"), + arg.ok_or_else(|| expression_error(pack_id, path, "unary expression requires arg"))?, + )?; + Ok(constructor(Box::new(arg))) +} + +fn expression_error(pack_id: &str, path: &JsonPointer, reason: impl Into) -> PackError { + PackError::ExpressionCompile { + pack_id: pack_id.to_owned(), + path: path.clone(), + reason: reason.into(), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{compile_expr, compile_query_ref, CompiledExpr}; + use crate::kernel::JsonPointer; + + #[test] + fn compiles_nested_expressions() { + let expr = compile_expr( + "generic/lift-v2", + &JsonPointer::parse("/lift_score").expect("pointer"), + &json!({ + "op": "add", + "args": [ + {"op": "const_int", "value": 1}, + {"op": "field", "path": "/touch/edit_files"} + ] + }), + ) + .expect("expr"); + + assert!(matches!(expr, CompiledExpr::Add(args) if args.len() == 2)); + } + + #[test] + fn rejects_bad_pointer_and_unknown_query_ref() { + let expr_error = compile_expr( + "generic/lift-v2", + &JsonPointer::parse("/lift_score").expect("pointer"), + &json!({"op": "field", "path": "touch/edit_files"}), + ) + .expect_err("bad pointer should fail"); + assert!(matches!( + expr_error, + crate::pack::error::PackError::ExpressionCompile { .. } + )); + + let query_error = compile_query_ref( + "generic/policy", + &JsonPointer::parse("/rules/0/query").expect("pointer"), + &json!({"pack": "not-a-ref", "id": "use_statement"}), + ) + .expect_err("bad query ref should fail"); + assert!(matches!( + query_error, + crate::pack::error::PackError::InvalidPackRef { .. } + )); + } +} diff --git a/crates/lift/src/pack/mod.rs b/crates/lift/src/pack/mod.rs new file mode 100644 index 000000000..f6e7ae11a --- /dev/null +++ b/crates/lift/src/pack/mod.rs @@ -0,0 +1,57 @@ +//! Internal pack compiler seam. + +#![allow(dead_code)] +#![allow(unused_imports)] + +pub(crate) mod builtin; +pub(crate) mod compiler; +pub(crate) mod diagnostics; +pub(crate) mod error; +pub(crate) mod expr; +pub(crate) mod names; +pub(crate) mod refs; +pub(crate) mod schema; +pub(crate) mod source; + +pub(crate) mod compiled; +pub(crate) mod raw; + +#[cfg(not(test))] +pub(crate) use crate::kernel::{BoundaryId, ComponentId}; +pub(crate) use compiled::{ + BoundaryCountingMode, CompiledAnalysisDefaults, CompiledBoundary, CompiledBoundaryTaxonomy, + CompiledComponent, CompiledComponentMap, CompiledConfidenceModel, CompiledConfidenceRule, + CompiledGlobMatcher, CompiledMissingInputRule, CompiledPackHeader, CompiledPackSet, + CompiledPathClasses, CompiledProfile, CompiledProfileApps, CompiledProfileIncludes, + CompiledProfileScore, CompiledProfileTopology, CompiledQueryCapture, CompiledQueryDef, + CompiledQueryPack, CompiledRecipeDef, CompiledRecipePack, CompiledRecipeTransform, + CompiledRuleDef, CompiledRuleEmit, CompiledRulePack, CompiledRuleScope, CompiledScoreModel, + CompiledTriggerRule, ComponentCountingMode, QueryEngineKind, ReservedPathClass, + ResolvedProfileTopology, +}; +pub(crate) use compiler::PackCompiler; +pub(crate) use diagnostics::{PackDiagnostic, PackLocation, PackRelatedLocation}; +pub(crate) use error::{PackError, PackResult}; +pub(crate) use expr::{compile_expr, compile_query_ref, CompiledExpr, CompiledQueryRef}; +pub(crate) use names::{AppName, LanguageId, PackName}; +pub(crate) use raw::PackKind; +pub(crate) use refs::{PackFileRef, PackRef}; +pub(crate) use schema::{ + PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_FILE, PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_JSON, PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_VERSION, + PACK_COMMON_V1_SCHEMA_FILE, PACK_COMMON_V1_SCHEMA_ID, PACK_COMMON_V1_SCHEMA_JSON, + PACK_COMMON_V1_SCHEMA_VERSION, PACK_COMPONENT_MAP_V1_SCHEMA_FILE, + PACK_COMPONENT_MAP_V1_SCHEMA_ID, PACK_COMPONENT_MAP_V1_SCHEMA_JSON, + PACK_COMPONENT_MAP_V1_SCHEMA_VERSION, PACK_PROFILE_V1_SCHEMA_FILE, PACK_PROFILE_V1_SCHEMA_ID, + PACK_PROFILE_V1_SCHEMA_JSON, PACK_PROFILE_V1_SCHEMA_VERSION, PACK_QUERY_PACK_V1_SCHEMA_FILE, + PACK_QUERY_PACK_V1_SCHEMA_ID, PACK_QUERY_PACK_V1_SCHEMA_JSON, + PACK_QUERY_PACK_V1_SCHEMA_VERSION, PACK_RECIPE_PACK_V1_SCHEMA_FILE, + PACK_RECIPE_PACK_V1_SCHEMA_ID, PACK_RECIPE_PACK_V1_SCHEMA_JSON, + PACK_RECIPE_PACK_V1_SCHEMA_VERSION, PACK_RULE_PACK_V1_SCHEMA_FILE, PACK_RULE_PACK_V1_SCHEMA_ID, + PACK_RULE_PACK_V1_SCHEMA_JSON, PACK_RULE_PACK_V1_SCHEMA_VERSION, + PACK_SCORE_MODEL_V1_SCHEMA_FILE, PACK_SCORE_MODEL_V1_SCHEMA_ID, + PACK_SCORE_MODEL_V1_SCHEMA_JSON, PACK_SCORE_MODEL_V1_SCHEMA_VERSION, +}; +pub(crate) use source::{PackFormat, PackOrigin, PackSource}; +#[cfg(test)] +pub(crate) use substrate_lift::kernel::{BoundaryId, ComponentId}; diff --git a/crates/lift/src/pack/names.rs b/crates/lift/src/pack/names.rs new file mode 100644 index 000000000..6cbe31f71 --- /dev/null +++ b/crates/lift/src/pack/names.rs @@ -0,0 +1,208 @@ +//! Pack-local name contracts. + +use serde::{Deserialize, Serialize}; + +use crate::pack::error::{PackError, PackResult}; + +/// Canonical pack identifier. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct PackName(String); + +impl PackName { + /// Parses and validates a pack name. + pub(crate) fn parse(input: &str) -> PackResult { + if is_valid_pack_name(input) { + Ok(Self(input.to_owned())) + } else { + Err(PackError::InvalidPackName { + input: input.to_owned(), + }) + } + } + + /// Returns the canonical string form. + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for PackName { + type Error = PackError; + + fn try_from(value: String) -> PackResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: PackName) -> Self { + value.0 + } +} + +/// App-level entrypoint name used by profiles. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct AppName(String); + +impl AppName { + /// Parses and validates an app name. + pub(crate) fn parse(input: &str) -> PackResult { + if is_valid_label(input) { + Ok(Self(input.to_owned())) + } else { + Err(PackError::SchemaViolation { + origin: input.to_owned(), + schema_id: "pack.app_name", + diagnostics: Vec::new(), + }) + } + } + + /// Returns the app name string. + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for AppName { + type Error = PackError; + + fn try_from(value: String) -> PackResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: AppName) -> Self { + value.0 + } +} + +/// Supported analysis language identifiers for profile defaults. +#[derive( + Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, Default, +)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LanguageId { + Json, + Toml, + Yaml, + #[default] + Rust, + Python, + Javascript, + Typescript, +} + +impl LanguageId { + /// Parses a language identifier from its canonical string form. + pub(crate) fn parse(input: &str) -> PackResult { + match input { + "json" => Ok(Self::Json), + "toml" => Ok(Self::Toml), + "yaml" => Ok(Self::Yaml), + "rust" => Ok(Self::Rust), + "python" => Ok(Self::Python), + "javascript" => Ok(Self::Javascript), + "typescript" => Ok(Self::Typescript), + _ => Err(PackError::SchemaViolation { + origin: input.to_owned(), + schema_id: "pack.language_id", + diagnostics: Vec::new(), + }), + } + } + + /// Returns the canonical string form. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Json => "json", + Self::Toml => "toml", + Self::Yaml => "yaml", + Self::Rust => "rust", + Self::Python => "python", + Self::Javascript => "javascript", + Self::Typescript => "typescript", + } + } +} + +fn is_valid_pack_name(input: &str) -> bool { + let mut saw_segment = false; + for segment in input.split('/') { + if segment.is_empty() || !is_valid_segment(segment) { + return false; + } + saw_segment = true; + } + saw_segment + && !input.starts_with('/') + && !input.ends_with('/') + && input.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'/' | b'.' | b'_' | b'-') + }) +} + +fn is_valid_segment(segment: &str) -> bool { + let mut chars = segment.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| { + ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '.' | '_' | '-') + }) + && !segment.ends_with(['.', '_', '-']) + && !segment.contains("..") + && !segment.contains("__") + && !segment.contains("--") + && !segment.contains("._") + && !segment.contains(".-") +} + +fn is_valid_label(input: &str) -> bool { + let mut chars = input.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_lowercase()) + && chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} + +#[cfg(test)] +mod tests { + use super::{AppName, LanguageId, PackName}; + + #[test] + fn pack_name_accepts_valid_values() { + for name in [ + "generic", + "generic/default", + "substrate/default", + "rust/core", + ] { + let parsed = PackName::parse(name).expect("pack name"); + assert_eq!(parsed.as_str(), name); + } + } + + #[test] + fn pack_name_rejects_invalid_values() { + for name in [ + "Generic", + "/generic", + "generic/", + "generic//default", + "generic default", + ] { + assert!(PackName::parse(name).is_err(), "{name} should fail"); + } + } + + #[test] + fn app_name_and_language_id_parse() { + assert_eq!(AppName::parse("score").expect("app").as_str(), "score"); + assert_eq!( + LanguageId::parse("typescript").expect("lang").as_str(), + "typescript" + ); + } +} diff --git a/crates/lift/src/pack/raw/boundary_taxonomy.rs b/crates/lift/src/pack/raw/boundary_taxonomy.rs new file mode 100644 index 000000000..11118d8a1 --- /dev/null +++ b/crates/lift/src/pack/raw/boundary_taxonomy.rs @@ -0,0 +1,45 @@ +//! Raw boundary taxonomy pack document shape. + +use serde::{Deserialize, Serialize}; + +use crate::pack::raw::PackKind; + +/// Raw counting configuration for boundary taxonomy packs. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawBoundaryTaxonomyCounting { + pub mode: RawBoundaryTaxonomyCountingMode, +} + +/// Supported counting modes for raw boundary taxonomy packs. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum RawBoundaryTaxonomyCountingMode { + DistinctMinusOne, +} + +/// Raw boundary entry keyed by a pack-local boundary id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawBoundaryEntry { + pub id: String, + pub label: String, + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub exclude: Vec, +} + +/// Raw seam-1 boundary taxonomy document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawBoundaryTaxonomy { + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub counting: RawBoundaryTaxonomyCounting, + #[serde(default)] + pub boundaries: Vec, +} diff --git a/crates/lift/src/pack/raw/common.rs b/crates/lift/src/pack/raw/common.rs new file mode 100644 index 000000000..b54a42ac6 --- /dev/null +++ b/crates/lift/src/pack/raw/common.rs @@ -0,0 +1,41 @@ +//! Shared raw profile sections. + +use serde::{Deserialize, Serialize}; + +/// Supported pack kinds in the seam-1 compiler domain. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PackKind { + Profile, + BoundaryTaxonomy, + ComponentMap, + ScoreModel, + RulePack, + QueryPack, + RecipePack, +} + +/// Raw apps section for a profile. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawProfileApps { + pub enabled: Option>, + pub default: Option, +} + +/// Raw analysis defaults section for a profile. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawProfileAnalysis { + pub languages: Option>, + pub follow_symlinks: Option, + pub max_scope_depth: Option, +} + +/// Raw list section for rule/query/recipe includes. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawIncludeSection { + #[serde(default)] + pub packs: Vec, +} diff --git a/crates/lift/src/pack/raw/component_map.rs b/crates/lift/src/pack/raw/component_map.rs new file mode 100644 index 000000000..29d451eb1 --- /dev/null +++ b/crates/lift/src/pack/raw/component_map.rs @@ -0,0 +1,47 @@ +//! Raw component map pack document shape. + +use serde::{Deserialize, Serialize}; + +use crate::pack::raw::PackKind; + +/// Raw counting configuration for component map packs. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawComponentMapCounting { + pub mode: RawComponentMapCountingMode, +} + +/// Supported counting modes for raw component map packs. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum RawComponentMapCountingMode { + Distinct, +} + +/// Raw component entry keyed by a pack-local component id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawComponentEntry { + pub id: String, + pub label: String, + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub exclude: Vec, + #[serde(default)] + pub tags: Vec, +} + +/// Raw seam-1 component map document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawComponentMap { + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub counting: RawComponentMapCounting, + #[serde(default)] + pub components: Vec, +} diff --git a/crates/lift/src/pack/raw/mod.rs b/crates/lift/src/pack/raw/mod.rs new file mode 100644 index 000000000..e33339059 --- /dev/null +++ b/crates/lift/src/pack/raw/mod.rs @@ -0,0 +1,29 @@ +//! Raw pack document shapes. + +mod boundary_taxonomy; +mod common; +mod component_map; +mod profile; +mod query_pack; +mod recipe_pack; +mod rule_pack; +mod score_model; + +pub(crate) use boundary_taxonomy::{ + RawBoundaryEntry, RawBoundaryTaxonomy, RawBoundaryTaxonomyCounting, + RawBoundaryTaxonomyCountingMode, +}; +pub(crate) use common::{PackKind, RawIncludeSection, RawProfileAnalysis, RawProfileApps}; +pub(crate) use component_map::{ + RawComponentEntry, RawComponentMap, RawComponentMapCounting, RawComponentMapCountingMode, +}; +pub(crate) use profile::{RawProfile, RawProfileScore, RawProfileTopology}; +pub(crate) use query_pack::{RawQueryCapture, RawQueryDef, RawQueryPack}; +pub(crate) use recipe_pack::{RawRecipeDef, RawRecipePack, RawRecipeQueryRef, RawRecipeTransform}; +pub(crate) use rule_pack::{ + RawReservedPathClass, RawRuleDef, RawRuleEmit, RawRulePack, RawRuleQueryRef, RawRuleScope, +}; +pub(crate) use score_model::{ + RawScoreConfidenceModel, RawScoreConfidenceRule, RawScoreMissingInputRule, RawScoreModel, + RawScoreTriggerRule, +}; diff --git a/crates/lift/src/pack/raw/profile.rs b/crates/lift/src/pack/raw/profile.rs new file mode 100644 index 000000000..bf9902639 --- /dev/null +++ b/crates/lift/src/pack/raw/profile.rs @@ -0,0 +1,38 @@ +//! Raw profile document shape. + +use serde::{Deserialize, Serialize}; + +use crate::pack::raw::{PackKind, RawIncludeSection, RawProfileAnalysis, RawProfileApps}; + +/// Raw topology references inside a profile. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawProfileTopology { + pub boundary_taxonomy: Option, + pub component_map: Option, +} + +/// Raw score configuration inside a profile. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawProfileScore { + pub model: Option, +} + +/// Raw seam-1 profile document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawProfile { + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub apps: Option, + pub analysis: Option, + pub topology: Option, + pub score: Option, + pub rules: Option, + pub queries: Option, + pub recipes: Option, +} diff --git a/crates/lift/src/pack/raw/query_pack.rs b/crates/lift/src/pack/raw/query_pack.rs new file mode 100644 index 000000000..de508f04b --- /dev/null +++ b/crates/lift/src/pack/raw/query_pack.rs @@ -0,0 +1,39 @@ +//! Raw query pack document shape. + +use serde::{Deserialize, Serialize}; + +use crate::pack::raw::PackKind; + +/// Raw query capture metadata. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawQueryCapture { + pub name: String, + pub required: bool, +} + +/// Raw query definition keyed by a pack-local query id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawQueryDef { + pub id: String, + pub summary: Option, + pub pattern: String, + pub captures: Vec, +} + +/// Raw seam-1 query-pack document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawQueryPack { + #[serde(rename = "$schema")] + pub schema: String, + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub language: String, + pub engine: String, + pub queries: Vec, +} diff --git a/crates/lift/src/pack/raw/recipe_pack.rs b/crates/lift/src/pack/raw/recipe_pack.rs new file mode 100644 index 000000000..adf158de9 --- /dev/null +++ b/crates/lift/src/pack/raw/recipe_pack.rs @@ -0,0 +1,44 @@ +//! Raw recipe-pack document shape. + +use serde::{Deserialize, Serialize}; + +use crate::pack::raw::PackKind; + +/// Raw query reference preserved structurally until bundle resolution. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRecipeQueryRef { + pub pack: String, + pub id: String, +} + +/// Raw recipe transform preserved in source order. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum RawRecipeTransform { + ReplaceCaptureText { capture: String, text: String }, +} + +/// Raw recipe definition keyed by a pack-local recipe id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRecipeDef { + pub id: String, + pub summary: Option, + pub query: RawRecipeQueryRef, + pub transforms: Vec, +} + +/// Raw seam-1 recipe-pack document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRecipePack { + #[serde(rename = "$schema")] + pub schema: String, + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub recipes: Vec, +} diff --git a/crates/lift/src/pack/raw/rule_pack.rs b/crates/lift/src/pack/raw/rule_pack.rs new file mode 100644 index 000000000..917e64528 --- /dev/null +++ b/crates/lift/src/pack/raw/rule_pack.rs @@ -0,0 +1,70 @@ +//! Raw rule-pack document shape. + +use serde::{Deserialize, Serialize}; + +use crate::kernel::Severity; +use crate::pack::names::LanguageId; +use crate::pack::raw::PackKind; + +/// Raw reserved path-class vocabulary for rule scopes. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum RawReservedPathClass { + Test, + Docs, + Ci, + Migration, + Security, + PublicApi, + Generated, + Vendor, +} + +/// Raw query reference preserved structurally until bundle resolution. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRuleQueryRef { + pub pack: String, + pub id: String, +} + +/// Optional structural scope for a rule definition. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRuleScope { + pub languages: Option>, + pub path_classes: Option>, +} + +/// Raw emit actions preserved in source order. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum RawRuleEmit { + Finding { code: String, message: String }, +} + +/// Raw rule definition keyed by a pack-local rule id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRuleDef { + pub id: String, + pub summary: Option, + pub severity: Severity, + pub scope: Option, + pub query: RawRuleQueryRef, + pub emit: Vec, +} + +/// Raw seam-1 rule-pack document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawRulePack { + #[serde(rename = "$schema")] + pub schema: String, + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub rules: Vec, +} diff --git a/crates/lift/src/pack/raw/score_model.rs b/crates/lift/src/pack/raw/score_model.rs new file mode 100644 index 000000000..d2df8747f --- /dev/null +++ b/crates/lift/src/pack/raw/score_model.rs @@ -0,0 +1,63 @@ +//! Raw score-model pack document shape. + +use serde::{Deserialize, Serialize}; + +use crate::pack::raw::PackKind; + +/// Raw trigger rule keyed by a pack-local trigger id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawScoreTriggerRule { + pub id: String, + pub when: serde_json::Value, +} + +/// Raw confidence rule keyed by a pack-local confidence-rule id. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawScoreConfidenceRule { + pub id: String, + pub when: serde_json::Value, + pub set: String, + #[serde(default)] + pub causes: Vec, +} + +/// Raw confidence model for a score-model pack. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawScoreConfidenceModel { + #[serde(rename = "default")] + pub default_level: String, + #[serde(default)] + pub rules: Vec, +} + +/// Raw missing-input rule keyed by the field it guards. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawScoreMissingInputRule { + pub field: String, + pub when: serde_json::Value, +} + +/// Raw seam-1 score-model document. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawScoreModel { + #[serde(rename = "$schema")] + pub schema: Option, + pub kind: PackKind, + pub version: u32, + pub id: String, + pub name: String, + pub description: Option, + pub vector_version: u32, + pub lift_score: serde_json::Value, + pub estimated_slices: serde_json::Value, + #[serde(default)] + pub triggers: Vec, + pub confidence: RawScoreConfidenceModel, + #[serde(default)] + pub missing_input_rules: Vec, +} diff --git a/crates/lift/src/pack/refs.rs b/crates/lift/src/pack/refs.rs new file mode 100644 index 000000000..39af0f77f --- /dev/null +++ b/crates/lift/src/pack/refs.rs @@ -0,0 +1,186 @@ +//! Pack reference contracts. + +use serde::{Deserialize, Serialize}; + +use crate::pack::error::{PackError, PackResult}; +use crate::pack::names::PackName; + +/// Logical relative file reference inside a pack source tree. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) struct PackFileRef(String); + +impl PackFileRef { + /// Parses and validates a file reference payload without the `file:` prefix. + pub(crate) fn parse(input: &str) -> PackResult { + validate_pack_file_ref(input)?; + Ok(Self(input.to_owned())) + } + + /// Returns the canonical string form. + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for PackFileRef { + type Error = PackError; + + fn try_from(value: String) -> PackResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: PackFileRef) -> Self { + value.0 + } +} + +/// Cross-pack reference. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub(crate) enum PackRef { + Builtin(PackName), + File(PackFileRef), +} + +impl PackRef { + /// Parses and validates a pack reference. + pub(crate) fn parse(input: &str) -> PackResult { + if let Some(name) = input.strip_prefix("builtin:") { + return Ok(Self::Builtin(PackName::parse(name)?)); + } + if let Some(path) = input.strip_prefix("file:") { + return Ok(Self::File(PackFileRef::parse(path)?)); + } + Err(PackError::InvalidPackRef { + input: input.to_owned(), + }) + } + + /// Returns the canonical string form. + pub(crate) fn as_str(&self) -> String { + match self { + Self::Builtin(name) => format!("builtin:{}", name.as_str()), + Self::File(path) => format!("file:{}", path.as_str()), + } + } +} + +impl TryFrom for PackRef { + type Error = PackError; + + fn try_from(value: String) -> PackResult { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: PackRef) -> Self { + value.as_str() + } +} + +fn validate_pack_file_ref(input: &str) -> PackResult<()> { + if input.is_empty() + || input.starts_with('/') + || input.ends_with('/') + || input.contains('\\') + || has_windows_drive_prefix(input) + { + return Err(PackError::InvalidPackRef { + input: format!("file:{input}"), + }); + } + + for segment in input.split('/') { + if segment.is_empty() + || matches!(segment, "." | "..") + || !is_valid_pack_file_segment(segment) + { + return Err(PackError::InvalidPackRef { + input: format!("file:{input}"), + }); + } + } + + Ok(()) +} + +fn has_windows_drive_prefix(input: &str) -> bool { + let bytes = input.as_bytes(); + bytes.len() >= 3 && bytes[1] == b':' && bytes[2] == b'/' +} + +fn is_valid_pack_file_segment(segment: &str) -> bool { + let mut bytes = segment.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + + is_pack_file_segment_start(first) && bytes.all(is_pack_file_segment_continue) +} + +fn is_pack_file_segment_start(byte: u8) -> bool { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' +} + +fn is_pack_file_segment_continue(byte: u8) -> bool { + is_pack_file_segment_start(byte) || matches!(byte, b'.' | b'_' | b'-') +} + +#[cfg(test)] +mod tests { + use super::{PackFileRef, PackRef}; + + #[test] + fn parses_builtin_and_file_refs() { + assert!(matches!( + PackRef::parse("builtin:generic/default").expect("builtin"), + PackRef::Builtin(_) + )); + for input in [ + "file:rules/security.v1.json", + "file:_profiles/default.toml", + "file:rules/0-security.toml", + ] { + assert!(matches!( + PackRef::parse(input).expect("file"), + PackRef::File(_) + )); + } + for input in [ + "profiles/default.toml", + "_profiles/default.toml", + "rules/0-security.toml", + ] { + assert_eq!(PackFileRef::parse(input).expect("path").as_str(), input); + } + } + + #[test] + fn rejects_absolute_or_traversal_refs() { + for input in [ + "file:/rules/security.v1.json", + "file:./rules/security.v1.json", + "file:../rules/security.v1.json", + "file:rules//security.v1.json", + "file:C:/rules/security.v1.json", + ] { + assert!(PackRef::parse(input).is_err(), "{input} should fail"); + } + } + + #[test] + fn rejects_invalid_segment_shapes() { + for input in [ + "file:-bad.toml", + "file:.bad.toml", + "file:rules/-bad.toml", + "file:rules/.bad.toml", + ] { + assert!(PackRef::parse(input).is_err(), "{input} should fail"); + } + } +} diff --git a/crates/lift/src/pack/schema.rs b/crates/lift/src/pack/schema.rs new file mode 100644 index 000000000..39f94c0c8 --- /dev/null +++ b/crates/lift/src/pack/schema.rs @@ -0,0 +1,115 @@ +//! Embedded pack schema metadata. + +#![allow(dead_code)] + +/// The schema identifier for seam-1 pack common definitions. +pub(crate) const PACK_COMMON_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/common.v1.json"; + +/// The current seam-1 pack common schema version. +pub(crate) const PACK_COMMON_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 pack common schema. +pub(crate) const PACK_COMMON_V1_SCHEMA_FILE: &str = "common.v1.json"; + +/// The embedded seam-1 pack common schema source. +pub(crate) const PACK_COMMON_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/common.v1.json"); + +/// The schema identifier for seam-1 profile packs. +pub(crate) const PACK_PROFILE_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/profile.v1.json"; + +/// The current seam-1 profile schema version. +pub(crate) const PACK_PROFILE_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 profile schema. +pub(crate) const PACK_PROFILE_V1_SCHEMA_FILE: &str = "profile.v1.json"; + +/// The embedded seam-1 profile schema source. +pub(crate) const PACK_PROFILE_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/profile.v1.json"); + +/// The schema identifier for seam-1 boundary taxonomy packs. +pub(crate) const PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/boundary_taxonomy.v1.json"; + +/// The current seam-1 boundary taxonomy schema version. +pub(crate) const PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 boundary taxonomy schema. +pub(crate) const PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_FILE: &str = "boundary_taxonomy.v1.json"; + +/// The embedded seam-1 boundary taxonomy schema source. +pub(crate) const PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/boundary_taxonomy.v1.json"); + +/// The schema identifier for seam-1 component map packs. +pub(crate) const PACK_COMPONENT_MAP_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/component_map.v1.json"; + +/// The current seam-1 component map schema version. +pub(crate) const PACK_COMPONENT_MAP_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 component map schema. +pub(crate) const PACK_COMPONENT_MAP_V1_SCHEMA_FILE: &str = "component_map.v1.json"; + +/// The embedded seam-1 component map schema source. +pub(crate) const PACK_COMPONENT_MAP_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/component_map.v1.json"); + +/// The schema identifier for seam-1 score-model packs. +pub(crate) const PACK_SCORE_MODEL_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/score_model.v1.json"; + +/// The current seam-1 score-model schema version. +pub(crate) const PACK_SCORE_MODEL_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 score-model schema. +pub(crate) const PACK_SCORE_MODEL_V1_SCHEMA_FILE: &str = "score_model.v1.json"; + +/// The embedded seam-1 score-model schema source. +pub(crate) const PACK_SCORE_MODEL_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/score_model.v1.json"); + +/// The schema identifier for seam-1 query-pack packs. +pub(crate) const PACK_QUERY_PACK_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/query_pack.v1.json"; + +/// The current seam-1 query-pack schema version. +pub(crate) const PACK_QUERY_PACK_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 query-pack schema. +pub(crate) const PACK_QUERY_PACK_V1_SCHEMA_FILE: &str = "query_pack.v1.json"; + +/// The embedded seam-1 query-pack schema source. +pub(crate) const PACK_QUERY_PACK_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/query_pack.v1.json"); + +/// The schema identifier for seam-1 rule-pack packs. +pub(crate) const PACK_RULE_PACK_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json"; + +/// The current seam-1 rule-pack schema version. +pub(crate) const PACK_RULE_PACK_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 rule-pack schema. +pub(crate) const PACK_RULE_PACK_V1_SCHEMA_FILE: &str = "rule_pack.v1.json"; + +/// The embedded seam-1 rule-pack schema source. +pub(crate) const PACK_RULE_PACK_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/rule_pack.v1.json"); + +/// The schema identifier for seam-1 recipe-pack packs. +pub(crate) const PACK_RECIPE_PACK_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json"; + +/// The current seam-1 recipe-pack schema version. +pub(crate) const PACK_RECIPE_PACK_V1_SCHEMA_VERSION: u32 = 1; + +/// The filename for the seam-1 recipe-pack schema. +pub(crate) const PACK_RECIPE_PACK_V1_SCHEMA_FILE: &str = "recipe_pack.v1.json"; + +/// The embedded seam-1 recipe-pack schema source. +pub(crate) const PACK_RECIPE_PACK_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/pack/recipe_pack.v1.json"); diff --git a/crates/lift/src/pack/source.rs b/crates/lift/src/pack/source.rs new file mode 100644 index 000000000..8aadb72c7 --- /dev/null +++ b/crates/lift/src/pack/source.rs @@ -0,0 +1,57 @@ +//! Source descriptors for pack compilation. + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; + +/// Wire format for a pack source. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PackFormat { + Json, + Toml, +} + +/// Diagnostic/reporting identity for a pack source. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum PackOrigin { + Builtin { logical_name: String }, + File { display_path: String }, + Inline { logical_name: String }, +} + +impl PackOrigin { + /// Returns a stable display string for errors and diagnostics. + pub(crate) fn display(&self) -> String { + match self { + Self::Builtin { logical_name } => format!("builtin:{logical_name}"), + Self::File { display_path } => display_path.clone(), + Self::Inline { logical_name } => format!("inline:{logical_name}"), + } + } + + /// Returns true when the source is file-backed. + pub(crate) fn is_file_backed(&self) -> bool { + matches!(self, Self::File { .. }) + } +} + +/// Selects bytes to feed through the pack compiler. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum PackSource { + Builtin { + logical_name: &'static str, + format: PackFormat, + bytes: &'static [u8], + }, + File { + path: PathBuf, + format_hint: Option, + }, + Inline { + logical_name: String, + format: PackFormat, + bytes: Vec, + }, +} diff --git a/crates/lift/src/patch/mod.rs b/crates/lift/src/patch/mod.rs new file mode 100644 index 000000000..8b0e8b9bc --- /dev/null +++ b/crates/lift/src/patch/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future patch planning seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/query/mod.rs b/crates/lift/src/query/mod.rs new file mode 100644 index 000000000..c6b2da121 --- /dev/null +++ b/crates/lift/src/query/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future query seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/src/repo/blob.rs b/crates/lift/src/repo/blob.rs new file mode 100644 index 000000000..3e0941109 --- /dev/null +++ b/crates/lift/src/repo/blob.rs @@ -0,0 +1,70 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use crate::kernel::{FileId, Fingerprint, RepoPath}; +use crate::repo::{RepoError, RepoResult}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BlobRecord { + pub file_id: FileId, + pub path: RepoPath, + pub blob_fingerprint: Fingerprint, + pub size_bytes: u64, + bytes: Arc<[u8]>, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct BlobStore { + by_path: BTreeMap, +} + +impl BlobRecord { + pub(crate) fn from_bytes( + file_id: FileId, + path: RepoPath, + blob_fingerprint: Fingerprint, + size_bytes: u64, + bytes: Vec, + ) -> Self { + Self { + file_id, + path, + blob_fingerprint, + size_bytes, + bytes: Arc::from(bytes), + } + } + + pub(crate) fn bytes(&self) -> &[u8] { + self.bytes.as_ref() + } +} + +impl BlobStore { + pub(crate) fn from_records(records: Vec) -> Self { + let by_path = records + .into_iter() + .map(|record| (record.path.clone(), record)) + .collect(); + Self { by_path } + } + + pub(crate) fn contains(&self, path: &RepoPath) -> bool { + self.by_path.contains_key(path) + } + + pub(crate) fn get(&self, path: &RepoPath) -> Option<&BlobRecord> { + self.by_path.get(path) + } + + pub(crate) fn read_bytes(&self, path: &RepoPath) -> RepoResult<&[u8]> { + self.by_path + .get(path) + .map(BlobRecord::bytes) + .ok_or_else(|| RepoError::MissingBlob { path: path.clone() }) + } + + pub(crate) fn len(&self) -> usize { + self.by_path.len() + } +} diff --git a/crates/lift/src/repo/diagnostics.rs b/crates/lift/src/repo/diagnostics.rs new file mode 100644 index 000000000..46a5d6cfd --- /dev/null +++ b/crates/lift/src/repo/diagnostics.rs @@ -0,0 +1,73 @@ +use std::cmp::Ordering; + +use serde::{Deserialize, Serialize}; + +use crate::kernel::{DiagnosticCode, RepoPath, Severity}; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct RepoLocation { + pub display_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_path: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub(crate) struct RepoRelatedLocation { + pub location: RepoLocation, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub(crate) struct RepoDiagnostic { + pub code: DiagnosticCode, + pub severity: Severity, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub related: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} + +impl Ord for RepoDiagnostic { + fn cmp(&self, other: &Self) -> Ordering { + diagnostic_sort_key(self) + .cmp(&diagnostic_sort_key(other)) + .then_with(|| self.subject.cmp(&other.subject)) + .then_with(|| self.related.cmp(&other.related)) + .then_with(|| self.help.cmp(&other.help)) + } +} + +impl PartialOrd for RepoDiagnostic { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +type RepoDiagnosticSortKey<'a> = (u8, Option<&'a str>, Option<&'a str>, &'a str, &'a str); + +fn diagnostic_sort_key(diagnostic: &RepoDiagnostic) -> RepoDiagnosticSortKey<'_> { + ( + severity_rank(diagnostic.severity), + diagnostic + .subject + .as_ref() + .map(|subject| subject.display_path.as_str()), + diagnostic + .subject + .as_ref() + .and_then(|subject| subject.repo_path.as_ref().map(RepoPath::as_str)), + diagnostic.code.as_str(), + diagnostic.message.as_str(), + ) +} + +fn severity_rank(severity: Severity) -> u8 { + match severity { + Severity::Error => 0, + Severity::Warning => 1, + Severity::Info => 2, + } +} diff --git a/crates/lift/src/repo/diff.rs b/crates/lift/src/repo/diff.rs new file mode 100644 index 000000000..6f2d1c661 --- /dev/null +++ b/crates/lift/src/repo/diff.rs @@ -0,0 +1,156 @@ +use std::cmp::Ordering; + +use serde::Serialize; + +use crate::kernel::{sha256_canonical_json, Fingerprint, RepoPath}; +use crate::repo::{InventoryEntry, RepoSnapshot}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DiffKind { + Added, + Modified, + Removed, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DiffEntry { + pub path: RepoPath, + pub kind: DiffKind, + pub before: Option, + pub after: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoDiff { + pub base_fingerprint: Fingerprint, + pub head_fingerprint: Fingerprint, + pub entries: Vec, + pub fingerprint: Fingerprint, +} + +pub(crate) fn build_diff(base: &RepoSnapshot, head: &RepoSnapshot) -> RepoDiff { + let mut base_entries = base.inventory.iter().peekable(); + let mut head_entries = head.inventory.iter().peekable(); + let mut entries = Vec::new(); + + loop { + match (base_entries.peek(), head_entries.peek()) { + (Some(base_entry), Some(head_entry)) => match base_entry.path.cmp(&head_entry.path) { + Ordering::Less => { + let removed = base_entries + .next() + .expect("peeked base inventory entry should exist"); + entries.push(DiffEntry { + path: removed.path.clone(), + kind: DiffKind::Removed, + before: Some(removed.clone()), + after: None, + }); + } + Ordering::Equal => { + let before = base_entries + .next() + .expect("peeked base inventory entry should exist"); + let after = head_entries + .next() + .expect("peeked head inventory entry should exist"); + + if before.blob_fingerprint != after.blob_fingerprint { + entries.push(DiffEntry { + path: before.path.clone(), + kind: DiffKind::Modified, + before: Some(before.clone()), + after: Some(after.clone()), + }); + } + } + Ordering::Greater => { + let added = head_entries + .next() + .expect("peeked head inventory entry should exist"); + entries.push(DiffEntry { + path: added.path.clone(), + kind: DiffKind::Added, + before: None, + after: Some(added.clone()), + }); + } + }, + (Some(_), None) => { + let removed = base_entries + .next() + .expect("peeked base inventory entry should exist"); + entries.push(DiffEntry { + path: removed.path.clone(), + kind: DiffKind::Removed, + before: Some(removed.clone()), + after: None, + }); + } + (None, Some(_)) => { + let added = head_entries + .next() + .expect("peeked head inventory entry should exist"); + entries.push(DiffEntry { + path: added.path.clone(), + kind: DiffKind::Added, + before: None, + after: Some(added.clone()), + }); + } + (None, None) => break, + } + } + + let base_fingerprint = base.fingerprint.clone(); + let head_fingerprint = head.fingerprint.clone(); + let fingerprint = fingerprint_diff(&base_fingerprint, &head_fingerprint, &entries); + + RepoDiff { + base_fingerprint, + head_fingerprint, + entries, + fingerprint, + } +} + +fn fingerprint_diff( + base_fingerprint: &Fingerprint, + head_fingerprint: &Fingerprint, + entries: &[DiffEntry], +) -> Fingerprint { + #[derive(Serialize)] + struct DiffEntryRecord<'a> { + path: &'a RepoPath, + kind: DiffKind, + before_blob_fingerprint: Option<&'a Fingerprint>, + after_blob_fingerprint: Option<&'a Fingerprint>, + } + + #[derive(Serialize)] + struct DiffFingerprintDocument<'a> { + version: u32, + base_fingerprint: &'a Fingerprint, + head_fingerprint: &'a Fingerprint, + entries: Vec>, + } + + let entries = entries + .iter() + .map(|entry| DiffEntryRecord { + path: &entry.path, + kind: entry.kind, + before_blob_fingerprint: entry.before.as_ref().map(|before| &before.blob_fingerprint), + after_blob_fingerprint: entry.after.as_ref().map(|after| &after.blob_fingerprint), + }) + .collect::>(); + + sha256_canonical_json(&DiffFingerprintDocument { + version: 1, + base_fingerprint, + head_fingerprint, + entries, + }) + .expect("repo diff fingerprint serialization should succeed") +} diff --git a/crates/lift/src/repo/error.rs b/crates/lift/src/repo/error.rs new file mode 100644 index 000000000..7e9bc8477 --- /dev/null +++ b/crates/lift/src/repo/error.rs @@ -0,0 +1,85 @@ +use crate::kernel::RepoPath; + +pub(crate) type RepoResult = Result; + +#[derive(Debug, thiserror::Error, Clone, Eq, PartialEq)] +pub(crate) enum RepoError { + #[error("repo start path does not exist")] + StartPathNotFound { path: String }, + + #[error("repo root could not be detected")] + RootNotFound { + start_path: String, + markers: Vec, + }, + + #[error("repo root is not a directory")] + RootNotDirectory { path: String }, + + #[error("repo I/O failure")] + Io { + op: &'static str, + path: String, + reason: String, + }, + + #[error("invalid root marker")] + InvalidRootMarker { input: String }, + + #[error("invalid repo-relative path derived from filesystem entry")] + InvalidRepoPath { + display_path: String, + reason: String, + }, + + #[error("encountered non-utf8 filesystem path")] + NonUtf8Path { display_path: String }, + + #[error("ignore glob compile failure")] + IgnoreGlobCompile { pattern: String, reason: String }, + + #[error("git repository open failure")] + GitOpen { path: String, reason: String }, + + #[error("git revision could not be resolved")] + GitRevisionResolve { + rev: String, + repo_root: String, + reason: String, + }, + + #[error("git revision did not resolve to a tree-compatible object")] + GitRevisionObjectKind { + rev: String, + actual_kind: String, + expected_kind: &'static str, + }, + + #[error("git object lookup failure")] + GitObjectLookup { object_id: String, reason: String }, + + #[error("git blob did not decode to utf-8 symlink target")] + GitSymlinkTargetInvalidUtf8 { rev: String, path: RepoPath }, + + #[error("symlink follow escaped the repository root")] + SymlinkTargetEscape { path: RepoPath, target: String }, + + #[error("symlink follow target does not exist")] + SymlinkTargetDangling { path: RepoPath, target: String }, + + #[error("symlink follow detected a loop")] + SymlinkTargetLoop { path: RepoPath, target: String }, + + #[error("symlink follow target is a directory")] + SymlinkTargetDirectory { path: RepoPath, target: String }, + + #[error("file exceeds configured max_file_bytes")] + FileTooLarge { + display_path: String, + size_bytes: u64, + max_file_bytes: u64, + }, + + #[error("blob not present in snapshot")] + MissingBlob { path: RepoPath }, +} diff --git a/crates/lift/src/repo/ignore.rs b/crates/lift/src/repo/ignore.rs new file mode 100644 index 000000000..d811a854d --- /dev/null +++ b/crates/lift/src/repo/ignore.rs @@ -0,0 +1,152 @@ +use std::path::Path; + +use globset::{Glob, GlobSet, GlobSetBuilder}; + +use crate::kernel::RepoPath; +use crate::repo::{RepoError, RepoResult}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum SymlinkPolicy { + Skip, + Follow, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum NonUtf8PathPolicy { + Error, + Skip, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum LargeFilePolicy { + Error, + Skip, +} + +#[derive( + Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, serde::Serialize, serde::Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub(crate) enum WellKnownExclude { + RustTarget, + NodeModules, + PythonHiddenVenv, + PythonVenv, + PythonPycache, + WebDist, + WebBuild, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SnapshotOptions { + pub symlink_policy: SymlinkPolicy, + pub well_known_excludes: Vec, + pub exclude_globs: Vec, + pub non_utf8_path_policy: NonUtf8PathPolicy, + pub max_file_bytes: Option, + pub large_file_policy: LargeFilePolicy, +} + +impl Default for SnapshotOptions { + fn default() -> Self { + Self { + symlink_policy: SymlinkPolicy::Skip, + well_known_excludes: Vec::new(), + exclude_globs: Vec::new(), + non_utf8_path_policy: NonUtf8PathPolicy::Error, + max_file_bytes: None, + large_file_policy: LargeFilePolicy::Error, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct CompiledIgnoreSet { + patterns: Vec, + matcher: GlobSet, + well_known_excludes: Vec, +} + +impl PartialEq for CompiledIgnoreSet { + fn eq(&self, other: &Self) -> bool { + self.patterns == other.patterns && self.well_known_excludes == other.well_known_excludes + } +} + +impl Eq for CompiledIgnoreSet {} + +impl CompiledIgnoreSet { + pub(crate) fn compile(options: &SnapshotOptions) -> RepoResult { + let mut builder = GlobSetBuilder::new(); + for pattern in &options.exclude_globs { + let glob = Glob::new(pattern).map_err(|error| RepoError::IgnoreGlobCompile { + pattern: pattern.clone(), + reason: error.to_string(), + })?; + builder.add(glob); + } + + let matcher = builder + .build() + .map_err(|error| RepoError::IgnoreGlobCompile { + pattern: "".to_owned(), + reason: error.to_string(), + })?; + Ok(Self { + patterns: options.exclude_globs.clone(), + matcher, + well_known_excludes: options.well_known_excludes.clone(), + }) + } + + pub(crate) fn is_ignored(&self, repo_path: &RepoPath, is_dir: bool) -> bool { + if is_intrinsic_git_path(repo_path) { + return true; + } + if self + .well_known_excludes + .iter() + .any(|exclude| exclude.matches(repo_path, is_dir)) + { + return true; + } + + let path = Path::new(repo_path.as_str()); + self.matcher.is_match(path) + || (is_dir + && self + .matcher + .is_match(Path::new(&format!("{}/", repo_path.as_str())))) + } +} + +fn is_intrinsic_git_path(repo_path: &RepoPath) -> bool { + repo_path.as_str() == ".git" || repo_path.as_str().starts_with(".git/") +} + +impl WellKnownExclude { + pub(crate) fn canonical_dir(self) -> &'static str { + match self { + WellKnownExclude::RustTarget => "target", + WellKnownExclude::NodeModules => "node_modules", + WellKnownExclude::PythonHiddenVenv => ".venv", + WellKnownExclude::PythonVenv => "venv", + WellKnownExclude::PythonPycache => "__pycache__", + WellKnownExclude::WebDist => "dist", + WellKnownExclude::WebBuild => "build", + } + } + + fn matches(self, repo_path: &RepoPath, is_dir: bool) -> bool { + let segments = repo_path.as_str().split('/').collect::>(); + let directory_segments = if is_dir { + &segments[..] + } else { + &segments[..segments.len().saturating_sub(1)] + }; + + directory_segments + .iter() + .any(|segment| *segment == self.canonical_dir()) + } +} diff --git a/crates/lift/src/repo/inventory.rs b/crates/lift/src/repo/inventory.rs new file mode 100644 index 000000000..fe43a2c62 --- /dev/null +++ b/crates/lift/src/repo/inventory.rs @@ -0,0 +1,84 @@ +use std::collections::BTreeMap; + +use serde::Serialize; + +use crate::kernel::{sha256_canonical_json, FileId, Fingerprint, RepoPath}; +use crate::repo::{RepoError, RepoResult}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, serde::Deserialize)] +pub(crate) struct InventoryEntry { + pub file_id: FileId, + pub path: RepoPath, + pub blob_fingerprint: Fingerprint, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Inventory { + pub entries: BTreeMap, + pub fingerprint: Fingerprint, +} + +impl Inventory { + pub(crate) fn from_entries(entries: Vec) -> RepoResult { + let entries = entries + .into_iter() + .map(|entry| (entry.path.clone(), entry)) + .collect::>(); + let fingerprint = fingerprint_entries(entries.values())?; + Ok(Self { + entries, + fingerprint, + }) + } + + pub(crate) fn get(&self, path: &RepoPath) -> Option<&InventoryEntry> { + self.entries.get(path) + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn iter(&self) -> impl Iterator { + self.entries.values() + } +} + +pub(crate) fn fingerprint_entries<'a>( + entries: impl IntoIterator, +) -> RepoResult { + #[derive(Serialize)] + struct FileRecord<'a> { + path: &'a RepoPath, + blob_fingerprint: &'a Fingerprint, + size_bytes: u64, + } + + #[derive(Serialize)] + struct FingerprintDocument<'a> { + version: u32, + files: Vec>, + } + + let files = entries + .into_iter() + .map(|entry| FileRecord { + path: &entry.path, + blob_fingerprint: &entry.blob_fingerprint, + size_bytes: entry.size_bytes, + }) + .collect::>(); + + sha256_canonical_json(&FingerprintDocument { version: 1, files }).map_err(|error| { + RepoError::Io { + op: "fingerprint_inventory", + path: "".to_owned(), + reason: error.to_string(), + } + }) +} diff --git a/crates/lift/src/repo/mod.rs b/crates/lift/src/repo/mod.rs new file mode 100644 index 000000000..1904d72b8 --- /dev/null +++ b/crates/lift/src/repo/mod.rs @@ -0,0 +1,35 @@ +//! Internal repo substrate seam. + +#![allow(dead_code)] + +pub(crate) mod blob; +pub(crate) mod diagnostics; +pub(crate) mod diff; +pub(crate) mod error; +pub(crate) mod ignore; +pub(crate) mod inventory; +pub(crate) mod root; +pub(crate) mod schema; +pub(crate) mod snapshot; + +#[allow(unused_imports)] +pub(crate) use blob::{BlobRecord, BlobStore}; +#[allow(unused_imports)] +pub(crate) use diagnostics::{RepoDiagnostic, RepoLocation, RepoRelatedLocation}; +#[allow(unused_imports)] +pub(crate) use diff::{build_diff, DiffEntry, DiffKind, RepoDiff}; +#[allow(unused_imports)] +pub(crate) use error::{RepoError, RepoResult}; +#[allow(unused_imports)] +pub(crate) use ignore::{ + CompiledIgnoreSet, LargeFilePolicy, NonUtf8PathPolicy, SnapshotOptions, SymlinkPolicy, + WellKnownExclude, +}; +#[allow(unused_imports)] +pub(crate) use inventory::{Inventory, InventoryEntry}; +#[allow(unused_imports)] +pub(crate) use root::{RepoRoot, RepoRootDetectionOptions, RootMarker}; +#[allow(unused_imports)] +pub(crate) use snapshot::{ + materialize_snapshot, RepoSnapshot, SnapshotRequest, SnapshotSource, SnapshotStats, +}; diff --git a/crates/lift/src/repo/root.rs b/crates/lift/src/repo/root.rs new file mode 100644 index 000000000..f2a14e04c --- /dev/null +++ b/crates/lift/src/repo/root.rs @@ -0,0 +1,151 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::repo::{RepoError, RepoResult}; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct RootMarker(String); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoRootDetectionOptions { + pub markers: BTreeSet, + pub ceiling_dir: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoRoot { + absolute_path: PathBuf, +} + +impl RootMarker { + pub(crate) fn parse(input: &str) -> RepoResult { + if input.is_empty() + || input.contains('/') + || input.contains('\\') + || input == "." + || input == ".." + { + return Err(RepoError::InvalidRootMarker { + input: input.to_owned(), + }); + } + + Ok(Self(input.to_owned())) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl RepoRootDetectionOptions { + pub(crate) fn git_default() -> Self { + let mut markers = BTreeSet::new(); + markers.insert(RootMarker::parse(".git").expect("default root marker should parse")); + Self { + markers, + ceiling_dir: None, + } + } +} + +impl RepoRoot { + pub(crate) fn from_dir(path: &Path) -> RepoResult { + if !path.exists() { + return Err(RepoError::StartPathNotFound { + path: path.display().to_string(), + }); + } + + let absolute_path = canonicalize_path(path, "canonicalize_repo_root")?; + if !absolute_path.is_dir() { + return Err(RepoError::RootNotDirectory { + path: absolute_path.display().to_string(), + }); + } + + Ok(Self { absolute_path }) + } + + pub(crate) fn as_path(&self) -> &Path { + &self.absolute_path + } + + pub(crate) fn display(&self) -> String { + self.absolute_path.display().to_string() + } +} + +pub(crate) fn detect_repo_root( + start_path: &Path, + options: &RepoRootDetectionOptions, +) -> RepoResult { + if !start_path.exists() { + return Err(RepoError::StartPathNotFound { + path: start_path.display().to_string(), + }); + } + + let start_absolute = canonicalize_path(start_path, "canonicalize_start_path")?; + let start_dir = if start_absolute.is_dir() { + start_absolute + } else { + start_absolute + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| RepoError::RootNotFound { + start_path: start_path.display().to_string(), + markers: marker_names(options), + })? + }; + + let ceiling_dir = options + .ceiling_dir + .as_deref() + .map(|path| canonicalize_path(path, "canonicalize_ceiling_dir")) + .transpose()?; + + let mut current = start_dir.as_path(); + loop { + if directory_matches_any_marker(current, &options.markers) { + return RepoRoot::from_dir(current); + } + + if Some(current) == ceiling_dir.as_deref() { + break; + } + + let Some(parent) = current.parent() else { + break; + }; + current = parent; + } + + Err(RepoError::RootNotFound { + start_path: start_path.display().to_string(), + markers: marker_names(options), + }) +} + +fn canonicalize_path(path: &Path, op: &'static str) -> RepoResult { + fs::canonicalize(path).map_err(|error| RepoError::Io { + op, + path: path.display().to_string(), + reason: error.to_string(), + }) +} + +fn directory_matches_any_marker(directory: &Path, markers: &BTreeSet) -> bool { + markers + .iter() + .any(|marker| directory.join(marker.as_str()).exists()) +} + +fn marker_names(options: &RepoRootDetectionOptions) -> Vec { + options + .markers + .iter() + .map(|marker| marker.as_str().to_owned()) + .collect() +} diff --git a/crates/lift/src/repo/schema.rs b/crates/lift/src/repo/schema.rs new file mode 100644 index 000000000..356bbc925 --- /dev/null +++ b/crates/lift/src/repo/schema.rs @@ -0,0 +1,21 @@ +#![allow(dead_code)] + +pub(crate) const SNAPSHOT_MANIFEST_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/repo/snapshot_manifest.v1.json"; + +pub(crate) const SNAPSHOT_MANIFEST_V1_SCHEMA_VERSION: u32 = 1; + +pub(crate) const SNAPSHOT_MANIFEST_V1_SCHEMA_FILE: &str = "snapshot_manifest.v1.json"; + +pub(crate) const SNAPSHOT_MANIFEST_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/repo/snapshot_manifest.v1.json"); + +pub(crate) const DIFF_MANIFEST_V1_SCHEMA_ID: &str = + "https://schemas.substrate.dev/lift/repo/diff_manifest.v1.json"; + +pub(crate) const DIFF_MANIFEST_V1_SCHEMA_VERSION: u32 = 1; + +pub(crate) const DIFF_MANIFEST_V1_SCHEMA_FILE: &str = "diff_manifest.v1.json"; + +pub(crate) const DIFF_MANIFEST_V1_SCHEMA_JSON: &str = + include_str!("../../schemas/repo/diff_manifest.v1.json"); diff --git a/crates/lift/src/repo/snapshot.rs b/crates/lift/src/repo/snapshot.rs new file mode 100644 index 000000000..0d613ce7f --- /dev/null +++ b/crates/lift/src/repo/snapshot.rs @@ -0,0 +1,869 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +use gix::bstr::ByteSlice; +use gix::object::tree::EntryKind; +use serde::Serialize; +use walkdir::{DirEntry, WalkDir}; + +use crate::kernel::{sha256_bytes, DiagnosticCode, FileId, KernelError, RepoPath, Severity}; +use crate::repo::blob::{BlobRecord, BlobStore}; +use crate::repo::diagnostics::{RepoDiagnostic, RepoLocation}; +use crate::repo::ignore::{ + CompiledIgnoreSet, LargeFilePolicy, NonUtf8PathPolicy, SnapshotOptions, SymlinkPolicy, +}; +use crate::repo::inventory::{Inventory, InventoryEntry}; +use crate::repo::root::{detect_repo_root, RepoRoot}; +use crate::repo::{RepoError, RepoResult}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum SnapshotSource { + Worktree, + GitRev { rev: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SnapshotRequest { + pub root: RepoRoot, + pub source: SnapshotSource, + pub options: SnapshotOptions, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, serde::Deserialize)] +pub(crate) struct SnapshotStats { + pub file_count: u64, + pub total_bytes: u64, + pub skipped_by_ignore: u64, + pub skipped_symlinks: u64, + pub skipped_non_utf8_paths: u64, + pub skipped_large_files: u64, + pub skipped_unsupported_file_kinds: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RepoSnapshot { + pub source: SnapshotSource, + pub inventory: Inventory, + pub blob_store: BlobStore, + pub fingerprint: crate::kernel::Fingerprint, + pub diagnostics: Vec, + pub stats: SnapshotStats, + + root: RepoRoot, +} + +impl RepoSnapshot { + pub(crate) fn root(&self) -> &RepoRoot { + &self.root + } + + pub(crate) fn entry(&self, path: &RepoPath) -> Option<&InventoryEntry> { + self.inventory.get(path) + } + + pub(crate) fn read_bytes(&self, path: &RepoPath) -> RepoResult<&[u8]> { + self.blob_store.read_bytes(path) + } +} + +pub(crate) fn materialize_snapshot(request: &SnapshotRequest) -> RepoResult { + let compiled_ignores = CompiledIgnoreSet::compile(&request.options)?; + let mut assembly = SnapshotAssembly::new(request); + match &request.source { + SnapshotSource::Worktree => { + materialize_worktree_snapshot(request, &compiled_ignores, &mut assembly)? + } + SnapshotSource::GitRev { rev } => { + materialize_gitrev_snapshot(request, rev, &compiled_ignores, &mut assembly)? + } + } + assembly.finalize() +} + +pub(crate) fn detect_root_and_materialize( + start_path: &Path, + options: &crate::repo::RepoRootDetectionOptions, + snapshot_options: SnapshotOptions, +) -> RepoResult { + let root = detect_repo_root(start_path, options)?; + materialize_snapshot(&SnapshotRequest { + root, + source: SnapshotSource::Worktree, + options: snapshot_options, + }) +} + +fn materialize_worktree_snapshot( + request: &SnapshotRequest, + compiled_ignores: &CompiledIgnoreSet, + assembly: &mut SnapshotAssembly, +) -> RepoResult<()> { + let mut walker = WalkDir::new(request.root.as_path()) + .follow_links(false) + .sort_by_file_name() + .into_iter(); + + while let Some(entry) = walker.next() { + let entry = entry.map_err(|error| RepoError::Io { + op: "walk_repo", + path: error + .path() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| request.root.display()), + reason: error.to_string(), + })?; + + if entry.path() == request.root.as_path() { + continue; + } + + let path_kind = EntryPathKind::from_entry(&entry); + let display_path = entry.path().display().to_string(); + let repo_path = match repo_path_from_entry(request.root.as_path(), entry.path()) { + Ok(repo_path) => repo_path, + Err(error) => match request.options.non_utf8_path_policy { + NonUtf8PathPolicy::Error => return Err(error), + NonUtf8PathPolicy::Skip => { + assembly.record_non_utf8_skip(display_path); + if path_kind.is_dir() { + walker.skip_current_dir(); + } + continue; + } + }, + }; + + if compiled_ignores.is_ignored(&repo_path, path_kind.is_dir()) { + assembly.record_ignore_skip(); + if path_kind.is_dir() { + walker.skip_current_dir(); + } + continue; + } + + if path_kind.is_dir() { + continue; + } + + if path_kind.is_symlink() { + match request.options.symlink_policy { + SymlinkPolicy::Skip => { + assembly.record_symlink_skip(display_path, repo_path); + continue; + } + SymlinkPolicy::Follow => { + let resolved = + resolve_worktree_symlink(request.root.as_path(), entry.path(), &repo_path)?; + if followed_target_is_ignored(compiled_ignores, &resolved.repo_path) { + assembly.record_ignore_skip(); + continue; + } + if assembly.enforce_file_size_policy( + request, + &resolved.display_path, + &repo_path, + resolved.size_bytes, + )? { + continue; + } + let bytes = fs::read(&resolved.fs_path).map_err(|error| RepoError::Io { + op: "read_file", + path: resolved.fs_path.display().to_string(), + reason: error.to_string(), + })?; + assembly.push_file(repo_path, bytes); + continue; + } + } + } + + if !path_kind.is_file() { + assembly.record_unsupported_kind(display_path, repo_path); + continue; + } + + let metadata = entry.metadata().map_err(|error| RepoError::Io { + op: "read_metadata", + path: entry.path().display().to_string(), + reason: error.to_string(), + })?; + let size_bytes = metadata.len(); + if assembly.enforce_file_size_policy(request, &display_path, &repo_path, size_bytes)? { + continue; + } + + let bytes = fs::read(entry.path()).map_err(|error| RepoError::Io { + op: "read_file", + path: entry.path().display().to_string(), + reason: error.to_string(), + })?; + assembly.push_file(repo_path, bytes); + } + + Ok(()) +} + +fn materialize_gitrev_snapshot( + request: &SnapshotRequest, + rev: &str, + compiled_ignores: &CompiledIgnoreSet, + assembly: &mut SnapshotAssembly, +) -> RepoResult<()> { + let repo = gix::open(request.root.as_path()).map_err(|error| RepoError::GitOpen { + path: request.root.display(), + reason: error.to_string(), + })?; + let object = repo + .rev_parse_single(rev.as_bytes().as_bstr()) + .map_err(|error| RepoError::GitRevisionResolve { + rev: rev.to_owned(), + repo_root: request.root.display(), + reason: error.to_string(), + })? + .object() + .map_err(|error| RepoError::GitRevisionResolve { + rev: rev.to_owned(), + repo_root: request.root.display(), + reason: error.to_string(), + })?; + let actual_kind = object.kind.to_string(); + let root_tree = object + .peel_to_tree() + .map_err(|_| RepoError::GitRevisionObjectKind { + rev: rev.to_owned(), + actual_kind, + expected_kind: "tree", + })?; + + walk_git_tree( + request, + &root_tree, + &root_tree, + None, + rev, + compiled_ignores, + assembly, + ) +} + +fn walk_git_tree( + request: &SnapshotRequest, + root_tree: &gix::Tree<'_>, + tree: &gix::Tree<'_>, + parent_repo_path: Option<&RepoPath>, + rev: &str, + compiled_ignores: &CompiledIgnoreSet, + assembly: &mut SnapshotAssembly, +) -> RepoResult<()> { + for entry in tree.iter() { + let entry = entry.map_err(|error| RepoError::GitObjectLookup { + object_id: tree.id.to_string(), + reason: error.to_string(), + })?; + let repo_path = match repo_path_from_git_entry(parent_repo_path, entry.inner.filename) { + Ok(repo_path) => repo_path, + Err(error) => match request.options.non_utf8_path_policy { + NonUtf8PathPolicy::Error => return Err(error), + NonUtf8PathPolicy::Skip => { + assembly.record_non_utf8_skip(format!("{rev}:{}", entry.inner.filename)); + continue; + } + }, + }; + let entry_kind = entry.inner.mode.kind(); + + if compiled_ignores.is_ignored(&repo_path, entry_kind == EntryKind::Tree) { + assembly.record_ignore_skip(); + continue; + } + + match entry_kind { + EntryKind::Tree => { + let object = + tree.repo + .find_object(entry.inner.oid.to_owned()) + .map_err(|error| RepoError::GitObjectLookup { + object_id: entry.inner.oid.to_string(), + reason: error.to_string(), + })?; + let child_tree = object.into_tree(); + walk_git_tree( + request, + root_tree, + &child_tree, + Some(&repo_path), + rev, + compiled_ignores, + assembly, + )?; + } + EntryKind::Blob | EntryKind::BlobExecutable => { + let object = + tree.repo + .find_object(entry.inner.oid.to_owned()) + .map_err(|error| RepoError::GitObjectLookup { + object_id: entry.inner.oid.to_string(), + reason: error.to_string(), + })?; + let bytes = object.data.to_vec(); + let display_path = format!("{rev}:{}", repo_path.as_str()); + if assembly.enforce_file_size_policy( + request, + &display_path, + &repo_path, + bytes.len() as u64, + )? { + continue; + } + assembly.push_file(repo_path, bytes); + } + EntryKind::Link => match request.options.symlink_policy { + SymlinkPolicy::Skip => { + assembly + .record_symlink_skip(format!("{rev}:{}", repo_path.as_str()), repo_path); + } + SymlinkPolicy::Follow => { + let object = + tree.repo + .find_object(entry.inner.oid.to_owned()) + .map_err(|error| RepoError::GitObjectLookup { + object_id: entry.inner.oid.to_string(), + reason: error.to_string(), + })?; + let target = std::str::from_utf8(&object.data).map_err(|_| { + RepoError::GitSymlinkTargetInvalidUtf8 { + rev: rev.to_owned(), + path: repo_path.clone(), + } + })?; + let resolved = resolve_git_symlink( + root_tree, + rev, + &repo_path, + target.trim_end_matches('\0'), + )?; + if followed_target_is_ignored(compiled_ignores, &resolved.repo_path) { + assembly.record_ignore_skip(); + continue; + } + let object = tree.repo.find_object(resolved.object_id).map_err(|error| { + RepoError::GitObjectLookup { + object_id: resolved.object_id.to_string(), + reason: error.to_string(), + } + })?; + let bytes = object.data.to_vec(); + if assembly.enforce_file_size_policy( + request, + &resolved.display_path, + &repo_path, + bytes.len() as u64, + )? { + continue; + } + assembly.push_file(repo_path, bytes); + } + }, + _ => { + assembly + .record_unsupported_kind(format!("{rev}:{}", repo_path.as_str()), repo_path); + } + } + } + + Ok(()) +} + +struct SnapshotAssembly { + request: SnapshotRequest, + inventory_entries: Vec, + blob_records: Vec, + diagnostics: Vec, + stats: SnapshotStats, +} + +impl SnapshotAssembly { + fn new(request: &SnapshotRequest) -> Self { + Self { + request: request.clone(), + inventory_entries: Vec::new(), + blob_records: Vec::new(), + diagnostics: Vec::new(), + stats: SnapshotStats::default(), + } + } + + fn record_ignore_skip(&mut self) { + self.stats.skipped_by_ignore += 1; + } + + fn record_symlink_skip(&mut self, display_path: String, repo_path: RepoPath) { + self.stats.skipped_symlinks += 1; + self.diagnostics.push(repo_diagnostic( + "repo.snapshot.symlink_skipped", + Severity::Warning, + "skipped symlink during repo snapshot materialization".to_owned(), + Some(RepoLocation { + display_path, + repo_path: Some(repo_path), + }), + )); + } + + fn record_non_utf8_skip(&mut self, display_path: String) { + self.stats.skipped_non_utf8_paths += 1; + self.diagnostics.push(repo_diagnostic( + "repo.snapshot.non_utf8_path", + Severity::Warning, + "skipped non-utf8 filesystem path".to_owned(), + Some(RepoLocation { + display_path, + repo_path: None, + }), + )); + } + + fn record_unsupported_kind(&mut self, display_path: String, repo_path: RepoPath) { + self.stats.skipped_unsupported_file_kinds += 1; + self.diagnostics.push(repo_diagnostic( + "repo.snapshot.unsupported_file_kind", + Severity::Warning, + "skipped unsupported filesystem entry kind".to_owned(), + Some(RepoLocation { + display_path, + repo_path: Some(repo_path), + }), + )); + } + + fn enforce_file_size_policy( + &mut self, + request: &SnapshotRequest, + display_path: &str, + repo_path: &RepoPath, + size_bytes: u64, + ) -> RepoResult { + let Some(max_file_bytes) = request.options.max_file_bytes else { + return Ok(false); + }; + if size_bytes <= max_file_bytes { + return Ok(false); + } + + match request.options.large_file_policy { + LargeFilePolicy::Error => Err(RepoError::FileTooLarge { + display_path: display_path.to_owned(), + size_bytes, + max_file_bytes, + }), + LargeFilePolicy::Skip => { + self.stats.skipped_large_files += 1; + self.diagnostics.push(repo_diagnostic( + "repo.snapshot.file_too_large", + Severity::Warning, + format!( + "skipped file larger than configured max_file_bytes ({max_file_bytes})" + ), + Some(RepoLocation { + display_path: display_path.to_owned(), + repo_path: Some(repo_path.clone()), + }), + )); + Ok(true) + } + } + } + + fn push_file(&mut self, repo_path: RepoPath, bytes: Vec) { + let size_bytes = bytes.len() as u64; + let blob_fingerprint = sha256_bytes(&bytes); + let file_id = FileId::from_identity(&format!("repo\0file\0v1\0{}", repo_path.as_str())); + + self.inventory_entries.push(InventoryEntry { + file_id: file_id.clone(), + path: repo_path.clone(), + blob_fingerprint: blob_fingerprint.clone(), + size_bytes, + }); + self.blob_records.push(BlobRecord::from_bytes( + file_id, + repo_path, + blob_fingerprint, + size_bytes, + bytes, + )); + self.stats.file_count += 1; + self.stats.total_bytes += size_bytes; + } + + fn finalize(mut self) -> RepoResult { + self.diagnostics.sort(); + let inventory = Inventory::from_entries(self.inventory_entries)?; + let blob_store = BlobStore::from_records(self.blob_records); + if blob_store.len() != inventory.len() { + return Err(RepoError::Io { + op: "assemble_snapshot", + path: self.request.root.display(), + reason: "inventory/blob store length mismatch".to_owned(), + }); + } + + Ok(RepoSnapshot { + source: self.request.source.clone(), + fingerprint: inventory.fingerprint.clone(), + inventory, + blob_store, + diagnostics: self.diagnostics, + stats: self.stats, + root: self.request.root, + }) + } +} + +struct ResolvedWorktreeTarget { + fs_path: PathBuf, + repo_path: RepoPath, + display_path: String, + size_bytes: u64, +} + +struct ResolvedGitTarget { + object_id: gix::ObjectId, + repo_path: RepoPath, + display_path: String, +} + +fn resolve_worktree_symlink( + root: &Path, + observed_path: &Path, + observed_repo_path: &RepoPath, +) -> RepoResult { + let mut current = observed_path.to_path_buf(); + let mut visited = BTreeSet::from([observed_repo_path.as_str().to_owned()]); + + loop { + let metadata = fs::symlink_metadata(¤t).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + RepoError::SymlinkTargetDangling { + path: observed_repo_path.clone(), + target: current.display().to_string(), + } + } else { + RepoError::Io { + op: "symlink_metadata", + path: current.display().to_string(), + reason: error.to_string(), + } + } + })?; + + if metadata.file_type().is_symlink() { + let raw_target = fs::read_link(¤t).map_err(|error| RepoError::Io { + op: "read_link", + path: current.display().to_string(), + reason: error.to_string(), + })?; + let next = normalize_follow_target(root, current.parent().unwrap_or(root), &raw_target) + .ok_or_else(|| RepoError::SymlinkTargetEscape { + path: observed_repo_path.clone(), + target: raw_target.display().to_string(), + })?; + let next_repo_path = repo_path_from_entry(root, &next)?; + if !visited.insert(next_repo_path.as_str().to_owned()) { + return Err(RepoError::SymlinkTargetLoop { + path: observed_repo_path.clone(), + target: raw_target.display().to_string(), + }); + } + current = next; + continue; + } + + if metadata.is_dir() { + return Err(RepoError::SymlinkTargetDirectory { + path: observed_repo_path.clone(), + target: current.display().to_string(), + }); + } + if !metadata.is_file() { + return Err(RepoError::SymlinkTargetDangling { + path: observed_repo_path.clone(), + target: current.display().to_string(), + }); + } + + let repo_path = repo_path_from_entry(root, ¤t)?; + return Ok(ResolvedWorktreeTarget { + fs_path: current.clone(), + repo_path, + display_path: current.display().to_string(), + size_bytes: metadata.len(), + }); + } +} + +fn resolve_git_symlink( + root_tree: &gix::Tree<'_>, + rev: &str, + observed_repo_path: &RepoPath, + initial_target: &str, +) -> RepoResult { + let mut current_target = initial_target.to_owned(); + let mut current_repo_path = + normalize_repo_symlink_target(observed_repo_path.parent(), ¤t_target).map_err( + |reason| RepoError::SymlinkTargetEscape { + path: observed_repo_path.clone(), + target: reason, + }, + )?; + let mut visited = BTreeSet::from([observed_repo_path.as_str().to_owned()]); + + loop { + if !visited.insert(current_repo_path.as_str().to_owned()) { + return Err(RepoError::SymlinkTargetLoop { + path: observed_repo_path.clone(), + target: current_target.clone(), + }); + } + + let entry = lookup_git_entry(root_tree, ¤t_repo_path)?.ok_or_else(|| { + RepoError::SymlinkTargetDangling { + path: observed_repo_path.clone(), + target: current_target.clone(), + } + })?; + + match entry.mode().kind() { + EntryKind::Tree => { + return Err(RepoError::SymlinkTargetDirectory { + path: observed_repo_path.clone(), + target: current_target.clone(), + }); + } + EntryKind::Blob | EntryKind::BlobExecutable => { + return Ok(ResolvedGitTarget { + object_id: entry.object_id().to_owned(), + repo_path: current_repo_path.clone(), + display_path: format!("{rev}:{}", current_repo_path.as_str()), + }); + } + EntryKind::Link => { + let object = entry.object().map_err(|error| RepoError::GitObjectLookup { + object_id: entry.object_id().to_string(), + reason: error.to_string(), + })?; + current_target = std::str::from_utf8(&object.data) + .map_err(|_| RepoError::GitSymlinkTargetInvalidUtf8 { + rev: rev.to_owned(), + path: current_repo_path.clone(), + })? + .trim_end_matches('\0') + .to_owned(); + current_repo_path = + normalize_repo_symlink_target(current_repo_path.parent(), ¤t_target) + .map_err(|reason| RepoError::SymlinkTargetEscape { + path: observed_repo_path.clone(), + target: reason, + })?; + } + _ => { + return Err(RepoError::SymlinkTargetDangling { + path: observed_repo_path.clone(), + target: current_target, + }); + } + } + } +} + +fn lookup_git_entry<'repo>( + root_tree: &gix::Tree<'repo>, + repo_path: &RepoPath, +) -> RepoResult>> { + root_tree + .lookup_entry(repo_path.as_str().split('/')) + .map_err(|error| RepoError::GitObjectLookup { + object_id: root_tree.id.to_string(), + reason: error.to_string(), + }) +} + +fn followed_target_is_ignored(compiled_ignores: &CompiledIgnoreSet, repo_path: &RepoPath) -> bool { + let mut ancestor = repo_path.parent(); + while let Some(directory) = ancestor { + if compiled_ignores.is_ignored(&directory, true) { + return true; + } + ancestor = directory.parent(); + } + + compiled_ignores.is_ignored(repo_path, false) +} + +fn normalize_repo_symlink_target( + base_parent: Option, + target: &str, +) -> Result { + if target.starts_with('/') { + return Err(target.to_owned()); + } + + let mut segments = base_parent + .as_ref() + .map(|path| { + path.as_str() + .split('/') + .map(ToOwned::to_owned) + .collect::>() + }) + .unwrap_or_default(); + + for component in target.split('/') { + match component { + "" | "." => {} + ".." => { + if segments.pop().is_none() { + return Err(target.to_owned()); + } + } + other => segments.push(other.to_owned()), + } + } + + let joined = segments.join("/"); + RepoPath::parse(&joined).map_err(|_| target.to_owned()) +} + +fn normalize_follow_target(root: &Path, current_parent: &Path, target: &Path) -> Option { + if target.is_absolute() { + return None; + } + + normalize_absolute_path(¤t_parent.join(target)) + .filter(|candidate| candidate.starts_with(root)) +} + +fn normalize_absolute_path(path: &Path) -> Option { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + if !normalized.pop() { + return None; + } + } + Component::Normal(part) => normalized.push(part), + } + } + Some(normalized) +} + +#[derive(Clone, Copy)] +enum EntryPathKind { + Dir, + File, + Symlink, + Other, +} + +impl EntryPathKind { + fn from_entry(entry: &DirEntry) -> Self { + let file_type = entry.file_type(); + if file_type.is_dir() { + Self::Dir + } else if file_type.is_file() { + Self::File + } else if file_type.is_symlink() { + Self::Symlink + } else { + Self::Other + } + } + + fn is_dir(self) -> bool { + matches!(self, Self::Dir) + } + + fn is_file(self) -> bool { + matches!(self, Self::File) + } + + fn is_symlink(self) -> bool { + matches!(self, Self::Symlink) + } +} + +fn repo_path_from_git_entry( + parent: Option<&RepoPath>, + filename: &gix::bstr::BStr, +) -> RepoResult { + let segment = filename.to_str().map_err(|_| RepoError::NonUtf8Path { + display_path: filename.to_string(), + })?; + let path = match parent { + Some(parent) => format!("{}/{}", parent.as_str(), segment), + None => segment.to_owned(), + }; + RepoPath::parse(&path).map_err(|error| map_kernel_repo_path_error(Path::new(&path), error)) +} + +fn repo_path_from_entry(root: &Path, path: &Path) -> RepoResult { + let relative = path + .strip_prefix(root) + .map_err(|error| RepoError::InvalidRepoPath { + display_path: path.display().to_string(), + reason: error.to_string(), + })?; + + let mut segments = Vec::new(); + for component in relative.components() { + let std::path::Component::Normal(part) = component else { + return Err(RepoError::InvalidRepoPath { + display_path: path.display().to_string(), + reason: "path contained a non-normal component".to_owned(), + }); + }; + let Some(segment) = part.to_str() else { + return Err(RepoError::NonUtf8Path { + display_path: path.display().to_string(), + }); + }; + segments.push(segment); + } + + RepoPath::parse(&segments.join("/")).map_err(|error| map_kernel_repo_path_error(path, error)) +} + +fn map_kernel_repo_path_error(path: &Path, error: KernelError) -> RepoError { + match error { + KernelError::InvalidRepoPath { reason, .. } => RepoError::InvalidRepoPath { + display_path: path.display().to_string(), + reason, + }, + other => RepoError::InvalidRepoPath { + display_path: path.display().to_string(), + reason: other.to_string(), + }, + } +} + +fn repo_diagnostic( + code: &'static str, + severity: Severity, + message: String, + subject: Option, +) -> RepoDiagnostic { + RepoDiagnostic { + code: DiagnosticCode::parse(code).expect("repo diagnostic code should be valid"), + severity, + message, + subject, + related: Vec::new(), + help: None, + } +} diff --git a/crates/lift/src/runtime_bootstrap_tests.rs b/crates/lift/src/runtime_bootstrap_tests.rs new file mode 100644 index 000000000..770fa06f0 --- /dev/null +++ b/crates/lift/src/runtime_bootstrap_tests.rs @@ -0,0 +1,304 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::app::runtime::{bootstrap_profile, ProfileBootstrap}; +use crate::pack::{builtin, PackCompiler, PackError, PackFormat, PackName, PackSource}; + +#[test] +fn bootstrap_builtin_profile_returns_profile_bootstrap() { + let bootstrap = bootstrap_profile( + builtin::profile_source("generic/default").expect("builtin profile source"), + ) + .expect("builtin bootstrap should succeed"); + + assert_eq!( + bootstrap.bundle.profile.header.id.as_str(), + "generic/default" + ); + assert!(bootstrap.bundle.boundary_taxonomy.is_some()); + assert!(bootstrap.bundle.component_map.is_some()); + assert!(bootstrap.bundle.score_model.is_none()); + assert!(bootstrap.bundle.query_packs.is_empty()); + assert!(bootstrap.bundle.rule_packs.is_empty()); + assert!(bootstrap.bundle.recipe_packs.is_empty()); +} + +#[test] +fn bootstrap_file_backed_advanced_profile_returns_profile_bootstrap() { + let bootstrap = bootstrap_profile(PackSource::File { + path: crate_root().join("fixtures/pack/valid/bundle/profile_advanced_file_backed.toml"), + format_hint: None, + }) + .expect("file-backed bootstrap should succeed"); + + assert_eq!( + bootstrap.bundle.profile.header.id.as_str(), + "acme/advanced-bundle" + ); + assert_eq!( + bootstrap + .bundle + .score_model + .as_ref() + .expect("score model") + .header + .id + .as_str(), + "acme/lift-v2" + ); + assert!(bootstrap + .bundle + .query_packs + .contains_key(&PackName::parse("acme/rust-core").expect("query pack name"))); + assert!(bootstrap + .bundle + .rule_packs + .contains_key(&PackName::parse("acme/policy").expect("rule pack name"))); + assert!(bootstrap + .bundle + .recipe_packs + .contains_key(&PackName::parse("acme/core-recipes").expect("recipe pack name"))); +} + +#[test] +fn bootstrap_inline_profile_returns_profile_bootstrap() { + let bootstrap = bootstrap_profile(PackSource::Inline { + logical_name: "inline-generic-default".to_owned(), + format: PackFormat::Toml, + bytes: load_bytes("fixtures/pack/canonical/generic_default_order_b.toml"), + }) + .expect("inline bootstrap should succeed"); + + assert_eq!( + bootstrap.bundle.profile.header.id.as_str(), + "generic/default" + ); + assert!(bootstrap.bundle.boundary_taxonomy.is_some()); + assert!(bootstrap.bundle.component_map.is_some()); + assert!(bootstrap.bundle.score_model.is_none()); + assert!(bootstrap.bundle.query_packs.is_empty()); + assert!(bootstrap.bundle.rule_packs.is_empty()); + assert!(bootstrap.bundle.recipe_packs.is_empty()); +} + +#[test] +fn bootstrap_propagates_profile_compile_errors_without_translation() { + let source = PackSource::Inline { + logical_name: "schema-violation".to_owned(), + format: PackFormat::Toml, + bytes: load_bytes("fixtures/pack/invalid/profile_schema_violation.toml"), + }; + + let compiler = PackCompiler::new(); + let expected = compiler + .compile_profile(source.clone()) + .expect_err("compile_profile should fail"); + let actual = bootstrap_profile(source).expect_err("bootstrap should fail"); + + assert_eq!(actual, expected); +} + +#[test] +fn bootstrap_propagates_bundle_resolution_errors_without_translation() { + let compiler = PackCompiler::new(); + + let unknown_source = write_temp_profile( + "runtime-bootstrap-unknown-ref", + r#"kind = "profile" +version = 1 +id = "acme/runtime-bootstrap-unknown-ref" +name = "Runtime bootstrap unknown ref" + +[queries] +packs = ["file:queries/missing.json"] +"#, + ); + let unknown_expected = compiler + .resolve_profile_pack_set( + &compiler + .compile_profile(unknown_source.clone()) + .expect("unknown-ref profile should compile"), + ) + .expect_err("bundle resolution should fail with missing ref"); + let unknown_actual = bootstrap_profile(unknown_source).expect_err("bootstrap should fail"); + assert_eq!(unknown_actual, unknown_expected); + assert!(matches!( + unknown_actual, + PackError::UnknownPackReference { .. } + )); + + let mismatch_dir = unique_temp_dir("runtime-bootstrap-ref-kind-mismatch"); + write_text( + &mismatch_dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "acme/runtime-bootstrap-ref-kind-mismatch" +name = "Runtime bootstrap ref kind mismatch" + +[queries] +packs = ["file:generic_policy.json"] +"#, + ); + write_text( + &mismatch_dir.join("generic_policy.json"), + &rule_pack_json("file:queries/rust_core.json"), + ); + let mismatch_source = PackSource::File { + path: mismatch_dir.join("profile.toml"), + format_hint: None, + }; + let mismatch_expected = compiler + .resolve_profile_pack_set( + &compiler + .compile_profile(mismatch_source.clone()) + .expect("mismatch profile should compile"), + ) + .expect_err("bundle resolution should fail with ref kind mismatch"); + let mismatch_actual = bootstrap_profile(mismatch_source).expect_err("bootstrap should fail"); + assert_eq!(mismatch_actual, mismatch_expected); + assert!(matches!(mismatch_actual, PackError::RefKindMismatch { .. })); +} + +#[test] +fn profile_bootstrap_from_pack_set_is_pure_after_source_cleanup() { + let compiler = PackCompiler::new(); + let source_root = unique_temp_dir("runtime-bootstrap-pure-source"); + copy_dir_recursive( + &crate_root().join("fixtures/pack/valid/bundle"), + &source_root, + ); + + let profile = compiler + .compile_profile(PackSource::File { + path: source_root.join("profile_advanced_file_backed.toml"), + format_hint: None, + }) + .expect("profile should compile"); + let bundle = compiler + .resolve_profile_pack_set(&profile) + .expect("bundle should resolve"); + + fs::remove_dir_all(&source_root).expect("source tree should be removable"); + + let bootstrap = ProfileBootstrap::from_pack_set(bundle); + + assert_eq!( + bootstrap.bundle.profile.header.id.as_str(), + "acme/advanced-bundle" + ); + assert_eq!( + bootstrap + .bundle + .score_model + .as_ref() + .expect("score model after cleanup") + .header + .id + .as_str(), + "acme/lift-v2" + ); + assert_eq!(bootstrap.bundle.query_packs.len(), 1); + assert_eq!(bootstrap.bundle.rule_packs.len(), 1); + assert_eq!(bootstrap.bundle.recipe_packs.len(), 1); + assert!(bootstrap + .bundle + .diagnostics + .iter() + .all(|diagnostic| !diagnostic.message.is_empty())); +} + +fn load_bytes(relative: &str) -> Vec { + fs::read(crate_root().join(relative)) + .unwrap_or_else(|err| panic!("failed to read fixture {relative}: {err}")) +} + +fn write_temp_profile(label: &str, contents: &str) -> PackSource { + let dir = unique_temp_dir(label); + let path = dir.join("profile.toml"); + write_text(&path, contents); + PackSource::File { + path, + format_hint: None, + } +} + +fn rule_pack_json(query_pack_ref: &str) -> String { + format!( + r#"{{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "acme/policy", + "name": "Policy rules", + "rules": [ + {{ + "id": "architecture.cross_boundary_import", + "severity": "warning", + "query": {{ + "pack": "{query_pack_ref}", + "id": "use_statement" + }}, + "emit": [ + {{ + "kind": "finding", + "code": "architecture.cross_boundary_import", + "message": "Import crosses boundary" + }} + ] + }} + ] +}}"# + ) +} + +fn write_text(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create parent dirs"); + } + fs::write(path, contents) + .unwrap_or_else(|err| panic!("failed to write {}: {err}", path.display())); +} + +fn copy_dir_recursive(source: &Path, target: &Path) { + fs::create_dir_all(target) + .unwrap_or_else(|err| panic!("failed to create {}: {err}", target.display())); + for entry in fs::read_dir(source) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", source.display())) + { + let entry = entry.unwrap_or_else(|err| panic!("failed to read dir entry: {err}")); + let entry_type = entry + .file_type() + .unwrap_or_else(|err| panic!("failed to stat {}: {err}", entry.path().display())); + let destination = target.join(entry.file_name()); + if entry_type.is_dir() { + copy_dir_recursive(&entry.path(), &destination); + } else { + fs::copy(entry.path(), &destination).unwrap_or_else(|err| { + panic!( + "failed to copy {} to {}: {err}", + entry.path().display(), + destination.display() + ) + }); + } + } +} + +fn unique_temp_dir(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_nanos(); + let dir = std::env::temp_dir().join(format!("substrate-lift-{label}-{nonce}")); + fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +fn crate_root() -> PathBuf { + static CWD_LOCK: OnceLock> = OnceLock::new(); + let lock = CWD_LOCK.get_or_init(|| Mutex::new(())); + let _guard: MutexGuard<'_, ()> = lock.lock().expect("lock"); + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} diff --git a/crates/lift/src/topo/mod.rs b/crates/lift/src/topo/mod.rs new file mode 100644 index 000000000..f0fcf6762 --- /dev/null +++ b/crates/lift/src/topo/mod.rs @@ -0,0 +1,4 @@ +//! Reserved for the future topology seam. No runtime logic in commit 1. + +#[allow(dead_code)] +pub(crate) struct ReservedForFutureSeam; diff --git a/crates/lift/tests/cli_help.rs b/crates/lift/tests/cli_help.rs new file mode 100644 index 000000000..30dfd63d0 --- /dev/null +++ b/crates/lift/tests/cli_help.rs @@ -0,0 +1,63 @@ +#![allow(unused_crate_dependencies)] + +use assert_cmd::Command; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates::prelude::*; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +#[test] +fn top_level_help_lists_required_commands() { + Command::cargo_bin("lift") + .expect("lift binary should be buildable for tests") + .arg("--help") + .assert() + .success() + .stdout( + predicate::str::contains("score") + .and(predicate::str::contains("impact")) + .and(predicate::str::contains("policy")) + .and(predicate::str::contains("contract")) + .and(predicate::str::contains("context")) + .and(predicate::str::contains("index")) + .and(predicate::str::contains("query")) + .and(predicate::str::contains("rewrite")) + .and(predicate::str::contains("estimate").not()) + .and(predicate::str::contains("analyze").not()) + .and(predicate::str::contains("explain").not()) + .and(predicate::str::contains("validate").not()) + .and(predicate::str::contains("print-schema").not()) + .and(predicate::str::contains("print-model").not()), + ); +} + +#[test] +fn representative_nested_help_invocations_exit_zero() { + let cases = [ + (["score", "--help"].as_slice(), "Usage: lift score"), + (["impact", "--help"].as_slice(), "Usage: lift impact"), + (["policy", "--help"].as_slice(), "Usage: lift policy"), + (["contract", "--help"].as_slice(), "Usage: lift contract"), + (["context", "--help"].as_slice(), "Usage: lift context"), + (["index", "--help"].as_slice(), "Usage: lift index"), + (["query", "--help"].as_slice(), "Usage: lift query"), + (["rewrite", "--help"].as_slice(), "Usage: lift rewrite"), + ]; + + for (args, expected) in cases { + Command::cargo_bin("lift") + .expect("lift binary should be buildable for tests") + .args(args) + .assert() + .success() + .stdout(predicate::str::contains(expected)); + } +} diff --git a/crates/lift/tests/compile_matrix.rs b/crates/lift/tests/compile_matrix.rs new file mode 100644 index 000000000..d70af8dc2 --- /dev/null +++ b/crates/lift/tests/compile_matrix.rs @@ -0,0 +1,72 @@ +#![allow(unused_crate_dependencies)] + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use thiserror as _; +use toml as _; +use walkdir as _; + +#[cfg(feature = "cli")] +#[test] +fn run_cli_is_exposed_when_cli_feature_is_enabled() { + let _entrypoint: fn() -> Result<(), substrate_lift::error::LiftError> = substrate_lift::run_cli; +} + +#[test] +fn crate_compiles_without_default_features() { + let workspace_root = workspace_root(); + let target_dir = workspace_root.join("target/lift-seam0-no-default-features"); + let output = Command::new(env!("CARGO")) + .current_dir(&workspace_root) + .env("CARGO_TARGET_DIR", &target_dir) + .args(["check", "-p", "substrate-lift", "--no-default-features"]) + .output() + .expect("cargo check should run"); + + assert!( + output.status.success(), + "cargo check -p substrate-lift --no-default-features failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +#[test] +fn binary_is_gated_behind_cli_feature_in_manifest() { + let manifest_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("Cargo.toml"); + let manifest = fs::read_to_string(&manifest_path).expect("crate Cargo.toml should be readable"); + + assert!( + manifest.contains("[[bin]]"), + "expected a binary target declaration in {}", + manifest_path.display(), + ); + assert!( + manifest.contains("name = \"lift\""), + "expected lift binary declaration in {}", + manifest_path.display(), + ); + assert!( + manifest.contains("required-features = [\"cli\"]"), + "expected lift binary to be gated by required-features = [\"cli\"] in {}", + manifest_path.display(), + ); +} + +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("crate should live under crates/") + .to_path_buf() +} diff --git a/crates/lift/tests/kernel_identity_schema.rs b/crates/lift/tests/kernel_identity_schema.rs new file mode 100644 index 000000000..a44c85b37 --- /dev/null +++ b/crates/lift/tests/kernel_identity_schema.rs @@ -0,0 +1,296 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema::Validator; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json::Value; +use sha2 as _; +use substrate_lift::kernel::{ + sha256_bytes, sha256_canonical_json, ByteSpan, Diagnostic, Fingerprint, JsonPointer, Locator, + QueryId, RepoPath, StableId, PRIMITIVES_V1_SCHEMA_FILE, PRIMITIVES_V1_SCHEMA_ID, + PRIMITIVES_V1_SCHEMA_JSON, PRIMITIVES_V1_SCHEMA_VERSION, +}; +use thiserror as _; +use toml as _; +use walkdir as _; + +#[test] +fn stable_id_from_identity_is_deterministic() { + let left = StableId::from_identity("file", "src/lib.rs"); + let right = StableId::from_identity("file", "src/lib.rs"); + + assert_eq!(left, right); + assert_eq!( + left.as_str(), + "file:sha256:13ec57d807bcff8c7ee2ffc89b3adfb999e5b183d93474f425ebbe2ce371c416" + ); +} + +#[test] +fn typed_ids_reject_wrong_kinds() { + let err = QueryId::parse( + "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ) + .expect_err("query ids must reject file ids"); + + match err { + substrate_lift::kernel::KernelError::InvalidStableId { + expected_kind: Some(kind), + .. + } => assert_eq!(kind, QueryId::KIND), + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn stable_id_and_fingerprint_round_trip_with_schema() { + let stable_id = StableId::parse( + "file:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ) + .expect("fixture stable id should parse"); + let fingerprint = Fingerprint::parse( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ) + .expect("fixture fingerprint should parse"); + + let stable_json = serde_json::to_value(&stable_id).expect("stable id should serialize"); + let fingerprint_json = + serde_json::to_value(&fingerprint).expect("fingerprint should serialize"); + + assert_schema_valid("stable_id", &stable_json); + assert_schema_valid("fingerprint", &fingerprint_json); + + let stable_roundtrip: StableId = + serde_json::from_value(stable_json).expect("stable id should deserialize"); + let fingerprint_roundtrip: Fingerprint = + serde_json::from_value(fingerprint_json).expect("fingerprint should deserialize"); + + assert_eq!(stable_roundtrip, stable_id); + assert_eq!(fingerprint_roundtrip, fingerprint); +} + +#[test] +fn canonical_json_is_deterministic_across_insertion_order() { + let a = load_json("fixtures/kernel/canonical/object_a.json"); + let b = load_json("fixtures/kernel/canonical/object_b.json"); + let expected = load_text("fixtures/kernel/canonical/expected.json"); + let expected = expected.trim_end(); + + let left = substrate_lift::kernel::canonical_json_string(&a).expect("canonical json"); + let right = substrate_lift::kernel::canonical_json_string(&b).expect("canonical json"); + + assert_eq!(left, expected); + assert_eq!(right, expected); + assert_eq!( + substrate_lift::kernel::canonical_json_bytes(&a).expect("canonical bytes"), + expected.as_bytes() + ); +} + +#[test] +fn canonical_json_hash_matches_fixture() { + let value = load_json("fixtures/kernel/canonical/object_a.json"); + let expected_hash = load_text("fixtures/kernel/canonical/expected_sha256.txt"); + let expected_hash = expected_hash.trim_end(); + let expected_json = load_text("fixtures/kernel/canonical/expected.json"); + let expected_json = expected_json.trim_end(); + + let fingerprint = sha256_canonical_json(&value).expect("canonical hash"); + assert_eq!(fingerprint.as_str(), format!("sha256:{expected_hash}")); + assert_eq!( + sha256_bytes(expected_json.as_bytes()).as_str(), + format!("sha256:{expected_hash}") + ); +} + +#[test] +fn valid_schema_fixtures_validate_and_round_trip() { + for (fixture, definition) in valid_fixture_cases() { + let instance = load_json(fixture); + assert_schema_valid(definition, &instance); + } + + let repo_path: RepoPath = + serde_json::from_value(load_json("fixtures/kernel/valid/repo_path.json")).expect("path"); + assert_eq!(repo_path.as_str(), "src/lib.rs"); + + let pointer: JsonPointer = + serde_json::from_value(load_json("fixtures/kernel/valid/json_pointer_nested.json")) + .expect("pointer"); + assert_eq!(pointer.as_str(), "/touch/crates_touched"); + + let span: ByteSpan = + serde_json::from_value(load_json("fixtures/kernel/valid/byte_span.json")).expect("span"); + assert_eq!(span.len(), 5); + + let locator: Locator = + serde_json::from_value(load_json("fixtures/kernel/valid/locator_with_pointer.json")) + .expect("locator"); + assert_eq!( + locator + .json_pointer + .as_ref() + .expect("pointer should exist") + .as_str(), + "/touch/crates_touched" + ); + + let diagnostic: Diagnostic = + serde_json::from_value(load_json("fixtures/kernel/valid/diagnostic_full.json")) + .expect("diagnostic"); + assert_eq!(diagnostic.code.as_str(), "kernel.schema.fixture_example"); + assert_eq!(diagnostic.related.len(), 1); +} + +#[test] +fn invalid_schema_fixtures_fail_validation() { + for (fixture, definition) in invalid_schema_fixture_cases() { + let instance = load_json(fixture); + assert_schema_invalid(definition, &instance); + } +} + +#[test] +fn runtime_only_invalid_fixtures_fail_deserialization() { + assert!(serde_json::from_value::(load_json( + "fixtures/kernel/invalid/json_pointer_malformed.json" + )) + .is_err()); + assert!(serde_json::from_value::(load_json( + "fixtures/kernel/invalid/byte_span_reversed.json" + )) + .is_err()); +} + +#[test] +fn embedded_schema_matches_schema_fixture_on_disk() { + let on_disk = load_text("schemas/kernel/primitives.v1.json"); + assert_eq!(PRIMITIVES_V1_SCHEMA_JSON, on_disk); + assert_eq!( + PRIMITIVES_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/kernel/primitives.v1.json" + ); + assert_eq!(PRIMITIVES_V1_SCHEMA_FILE, "primitives.v1.json"); + assert_eq!(PRIMITIVES_V1_SCHEMA_VERSION, 1); +} + +fn valid_fixture_cases() -> Vec<(&'static str, &'static str)> { + vec![ + ("fixtures/kernel/valid/repo_path.json", "repo_path"), + ( + "fixtures/kernel/valid/json_pointer_root.json", + "json_pointer", + ), + ( + "fixtures/kernel/valid/json_pointer_nested.json", + "json_pointer", + ), + ("fixtures/kernel/valid/stable_id.json", "stable_id"), + ("fixtures/kernel/valid/fingerprint.json", "fingerprint"), + ("fixtures/kernel/valid/byte_span.json", "byte_span"), + ("fixtures/kernel/valid/locator_path_only.json", "locator"), + ("fixtures/kernel/valid/locator_with_span.json", "locator"), + ("fixtures/kernel/valid/locator_with_pointer.json", "locator"), + ( + "fixtures/kernel/valid/diagnostic_minimal.json", + "diagnostic", + ), + ("fixtures/kernel/valid/diagnostic_full.json", "diagnostic"), + ] +} + +fn invalid_schema_fixture_cases() -> Vec<(&'static str, &'static str)> { + vec![ + ( + "fixtures/kernel/invalid/repo_path_absolute.json", + "repo_path", + ), + ( + "fixtures/kernel/invalid/repo_path_backslash.json", + "repo_path", + ), + ( + "fixtures/kernel/invalid/repo_path_dot_segment.json", + "repo_path", + ), + ( + "fixtures/kernel/invalid/stable_id_bad_prefix.json", + "stable_id", + ), + ( + "fixtures/kernel/invalid/stable_id_bad_hex.json", + "stable_id", + ), + ( + "fixtures/kernel/invalid/fingerprint_bad_prefix.json", + "fingerprint", + ), + ( + "fixtures/kernel/invalid/fingerprint_bad_hex.json", + "fingerprint", + ), + ( + "fixtures/kernel/invalid/diagnostic_code_missing_namespace.json", + "diagnostic", + ), + ( + "fixtures/kernel/invalid/related_location_empty_message.json", + "related_location", + ), + ] +} + +fn assert_schema_valid(definition: &str, instance: &Value) { + let validator = validator_for(definition); + if let Err(error) = validator.validate(instance) { + panic!("expected fixture to validate against {definition}: {error}"); + } +} + +fn assert_schema_invalid(definition: &str, instance: &Value) { + let validator = validator_for(definition); + assert!( + validator.validate(instance).is_err(), + "expected fixture to fail validation for {definition}" + ); +} + +fn validator_for(definition: &str) -> Validator { + let schema = load_json("schemas/kernel/primitives.v1.json"); + let wrapped = serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": format!("#/$defs/{definition}"), + "$defs": schema["$defs"].clone(), + }); + + jsonschema::validator_for(&wrapped).expect("schema should compile") +} + +fn load_json(relative: &str) -> Value { + let path = crate_root().join(relative); + let contents = fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }); + + serde_json::from_str(&contents).unwrap_or_else(|err| { + panic!("failed to parse {} as JSON: {err}", path.display()); + }) +} + +fn load_text(relative: &str) -> String { + let path = crate_root().join(relative); + fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }) +} + +fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} diff --git a/crates/lift/tests/lang_cache.rs b/crates/lift/tests/lang_cache.rs new file mode 100644 index 000000000..69ca73ba8 --- /dev/null +++ b/crates/lift/tests/lang_cache.rs @@ -0,0 +1,637 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::BTreeSet; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde::Deserialize; +use serde_jcs as _; +use serde_json::json; +use sha2 as _; +use substrate_lift as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/lang/mod.rs"] +mod lang; +#[path = "../src/pack/mod.rs"] +mod pack; +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use kernel::RepoPath; +use lang::{ + AdapterDescriptor, AdapterName, AdapterParseOutput, AdapterParseResult, CachedParseOutcome, + InMemoryParseCache, LangError, LangResult, LanguageAdapter, LanguageRegistryBuilder, + ParseCache, ParseCacheKey, ParseCacheLookup, ParseDriver, ParseInput, ParseRequest, ParseScope, + PLATFORM_CACHE_VERSION, +}; + +#[derive(Clone, Debug, Deserialize)] +struct FakeDocument { + #[serde(default)] + fail: bool, + #[serde(default)] + symbols: Vec, +} + +#[derive(Clone, Debug)] +struct FakeAdapter { + descriptor: AdapterDescriptor, + capabilities: lang::AdapterCapabilities, + suffix: &'static str, + panic_recognizes: BTreeSet, +} + +impl FakeAdapter { + fn new(name: &str, language: &str, suffix: &'static str, version: &str) -> Self { + Self { + descriptor: AdapterDescriptor { + name: AdapterName::parse(name).expect("adapter name"), + language: pack::LanguageId::parse(language).expect("language"), + version: version.to_owned(), + }, + capabilities: lang::AdapterCapabilities::default(), + suffix, + panic_recognizes: BTreeSet::new(), + } + } + + fn with_capabilities(mut self, capabilities: lang::AdapterCapabilities) -> Self { + self.capabilities = capabilities; + self + } + + fn with_recognize_panic(mut self, path: &str) -> Self { + self.panic_recognizes.insert(path.to_owned()); + self + } +} + +impl LanguageAdapter for FakeAdapter { + fn descriptor(&self) -> AdapterDescriptor { + self.descriptor.clone() + } + + fn capabilities(&self) -> lang::AdapterCapabilities { + self.capabilities.clone() + } + + fn recognizes(&self, input: &ParseInput<'_>) -> bool { + if self.panic_recognizes.contains(input.path.as_str()) { + panic!("recognize panic for {}", input.path.as_str()); + } + input.path.as_str().ends_with(self.suffix) + } + + fn parse(&self, input: &ParseInput<'_>) -> AdapterParseResult { + let document: FakeDocument = match serde_json::from_slice(input.bytes) { + Ok(document) => document, + Err(_) => { + return AdapterParseResult::Failed { + diagnostics: Vec::new(), + }; + } + }; + + if document.fail { + return AdapterParseResult::Failed { + diagnostics: Vec::new(), + }; + } + + AdapterParseResult::Parsed(AdapterParseOutput { + symbols: document.symbols, + edges: Vec::new(), + surface_markers: Vec::new(), + diagnostics: Vec::new(), + }) + } +} + +struct FailingCache; + +impl ParseCache for FailingCache { + fn get(&self, _key: &ParseCacheKey) -> LangResult { + Err(LangError::CacheInvariant { + reason: "simulated cache get failure".to_owned(), + }) + } + + fn put(&self, _key: ParseCacheKey, _value: CachedParseOutcome) -> LangResult<()> { + Ok(()) + } +} + +fn registry(adapters: Vec) -> lang::LanguageRegistry { + let mut builder = LanguageRegistryBuilder::new(); + for adapter in adapters { + builder = builder.register(adapter).expect("register adapter"); + } + builder.build().expect("build registry") +} + +fn write_repo_files(temp: &repo_support::TempDir, files: &[(&str, &[u8])]) { + repo_support::write_file(&temp.path().join(".git/HEAD"), b"ref: refs/heads/main\n"); + for (path, bytes) in files { + repo_support::write_file(&temp.path().join(path), bytes); + } +} + +fn snapshot_with_files(files: &[(&str, &[u8])]) -> (repo_support::TempDir, repo::RepoSnapshot) { + let temp = repo_support::TempDir::new("lang-cache"); + write_repo_files(&temp, files); + let snapshot = repo_support::materialize(temp.path(), repo_support::default_snapshot_options()); + (temp, snapshot) +} + +fn parse_with_driver( + driver: &ParseDriver, + snapshot: &repo::RepoSnapshot, + request: ParseRequest, +) -> lang::ParseSet { + driver + .parse_snapshot(snapshot, &request) + .expect("parse snapshot") +} + +fn path_set(paths: &[&str]) -> BTreeSet { + paths + .iter() + .map(|path| RepoPath::parse(path).expect("repo path")) + .collect() +} + +fn language_set(languages: &[&str]) -> BTreeSet { + languages + .iter() + .map(|language| pack::LanguageId::parse(language).expect("language")) + .collect() +} + +fn fake_symbol(local_key: &str, name: &str, start: u64, end: u64) -> serde_json::Value { + json!({ + "local_key": local_key, + "kind": "function", + "name": name, + "path": [name], + "span": { + "start_byte": start, + "end_byte": end + }, + "visibility": "public" + }) +} + +fn assert_same_payload_except_stats(left: &lang::ParseSet, right: &lang::ParseSet) { + assert_eq!(left.snapshot_fingerprint, right.snapshot_fingerprint); + assert_eq!(left.request, right.request); + assert_eq!(left.request_fingerprint, right.request_fingerprint); + assert_eq!(left.units, right.units); + assert_eq!(left.failed, right.failed); + assert_eq!(left.skipped, right.skipped); + assert_eq!(left.missing_languages, right.missing_languages); + assert_eq!(left.diagnostics, right.diagnostics); +} + +#[test] +fn in_memory_cache_hits_on_second_equivalent_run_except_stats() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let cache = InMemoryParseCache::default(); + let driver = ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + cache.clone(), + ); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + + let first = parse_with_driver(&driver, &snapshot, request.clone()); + let second = parse_with_driver(&driver, &snapshot, request); + + assert_same_payload_except_stats(&first, &second); + assert_eq!(first.stats.cache_hits, 0); + assert_eq!(first.stats.cache_misses, 1); + assert_eq!(second.stats.cache_hits, 1); + assert_eq!(second.stats.cache_misses, 0); + assert_eq!(cache.len().expect("cache length"), 1); +} + +#[test] +fn phase_c_capabilities_do_not_change_cache_keys_or_parse_stats() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let cache = InMemoryParseCache::default(); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + + let baseline = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + cache.clone(), + ), + &snapshot, + request.clone(), + ); + let phase_c = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + ) + .with_capabilities(lang::AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: true, + query_engines: [lang::QueryEngineKind::TreeSitter].into_iter().collect(), + })]), + cache.clone(), + ), + &snapshot, + request, + ); + + assert_same_payload_except_stats(&baseline, &phase_c); + assert_eq!(baseline.stats.considered_files, 1); + assert_eq!(baseline.stats.parsed_units, 1); + assert_eq!(baseline.stats.failed_units, 0); + assert_eq!(baseline.stats.skipped_no_adapter, 0); + assert_eq!(baseline.stats.skipped_missing_paths, 0); + assert_eq!(baseline.stats.missing_requested_languages, 0); + assert_eq!(baseline.stats.diagnostic_count, 0); + assert_eq!(baseline.stats.cache_hits, 0); + assert_eq!(baseline.stats.cache_misses, 1); + assert_eq!( + phase_c.stats.considered_files, + baseline.stats.considered_files + ); + assert_eq!(phase_c.stats.parsed_units, baseline.stats.parsed_units); + assert_eq!(phase_c.stats.failed_units, baseline.stats.failed_units); + assert_eq!( + phase_c.stats.skipped_no_adapter, + baseline.stats.skipped_no_adapter + ); + assert_eq!( + phase_c.stats.skipped_missing_paths, + baseline.stats.skipped_missing_paths + ); + assert_eq!( + phase_c.stats.missing_requested_languages, + baseline.stats.missing_requested_languages + ); + assert_eq!( + phase_c.stats.diagnostic_count, + baseline.stats.diagnostic_count + ); + assert_eq!(phase_c.stats.cache_hits, 1); + assert_eq!(phase_c.stats.cache_misses, 0); + assert_eq!(cache.len().expect("cache length"), 1); +} + +#[test] +fn cache_miss_when_blob_fingerprint_changes() { + let original = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let updated = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "beta", 0, 4)] + })) + .expect("json"); + let (_left_temp, left_snapshot) = snapshot_with_files(&[("src/app.fake.json", &original)]); + let (_right_temp, right_snapshot) = snapshot_with_files(&[("src/app.fake.json", &updated)]); + let cache = InMemoryParseCache::default(); + let driver = ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + cache.clone(), + ); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + + let first = parse_with_driver(&driver, &left_snapshot, request.clone()); + let second = parse_with_driver(&driver, &right_snapshot, request); + + assert_eq!(first.stats.cache_misses, 1); + assert_eq!(second.stats.cache_hits, 0); + assert_eq!(second.stats.cache_misses, 1); + assert_ne!( + first.units[0].blob_fingerprint, + second.units[0].blob_fingerprint + ); + assert_eq!(cache.len().expect("cache length"), 2); +} + +#[test] +fn cache_miss_when_adapter_version_changes() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let cache = InMemoryParseCache::default(); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + + let first = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + cache.clone(), + ), + &snapshot, + request.clone(), + ); + let second = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "2.0.0", + )]), + cache.clone(), + ), + &snapshot, + request, + ); + + assert_eq!(first.stats.cache_misses, 1); + assert_eq!(second.stats.cache_hits, 0); + assert_eq!(second.stats.cache_misses, 1); + assert_eq!(second.units[0].adapter_version, "2.0.0"); + assert_eq!(cache.len().expect("cache length"), 2); +} + +#[test] +fn cache_miss_when_platform_cache_version_changes() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + let seed_registry = registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]); + let seed = parse_with_driver(&ParseDriver::new(seed_registry), &snapshot, request.clone()); + + let cache = InMemoryParseCache::default(); + let entry = snapshot + .entry(&RepoPath::parse("src/app.fake.json").expect("repo path")) + .expect("snapshot entry should exist"); + let descriptor = registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]) + .adapter_for_language(pack::LanguageId::Json) + .expect("adapter should exist") + .descriptor(); + let bytes = snapshot + .read_bytes(&entry.path) + .expect("snapshot bytes should exist"); + let input = ParseInput { + path: &entry.path, + file_id: &entry.file_id, + blob_fingerprint: &entry.blob_fingerprint, + bytes, + }; + cache + .put( + ParseCacheKey::with_platform_cache_version(&input, &descriptor, "stale-platform-v0"), + CachedParseOutcome::Parsed(seed.units[0].clone()), + ) + .expect("seed cache entry"); + + let current = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + cache.clone(), + ), + &snapshot, + request, + ); + + assert_eq!(PLATFORM_CACHE_VERSION, "phase_b_v1"); + assert_eq!(current.stats.cache_hits, 0); + assert_eq!(current.stats.cache_misses, 1); + assert_eq!(cache.len().expect("cache length"), 2); +} + +#[test] +fn cache_reuses_failed_parse_outcomes() { + let bytes = serde_json::to_vec(&json!({ + "fail": true + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let cache = InMemoryParseCache::default(); + let driver = ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + cache.clone(), + ); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + + let first = parse_with_driver(&driver, &snapshot, request.clone()); + let second = parse_with_driver(&driver, &snapshot, request); + + assert!(first.units.is_empty()); + assert_eq!(first.failed.len(), 1); + assert_same_payload_except_stats(&first, &second); + assert_eq!(first.stats.cache_misses, 1); + assert_eq!(second.stats.cache_hits, 1); + assert_eq!(cache.len().expect("cache length"), 1); +} + +#[test] +fn cache_does_not_cache_skipped_or_missing_language_records() { + let matching = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let missing_cache = InMemoryParseCache::default(); + let (_missing_temp, missing_snapshot) = + snapshot_with_files(&[("src/app.fake.json", &matching)]); + let missing = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + missing_cache.clone(), + ), + &missing_snapshot, + ParseRequest { + languages: language_set(&["rust"]), + scope: ParseScope::Snapshot, + }, + ); + + assert_eq!(missing.stats.cache_hits, 0); + assert_eq!(missing.stats.cache_misses, 0); + assert_eq!(missing_cache.len().expect("cache length"), 0); + assert_eq!(missing.missing_languages.len(), 1); + + let skip_cache = InMemoryParseCache::default(); + let (_skip_temp, skip_snapshot) = snapshot_with_files(&[ + ("src/plain.txt", b"plain text"), + ("src/app.fake.json", &matching), + ]); + let skipped = parse_with_driver( + &ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + skip_cache.clone(), + ), + &skip_snapshot, + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Paths(path_set(&["src/plain.txt"])), + }, + ); + + assert_eq!(skipped.stats.cache_hits, 0); + assert_eq!(skipped.stats.cache_misses, 0); + assert_eq!(skip_cache.len().expect("cache length"), 0); + assert_eq!(skipped.skipped.len(), 1); + assert_eq!( + skipped.skipped[0].reason, + lang::SkippedReason::NoMatchingAdapter + ); +} + +#[test] +fn cache_backend_failures_surface_as_cache_invariant_errors() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let driver = ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + )]), + FailingCache, + ); + + let error = driver + .parse_snapshot( + &snapshot, + &ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ) + .expect_err("cache get failure should surface"); + + assert_eq!( + error, + LangError::CacheInvariant { + reason: "simulated cache get failure".to_owned(), + } + ); +} + +#[test] +fn recognize_panic_behavior_remains_deterministic_and_uncached() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let cache = InMemoryParseCache::default(); + let driver = ParseDriver::with_cache( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + ) + .with_recognize_panic("src/app.fake.json")]), + cache.clone(), + ); + let request = ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }; + + let first = parse_with_driver(&driver, &snapshot, request.clone()); + let second = parse_with_driver(&driver, &snapshot, request); + + assert_eq!(first, second); + assert_eq!(first.stats.cache_hits, 0); + assert_eq!(first.stats.cache_misses, 0); + assert_eq!(second.stats.cache_hits, 0); + assert_eq!(second.stats.cache_misses, 0); + assert_eq!(cache.len().expect("cache length"), 0); +} diff --git a/crates/lift/tests/lang_capabilities.rs b/crates/lift/tests/lang_capabilities.rs new file mode 100644 index 000000000..92ee3efec --- /dev/null +++ b/crates/lift/tests/lang_capabilities.rs @@ -0,0 +1,95 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::BTreeSet; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde_json as _; +use sha2 as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} + +#[path = "../src/pack/mod.rs"] +mod pack; + +#[path = "../src/lang/mod.rs"] +mod lang; + +#[path = "../src/repo/mod.rs"] +mod repo; + +use lang::{ + AdapterCapabilities, AdapterDescriptor, AdapterName, AdapterParseResult, LanguageAdapter, + ParseInput, QueryEngineKind, +}; + +struct TestAdapter { + descriptor: AdapterDescriptor, +} + +impl TestAdapter { + fn new() -> Self { + Self { + descriptor: AdapterDescriptor { + name: AdapterName::parse("builtin.capabilities").expect("adapter name"), + language: pack::LanguageId::Rust, + version: "1.0.0".to_owned(), + }, + } + } +} + +impl LanguageAdapter for TestAdapter { + fn descriptor(&self) -> AdapterDescriptor { + self.descriptor.clone() + } + + fn recognizes(&self, _input: &ParseInput<'_>) -> bool { + false + } + + fn parse(&self, _input: &ParseInput<'_>) -> AdapterParseResult { + AdapterParseResult::Failed { + diagnostics: Vec::new(), + } + } +} + +#[test] +fn default_adapter_capabilities_are_empty_and_backwards_compatible() { + let adapter = TestAdapter::new(); + + assert_eq!(adapter.capabilities(), AdapterCapabilities::default()); + assert_eq!( + serde_json::from_str::("{}").expect("deserialize empty object"), + AdapterCapabilities::default() + ); +} + +#[test] +fn adapter_capabilities_serde_round_trip_is_canonical() { + let capabilities = AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: true, + query_engines: BTreeSet::from([QueryEngineKind::TreeSitter]), + }; + + let canonical = + kernel::canonical_json_string(&capabilities).expect("serialize canonical capabilities"); + assert_eq!( + canonical, + r#"{"emits_local_edges":true,"emits_surface_markers":true,"query_engines":["tree_sitter"]}"# + ); + + let reparsed: AdapterCapabilities = + serde_json::from_str(&canonical).expect("deserialize canonical capabilities"); + assert_eq!(reparsed, capabilities); +} diff --git a/crates/lift/tests/lang_consumer.rs b/crates/lift/tests/lang_consumer.rs new file mode 100644 index 000000000..6cc609327 --- /dev/null +++ b/crates/lift/tests/lang_consumer.rs @@ -0,0 +1,117 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::BTreeSet; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde_jcs as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} + +#[path = "../src/pack/mod.rs"] +mod pack; +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +include!("support/lang_support.rs"); + +fn config_key_inventory(parse_set: &lang::ParseSet) -> Vec { + let mut inventory = parse_set + .units + .iter() + .flat_map(|unit| { + unit.symbols + .iter() + .filter(|symbol| symbol.kind == lang::SymbolKind::ConfigKey) + .map(|symbol| format!("{}::{}", unit.path.as_str(), symbol.path.join("."))) + }) + .collect::>(); + inventory.sort(); + inventory +} + +fn edge_kinds(parse_set: &lang::ParseSet) -> BTreeSet { + parse_set + .units + .iter() + .flat_map(|unit| unit.edges.iter().map(|edge| edge.kind)) + .collect() +} + +#[test] +fn toml_proof_slice_is_stable_across_repeat_runs() { + let (_left_temp, _left_snapshot, left) = parse_fixture_case("repeat_run_repo"); + let (_right_temp, _right_snapshot, right) = parse_fixture_case("repeat_run_repo"); + + assert_eq!(left, right); + assert_eq!(left.stats.parsed_units, 2); + assert!(left.failed.is_empty()); + + let kinds = edge_kinds(&left); + assert!(kinds.contains(&lang::LocalEdgeKind::Contains)); + assert!(kinds.contains(&lang::LocalEdgeKind::ConfigRef)); +} + +#[test] +fn hostile_malformed_toml_becomes_failed_parse_instead_of_crash() { + let (_temp, _snapshot, parsed) = parse_fixture_case("malformed_repo"); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert_eq!(parsed.failed[0].path.as_str(), "configs/hostile.toml"); + assert_eq!(parsed.failed[0].language, pack::LanguageId::Toml); + assert!(parsed.failed[0] + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code.as_str() == "lang.toml.parse_failed")); + assert_eq!(parsed.stats.failed_units, 1); +} + +#[test] +fn consumer_derives_deterministic_config_key_inventory_from_parse_set_without_rereading_raw_bytes() +{ + let (temp, _snapshot, parsed) = parse_fixture_case("valid_repo"); + + let before = config_key_inventory(&parsed); + + repo_support::write_file( + &temp.path().join("configs/app.toml"), + br#" +[server] +port = 1 +"#, + ); + std::fs::remove_file(temp.path().join("configs/feature_flags.toml")) + .expect("feature flag fixture should delete"); + + let after = config_key_inventory(&parsed); + + assert_eq!(after, before); + assert_eq!( + after, + vec![ + "configs/app.toml::name".to_owned(), + "configs/app.toml::profiles".to_owned(), + "configs/app.toml::profiles.primary".to_owned(), + "configs/app.toml::profiles.primary.database_url".to_owned(), + "configs/app.toml::profiles.primary.max_connections".to_owned(), + "configs/app.toml::server".to_owned(), + "configs/app.toml::server.port".to_owned(), + "configs/app.toml::server.profile".to_owned(), + "configs/feature_flags.toml::flags".to_owned(), + "configs/feature_flags.toml::flags.enable_metrics".to_owned(), + "configs/feature_flags.toml::flags.enable_search".to_owned(), + ] + ); +} diff --git a/crates/lift/tests/lang_parse.rs b/crates/lift/tests/lang_parse.rs new file mode 100644 index 000000000..8e3fe69ea --- /dev/null +++ b/crates/lift/tests/lang_parse.rs @@ -0,0 +1,650 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::BTreeSet; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde::Deserialize; +use serde_jcs as _; +use serde_json::json; +use sha2 as _; +use substrate_lift as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/lang/mod.rs"] +mod lang; +#[path = "../src/pack/mod.rs"] +mod pack; +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use kernel::{Diagnostic, RepoPath}; +use lang::{ + AdapterDescriptor, AdapterName, AdapterParseOutput, AdapterParseResult, LanguageAdapter, + LanguageRegistryBuilder, LocalEdgeDraft, LocalSymbolDraft, ParseDriver, ParseInput, + ParseRequest, ParseScope, SurfaceMarkerDraft, +}; + +#[derive(Clone, Debug, Deserialize)] +struct FakeDocument { + #[serde(default)] + fail: bool, + #[serde(default)] + symbols: Vec, + #[serde(default)] + edges: Vec, + #[serde(default)] + surface_markers: Vec, + #[serde(default)] + diagnostics: Vec, +} + +#[derive(Clone, Debug)] +struct FakeAdapter { + descriptor: AdapterDescriptor, + suffix: &'static str, + panic_recognizes: BTreeSet, + panic_parses: BTreeSet, +} + +impl FakeAdapter { + fn new(name: &str, language: &str, suffix: &'static str) -> Self { + Self { + descriptor: AdapterDescriptor { + name: AdapterName::parse(name).expect("adapter name"), + language: pack::LanguageId::parse(language).expect("language"), + version: "1.0.0".to_owned(), + }, + suffix, + panic_recognizes: BTreeSet::new(), + panic_parses: BTreeSet::new(), + } + } + + fn with_recognize_panic(mut self, path: &str) -> Self { + self.panic_recognizes.insert(path.to_owned()); + self + } + + fn with_parse_panic(mut self, path: &str) -> Self { + self.panic_parses.insert(path.to_owned()); + self + } +} + +impl LanguageAdapter for FakeAdapter { + fn descriptor(&self) -> AdapterDescriptor { + self.descriptor.clone() + } + + fn recognizes(&self, input: &ParseInput<'_>) -> bool { + if self.panic_recognizes.contains(input.path.as_str()) { + panic!("recognize panic for {}", input.path.as_str()); + } + input.path.as_str().ends_with(self.suffix) + } + + fn parse(&self, input: &ParseInput<'_>) -> AdapterParseResult { + if self.panic_parses.contains(input.path.as_str()) { + panic!("parse panic for {}", input.path.as_str()); + } + + let document: FakeDocument = match serde_json::from_slice(input.bytes) { + Ok(document) => document, + Err(_) => { + return AdapterParseResult::Failed { + diagnostics: Vec::new(), + }; + } + }; + + if document.fail { + return AdapterParseResult::Failed { + diagnostics: document.diagnostics, + }; + } + + AdapterParseResult::Parsed(AdapterParseOutput { + symbols: document.symbols, + edges: document.edges, + surface_markers: document.surface_markers, + diagnostics: document.diagnostics, + }) + } +} + +fn registry(adapters: Vec) -> lang::LanguageRegistry { + let mut builder = LanguageRegistryBuilder::new(); + for adapter in adapters { + builder = builder.register(adapter).expect("register adapter"); + } + builder.build().expect("build registry") +} + +fn write_repo_files(temp: &repo_support::TempDir, files: &[(&str, &[u8])]) { + repo_support::write_file(&temp.path().join(".git/HEAD"), b"ref: refs/heads/main\n"); + for (path, bytes) in files { + repo_support::write_file(&temp.path().join(path), bytes); + } +} + +fn snapshot_with_files(files: &[(&str, &[u8])]) -> (repo_support::TempDir, repo::RepoSnapshot) { + let temp = repo_support::TempDir::new("lang-parse"); + write_repo_files(&temp, files); + let snapshot = repo_support::materialize(temp.path(), repo_support::default_snapshot_options()); + (temp, snapshot) +} + +fn parse_snapshot( + snapshot: &repo::RepoSnapshot, + registry: lang::LanguageRegistry, + request: ParseRequest, +) -> lang::ParseSet { + ParseDriver::new(registry) + .parse_snapshot(snapshot, &request) + .expect("parse snapshot") +} + +fn with_silenced_panic_hook(work: impl FnOnce() -> T) -> T { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = work(); + std::panic::set_hook(hook); + result +} + +fn path_set(paths: &[&str]) -> BTreeSet { + paths + .iter() + .map(|path| RepoPath::parse(path).expect("repo path")) + .collect() +} + +fn language_set(languages: &[&str]) -> BTreeSet { + languages + .iter() + .map(|language| pack::LanguageId::parse(language).expect("language")) + .collect() +} + +fn fake_symbol(local_key: &str, name: &str, start: u64, end: u64) -> serde_json::Value { + json!({ + "local_key": local_key, + "kind": "function", + "name": name, + "path": [name], + "span": { + "start_byte": start, + "end_byte": end + }, + "visibility": "public" + }) +} + +fn fake_edge(source: &str, target: &str, start: u64, end: u64) -> serde_json::Value { + json!({ + "kind": "call", + "source": { + "Symbol": { + "local_key": source + } + }, + "target": { + "LocalSymbol": { + "local_key": target + } + }, + "span": { + "start_byte": start, + "end_byte": end + } + }) +} + +fn fake_marker(local_key: &str, start: u64, end: u64) -> serde_json::Value { + json!({ + "kind": "entry_point", + "symbol_local_key": local_key, + "span": { + "start_byte": start, + "end_byte": end + }, + "label": "entry" + }) +} + +fn fake_diagnostic(code: &str, path: &str, start: u64, end: u64) -> serde_json::Value { + json!({ + "code": code, + "severity": "warning", + "message": "adapter note", + "subject": { + "path": path, + "span": { + "start_byte": start, + "end_byte": end + } + } + }) +} + +#[test] +fn snapshot_parse_is_deterministic_and_uses_snapshot_bytes_only() { + let original_json = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let original_yaml = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("cfg", "beta", 0, 4)] + })) + .expect("json"); + let (temp, snapshot) = snapshot_with_files(&[ + ("src/b.fake.json", &original_json), + ("src/a.fake.yaml", &original_yaml), + ]); + + repo_support::write_file( + &temp.path().join("src/b.fake.json"), + serde_json::to_string(&json!({ + "symbols": [fake_symbol("main", "mutated", 0, 7)] + })) + .expect("json") + .as_bytes(), + ); + + let request = ParseRequest { + languages: BTreeSet::new(), + scope: ParseScope::Snapshot, + }; + let left = parse_snapshot( + &snapshot, + registry(vec![ + FakeAdapter::new("builtin.fake_yaml", "yaml", ".fake.yaml"), + FakeAdapter::new("builtin.fake_json", "json", ".fake.json"), + ]), + request.clone(), + ); + let right = parse_snapshot( + &snapshot, + registry(vec![ + FakeAdapter::new("builtin.fake_json", "json", ".fake.json"), + FakeAdapter::new("builtin.fake_yaml", "yaml", ".fake.yaml"), + ]), + request, + ); + + assert_eq!(left, right); + assert_eq!(left.units.len(), 2); + assert_eq!(left.units[0].path.as_str(), "src/a.fake.yaml"); + assert_eq!(left.units[1].path.as_str(), "src/b.fake.json"); + assert_eq!(left.units[1].symbols[0].name.as_deref(), Some("alpha")); + assert_eq!(left.stats.parsed_units, 2); + assert_eq!(left.stats.considered_files, 2); + assert_eq!(left.stats.failed_units, 0); +} + +#[test] +fn empty_language_requests_follow_registry_adapter_name_order() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/shared.fake", &bytes)]); + let request = ParseRequest { + languages: BTreeSet::new(), + scope: ParseScope::Snapshot, + }; + + let left = parse_snapshot( + &snapshot, + registry(vec![ + FakeAdapter::new("builtin.zulu", "json", ".fake").with_parse_panic("src/shared.fake"), + FakeAdapter::new("builtin.alpha", "yaml", ".fake"), + ]), + request.clone(), + ); + let right = parse_snapshot( + &snapshot, + registry(vec![ + FakeAdapter::new("builtin.alpha", "yaml", ".fake"), + FakeAdapter::new("builtin.zulu", "json", ".fake").with_parse_panic("src/shared.fake"), + ]), + request, + ); + + assert_eq!(left, right); + assert_eq!(left.units.len(), 1); + assert!(left.failed.is_empty()); + assert_eq!(left.units[0].language, pack::LanguageId::Yaml); + assert_eq!(left.units[0].path.as_str(), "src/shared.fake"); +} + +#[test] +fn snapshot_scope_counts_no_matching_files_in_stats_only() { + let matching = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[ + ("src/match.fake.json", &matching), + ("README.txt", b"plain text"), + ]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ); + + assert_eq!(parsed.units.len(), 1); + assert!(parsed.skipped.is_empty()); + assert_eq!(parsed.stats.considered_files, 2); + assert_eq!(parsed.stats.skipped_no_adapter, 1); +} + +#[test] +fn explicit_paths_emit_skips_for_missing_paths_and_no_matching_adapter() { + let matching = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[ + ("src/match.fake.json", &matching), + ("src/plain.txt", b"plain text"), + ]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Paths(path_set(&["src/plain.txt", "src/missing.txt"])), + }, + ); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.skipped.len(), 2); + assert_eq!(parsed.skipped[0].path.as_str(), "src/missing.txt"); + assert_eq!( + parsed.skipped[0].reason, + lang::SkippedReason::PathNotInSnapshot + ); + assert!(parsed.skipped[0].file_id.is_none()); + assert_eq!(parsed.skipped[1].path.as_str(), "src/plain.txt"); + assert_eq!( + parsed.skipped[1].reason, + lang::SkippedReason::NoMatchingAdapter + ); + assert!(parsed.skipped[1].file_id.is_some()); + assert_eq!(parsed.stats.skipped_missing_paths, 1); + assert_eq!(parsed.stats.skipped_no_adapter, 1); + assert_eq!(parsed.stats.considered_files, 1); +} + +#[test] +fn empty_explicit_scope_returns_a_deterministic_empty_parse_set() { + let matching = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/match.fake.json", &matching)]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Paths(BTreeSet::new()), + }, + ); + + assert!(parsed.units.is_empty()); + assert!(parsed.failed.is_empty()); + assert!(parsed.skipped.is_empty()); + assert_eq!(parsed.stats.considered_files, 0); + assert_eq!(parsed.stats.parsed_units, 0); + assert_eq!(parsed.stats.failed_units, 0); + assert_eq!(parsed.stats.skipped_no_adapter, 0); +} + +#[test] +fn missing_requested_languages_surface_run_level_records_and_diagnostics() { + let matching = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/match.fake.json", &matching)]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json", "rust"]), + scope: ParseScope::Snapshot, + }, + ); + + assert_eq!(parsed.units.len(), 1); + assert_eq!(parsed.missing_languages.len(), 1); + assert_eq!(parsed.missing_languages[0].language, pack::LanguageId::Rust); + assert_eq!(parsed.diagnostics.len(), 1); + assert_eq!( + parsed.diagnostics[0].code.as_str(), + "lang.parse.missing_registered_adapter" + ); + assert_eq!(parsed.stats.missing_requested_languages, 1); +} + +#[test] +fn adapter_parse_failures_become_failed_parse_records_with_deterministic_diagnostics() { + let (_temp, snapshot) = snapshot_with_files(&[("src/malformed.fake.json", b"\xff\xfe\xfd")]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert_eq!(parsed.failed[0].diagnostics.len(), 1); + assert_eq!( + parsed.failed[0].diagnostics[0].code.as_str(), + "lang.parse.failed" + ); +} + +#[test] +fn duplicate_local_keys_become_failed_parse_records() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [ + fake_symbol("dup", "alpha", 0, 5), + fake_symbol("dup", "beta", 6, 10) + ] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/dup.fake.json", &bytes)]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert!(parsed.failed[0] + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code.as_str() == "lang.parse.duplicate_local_key")); +} + +#[test] +fn unresolved_local_and_marker_refs_become_failed_parse_records() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)], + "edges": [fake_edge("missing_source", "main", 0, 1)], + "surface_markers": [fake_marker("missing_marker", 0, 1)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/unresolved.fake.json", &bytes)]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert!(parsed.failed[0] + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code.as_str() == "lang.parse.unresolved_local_ref")); + assert!(parsed.failed[0] + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code.as_str() == "lang.parse.unresolved_marker_ref")); +} + +#[test] +fn invalid_spans_become_failed_parse_records() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 999)], + "diagnostics": [fake_diagnostic("lang.fake.note", "src/invalid.fake.json", 0, 999)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/invalid.fake.json", &bytes)]); + + let parsed = parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + )]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert!(parsed.failed[0] + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code.as_str() == "lang.parse.invalid_span")); +} + +#[test] +fn adapter_panics_are_contained_as_failed_parse_records() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/panic.fake.json", &bytes)]); + + let parsed = with_silenced_panic_hook(|| { + parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + ) + .with_parse_panic("src/panic.fake.json")]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ) + }); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert_eq!( + parsed.failed[0].diagnostics[0].code.as_str(), + "lang.parse.adapter_panic" + ); +} + +#[test] +fn recognize_panics_are_contained_as_failed_parse_records() { + let bytes = serde_json::to_vec(&json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/recognize.fake.json", &bytes)]); + + let parsed = with_silenced_panic_hook(|| { + parse_snapshot( + &snapshot, + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + ) + .with_recognize_panic("src/recognize.fake.json")]), + ParseRequest { + languages: language_set(&["json"]), + scope: ParseScope::Snapshot, + }, + ) + }); + + assert!(parsed.units.is_empty()); + assert_eq!(parsed.failed.len(), 1); + assert_eq!( + parsed.failed[0].diagnostics[0].code.as_str(), + "lang.parse.adapter_panic" + ); +} diff --git a/crates/lift/tests/lang_registry.rs b/crates/lift/tests/lang_registry.rs new file mode 100644 index 000000000..174e87c75 --- /dev/null +++ b/crates/lift/tests/lang_registry.rs @@ -0,0 +1,364 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::BTreeSet; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} + +#[path = "../src/pack/mod.rs"] +mod pack; + +#[path = "../src/repo/mod.rs"] +mod repo; + +#[path = "../src/lang/mod.rs"] +mod lang; + +use lang::{ + AdapterCapabilities, AdapterDescriptor, AdapterName, AdapterParseResult, LangError, + LanguageAdapter, LanguageRegistryBuilder, ParseInput, QueryEngineKind, +}; + +struct TestAdapter { + descriptor: AdapterDescriptor, + capabilities: AdapterCapabilities, +} + +impl TestAdapter { + fn new(name: &str, language: pack::LanguageId, version: &str) -> Self { + Self::with_capabilities(name, language, version, AdapterCapabilities::default()) + } + + fn with_capabilities( + name: &str, + language: pack::LanguageId, + version: &str, + capabilities: AdapterCapabilities, + ) -> Self { + Self { + descriptor: AdapterDescriptor { + name: AdapterName::parse(name).expect("adapter name should parse"), + language, + version: version.to_owned(), + }, + capabilities, + } + } +} + +impl LanguageAdapter for TestAdapter { + fn descriptor(&self) -> AdapterDescriptor { + self.descriptor.clone() + } + + fn capabilities(&self) -> AdapterCapabilities { + self.capabilities.clone() + } + + fn recognizes(&self, _input: &ParseInput<'_>) -> bool { + true + } + + fn parse(&self, _input: &ParseInput<'_>) -> AdapterParseResult { + AdapterParseResult::Failed { + diagnostics: Vec::new(), + } + } +} + +#[test] +fn built_in_registry_remains_deterministic_and_empty_before_seam4() { + let first = lang::registry::built_in_registry().expect("built-in registry should build"); + let second = lang::registry::built_in_registry().expect("built-in registry should build"); + + assert_eq!(first.descriptors(), second.descriptors()); + assert!(first.descriptors().is_empty()); + assert!(second.descriptors().is_empty()); + assert!(first.adapter_for_language(pack::LanguageId::Rust).is_none()); + assert!(second + .adapter_for_language(pack::LanguageId::Rust) + .is_none()); +} + +#[test] +fn registry_exposes_capabilities_in_language_scoped_lookup() { + let rust_capabilities = AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: false, + query_engines: BTreeSet::from([QueryEngineKind::TreeSitter]), + }; + let registry = LanguageRegistryBuilder::new() + .register(TestAdapter::with_capabilities( + "builtin.alpha", + pack::LanguageId::Rust, + "1.0.0", + rust_capabilities.clone(), + )) + .expect("rust adapter should register") + .register(TestAdapter::with_capabilities( + "builtin.beta", + pack::LanguageId::Json, + "2.0.0", + AdapterCapabilities { + emits_local_edges: false, + emits_surface_markers: true, + query_engines: BTreeSet::new(), + }, + )) + .expect("json adapter should register") + .build() + .expect("registry should build"); + + assert_eq!( + registry.capabilities_for_language(pack::LanguageId::Rust), + Some(rust_capabilities), + ); + assert_eq!( + registry.capabilities_for_language(pack::LanguageId::Typescript), + None, + ); +} + +#[test] +fn supported_query_engines_lookup_is_language_scoped() { + let registry = LanguageRegistryBuilder::new() + .register(TestAdapter::with_capabilities( + "builtin.alpha", + pack::LanguageId::Rust, + "1.0.0", + AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: false, + query_engines: BTreeSet::from([QueryEngineKind::TreeSitter]), + }, + )) + .expect("rust adapter should register") + .register(TestAdapter::with_capabilities( + "builtin.beta", + pack::LanguageId::Json, + "2.0.0", + AdapterCapabilities::default(), + )) + .expect("json adapter should register") + .build() + .expect("registry should build"); + + assert_eq!( + registry.supported_query_engines_for_language(pack::LanguageId::Rust), + BTreeSet::from([QueryEngineKind::TreeSitter]), + ); + assert_eq!( + registry.supported_query_engines_for_language(pack::LanguageId::Json), + BTreeSet::new(), + ); + assert_eq!( + registry.supported_query_engines_for_language(pack::LanguageId::Typescript), + BTreeSet::new(), + ); +} + +#[test] +fn duplicate_adapter_name_is_rejected() { + let registration = LanguageRegistryBuilder::new() + .register(TestAdapter::new( + "builtin.alpha", + pack::LanguageId::Rust, + "1.0.0", + )) + .expect("first adapter should register") + .register(TestAdapter::new( + "builtin.alpha", + pack::LanguageId::Json, + "1.0.0", + )); + let error = match registration { + Ok(_) => panic!("duplicate name should fail"), + Err(error) => error, + }; + + assert_eq!( + error, + LangError::DuplicateAdapterName { + name: "builtin.alpha".to_owned(), + } + ); +} + +#[test] +fn duplicate_language_registration_is_rejected() { + let registration = LanguageRegistryBuilder::new() + .register(TestAdapter::new( + "builtin.alpha", + pack::LanguageId::Rust, + "1.0.0", + )) + .expect("first adapter should register") + .register(TestAdapter::new( + "builtin.beta", + pack::LanguageId::Rust, + "1.0.0", + )); + let error = match registration { + Ok(_) => panic!("duplicate language should fail"), + Err(error) => error, + }; + + assert_eq!( + error, + LangError::DuplicateLanguageAdapter { + language: pack::LanguageId::Rust, + existing: "builtin.alpha".to_owned(), + duplicate: "builtin.beta".to_owned(), + } + ); +} + +#[test] +fn descriptor_order_is_unchanged_by_capability_metadata() { + let forward = LanguageRegistryBuilder::new() + .register(TestAdapter::with_capabilities( + "builtin.zeta", + pack::LanguageId::Json, + "1.0.0", + AdapterCapabilities { + emits_local_edges: false, + emits_surface_markers: true, + query_engines: BTreeSet::new(), + }, + )) + .expect("zeta should register") + .register(TestAdapter::with_capabilities( + "builtin.alpha", + pack::LanguageId::Rust, + "1.1.0", + AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: false, + query_engines: BTreeSet::from([QueryEngineKind::TreeSitter]), + }, + )) + .expect("alpha should register") + .register(TestAdapter::with_capabilities( + "builtin.gamma", + pack::LanguageId::Typescript, + "1.2.0", + AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: true, + query_engines: BTreeSet::from([QueryEngineKind::TreeSitter]), + }, + )) + .expect("gamma should register") + .build() + .expect("registry should build"); + + let reverse = LanguageRegistryBuilder::new() + .register(TestAdapter::with_capabilities( + "builtin.gamma", + pack::LanguageId::Typescript, + "1.2.0", + AdapterCapabilities { + emits_local_edges: false, + emits_surface_markers: false, + query_engines: BTreeSet::new(), + }, + )) + .expect("gamma should register") + .register(TestAdapter::with_capabilities( + "builtin.alpha", + pack::LanguageId::Rust, + "1.1.0", + AdapterCapabilities::default(), + )) + .expect("alpha should register") + .register(TestAdapter::with_capabilities( + "builtin.zeta", + pack::LanguageId::Json, + "1.0.0", + AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: true, + query_engines: BTreeSet::from([QueryEngineKind::TreeSitter]), + }, + )) + .expect("zeta should register") + .build() + .expect("registry should build"); + + let expected_names = vec![ + "builtin.alpha".to_owned(), + "builtin.gamma".to_owned(), + "builtin.zeta".to_owned(), + ]; + + assert_eq!( + forward + .descriptors() + .iter() + .map(|descriptor| descriptor.name.as_str().to_owned()) + .collect::>(), + expected_names, + ); + assert_eq!(forward.descriptors(), reverse.descriptors()); +} + +#[test] +fn canonical_built_in_language_order_is_frozen() { + assert_eq!( + lang::registry::BUILT_IN_LANGUAGE_ORDER, + &[ + pack::LanguageId::Json, + pack::LanguageId::Toml, + pack::LanguageId::Yaml, + pack::LanguageId::Rust, + pack::LanguageId::Python, + pack::LanguageId::Javascript, + pack::LanguageId::Typescript, + ], + ); +} + +#[test] +fn adapter_lookup_returns_registered_language_adapter() { + let registry = LanguageRegistryBuilder::new() + .register(TestAdapter::new( + "builtin.alpha", + pack::LanguageId::Rust, + "1.0.0", + )) + .expect("rust adapter should register") + .register(TestAdapter::new( + "builtin.beta", + pack::LanguageId::Json, + "2.0.0", + )) + .expect("json adapter should register") + .build() + .expect("registry should build"); + + let rust_adapter = registry + .adapter_for_language(pack::LanguageId::Rust) + .expect("rust adapter should exist") + .descriptor(); + assert_eq!(rust_adapter.name.as_str(), "builtin.alpha"); + assert_eq!(rust_adapter.language, pack::LanguageId::Rust); + assert_eq!(rust_adapter.version, "1.0.0"); + + assert!(registry + .adapter_for_language(pack::LanguageId::Typescript) + .is_none()); +} diff --git a/crates/lift/tests/lang_schema.rs b/crates/lift/tests/lang_schema.rs new file mode 100644 index 000000000..4214b0222 --- /dev/null +++ b/crates/lift/tests/lang_schema.rs @@ -0,0 +1,539 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::{BTreeSet, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; + +use assert_cmd as _; +use gix as _; +use jsonschema::{Retrieve, Uri, Validator}; +use predicates as _; +use serde::Deserialize; +use serde_jcs as _; +use serde_json::{Map, Value}; +use sha2 as _; +use substrate_lift as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} + +#[path = "../src/pack/mod.rs"] +mod pack; + +#[path = "../src/repo/mod.rs"] +mod repo; + +#[path = "../src/lang/mod.rs"] +mod lang; + +#[path = "support/repo_support.rs"] +mod repo_support; + +#[derive(Clone, Debug, Deserialize)] +struct ParseManifest { + version: u32, + case: String, + #[serde(flatten)] + parse_set: lang::ParseSet, +} + +#[derive(Clone, Debug, Deserialize)] +struct FakeDocument { + #[serde(default)] + fail: bool, + #[serde(default)] + symbols: Vec, +} + +#[derive(Clone, Debug)] +struct FakeAdapter { + descriptor: lang::AdapterDescriptor, + capabilities: lang::AdapterCapabilities, + suffix: &'static str, +} + +impl FakeAdapter { + fn new(name: &str, language: &str, suffix: &'static str, version: &str) -> Self { + Self { + descriptor: lang::AdapterDescriptor { + name: lang::AdapterName::parse(name).expect("adapter name"), + language: pack::LanguageId::parse(language).expect("language"), + version: version.to_owned(), + }, + capabilities: lang::AdapterCapabilities::default(), + suffix, + } + } + + fn with_capabilities(mut self, capabilities: lang::AdapterCapabilities) -> Self { + self.capabilities = capabilities; + self + } +} + +impl lang::LanguageAdapter for FakeAdapter { + fn descriptor(&self) -> lang::AdapterDescriptor { + self.descriptor.clone() + } + + fn capabilities(&self) -> lang::AdapterCapabilities { + self.capabilities.clone() + } + + fn recognizes(&self, input: &lang::ParseInput<'_>) -> bool { + input.path.as_str().ends_with(self.suffix) + } + + fn parse(&self, input: &lang::ParseInput<'_>) -> lang::AdapterParseResult { + let document: FakeDocument = match serde_json::from_slice(input.bytes) { + Ok(document) => document, + Err(_) => { + return lang::AdapterParseResult::Failed { + diagnostics: Vec::new(), + }; + } + }; + + if document.fail { + return lang::AdapterParseResult::Failed { + diagnostics: Vec::new(), + }; + } + + lang::AdapterParseResult::Parsed(lang::AdapterParseOutput { + symbols: document.symbols, + edges: Vec::new(), + surface_markers: Vec::new(), + diagnostics: Vec::new(), + }) + } +} + +#[test] +fn embedded_lang_schemas_match_disk() { + assert_eq!( + lang::LANG_PARSE_MANIFEST_V1_SCHEMA_JSON, + load_text("schemas/lang/parse_manifest.v1.json") + ); + assert_eq!( + lang::LANG_PARSE_MANIFEST_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/lang/parse_manifest.v1.json" + ); + assert_eq!( + lang::LANG_PARSE_MANIFEST_V1_SCHEMA_FILE, + "parse_manifest.v1.json" + ); + assert_eq!(lang::LANG_PARSE_MANIFEST_V1_SCHEMA_VERSION, 1); + + assert_eq!( + lang::LANG_PARSE_MANIFEST_V2_SCHEMA_JSON, + load_text("schemas/lang/parse_manifest.v2.json") + ); + assert_eq!( + lang::LANG_PARSE_MANIFEST_V2_SCHEMA_ID, + "https://schemas.substrate.dev/lift/lang/parse_manifest.v2.json" + ); + assert_eq!( + lang::LANG_PARSE_MANIFEST_V2_SCHEMA_FILE, + "parse_manifest.v2.json" + ); + assert_eq!(lang::LANG_PARSE_MANIFEST_V2_SCHEMA_VERSION, 2); +} + +#[test] +fn lang_module_re_exports_v2_schema_constants() { + assert_eq!( + lang::LANG_PARSE_MANIFEST_V2_SCHEMA_JSON, + load_text("schemas/lang/parse_manifest.v2.json") + ); + assert_eq!( + lang::LANG_PARSE_MANIFEST_V2_SCHEMA_ID, + "https://schemas.substrate.dev/lift/lang/parse_manifest.v2.json" + ); + assert_eq!( + lang::LANG_PARSE_MANIFEST_V2_SCHEMA_FILE, + "parse_manifest.v2.json" + ); + assert_eq!(lang::LANG_PARSE_MANIFEST_V2_SCHEMA_VERSION, 2); +} + +#[test] +fn valid_lang_schema_v1_fixtures_validate_and_deserialize() { + for fixture in [ + "fixtures/lang/valid/snapshot_parse_manifest.json", + "fixtures/lang/valid/paths_scope_parse_manifest.json", + ] { + let instance = assert_schema_valid("schemas/lang/parse_manifest.v1.json", fixture); + let manifest: ParseManifest = + serde_json::from_value(instance.clone()).expect("fixture should deserialize"); + + assert_eq!(manifest.version, 1); + assert_manifest_invariants(&manifest, fixture); + assert!( + instance["stats"].get("cache_hits").is_none(), + "unexpected cache_hits in {fixture}" + ); + assert!( + instance["stats"].get("cache_misses").is_none(), + "unexpected cache_misses in {fixture}" + ); + } +} + +#[test] +fn valid_lang_schema_v2_fixtures_validate_and_deserialize() { + for (fixture, expected_hits, expected_misses) in [ + ( + "fixtures/lang/valid/snapshot_parse_manifest_v2.json", + 1_u64, + 2_u64, + ), + ( + "fixtures/lang/valid/paths_scope_parse_manifest_v2.json", + 0_u64, + 0_u64, + ), + ] { + let instance = assert_schema_valid("schemas/lang/parse_manifest.v2.json", fixture); + let manifest: ParseManifest = + serde_json::from_value(instance.clone()).expect("fixture should deserialize"); + + assert_eq!(manifest.version, 2); + assert_manifest_invariants(&manifest, fixture); + assert_eq!( + instance["stats"]["cache_hits"].as_u64(), + Some(expected_hits) + ); + assert_eq!( + instance["stats"]["cache_misses"].as_u64(), + Some(expected_misses) + ); + } +} + +#[test] +fn invalid_lang_schema_fixtures_fail_validation() { + for fixture in [ + "fixtures/lang/invalid/adapter_name_invalid.json", + "fixtures/lang/invalid/request_scope_missing_paths.json", + "fixtures/lang/invalid/reference_target_invalid_shape.json", + "fixtures/lang/invalid/top_level_unknown_field.json", + ] { + let instance = load_json(fixture); + assert_schema_invalid("schemas/lang/parse_manifest.v1.json", &instance, fixture); + } +} + +#[test] +fn phase_c_preserves_parse_manifest_v2_shape() { + let bytes = serde_json::to_vec(&serde_json::json!({ + "symbols": [fake_symbol("main", "alpha", 0, 5)] + })) + .expect("json"); + let (_temp, snapshot) = snapshot_with_files(&[("src/app.fake.json", &bytes)]); + let parse_set = parse_with_driver( + registry(vec![FakeAdapter::new( + "builtin.fake_json", + "json", + ".fake.json", + "1.0.0", + ) + .with_capabilities(lang::AdapterCapabilities { + emits_local_edges: true, + emits_surface_markers: true, + query_engines: [lang::QueryEngineKind::TreeSitter].into_iter().collect(), + })]), + &snapshot, + lang::ParseRequest { + languages: language_set(&["json"]), + scope: lang::ParseScope::Snapshot, + }, + ); + + let parse_set_json = serde_json::to_value(&parse_set).expect("parse set should serialize"); + assert_parse_set_shape(&parse_set_json); + assert!( + parse_set_json.get("capabilities").is_none(), + "parse_set should not serialize adapter capabilities" + ); + assert!( + parse_set_json["request"].get("query_engines").is_none(), + "request should not serialize adapter query engine metadata" + ); + assert!( + parse_set_json["units"][0].get("capabilities").is_none(), + "parsed units should not serialize adapter capabilities" + ); + assert!( + parse_set_json["units"][0].get("query_engines").is_none(), + "parsed units should not serialize adapter query engine metadata" + ); + assert!( + parse_set_json["units"][0] + .get("emits_local_edges") + .is_none(), + "parsed units should not serialize edge capability flags" + ); + assert!( + parse_set_json["units"][0] + .get("emits_surface_markers") + .is_none(), + "parsed units should not serialize surface marker capability flags" + ); + assert!( + parse_set_json["units"][0]["adapter"].is_string(), + "adapter field should remain the manifest string identifier" + ); + + let manifest = manifest_v2("phase_c_preserves_parse_manifest_v2_shape", &parse_set); + let validator = schema_validator("schemas/lang/parse_manifest.v2.json"); + if let Err(error) = validator.validate(&manifest) { + panic!("expected phase-c manifest to validate: {error}"); + } +} + +fn assert_manifest_invariants(manifest: &ParseManifest, fixture: &str) { + assert!( + !manifest.case.is_empty(), + "fixture case should not be empty" + ); + assert_eq!( + manifest + .parse_set + .request + .fingerprint() + .expect("request fingerprint"), + manifest.parse_set.request_fingerprint, + "request_fingerprint mismatch in {fixture}" + ); + + for unit in &manifest.parse_set.units { + assert_eq!( + unit.fingerprint().expect("unit fingerprint"), + unit.unit_fingerprint, + "unit_fingerprint mismatch in {fixture} for {}", + unit.path.as_str() + ); + } + + assert_eq!( + manifest.parse_set.stats.parsed_units, + manifest.parse_set.units.len() as u64, + "parsed_units mismatch in {fixture}" + ); + assert_eq!( + manifest.parse_set.stats.failed_units, + manifest.parse_set.failed.len() as u64, + "failed_units mismatch in {fixture}" + ); + assert_eq!( + manifest.parse_set.stats.missing_requested_languages, + manifest.parse_set.missing_languages.len() as u64, + "missing_requested_languages mismatch in {fixture}" + ); + + let expected_diagnostic_count = manifest.parse_set.diagnostics.len() as u64 + + manifest + .parse_set + .units + .iter() + .map(|unit| unit.diagnostics.len() as u64) + .sum::() + + manifest + .parse_set + .failed + .iter() + .map(|failed| failed.diagnostics.len() as u64) + .sum::(); + assert_eq!( + manifest.parse_set.stats.diagnostic_count, expected_diagnostic_count, + "diagnostic_count mismatch in {fixture}" + ); +} + +fn assert_schema_valid(schema: &str, fixture: &str) -> Value { + let instance = load_json(fixture); + let validator = schema_validator(schema); + if let Err(error) = validator.validate(&instance) { + panic!("expected fixture to validate: {fixture}: {error}"); + } + instance +} + +fn assert_schema_invalid(schema: &str, instance: &Value, fixture: &str) { + let validator = schema_validator(schema); + assert!( + validator.validate(instance).is_err(), + "expected fixture to fail schema validation: {fixture}" + ); +} + +fn schema_validator(schema: &str) -> Validator { + let root_schema = load_json(schema); + let kernel_schema = load_json("schemas/kernel/primitives.v1.json"); + let retriever = InMemoryRetriever { + schemas: HashMap::from([ + ( + kernel::PRIMITIVES_V1_SCHEMA_ID.to_owned(), + kernel_schema.clone(), + ), + ("../kernel/primitives.v1.json".to_owned(), kernel_schema), + ]), + }; + + jsonschema::draft202012::options() + .with_retriever(retriever) + .build(&root_schema) + .expect("schema should compile") +} + +#[derive(Clone, Debug)] +struct InMemoryRetriever { + schemas: HashMap, +} + +impl Retrieve for InMemoryRetriever { + fn retrieve( + &self, + uri: &Uri, + ) -> Result> { + self.schemas + .get(uri.as_str()) + .cloned() + .ok_or_else(|| format!("schema not found: {uri}").into()) + } +} + +fn load_json(relative: &str) -> Value { + let path = crate_root().join(relative); + let contents = fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }); + + serde_json::from_str(&contents).unwrap_or_else(|err| { + panic!("failed to parse {} as JSON: {err}", path.display()); + }) +} + +fn load_text(relative: &str) -> String { + let path = crate_root().join(relative); + fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }) +} + +fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} + +fn registry(adapters: Vec) -> lang::LanguageRegistry { + let mut builder = lang::LanguageRegistryBuilder::new(); + for adapter in adapters { + builder = builder.register(adapter).expect("register adapter"); + } + builder.build().expect("build registry") +} + +fn parse_with_driver( + registry: lang::LanguageRegistry, + snapshot: &repo::RepoSnapshot, + request: lang::ParseRequest, +) -> lang::ParseSet { + lang::ParseDriver::new(registry) + .parse_snapshot(snapshot, &request) + .expect("parse snapshot") +} + +fn snapshot_with_files(files: &[(&str, &[u8])]) -> (repo_support::TempDir, repo::RepoSnapshot) { + let temp = repo_support::TempDir::new("lang-schema"); + repo_support::write_file(&temp.path().join(".git/HEAD"), b"ref: refs/heads/main\n"); + for (path, bytes) in files { + repo_support::write_file(&temp.path().join(path), bytes); + } + let snapshot = repo_support::materialize(temp.path(), repo_support::default_snapshot_options()); + (temp, snapshot) +} + +fn language_set(languages: &[&str]) -> BTreeSet { + languages + .iter() + .map(|language| pack::LanguageId::parse(language).expect("language")) + .collect() +} + +fn fake_symbol(local_key: &str, name: &str, start: u64, end: u64) -> serde_json::Value { + serde_json::json!({ + "local_key": local_key, + "kind": "function", + "name": name, + "path": [name], + "span": { + "start_byte": start, + "end_byte": end + }, + "visibility": "public" + }) +} + +fn manifest_v2(case: &str, parse_set: &lang::ParseSet) -> Value { + let Value::Object(parse_set_fields) = + serde_json::to_value(parse_set).expect("parse set should serialize") + else { + panic!("parse set should serialize to an object"); + }; + let mut manifest = Map::from_iter([ + ("version".to_owned(), Value::from(2)), + ("case".to_owned(), Value::from(case.to_owned())), + ]); + manifest.extend(parse_set_fields); + Value::Object(manifest) +} + +fn assert_parse_set_shape(parse_set: &Value) { + let top_level_keys = parse_set + .as_object() + .expect("parse set should serialize to an object") + .keys() + .cloned() + .collect::>(); + assert_eq!( + top_level_keys, + BTreeSet::from([ + "diagnostics".to_owned(), + "failed".to_owned(), + "missing_languages".to_owned(), + "request".to_owned(), + "request_fingerprint".to_owned(), + "skipped".to_owned(), + "snapshot_fingerprint".to_owned(), + "stats".to_owned(), + "units".to_owned(), + ]) + ); + + let unit_keys = parse_set["units"][0] + .as_object() + .expect("parsed unit should serialize to an object") + .keys() + .cloned() + .collect::>(); + assert_eq!( + unit_keys, + BTreeSet::from([ + "adapter".to_owned(), + "adapter_version".to_owned(), + "blob_fingerprint".to_owned(), + "diagnostics".to_owned(), + "edges".to_owned(), + "file_id".to_owned(), + "language".to_owned(), + "path".to_owned(), + "surface_markers".to_owned(), + "symbols".to_owned(), + "unit_fingerprint".to_owned(), + ]) + ); +} diff --git a/crates/lift/tests/pack_bundle.rs b/crates/lift/tests/pack_bundle.rs new file mode 100644 index 000000000..bdf0789ca --- /dev/null +++ b/crates/lift/tests/pack_bundle.rs @@ -0,0 +1,470 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::{ + sha256_bytes, sha256_canonical_json, DiagnosticCode, Fingerprint, JsonPointer, QueryId, + RecipeId, RuleId, Severity, + }; +} +#[path = "../src/pack/mod.rs"] +mod pack; + +#[test] +fn bundle_resolution_pulls_transitive_query_closure() { + let compiler = pack::PackCompiler::new(); + let profile = compile_profile( + &compiler, + "fixtures/pack/valid/bundle/profile_advanced_file_backed.toml", + ); + + let bundle = compiler + .resolve_profile_pack_set(&profile) + .expect("bundle should resolve"); + + assert!(bundle.score_model.is_some()); + assert_eq!(bundle.query_packs.len(), 1); + assert_eq!(bundle.rule_packs.len(), 1); + assert_eq!(bundle.recipe_packs.len(), 1); + assert!(bundle + .query_packs + .contains_key(&pack::PackName::parse("acme/rust-core").expect("name"))); +} + +#[test] +fn bundle_resolution_dedupes_same_source_references() { + let compiler = pack::PackCompiler::new(); + let profile = compile_profile( + &compiler, + "fixtures/pack/valid/bundle/profile_advanced_dedupe.toml", + ); + + let bundle = compiler + .resolve_profile_pack_set(&profile) + .expect("bundle should resolve"); + + assert_eq!(bundle.query_packs.len(), 1); + assert_eq!(bundle.rule_packs.len(), 1); + assert_eq!(bundle.recipe_packs.len(), 1); +} + +#[test] +fn missing_bundle_ref_fails_with_unknown_pack_reference() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("bundle-missing"); + write_text( + &dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "acme/missing-bundle" +name = "Missing bundle" + +[queries] +packs = ["file:queries/missing.json"] +"#, + ); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: dir.join("profile.toml"), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_pack_set(&profile) + .expect_err("missing bundle ref should fail"); + + match error { + pack::PackError::UnknownPackReference { + referring_pack, + reference, + } => { + assert_eq!(referring_pack, "acme/missing-bundle"); + assert_eq!(reference, "file:queries/missing.json"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn wrong_kind_bundle_ref_fails_with_ref_kind_mismatch() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("bundle-wrong-kind"); + write_text( + &dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "acme/wrong-kind-bundle" +name = "Wrong kind bundle" + +[queries] +packs = ["file:generic_policy.json"] +"#, + ); + write_text( + &dir.join("generic_policy.json"), + &rule_pack_json("file:queries/rust_core.json"), + ); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: dir.join("profile.toml"), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_pack_set(&profile) + .expect_err("wrong kind should fail"); + + match error { + pack::PackError::RefKindMismatch { + reference, + expected, + actual, + } => { + assert_eq!(reference, "file:generic_policy.json"); + assert_eq!(expected, pack::PackKind::QueryPack); + assert_eq!(actual, pack::PackKind::RulePack); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn duplicate_pack_ids_across_origins_fail_bundle_resolution() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("bundle-duplicate"); + fs::create_dir_all(dir.join("queries")).expect("queries dir"); + fs::create_dir_all(dir.join("queries_dup")).expect("queries dup dir"); + write_text( + &dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "acme/duplicate-bundle" +name = "Duplicate bundle" + +[queries] +packs = ["file:queries/rust_core.json"] + +[rules] +packs = ["file:generic_policy.json"] +"#, + ); + write_text( + &dir.join("queries/rust_core.json"), + &query_pack_json("dup/core"), + ); + write_text( + &dir.join("queries_dup/rust_core.json"), + &query_pack_json("dup/core"), + ); + write_text( + &dir.join("generic_policy.json"), + &rule_pack_json("file:queries_dup/rust_core.json"), + ); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: dir.join("profile.toml"), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_pack_set(&profile) + .expect_err("duplicate pack ids should fail"); + + match error { + pack::PackError::DuplicatePackId { kind, id } => { + assert_eq!(kind, pack::PackKind::QueryPack); + assert_eq!(id, "dup/core"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn duplicate_pack_id_against_profile_fails_bundle_resolution() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("bundle-duplicate-profile-id"); + fs::create_dir_all(dir.join("queries")).expect("queries dir"); + write_text( + &dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "dup/shared" +name = "Duplicate bundle" + +[queries] +packs = ["file:queries/rust_core.json"] +"#, + ); + write_text( + &dir.join("queries/rust_core.json"), + &query_pack_json("dup/shared"), + ); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: dir.join("profile.toml"), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_pack_set(&profile) + .expect_err("duplicate profile id should fail"); + + match error { + pack::PackError::DuplicatePackId { kind, id } => { + assert_eq!(kind, pack::PackKind::QueryPack); + assert_eq!(id, "dup/shared"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn duplicate_pack_id_against_boundary_taxonomy_fails_bundle_resolution() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("bundle-duplicate-boundary-id"); + fs::create_dir_all(dir.join("queries")).expect("queries dir"); + fs::create_dir_all(dir.join("topology")).expect("topology dir"); + write_text( + &dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "acme/duplicate-boundary-bundle" +name = "Duplicate boundary bundle" + +[topology] +boundary_taxonomy = "file:topology/boundary-taxonomy.json" + +[queries] +packs = ["file:queries/rust_core.json"] +"#, + ); + write_text( + &dir.join("topology/boundary-taxonomy.json"), + &boundary_taxonomy_json("dup/shared"), + ); + write_text( + &dir.join("queries/rust_core.json"), + &query_pack_json("dup/shared"), + ); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: dir.join("profile.toml"), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_pack_set(&profile) + .expect_err("duplicate boundary taxonomy id should fail"); + + match error { + pack::PackError::DuplicatePackId { kind, id } => { + assert_eq!(kind, pack::PackKind::QueryPack); + assert_eq!(id, "dup/shared"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn duplicate_pack_id_against_component_map_fails_bundle_resolution() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("bundle-duplicate-component-id"); + fs::create_dir_all(dir.join("queries")).expect("queries dir"); + fs::create_dir_all(dir.join("topology")).expect("topology dir"); + write_text( + &dir.join("profile.toml"), + r#"kind = "profile" +version = 1 +id = "acme/duplicate-component-bundle" +name = "Duplicate component bundle" + +[topology] +component_map = "file:topology/component-map.json" + +[queries] +packs = ["file:queries/rust_core.json"] +"#, + ); + write_text( + &dir.join("topology/component-map.json"), + &component_map_json("dup/shared"), + ); + write_text( + &dir.join("queries/rust_core.json"), + &query_pack_json("dup/shared"), + ); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: dir.join("profile.toml"), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_pack_set(&profile) + .expect_err("duplicate component map id should fail"); + + match error { + pack::PackError::DuplicatePackId { kind, id } => { + assert_eq!(kind, pack::PackKind::QueryPack); + assert_eq!(id, "dup/shared"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +fn compile_profile(compiler: &pack::PackCompiler, relative: &str) -> pack::CompiledProfile { + compiler + .compile_profile(pack::PackSource::File { + path: crate_root().join(relative), + format_hint: None, + }) + .unwrap_or_else(|err| panic!("failed to compile profile {relative}: {err:?}")) +} + +fn query_pack_json(id: &str) -> String { + format!( + r#"{{ + "$schema": "https://schemas.substrate.dev/lift/pack/query_pack.v1.json", + "kind": "query_pack", + "version": 1, + "id": "{id}", + "name": "Query pack", + "language": "rust", + "engine": "tree_sitter", + "queries": [ + {{ + "id": "use_statement", + "pattern": "(use_declaration) @use", + "captures": [{{ "name": "use", "required": true }}] + }} + ] +}}"# + ) +} + +fn rule_pack_json(query_pack_ref: &str) -> String { + format!( + r#"{{ + "$schema": "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json", + "kind": "rule_pack", + "version": 1, + "id": "acme/policy", + "name": "Policy rules", + "rules": [ + {{ + "id": "architecture.cross_boundary_import", + "severity": "warning", + "query": {{ + "pack": "{query_pack_ref}", + "id": "use_statement" + }}, + "emit": [ + {{ + "kind": "finding", + "code": "architecture.cross_boundary_import", + "message": "Import crosses boundary" + }} + ] + }} + ] +}}"# + ) +} + +fn boundary_taxonomy_json(id: &str) -> String { + format!( + r#"{{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "{id}", + "name": "Boundary taxonomy", + "counting": {{ + "mode": "distinct_minus_one" + }}, + "boundaries": [ + {{ + "id": "runtime", + "label": "Runtime", + "include": ["services/runtime/**"] + }} + ] +}}"# + ) +} + +fn component_map_json(id: &str) -> String { + format!( + r#"{{ + "kind": "component_map", + "version": 1, + "id": "{id}", + "name": "Component map", + "counting": {{ + "mode": "distinct" + }}, + "components": [ + {{ + "id": "runtime", + "label": "Runtime", + "include": ["services/runtime/**"], + "tags": ["internal"] + }} + ] +}}"# + ) +} + +fn write_text(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create parent dirs"); + } + fs::write(path, contents).unwrap_or_else(|err| { + panic!("failed to write {}: {err}", path.display()); + }); +} + +fn unique_temp_dir(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_nanos(); + let dir = std::env::temp_dir().join(format!("substrate-lift-{label}-{nonce}")); + fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +fn crate_root() -> PathBuf { + static CWD_LOCK: OnceLock> = OnceLock::new(); + let lock = CWD_LOCK.get_or_init(|| Mutex::new(())); + let _guard: MutexGuard<'_, ()> = lock.lock().expect("lock"); + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} diff --git a/crates/lift/tests/pack_compile.rs b/crates/lift/tests/pack_compile.rs new file mode 100644 index 000000000..9220a9650 --- /dev/null +++ b/crates/lift/tests/pack_compile.rs @@ -0,0 +1,829 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::{ + sha256_bytes, sha256_canonical_json, DiagnosticCode, Fingerprint, JsonPointer, QueryId, + RecipeId, RuleId, Severity, + }; +} +#[path = "../src/pack/mod.rs"] +mod pack; + +#[test] +fn builtin_file_and_inline_profiles_compile_on_the_happy_path() { + let compiler = pack::PackCompiler::new(); + + let builtin = compiler + .compile_profile(pack::builtin::profile_source("generic/default").expect("builtin")) + .expect("builtin profile should compile"); + let file = compiler + .compile_profile(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/profile_file_backed.toml"), + format_hint: None, + }) + .expect("file profile should compile"); + let inline = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "inline-happy".to_owned(), + format: pack::PackFormat::Toml, + bytes: load_bytes("fixtures/pack/canonical/generic_default_order_b.toml"), + }) + .expect("inline profile should compile"); + + assert_eq!(builtin.header.id.as_str(), "generic/default"); + assert_eq!(file.header.id.as_str(), "acme/file-backed"); + assert_eq!(inline.header.id.as_str(), "generic/default"); + assert_eq!(file.header.origin.display(), file_fixture_display_path()); + assert!(matches!( + file.topology.boundary_taxonomy, + Some(pack::PackRef::File(_)) + )); + assert!(matches!(file.score.model, Some(pack::PackRef::File(_)))); + assert_eq!(file.includes.rule_packs.len(), 1); + assert!(file + .apps + .enabled + .contains(&pack::AppName::parse("score").expect("app"))); +} + +#[test] +fn standalone_topology_packs_compile_on_the_happy_path() { + let compiler = pack::PackCompiler::new(); + + let builtin_boundary = compiler + .compile_boundary_taxonomy( + pack::builtin::boundary_taxonomy_source("generic/boundaries").expect("builtin"), + ) + .expect("builtin boundary taxonomy should compile"); + let file_boundary = compiler + .compile_boundary_taxonomy(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/topology/boundary-taxonomy.json"), + format_hint: None, + }) + .expect("file boundary taxonomy should compile"); + let inline_boundary = compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: "inline-boundary".to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes("fixtures/pack/valid/generic_boundaries.json"), + }) + .expect("inline boundary taxonomy should compile"); + + assert_eq!(builtin_boundary.header.id.as_str(), "generic/boundaries"); + assert_eq!(file_boundary.header.id.as_str(), "acme/boundaries"); + assert_eq!(inline_boundary.header.id.as_str(), "generic/boundaries"); + assert_eq!(file_boundary.boundaries.len(), 2); + assert!(file_boundary + .boundaries + .values() + .next() + .expect("boundary") + .include_matcher + .is_match(Path::new("services/runtime/main.rs"))); + + let builtin_component = compiler + .compile_component_map( + pack::builtin::component_map_source("generic/components").expect("builtin"), + ) + .expect("builtin component map should compile"); + let file_component = compiler + .compile_component_map(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/topology/component-map.json"), + format_hint: None, + }) + .expect("file component map should compile"); + let inline_component = compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: "inline-component".to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes("fixtures/pack/valid/generic_components.json"), + }) + .expect("inline component map should compile"); + + assert_eq!(builtin_component.header.id.as_str(), "generic/components"); + assert_eq!(file_component.header.id.as_str(), "acme/components"); + assert_eq!(inline_component.header.id.as_str(), "generic/components"); + assert!(file_component + .components + .values() + .any(|component| component.tags.contains("public"))); +} + +#[test] +fn standalone_advanced_packs_compile_on_the_happy_path() { + let compiler = pack::PackCompiler::new(); + + let score = compiler + .compile_score_model(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/score/generic_lift_v2.json"), + format_hint: None, + }) + .expect("score model should compile"); + assert_eq!(score.header.id.as_str(), "generic/lift-v2"); + assert_eq!(score.vector_version, 2); + assert_eq!(score.triggers.len(), 1); + + let query = compiler + .compile_query_pack(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/queries/rust_core.json"), + format_hint: None, + }) + .expect("query pack should compile"); + assert_eq!(query.header.id.as_str(), "rust/core"); + assert_eq!(query.queries.len(), 1); + + let rule = compiler + .compile_rule_pack(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/rules/generic_policy.json"), + format_hint: None, + }) + .expect("rule pack should compile"); + let rule_def = rule.rules.values().next().expect("rule"); + assert_eq!(rule.header.id.as_str(), "generic/policy"); + assert_eq!(rule.rules.len(), 1); + assert_eq!(rule_def.query.pack.as_str(), "file:queries/rust_core.json"); + + let recipe = compiler + .compile_recipe_pack(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/recipes/generic_core_recipes.json"), + format_hint: None, + }) + .expect("recipe pack should compile"); + let recipe_def = recipe.recipes.values().next().expect("recipe"); + assert_eq!(recipe.header.id.as_str(), "generic/core-recipes"); + assert_eq!(recipe.recipes.len(), 1); + assert_eq!( + recipe_def.query.pack.as_str(), + "file:queries/rust_core.json" + ); +} + +#[test] +fn advanced_pack_compile_failures_are_typed() { + let compiler = pack::PackCompiler::new(); + + let bad_expr = compiler + .compile_score_model(pack::PackSource::File { + path: crate_root().join("fixtures/pack/invalid/score_model_bad_expr.json"), + format_hint: None, + }) + .expect_err("bad expr should fail"); + assert!(matches!( + bad_expr, + pack::PackError::ExpressionCompile { .. } + )); + + let duplicate_trigger = compiler + .compile_score_model(pack::PackSource::File { + path: crate_root().join("fixtures/pack/invalid/score_model_duplicate_trigger.json"), + format_hint: None, + }) + .expect_err("duplicate trigger should fail"); + assert_duplicate_entry( + duplicate_trigger, + pack::PackKind::ScoreModel, + "acme/duplicate-trigger", + "trigger", + "dup", + ); + + let duplicate_query = compiler + .compile_query_pack(pack::PackSource::File { + path: crate_root().join("fixtures/pack/invalid/query_pack_duplicate_id.json"), + format_hint: None, + }) + .expect_err("duplicate query ids should fail"); + assert_duplicate_entry( + duplicate_query, + pack::PackKind::QueryPack, + "acme/duplicate-query", + "query", + "dup", + ); + + let invalid_severity = compiler + .compile_rule_pack(pack::PackSource::File { + path: crate_root().join("fixtures/pack/invalid/rule_pack_invalid_severity.json"), + format_hint: None, + }) + .expect_err("invalid severity should fail"); + assert!(matches!( + invalid_severity, + pack::PackError::SchemaViolation { .. } + )); + + let invalid_transform = compiler + .compile_recipe_pack(pack::PackSource::File { + path: crate_root().join("fixtures/pack/invalid/recipe_pack_bad_transform.json"), + format_hint: None, + }) + .expect_err("invalid transform should fail"); + assert!(matches!( + invalid_transform, + pack::PackError::SchemaViolation { .. } + )); +} + +#[test] +fn unsupported_format_and_missing_file_hard_fail() { + let compiler = pack::PackCompiler::new(); + + let unsupported = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "inline-json".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{}"#.to_vec(), + }) + .expect_err("json input should fail in phase a"); + assert!(matches!( + unsupported, + pack::PackError::UnsupportedFormat { .. } + )); + + let missing = compiler + .compile_profile(pack::PackSource::File { + path: crate_root().join("fixtures/pack/invalid/missing-profile.toml"), + format_hint: None, + }) + .expect_err("missing file should fail"); + assert!(matches!(missing, pack::PackError::Io { .. })); +} + +#[test] +fn malformed_topology_json_reports_parse_failure() { + let compiler = pack::PackCompiler::new(); + + let boundary = compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: "bad-boundary-json".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{"kind":"boundary_taxonomy""#.to_vec(), + }) + .expect_err("malformed boundary taxonomy should fail"); + assert!(matches!(boundary, pack::PackError::ParseFailure { .. })); + + let component = compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: "bad-component-json".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{"kind":"component_map""#.to_vec(), + }) + .expect_err("malformed component map should fail"); + assert!(matches!(component, pack::PackError::ParseFailure { .. })); +} + +#[test] +fn topology_schema_violations_are_sorted_deterministically() { + let compiler = pack::PackCompiler::new(); + + let boundary = compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: "bad-boundary-schema".to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes("fixtures/pack/invalid/boundary_taxonomy_schema_violation.json"), + }) + .expect_err("invalid boundary schema should fail"); + assert_schema_violation_codes( + boundary, + "inline:bad-boundary-schema", + pack::PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + &[ + "/boundaries/0/id", + "/boundaries/0/include/0", + "/boundaries/0/label", + "/counting/mode", + "/name", + "/version", + ], + ); + + let component = compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: "bad-component-schema".to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes("fixtures/pack/invalid/component_map_schema_violation.json"), + }) + .expect_err("invalid component schema should fail"); + assert_schema_violation_codes( + component, + "inline:bad-component-schema", + pack::PACK_COMPONENT_MAP_V1_SCHEMA_ID, + &[ + "/components/0/id", + "/components/0/include/0", + "/components/0/label", + "/components/0/tags/0", + "/counting/mode", + "/name", + "/version", + ], + ); +} + +#[test] +fn topology_unknown_fields_report_pointer_addressable_schema_diagnostics() { + let compiler = pack::PackCompiler::new(); + + let boundary = compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: "boundary-unknown-field".to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes("fixtures/pack/invalid/boundary_taxonomy_unknown_field.json"), + }) + .expect_err("unknown boundary fields should fail"); + assert_schema_violation_with_schema( + boundary, + "inline:boundary-unknown-field", + pack::PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + &[ + ("/boundaries/0/owner", "pack.schema.unknown_field"), + ("/unexpected_root", "pack.schema.unknown_field"), + ], + ); + + let component = compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: "component-unknown-field".to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes("fixtures/pack/invalid/component_map_unknown_field.json"), + }) + .expect_err("unknown component fields should fail"); + assert_schema_violation_with_schema( + component, + "inline:component-unknown-field", + pack::PACK_COMPONENT_MAP_V1_SCHEMA_ID, + &[ + ("/components/0/weight", "pack.schema.unknown_field"), + ("/unexpected_root", "pack.schema.unknown_field"), + ], + ); +} + +#[test] +fn duplicate_topology_entry_ids_fail_with_typed_errors() { + let compiler = pack::PackCompiler::new(); + + let boundary = compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: "duplicate-boundary".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "acme/boundaries", + "name": "Dup boundaries", + "counting": { "mode": "distinct_minus_one" }, + "boundaries": [ + { "id": "runtime", "label": "Runtime", "include": ["services/runtime/**"] }, + { "id": "runtime", "label": "Runtime two", "include": ["services/runtime/v2/**"] } + ] +}"# + .to_vec(), + }) + .expect_err("duplicate boundary ids should fail"); + assert_duplicate_entry( + boundary, + pack::PackKind::BoundaryTaxonomy, + "acme/boundaries", + "boundary", + "runtime", + ); + + let component = compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: "duplicate-component".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{ + "kind": "component_map", + "version": 1, + "id": "acme/components", + "name": "Dup components", + "counting": { "mode": "distinct" }, + "components": [ + { "id": "api", "label": "API", "include": ["api/**"] }, + { "id": "api", "label": "API two", "include": ["api/v2/**"] } + ] +}"# + .to_vec(), + }) + .expect_err("duplicate component ids should fail"); + assert_duplicate_entry( + component, + pack::PackKind::ComponentMap, + "acme/components", + "component", + "api", + ); +} + +#[test] +fn invalid_topology_globs_fail_during_compile() { + let compiler = pack::PackCompiler::new(); + + let invalid_include = compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: "invalid-include".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{ + "kind": "boundary_taxonomy", + "version": 1, + "id": "acme/boundaries", + "name": "Bad include", + "counting": { "mode": "distinct_minus_one" }, + "boundaries": [ + { "id": "runtime", "label": "Runtime", "include": ["["] } + ] +}"# + .to_vec(), + }) + .expect_err("invalid include glob should fail"); + assert!(matches!( + invalid_include, + pack::PackError::GlobCompile { .. } + )); + + let invalid_exclude = compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: "invalid-exclude".to_owned(), + format: pack::PackFormat::Json, + bytes: br#"{ + "kind": "component_map", + "version": 1, + "id": "acme/components", + "name": "Bad exclude", + "counting": { "mode": "distinct" }, + "components": [ + { "id": "api", "label": "API", "include": ["api/**"], "exclude": ["["] } + ] +}"# + .to_vec(), + }) + .expect_err("invalid exclude glob should fail"); + assert!(matches!( + invalid_exclude, + pack::PackError::GlobCompile { .. } + )); +} + +#[test] +fn invalid_toml_reports_typed_diagnostics() { + let compiler = pack::PackCompiler::new(); + let error = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "invalid-toml".to_owned(), + format: pack::PackFormat::Toml, + bytes: load_bytes("fixtures/pack/invalid/profile_invalid_toml.toml"), + }) + .expect_err("invalid toml should fail"); + + match error { + pack::PackError::ParseFailure { + origin, + diagnostics, + } => { + assert_eq!(origin, "inline:invalid-toml"); + assert_eq!(diagnostics.len(), 1); + assert_eq!(diagnostics[0].code.as_str(), "pack.profile.invalid_toml"); + assert_eq!( + diagnostics[0] + .subject + .as_ref() + .map(|subject| subject.origin.display()), + Some(origin) + ); + assert!(!diagnostics[0].message.is_empty()); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn schema_violations_are_sorted_deterministically() { + let compiler = pack::PackCompiler::new(); + let error = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "schema-violation".to_owned(), + format: pack::PackFormat::Toml, + bytes: load_bytes("fixtures/pack/invalid/profile_schema_violation.toml"), + }) + .expect_err("schema violation should fail"); + + match error { + pack::PackError::SchemaViolation { + origin, + schema_id, + diagnostics, + } => { + assert_eq!(origin, "inline:schema-violation"); + assert_eq!(schema_id, pack::PACK_PROFILE_V1_SCHEMA_ID); + let actual: Vec<(String, String)> = diagnostics + .iter() + .map(|diagnostic| { + ( + diagnostic + .subject + .as_ref() + .and_then(|subject| subject.path.as_ref()) + .map(|path| path.as_str().to_owned()) + .expect("schema diagnostics should include paths"), + diagnostic.code.as_str().to_owned(), + ) + }) + .collect(); + + assert_eq!( + actual, + vec![ + ( + "/analysis/follow_symlinks".to_owned(), + "pack.schema.invalid_type".to_owned(), + ), + ( + "/analysis/languages/1".to_owned(), + "pack.schema.invalid_language_id".to_owned(), + ), + ( + "/analysis/max_scope_depth".to_owned(), + "pack.schema.invalid_type".to_owned(), + ), + ( + "/apps/default".to_owned(), + "pack.schema.invalid_app_name".to_owned(), + ), + ( + "/apps/enabled/1".to_owned(), + "pack.schema.invalid_app_name".to_owned(), + ), + ("/id".to_owned(), "pack.schema.invalid_pack_name".to_owned(),), + ("/kind".to_owned(), "pack.schema.invalid_kind".to_owned(),), + ( + "/version".to_owned(), + "pack.schema.invalid_version".to_owned(), + ), + ] + ); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn pack_refs_reject_absolute_and_traversal_inputs() { + for input in [ + "file:/rules/security.toml", + "file:./rules/security.toml", + "file:../rules/security.toml", + "file:rules//security.toml", + "file:C:/rules/security.toml", + "file:-bad.toml", + "file:.bad.toml", + "not-a-ref", + ] { + assert!( + pack::PackRef::parse(input).is_err(), + "{input} should be rejected" + ); + } +} + +#[test] +fn empty_profile_name_fails_schema_validation() { + let compiler = pack::PackCompiler::new(); + let error = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "empty-name".to_owned(), + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "generic/default" +name = "" +"# + .to_vec(), + }) + .expect_err("empty name should fail"); + + assert_schema_violation( + error, + "inline:empty-name", + &[("/name", "pack.schema.invalid_name")], + ); +} + +#[test] +fn malformed_file_ref_fails_schema_validation() { + let compiler = pack::PackCompiler::new(); + let error = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "bad-ref".to_owned(), + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "generic/default" +name = "Invalid ref" + +[rules] +packs = ["file:-bad.toml"] +"# + .to_vec(), + }) + .expect_err("invalid file ref should fail"); + + assert_schema_violation( + error, + "inline:bad-ref", + &[("/rules/packs/0", "pack.schema.invalid_pack_ref")], + ); +} + +#[test] +fn builtin_and_inline_sources_reject_file_refs_in_phase_a() { + let compiler = pack::PackCompiler::new(); + + let builtin_error = compiler + .compile_profile(pack::PackSource::Builtin { + logical_name: "bad-builtin", + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "generic/default" +name = "Invalid builtin" + +[score] +model = "file:score/generic_lift_v2.json" +"#, + }) + .expect_err("builtin file refs should fail"); + assert_rejects_file_ref_origin(builtin_error, "builtin:bad-builtin", "/score/model"); + + let inline_error = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "bad-inline".to_owned(), + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "generic/default" +name = "Invalid inline" + +[rules] +packs = ["file:rules/generic_policy.json"] +"# + .to_vec(), + }) + .expect_err("inline file refs should fail"); + assert_rejects_file_ref_origin(inline_error, "inline:bad-inline", "/rules/packs/0"); +} + +fn assert_rejects_file_ref_origin( + error: pack::PackError, + expected_origin: &str, + expected_path: &str, +) { + assert_schema_violation( + error, + expected_origin, + &[(expected_path, "pack.refs.file_requires_file_origin")], + ); +} + +fn assert_schema_violation( + error: pack::PackError, + expected_origin: &str, + expected: &[(&str, &str)], +) { + assert_schema_violation_with_schema(error, expected_origin, "", expected); +} + +fn assert_schema_violation_with_schema( + error: pack::PackError, + expected_origin: &str, + expected_schema: &str, + expected: &[(&str, &str)], +) { + match error { + pack::PackError::SchemaViolation { + origin, + schema_id, + diagnostics, + .. + } => { + assert_eq!(origin, expected_origin); + if !expected_schema.is_empty() { + assert_eq!(schema_id, expected_schema); + } + let actual: Vec<(String, String)> = diagnostics + .iter() + .map(|diagnostic| { + ( + diagnostic + .subject + .as_ref() + .and_then(|subject| subject.path.as_ref()) + .map(|path| path.as_str().to_owned()) + .unwrap_or_default(), + diagnostic.code.as_str().to_owned(), + ) + }) + .collect(); + let expected: Vec<(String, String)> = expected + .iter() + .map(|(path, code)| ((*path).to_owned(), (*code).to_owned())) + .collect(); + assert_eq!(actual, expected); + } + other => panic!("unexpected error: {other:?}"), + } +} + +fn assert_schema_violation_codes( + error: pack::PackError, + expected_origin: &str, + expected_schema: &str, + expected_paths: &[&str], +) { + match error { + pack::PackError::SchemaViolation { + origin, + schema_id, + diagnostics, + } => { + assert_eq!(origin, expected_origin); + assert_eq!(schema_id, expected_schema); + let actual: Vec = diagnostics + .iter() + .map(|diagnostic| { + diagnostic + .subject + .as_ref() + .and_then(|subject| subject.path.as_ref()) + .map(|path| path.as_str().to_owned()) + .expect("schema diagnostics should include paths") + }) + .collect(); + assert_eq!( + actual, + expected_paths + .iter() + .map(|path| (*path).to_owned()) + .collect::>() + ); + } + other => panic!("unexpected error: {other:?}"), + } +} + +fn assert_duplicate_entry( + error: pack::PackError, + expected_kind: pack::PackKind, + expected_pack_id: &str, + expected_entry_kind: &'static str, + expected_entry_id: &str, +) { + match error { + pack::PackError::DuplicateEntryId { + pack_kind, + pack_id, + entry_kind, + entry_id, + } => { + assert_eq!(pack_kind, expected_kind); + assert_eq!(pack_id, expected_pack_id); + assert_eq!(entry_kind, expected_entry_kind); + assert_eq!(entry_id, expected_entry_id); + } + other => panic!("unexpected error: {other:?}"), + } +} + +fn load_bytes(relative: &str) -> Vec { + let path = crate_root().join(relative); + fs::read(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }) +} + +fn file_fixture_display_path() -> String { + crate_root() + .join("fixtures/pack/valid/profile_file_backed.toml") + .display() + .to_string() +} + +fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} diff --git a/crates/lift/tests/pack_fingerprints.rs b/crates/lift/tests/pack_fingerprints.rs new file mode 100644 index 000000000..884c5b160 --- /dev/null +++ b/crates/lift/tests/pack_fingerprints.rs @@ -0,0 +1,475 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::{ + sha256_bytes, sha256_canonical_json, DiagnosticCode, Fingerprint, JsonPointer, QueryId, + RecipeId, RuleId, Severity, + }; +} +#[path = "../src/pack/mod.rs"] +mod pack; + +#[test] +fn semantic_fingerprint_is_stable_across_toml_key_reordering() { + let compiler = pack::PackCompiler::new(); + + let first = compile_file( + &compiler, + "fixtures/pack/canonical/generic_default_order_a.toml", + ); + let second = compile_inline( + &compiler, + "reordered", + "fixtures/pack/canonical/generic_default_order_b.toml", + ); + + assert_eq!( + first.header.semantic_fingerprint, + second.header.semantic_fingerprint + ); +} + +#[test] +fn omitted_optional_fields_normalize_to_the_same_semantic_output() { + let compiler = pack::PackCompiler::new(); + + let omitted = compile_file(&compiler, "fixtures/pack/canonical/minimal_omitted.toml"); + let explicit = compile_file( + &compiler, + "fixtures/pack/canonical/minimal_explicit_defaults.toml", + ); + + assert_eq!( + omitted.header.semantic_fingerprint, + explicit.header.semantic_fingerprint + ); + assert_ne!( + omitted.header.source_fingerprint, + explicit.header.source_fingerprint + ); + assert_eq!( + omitted.apps.default, + pack::AppName::parse("score").expect("default app") + ); + assert_eq!(omitted.apps.enabled.len(), 1); + assert!(omitted + .analysis + .languages + .contains(&pack::LanguageId::parse("rust").expect("language"))); + assert!(!omitted.analysis.follow_symlinks); + assert_eq!(omitted.analysis.max_scope_depth, 2); +} + +#[test] +fn source_fingerprint_differs_while_semantic_fingerprint_matches() { + let compiler = pack::PackCompiler::new(); + + let file = compile_file( + &compiler, + "fixtures/pack/canonical/generic_default_order_a.toml", + ); + let inline = compile_inline( + &compiler, + "same-semantics", + "fixtures/pack/canonical/generic_default_order_b.toml", + ); + + assert_eq!( + file.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_ne!( + file.header.source_fingerprint, + inline.header.source_fingerprint + ); +} + +#[test] +fn builtin_file_and_inline_sources_are_semantically_equivalent() { + let compiler = pack::PackCompiler::new(); + + let builtin = compiler + .compile_profile(pack::builtin::profile_source("generic/default").expect("builtin")) + .expect("builtin should compile"); + let file = compile_file( + &compiler, + "fixtures/pack/canonical/generic_default_order_a.toml", + ); + let inline = compile_inline( + &compiler, + "builtin-equivalent", + "fixtures/pack/canonical/generic_default_order_b.toml", + ); + + assert_eq!( + builtin.header.semantic_fingerprint, + file.header.semantic_fingerprint + ); + assert_eq!( + builtin.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_ne!( + builtin.header.source_fingerprint, + file.header.source_fingerprint + ); + assert_ne!( + builtin.header.source_fingerprint, + inline.header.source_fingerprint + ); +} + +#[test] +fn boundary_taxonomy_semantic_fingerprint_is_stable_across_json_key_reordering() { + let compiler = pack::PackCompiler::new(); + + let first = compile_boundary_file( + &compiler, + "fixtures/pack/canonical/boundary_taxonomy_order_a.json", + ); + let second = compile_boundary_inline( + &compiler, + "boundary-reordered", + "fixtures/pack/canonical/boundary_taxonomy_order_b.json", + ); + + assert_eq!( + first.header.semantic_fingerprint, + second.header.semantic_fingerprint + ); +} + +#[test] +fn boundary_taxonomy_source_fingerprint_differs_while_semantics_match() { + let compiler = pack::PackCompiler::new(); + + let file = compile_boundary_file( + &compiler, + "fixtures/pack/canonical/boundary_taxonomy_order_a.json", + ); + let inline = compile_boundary_inline( + &compiler, + "boundary-same-semantics", + "fixtures/pack/canonical/boundary_taxonomy_order_b.json", + ); + + assert_eq!( + file.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_ne!( + file.header.source_fingerprint, + inline.header.source_fingerprint + ); +} + +#[test] +fn builtin_file_and_inline_boundary_taxonomies_are_semantically_equivalent() { + let compiler = pack::PackCompiler::new(); + + let builtin = compiler + .compile_boundary_taxonomy( + pack::builtin::boundary_taxonomy_source("generic/boundaries").expect("builtin"), + ) + .expect("builtin boundary taxonomy should compile"); + let file = compile_boundary_file(&compiler, "fixtures/pack/valid/generic_boundaries.json"); + let inline = compile_boundary_inline( + &compiler, + "boundary-builtin-equivalent", + "fixtures/pack/valid/generic_boundaries.json", + ); + + assert_eq!( + builtin.header.semantic_fingerprint, + file.header.semantic_fingerprint + ); + assert_eq!( + builtin.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_ne!( + builtin.header.source_fingerprint, + file.header.source_fingerprint + ); +} + +#[test] +fn component_map_semantic_fingerprint_is_stable_across_json_key_reordering() { + let compiler = pack::PackCompiler::new(); + + let first = compile_component_file( + &compiler, + "fixtures/pack/canonical/component_map_order_a.json", + ); + let second = compile_component_inline( + &compiler, + "component-reordered", + "fixtures/pack/canonical/component_map_order_b.json", + ); + + assert_eq!( + first.header.semantic_fingerprint, + second.header.semantic_fingerprint + ); +} + +#[test] +fn component_map_source_fingerprint_differs_while_semantics_match() { + let compiler = pack::PackCompiler::new(); + + let file = compile_component_file( + &compiler, + "fixtures/pack/canonical/component_map_order_a.json", + ); + let inline = compile_component_inline( + &compiler, + "component-same-semantics", + "fixtures/pack/canonical/component_map_order_b.json", + ); + + assert_eq!( + file.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_ne!( + file.header.source_fingerprint, + inline.header.source_fingerprint + ); +} + +#[test] +fn builtin_file_and_inline_component_maps_are_semantically_equivalent() { + let compiler = pack::PackCompiler::new(); + + let builtin = compiler + .compile_component_map( + pack::builtin::component_map_source("generic/components").expect("builtin"), + ) + .expect("builtin component map should compile"); + let file = compile_component_file(&compiler, "fixtures/pack/valid/generic_components.json"); + let inline = compile_component_inline( + &compiler, + "component-builtin-equivalent", + "fixtures/pack/valid/generic_components.json", + ); + + assert_eq!( + builtin.header.semantic_fingerprint, + file.header.semantic_fingerprint + ); + assert_eq!( + builtin.header.semantic_fingerprint, + inline.header.semantic_fingerprint + ); + assert_ne!( + builtin.header.source_fingerprint, + file.header.source_fingerprint + ); + assert_ne!( + builtin.header.source_fingerprint, + inline.header.source_fingerprint + ); +} + +#[test] +fn score_model_semantic_fingerprint_is_stable_across_json_key_reordering() { + let compiler = pack::PackCompiler::new(); + + let first = compile_score_file( + &compiler, + "fixtures/pack/canonical/score_model_order_a.json", + ); + let second = compile_score_inline( + &compiler, + "score-reordered", + "fixtures/pack/canonical/score_model_order_b.json", + ); + + assert_eq!( + first.header.semantic_fingerprint, + second.header.semantic_fingerprint + ); + assert_ne!( + first.header.source_fingerprint, + second.header.source_fingerprint + ); +} + +#[test] +fn bundle_semantic_fingerprint_is_stable_across_include_order() { + let compiler = pack::PackCompiler::new(); + + let first = compile_bundle( + &compiler, + "fixtures/pack/canonical/bundle_order_a/profile.toml", + ); + let second = compile_bundle( + &compiler, + "fixtures/pack/canonical/bundle_order_b/profile.toml", + ); + + assert_eq!(first.semantic_fingerprint, second.semantic_fingerprint); +} + +#[test] +fn resolved_topology_fingerprint_is_stable_for_equivalent_profile_inputs() { + let compiler = pack::PackCompiler::new(); + + let builtin = compiler + .compile_profile(pack::builtin::profile_source("generic/default").expect("builtin")) + .expect("builtin profile should compile"); + let inline = compile_inline( + &compiler, + "resolved-topology-inline", + "fixtures/pack/canonical/generic_default_order_b.toml", + ); + + let builtin_resolved = compiler + .resolve_profile_topology(&builtin) + .expect("builtin topology should resolve"); + let inline_resolved = compiler + .resolve_profile_topology(&inline) + .expect("inline topology should resolve"); + + assert_eq!( + builtin_resolved.semantic_fingerprint, + inline_resolved.semantic_fingerprint + ); +} + +fn compile_file(compiler: &pack::PackCompiler, relative: &str) -> pack::CompiledProfile { + compiler + .compile_profile(pack::PackSource::File { + path: crate_root().join(relative), + format_hint: None, + }) + .unwrap_or_else(|err| panic!("failed to compile {relative}: {err:?}")) +} + +fn compile_inline( + compiler: &pack::PackCompiler, + logical_name: &str, + relative: &str, +) -> pack::CompiledProfile { + compiler + .compile_profile(pack::PackSource::Inline { + logical_name: logical_name.to_owned(), + format: pack::PackFormat::Toml, + bytes: load_bytes(relative), + }) + .unwrap_or_else(|err| panic!("failed to compile {relative} inline: {err:?}")) +} + +fn compile_boundary_file( + compiler: &pack::PackCompiler, + relative: &str, +) -> pack::CompiledBoundaryTaxonomy { + compiler + .compile_boundary_taxonomy(pack::PackSource::File { + path: crate_root().join(relative), + format_hint: None, + }) + .unwrap_or_else(|err| panic!("failed to compile boundary {relative}: {err:?}")) +} + +fn compile_score_file(compiler: &pack::PackCompiler, relative: &str) -> pack::CompiledScoreModel { + compiler + .compile_score_model(pack::PackSource::File { + path: crate_root().join(relative), + format_hint: None, + }) + .unwrap_or_else(|err| panic!("failed to compile score model {relative}: {err:?}")) +} + +fn compile_score_inline( + compiler: &pack::PackCompiler, + logical_name: &str, + relative: &str, +) -> pack::CompiledScoreModel { + compiler + .compile_score_model(pack::PackSource::Inline { + logical_name: logical_name.to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes(relative), + }) + .unwrap_or_else(|err| panic!("failed to compile score model {relative} inline: {err:?}")) +} + +fn compile_boundary_inline( + compiler: &pack::PackCompiler, + logical_name: &str, + relative: &str, +) -> pack::CompiledBoundaryTaxonomy { + compiler + .compile_boundary_taxonomy(pack::PackSource::Inline { + logical_name: logical_name.to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes(relative), + }) + .unwrap_or_else(|err| panic!("failed to compile boundary {relative} inline: {err:?}")) +} + +fn compile_component_file( + compiler: &pack::PackCompiler, + relative: &str, +) -> pack::CompiledComponentMap { + compiler + .compile_component_map(pack::PackSource::File { + path: crate_root().join(relative), + format_hint: None, + }) + .unwrap_or_else(|err| panic!("failed to compile component map {relative}: {err:?}")) +} + +fn compile_component_inline( + compiler: &pack::PackCompiler, + logical_name: &str, + relative: &str, +) -> pack::CompiledComponentMap { + compiler + .compile_component_map(pack::PackSource::Inline { + logical_name: logical_name.to_owned(), + format: pack::PackFormat::Json, + bytes: load_bytes(relative), + }) + .unwrap_or_else(|err| panic!("failed to compile component map {relative} inline: {err:?}")) +} + +fn compile_bundle(compiler: &pack::PackCompiler, relative: &str) -> pack::CompiledPackSet { + let profile = compiler + .compile_profile(pack::PackSource::File { + path: crate_root().join(relative), + format_hint: None, + }) + .unwrap_or_else(|err| panic!("failed to compile bundle profile {relative}: {err:?}")); + compiler + .resolve_profile_pack_set(&profile) + .unwrap_or_else(|err| panic!("failed to resolve bundle {relative}: {err:?}")) +} + +fn load_bytes(relative: &str) -> Vec { + let path = crate_root().join(relative); + fs::read(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }) +} + +fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} diff --git a/crates/lift/tests/pack_schema.rs b/crates/lift/tests/pack_schema.rs new file mode 100644 index 000000000..ff4101e11 --- /dev/null +++ b/crates/lift/tests/pack_schema.rs @@ -0,0 +1,323 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use assert_cmd as _; +use gix as _; +use jsonschema::{Retrieve, Uri, Validator}; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json::Value; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::{ + sha256_bytes, sha256_canonical_json, DiagnosticCode, Fingerprint, JsonPointer, QueryId, + RecipeId, RuleId, Severity, + }; +} +#[path = "../src/pack/mod.rs"] +mod pack; + +#[test] +fn embedded_pack_schemas_match_disk() { + assert_eq!( + pack::PACK_COMMON_V1_SCHEMA_JSON, + load_text("schemas/pack/common.v1.json") + ); + assert_eq!( + pack::PACK_PROFILE_V1_SCHEMA_JSON, + load_text("schemas/pack/profile.v1.json") + ); + assert_eq!( + pack::PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_JSON, + load_text("schemas/pack/boundary_taxonomy.v1.json") + ); + assert_eq!( + pack::PACK_COMPONENT_MAP_V1_SCHEMA_JSON, + load_text("schemas/pack/component_map.v1.json") + ); + assert_eq!( + pack::PACK_SCORE_MODEL_V1_SCHEMA_JSON, + load_text("schemas/pack/score_model.v1.json") + ); + assert_eq!( + pack::PACK_QUERY_PACK_V1_SCHEMA_JSON, + load_text("schemas/pack/query_pack.v1.json") + ); + assert_eq!( + pack::PACK_RULE_PACK_V1_SCHEMA_JSON, + load_text("schemas/pack/rule_pack.v1.json") + ); + assert_eq!( + pack::PACK_RECIPE_PACK_V1_SCHEMA_JSON, + load_text("schemas/pack/recipe_pack.v1.json") + ); + assert_eq!( + pack::PACK_COMMON_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/common.v1.json" + ); + assert_eq!( + pack::PACK_PROFILE_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/profile.v1.json" + ); + assert_eq!( + pack::PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/boundary_taxonomy.v1.json" + ); + assert_eq!( + pack::PACK_COMPONENT_MAP_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/component_map.v1.json" + ); + assert_eq!( + pack::PACK_SCORE_MODEL_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/score_model.v1.json" + ); + assert_eq!( + pack::PACK_QUERY_PACK_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/query_pack.v1.json" + ); + assert_eq!( + pack::PACK_RULE_PACK_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/rule_pack.v1.json" + ); + assert_eq!( + pack::PACK_RECIPE_PACK_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/pack/recipe_pack.v1.json" + ); + assert_eq!(pack::PACK_COMMON_V1_SCHEMA_FILE, "common.v1.json"); + assert_eq!(pack::PACK_PROFILE_V1_SCHEMA_FILE, "profile.v1.json"); + assert_eq!( + pack::PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_FILE, + "boundary_taxonomy.v1.json" + ); + assert_eq!( + pack::PACK_COMPONENT_MAP_V1_SCHEMA_FILE, + "component_map.v1.json" + ); + assert_eq!(pack::PACK_SCORE_MODEL_V1_SCHEMA_FILE, "score_model.v1.json"); + assert_eq!(pack::PACK_QUERY_PACK_V1_SCHEMA_FILE, "query_pack.v1.json"); + assert_eq!(pack::PACK_RULE_PACK_V1_SCHEMA_FILE, "rule_pack.v1.json"); + assert_eq!(pack::PACK_RECIPE_PACK_V1_SCHEMA_FILE, "recipe_pack.v1.json"); + assert_eq!(pack::PACK_COMMON_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_PROFILE_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_BOUNDARY_TAXONOMY_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_COMPONENT_MAP_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_SCORE_MODEL_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_QUERY_PACK_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_RULE_PACK_V1_SCHEMA_VERSION, 1); + assert_eq!(pack::PACK_RECIPE_PACK_V1_SCHEMA_VERSION, 1); +} + +#[test] +fn valid_pack_profile_schema_fixtures_validate_and_deserialize() { + for fixture in [ + "fixtures/pack/valid/profile_minimal.json", + "fixtures/pack/valid/profile_full.json", + ] { + let instance = assert_schema_valid("profile", fixture); + + let raw: pack::raw::RawProfile = + serde_json::from_value(instance).expect("fixture should deserialize"); + assert_eq!(raw.kind, pack::raw::PackKind::Profile); + } +} + +#[test] +fn valid_topology_schema_fixtures_validate_and_deserialize() { + let boundary: pack::raw::RawBoundaryTaxonomy = serde_json::from_value(assert_schema_valid( + "boundary_taxonomy", + "fixtures/pack/valid/generic_boundaries.json", + )) + .expect("boundary taxonomy should deserialize"); + assert_eq!(boundary.kind, pack::raw::PackKind::BoundaryTaxonomy); + + let components: pack::raw::RawComponentMap = serde_json::from_value(assert_schema_valid( + "component_map", + "fixtures/pack/valid/generic_components.json", + )) + .expect("component map should deserialize"); + assert_eq!(components.kind, pack::raw::PackKind::ComponentMap); +} + +#[test] +fn valid_advanced_schema_fixtures_validate_and_deserialize() { + let score: pack::raw::RawScoreModel = serde_json::from_value(assert_schema_valid( + "score_model", + "fixtures/pack/valid/score/generic_lift_v2.json", + )) + .expect("score model should deserialize"); + assert_eq!(score.kind, pack::raw::PackKind::ScoreModel); + + let query: pack::raw::RawQueryPack = serde_json::from_value(assert_schema_valid( + "query_pack", + "fixtures/pack/valid/queries/rust_core.json", + )) + .expect("query pack should deserialize"); + assert_eq!(query.kind, pack::raw::PackKind::QueryPack); + + let rule: pack::raw::RawRulePack = serde_json::from_value(assert_schema_valid( + "rule_pack", + "fixtures/pack/valid/rules/generic_policy.json", + )) + .expect("rule pack should deserialize"); + assert_eq!(rule.kind, pack::raw::PackKind::RulePack); + + let recipe: pack::raw::RawRecipePack = serde_json::from_value(assert_schema_valid( + "recipe_pack", + "fixtures/pack/valid/recipes/generic_core_recipes.json", + )) + .expect("recipe pack should deserialize"); + assert_eq!(recipe.kind, pack::raw::PackKind::RecipePack); +} + +#[test] +fn invalid_pack_profile_schema_fixtures_fail_validation() { + for fixture in [ + "fixtures/pack/invalid/profile_missing_name.json", + "fixtures/pack/invalid/profile_empty_name.json", + "fixtures/pack/invalid/profile_invalid_pack_ref.json", + "fixtures/pack/invalid/profile_invalid_pack_ref_leading_dash.json", + "fixtures/pack/invalid/profile_traversal_pack_ref.json", + "fixtures/pack/invalid/profile_unknown_field.json", + ] { + let instance = load_json(fixture); + assert_schema_invalid("profile", &instance); + } +} + +#[test] +fn invalid_topology_schema_fixtures_fail_validation() { + assert_schema_invalid( + "boundary_taxonomy", + &load_json("fixtures/pack/invalid/boundary_taxonomy_schema_violation.json"), + ); + assert_schema_invalid( + "boundary_taxonomy", + &load_json("fixtures/pack/invalid/boundary_taxonomy_unknown_field.json"), + ); + assert_schema_invalid( + "component_map", + &load_json("fixtures/pack/invalid/component_map_schema_violation.json"), + ); + assert_schema_invalid( + "component_map", + &load_json("fixtures/pack/invalid/component_map_unknown_field.json"), + ); +} + +#[test] +fn invalid_advanced_schema_fixtures_fail_validation() { + assert_schema_invalid( + "score_model", + &load_json("fixtures/pack/invalid/score_model_schema_violation.json"), + ); + assert_schema_invalid( + "query_pack", + &load_json("fixtures/pack/invalid/query_pack_schema_violation.json"), + ); + assert_schema_invalid( + "rule_pack", + &load_json("fixtures/pack/invalid/rule_pack_bad_query_ref.json"), + ); + assert_schema_invalid( + "rule_pack", + &load_json("fixtures/pack/invalid/rule_pack_invalid_severity.json"), + ); + assert_schema_invalid( + "recipe_pack", + &load_json("fixtures/pack/invalid/recipe_pack_bad_transform.json"), + ); +} + +fn assert_schema_valid(kind: &str, fixture: &str) -> Value { + let instance = load_json(fixture); + let validator = schema_validator(kind); + if let Err(error) = validator.validate(&instance) { + panic!("expected fixture to validate: {error}"); + } + instance +} + +fn assert_schema_invalid(kind: &str, instance: &Value) { + let validator = schema_validator(kind); + assert!( + validator.validate(instance).is_err(), + "expected fixture to fail schema validation" + ); +} + +fn schema_validator(kind: &str) -> Validator { + let schema_path = match kind { + "profile" => "schemas/pack/profile.v1.json", + "boundary_taxonomy" => "schemas/pack/boundary_taxonomy.v1.json", + "component_map" => "schemas/pack/component_map.v1.json", + "score_model" => "schemas/pack/score_model.v1.json", + "query_pack" => "schemas/pack/query_pack.v1.json", + "rule_pack" => "schemas/pack/rule_pack.v1.json", + "recipe_pack" => "schemas/pack/recipe_pack.v1.json", + other => panic!("unsupported schema kind {other}"), + }; + let root_schema = load_json(schema_path); + let common_schema = load_json("schemas/pack/common.v1.json"); + let retriever = InMemoryRetriever { + schemas: HashMap::from([ + ( + pack::PACK_COMMON_V1_SCHEMA_ID.to_owned(), + common_schema.clone(), + ), + ("common.v1.json".to_owned(), common_schema), + ]), + }; + + jsonschema::draft202012::options() + .with_retriever(retriever) + .build(&root_schema) + .expect("schema should compile") +} + +#[derive(Clone, Debug)] +struct InMemoryRetriever { + schemas: HashMap, +} + +impl Retrieve for InMemoryRetriever { + fn retrieve( + &self, + uri: &Uri, + ) -> Result> { + self.schemas + .get(uri.as_str()) + .cloned() + .ok_or_else(|| format!("schema not found: {uri}").into()) + } +} + +fn load_json(relative: &str) -> Value { + let path = crate_root().join(relative); + let contents = fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }); + + serde_json::from_str(&contents).unwrap_or_else(|err| { + panic!("failed to parse {} as JSON: {err}", path.display()); + }) +} + +fn load_text(relative: &str) -> String { + let path = crate_root().join(relative); + fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }) +} + +fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} diff --git a/crates/lift/tests/pack_topology.rs b/crates/lift/tests/pack_topology.rs new file mode 100644 index 000000000..48518bac6 --- /dev/null +++ b/crates/lift/tests/pack_topology.rs @@ -0,0 +1,354 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use assert_cmd as _; +use gix as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::{ + sha256_bytes, sha256_canonical_json, DiagnosticCode, Fingerprint, JsonPointer, QueryId, + RecipeId, RuleId, Severity, + }; +} +#[path = "../src/pack/mod.rs"] +mod pack; + +#[test] +fn builtin_profile_resolves_builtin_topology_refs() { + let compiler = pack::PackCompiler::new(); + let profile = compiler + .compile_profile(pack::builtin::profile_source("generic/default").expect("builtin")) + .expect("builtin profile"); + + let resolved = compiler + .resolve_profile_topology(&profile) + .expect("builtin topology should resolve"); + + let boundary = resolved + .boundary_taxonomy + .expect("builtin boundary taxonomy should be present"); + let component = resolved + .component_map + .expect("builtin component map should be present"); + + assert_eq!(boundary.header.id.as_str(), "generic/boundaries"); + assert_eq!(component.header.id.as_str(), "generic/components"); + assert_eq!( + boundary.header.origin.display(), + "builtin:generic/boundaries" + ); + assert_eq!( + component.header.origin.display(), + "builtin:generic/components" + ); +} + +#[test] +fn file_backed_profile_resolves_relative_topology_refs() { + let compiler = pack::PackCompiler::new(); + let profile = compiler + .compile_profile(pack::PackSource::File { + path: crate_root().join("fixtures/pack/valid/profile_file_backed.toml"), + format_hint: None, + }) + .expect("file-backed profile"); + + let resolved = compiler + .resolve_profile_topology(&profile) + .expect("file topology should resolve"); + + let boundary = resolved.boundary_taxonomy.expect("boundary taxonomy"); + let component = resolved.component_map.expect("component map"); + + assert_eq!(boundary.header.id.as_str(), "acme/boundaries"); + assert_eq!(component.header.id.as_str(), "acme/components"); + assert_eq!( + boundary.header.origin.display(), + boundary_fixture_display_path() + ); + assert_eq!( + component.header.origin.display(), + component_fixture_display_path() + ); +} + +#[test] +fn file_backed_profile_topology_resolution_is_stable_across_cwd_changes() { + let compiler = pack::PackCompiler::new(); + let cwd_guard = CwdGuard::acquire(); + cwd_guard.set_current_dir(crate_root()); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path: PathBuf::from("fixtures/pack/valid/profile_file_backed.toml"), + format_hint: None, + }) + .expect("file-backed profile from relative path"); + + cwd_guard.set_current_dir(std::env::temp_dir()); + + let resolved = compiler + .resolve_profile_topology(&profile) + .expect("topology resolution should ignore later cwd changes"); + + let boundary = resolved.boundary_taxonomy.expect("boundary taxonomy"); + let component = resolved.component_map.expect("component map"); + assert_eq!(boundary.header.id.as_str(), "acme/boundaries"); + assert_eq!(component.header.id.as_str(), "acme/components"); +} + +#[test] +fn topology_resolution_supports_zero_one_or_two_slots() { + let compiler = pack::PackCompiler::new(); + + let zero = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "zero-topology".to_owned(), + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "acme/zero" +name = "Zero topology" +"# + .to_vec(), + }) + .expect("zero topology profile"); + let zero_resolved = compiler + .resolve_profile_topology(&zero) + .expect("zero topology should resolve"); + assert!(zero_resolved.boundary_taxonomy.is_none()); + assert!(zero_resolved.component_map.is_none()); + + let one = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "one-topology".to_owned(), + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "acme/one" +name = "One topology" + +[topology] +boundary_taxonomy = "builtin:generic/boundaries" +"# + .to_vec(), + }) + .expect("one topology profile"); + let one_resolved = compiler + .resolve_profile_topology(&one) + .expect("one topology should resolve"); + assert!(one_resolved.boundary_taxonomy.is_some()); + assert!(one_resolved.component_map.is_none()); + + let two = compiler + .compile_profile(pack::builtin::profile_source("generic/default").expect("builtin")) + .expect("builtin profile"); + let two_resolved = compiler + .resolve_profile_topology(&two) + .expect("two topology slots should resolve"); + assert!(two_resolved.boundary_taxonomy.is_some()); + assert!(two_resolved.component_map.is_some()); +} + +#[test] +fn missing_topology_ref_fails_with_unknown_pack_reference() { + let compiler = pack::PackCompiler::new(); + let profile = compiler + .compile_profile(pack::PackSource::File { + path: write_temp_profile( + "missing-topology.toml", + r#"kind = "profile" +version = 1 +id = "acme/missing" +name = "Missing topology" + +[topology] +boundary_taxonomy = "file:topology/does-not-exist.json" +"#, + ), + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_topology(&profile) + .expect_err("missing topology ref should fail"); + + match error { + pack::PackError::UnknownPackReference { + referring_pack, + reference, + } => { + assert_eq!(referring_pack, "acme/missing"); + assert_eq!(reference, "file:topology/does-not-exist.json"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn topology_resolution_surfaces_filesystem_errors_from_try_exists() { + let compiler = pack::PackCompiler::new(); + let dir = unique_temp_dir("topology-io-error"); + let path = dir.join("io-error.toml"); + let blocking_segment = dir.join("topology"); + let expected_ref_path = dir.join("topology/component-map.json"); + + fs::write( + &path, + r#"kind = "profile" +version = 1 +id = "acme/io-error" +name = "Topology IO error" + +[topology] +component_map = "file:topology/component-map.json" +"#, + ) + .expect("write temp profile"); + fs::write(&blocking_segment, "not a directory").expect("write blocking segment"); + + let profile = compiler + .compile_profile(pack::PackSource::File { + path, + format_hint: None, + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_topology(&profile) + .expect_err("topology resolution should surface path I/O failures"); + + match error { + pack::PackError::Io { origin, reason } => { + assert_eq!(origin, expected_ref_path.display().to_string()); + assert!(!reason.is_empty(), "io reason should not be empty"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn wrong_kind_topology_ref_fails_with_ref_kind_mismatch() { + let compiler = pack::PackCompiler::new(); + let profile = compiler + .compile_profile(pack::PackSource::Inline { + logical_name: "wrong-kind".to_owned(), + format: pack::PackFormat::Toml, + bytes: br#"kind = "profile" +version = 1 +id = "acme/wrong-kind" +name = "Wrong kind" + +[topology] +boundary_taxonomy = "builtin:generic/components" +"# + .to_vec(), + }) + .expect("profile should compile"); + + let error = compiler + .resolve_profile_topology(&profile) + .expect_err("wrong kind should fail"); + + match error { + pack::PackError::RefKindMismatch { + reference, + expected, + actual, + } => { + assert_eq!(reference, "builtin:generic/components"); + assert_eq!(expected, pack::PackKind::BoundaryTaxonomy); + assert_eq!(actual, pack::PackKind::ComponentMap); + } + other => panic!("unexpected error: {other:?}"), + } +} + +fn write_temp_profile(name: &str, contents: &str) -> PathBuf { + let dir = unique_temp_dir(name); + let topology_dir = dir.join("topology"); + fs::create_dir_all(&topology_dir).expect("create topology dir"); + let path = dir.join(name); + fs::write(&path, contents).expect("write temp profile"); + path +} + +fn unique_temp_dir(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_nanos(); + let dir = std::env::temp_dir().join(format!("substrate-lift-{nanos}-{name}")); + fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +fn cwd_test_mutex() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) +} + +struct CwdGuard { + _lock: MutexGuard<'static, ()>, + previous: PathBuf, +} + +impl CwdGuard { + fn acquire() -> Self { + let lock = cwd_test_mutex().lock().expect("cwd mutex poisoned"); + let previous = std::env::current_dir().expect("capture current dir"); + Self { + _lock: lock, + previous, + } + } + + fn set_current_dir>(&self, path: P) { + std::env::set_current_dir(path).expect("set current dir"); + } +} + +impl Drop for CwdGuard { + fn drop(&mut self) { + std::env::set_current_dir(&self.previous).expect("restore current dir"); + } +} + +fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} + +fn boundary_fixture_display_path() -> String { + crate_root() + .join("fixtures/pack/valid/profile_file_backed.toml") + .parent() + .expect("fixture parent should exist") + .join("topology/boundary-taxonomy.json") + .display() + .to_string() +} + +fn component_fixture_display_path() -> String { + crate_root() + .join("fixtures/pack/valid/profile_file_backed.toml") + .parent() + .expect("fixture parent should exist") + .join("topology/component-map.json") + .display() + .to_string() +} diff --git a/crates/lift/tests/repo_diff.rs b/crates/lift/tests/repo_diff.rs new file mode 100644 index 000000000..d85d28c70 --- /dev/null +++ b/crates/lift/tests/repo_diff.rs @@ -0,0 +1,286 @@ +#![allow(unused_crate_dependencies)] + +use assert_cmd as _; +use gix as _; +use globset as _; +use predicates as _; +use serde_jcs as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +use std::fs; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use repo_support::{ + default_snapshot_options, materialize, materialize_basic_worktree_pair, write_file, +}; + +fn diff_paths(diff: &repo::RepoDiff) -> Vec { + diff.entries + .iter() + .map(|entry| entry.path.as_str().to_owned()) + .collect() +} + +#[test] +fn base_exhausted_first_marks_remaining_entries_as_added() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-base-exhausted", + |_| {}, + |head| { + write_file(&head.join("zzz/added-a.txt"), b"added-a"); + write_file(&head.join("zzz/added-b.txt"), b"added-b"); + }, + ); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + + assert_eq!( + diff_paths(&diff), + vec!["zzz/added-a.txt", "zzz/added-b.txt"] + ); + assert!(diff + .entries + .iter() + .all(|entry| entry.kind == repo::DiffKind::Added)); + assert!(diff.entries.iter().all(|entry| entry.before.is_none())); + assert!(diff.entries.iter().all(|entry| entry.after.is_some())); +} + +#[test] +fn head_exhausted_first_marks_remaining_entries_as_removed() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-head-exhausted", + |base| { + write_file(&base.join("target/cache.txt"), b"cached"); + }, + |head| { + fs::remove_file(head.join("src/lib.rs")).expect("src/lib.rs should be removable"); + }, + ); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + + assert_eq!(diff_paths(&diff), vec!["src/lib.rs", "target/cache.txt"]); + assert!(diff + .entries + .iter() + .all(|entry| entry.kind == repo::DiffKind::Removed)); + assert!(diff.entries.iter().all(|entry| entry.before.is_some())); + assert!(diff.entries.iter().all(|entry| entry.after.is_none())); +} + +#[test] +fn equal_path_and_equal_blob_entries_are_omitted() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-omit-equal", + |_| {}, + |head| { + write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"changed\"\n}\n", + ); + }, + ); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + let paths = diff_paths(&diff); + + assert_eq!(paths, vec!["src/lib.rs"]); + assert!(!paths.iter().any(|path| path == "Cargo.toml")); + assert!(!paths.iter().any(|path| path == "build/output.txt")); +} + +#[test] +fn equal_path_and_different_blob_produces_single_modified_entry() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-modified", + |_| {}, + |head| { + write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"modified\"\n}\n", + ); + }, + ); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + + assert_eq!(diff.entries.len(), 1); + let entry = &diff.entries[0]; + assert_eq!(entry.path.as_str(), "src/lib.rs"); + assert_eq!(entry.kind, repo::DiffKind::Modified); + assert!(entry.before.is_some()); + assert!(entry.after.is_some()); + + let before = entry + .before + .as_ref() + .expect("modified entry should have before"); + let after = entry + .after + .as_ref() + .expect("modified entry should have after"); + assert_eq!(before.path, after.path); + assert_ne!(before.blob_fingerprint, after.blob_fingerprint); +} + +#[test] +fn identical_snapshots_produce_empty_diff() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = + materialize_basic_worktree_pair("repo-diff-identical", |_| {}, |_| {}); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + + assert!(diff.entries.is_empty()); + assert_eq!(diff.base_fingerprint, diff.head_fingerprint); +} + +#[test] +fn rename_shaped_change_is_removed_plus_added() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-rename-shaped", + |_| {}, + |head| { + fs::rename(head.join("src/lib.rs"), head.join("src/renamed.rs")) + .expect("rename should succeed"); + }, + ); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + + assert_eq!(diff_paths(&diff), vec!["src/lib.rs", "src/renamed.rs"]); + assert_eq!(diff.entries[0].kind, repo::DiffKind::Removed); + assert_eq!(diff.entries[1].kind, repo::DiffKind::Added); + + let removed = diff.entries[0] + .before + .as_ref() + .expect("removed entry should have before"); + let added = diff.entries[1] + .after + .as_ref() + .expect("added entry should have after"); + assert_eq!(removed.blob_fingerprint, added.blob_fingerprint); +} + +#[test] +fn diff_entry_path_order_matches_lexical_repo_path_order() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-lexical-order", + |base| { + write_file(&base.join("target/cache.txt"), b"cached"); + }, + |head| { + write_file(&head.join("docs/new.txt"), b"new"); + write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"ordered\"\n}\n", + ); + write_file(&head.join("zeta/final.txt"), b"final"); + }, + ); + + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + let paths = diff_paths(&diff); + let mut sorted_paths = paths.clone(); + sorted_paths.sort(); + + assert_eq!( + paths, + vec![ + "docs/new.txt", + "src/lib.rs", + "target/cache.txt", + "zeta/final.txt", + ] + ); + assert_eq!(paths, sorted_paths); +} + +#[test] +fn repeated_build_diff_calls_are_fingerprint_stable() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = materialize_basic_worktree_pair( + "repo-diff-repeatable", + |_| {}, + |head| { + write_file(&head.join("docs/new.txt"), b"new"); + write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"repeatable\"\n}\n", + ); + }, + ); + + let first = repo::build_diff(&base_snapshot, &head_snapshot); + let second = repo::build_diff(&base_snapshot, &head_snapshot); + + assert_eq!(first, second); + assert_eq!(first.fingerprint, second.fingerprint); +} + +#[test] +fn equivalent_trees_under_different_temp_roots_produce_the_same_semantic_diff() { + let (_base_root_a, _head_root_a, base_snapshot_a, head_snapshot_a) = + materialize_basic_worktree_pair( + "repo-diff-semantic-a", + |_| {}, + |head| { + write_file(&head.join("docs/new.txt"), b"new"); + write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"semantic\"\n}\n", + ); + }, + ); + let (_base_root_b, _head_root_b, base_snapshot_b, head_snapshot_b) = + materialize_basic_worktree_pair( + "repo-diff-semantic-b", + |_| {}, + |head| { + write_file(&head.join("docs/new.txt"), b"new"); + write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"semantic\"\n}\n", + ); + }, + ); + + let diff_a = repo::build_diff(&base_snapshot_a, &head_snapshot_a); + let diff_b = repo::build_diff(&base_snapshot_b, &head_snapshot_b); + + assert_eq!(diff_a, diff_b); +} + +#[test] +fn prebuilt_snapshots_remain_stable_after_later_live_tree_mutation() { + let (base_root, head_root, base_snapshot, head_snapshot) = + materialize_basic_worktree_pair("repo-diff-prebuilt-stable", |_| {}, |_| {}); + + let original_diff = repo::build_diff(&base_snapshot, &head_snapshot); + assert!(original_diff.entries.is_empty()); + + write_file( + &base_root.path().join("Cargo.toml"), + b"[package]\nname = \"mutated-base\"\n", + ); + write_file(&head_root.path().join("docs/new.txt"), b"mutated-head"); + + let preserved_diff = repo::build_diff(&base_snapshot, &head_snapshot); + assert_eq!(preserved_diff, original_diff); + + let fresh_base = materialize(base_root.path(), default_snapshot_options()); + let fresh_head = materialize(head_root.path(), default_snapshot_options()); + let fresh_diff = repo::build_diff(&fresh_base, &fresh_head); + assert_ne!(fresh_diff, original_diff); +} diff --git a/crates/lift/tests/repo_fingerprints.rs b/crates/lift/tests/repo_fingerprints.rs new file mode 100644 index 000000000..b34ae6104 --- /dev/null +++ b/crates/lift/tests/repo_fingerprints.rs @@ -0,0 +1,143 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use repo_support::{copy_fixture_tree, default_snapshot_options, write_file, TempDir}; + +#[test] +fn identical_trees_under_different_absolute_roots_share_fingerprint() { + let left = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + "repo-fingerprint-left", + ); + let right = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + "repo-fingerprint-right", + ); + + let left_snapshot = repo_support::materialize(left.path(), default_snapshot_options()); + let right_snapshot = repo_support::materialize(right.path(), default_snapshot_options()); + + assert_eq!(left_snapshot.fingerprint, right_snapshot.fingerprint); +} + +#[test] +fn traversal_order_does_not_affect_inventory_order_or_fingerprint() { + let left = TempDir::new("repo-fingerprint-order-left"); + fs::create_dir_all(left.path().join(".git")).expect("git dir"); + write_file(&left.path().join("b.txt"), b"b"); + write_file(&left.path().join("a.txt"), b"a"); + + let right = TempDir::new("repo-fingerprint-order-right"); + fs::create_dir_all(right.path().join(".git")).expect("git dir"); + write_file(&right.path().join("a.txt"), b"a"); + write_file(&right.path().join("b.txt"), b"b"); + + let left_snapshot = repo_support::materialize(left.path(), default_snapshot_options()); + let right_snapshot = repo_support::materialize(right.path(), default_snapshot_options()); + + let left_paths = repo_support::inventory_paths(&left_snapshot); + let right_paths = repo_support::inventory_paths(&right_snapshot); + assert_eq!(left_paths, vec!["a.txt".to_owned(), "b.txt".to_owned()]); + assert_eq!(left_paths, right_paths); + assert_eq!(left_snapshot.fingerprint, right_snapshot.fingerprint); +} + +#[test] +fn changing_one_file_changes_its_blob_and_snapshot_fingerprint_only() { + let left = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + "repo-fingerprint-before", + ); + let right = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + "repo-fingerprint-after", + ); + write_file( + &right.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"changed\"\n}\n", + ); + + let left_snapshot = repo_support::materialize(left.path(), default_snapshot_options()); + let right_snapshot = repo_support::materialize(right.path(), default_snapshot_options()); + + let changed = crate::kernel::RepoPath::parse("src/lib.rs").expect("path"); + let unchanged = crate::kernel::RepoPath::parse("Cargo.toml").expect("path"); + assert_ne!( + left_snapshot + .entry(&changed) + .expect("entry") + .blob_fingerprint, + right_snapshot + .entry(&changed) + .expect("entry") + .blob_fingerprint + ); + assert_eq!( + left_snapshot + .entry(&unchanged) + .expect("entry") + .blob_fingerprint, + right_snapshot + .entry(&unchanged) + .expect("entry") + .blob_fingerprint + ); + assert_ne!(left_snapshot.fingerprint, right_snapshot.fingerprint); +} + +#[cfg(unix)] +#[test] +fn diagnostics_are_sorted_deterministically() { + use std::os::unix::fs::symlink; + + let temp = TempDir::new("repo-fingerprint-diagnostics"); + fs::create_dir_all(temp.path().join(".git")).expect("git dir"); + write_file(&temp.path().join("b-large.txt"), b"1234567890"); + write_file(&temp.path().join("a-target.txt"), b"target"); + symlink("a-target.txt", temp.path().join("a-link.txt")).expect("symlink"); + + let mut options = default_snapshot_options(); + options.max_file_bytes = Some(3); + options.large_file_policy = repo::LargeFilePolicy::Skip; + let snapshot = repo_support::materialize(temp.path(), options); + + let mut sorted = snapshot.diagnostics.clone(); + sorted.sort(); + assert_eq!(snapshot.diagnostics, sorted); + let codes = snapshot + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str().to_owned()) + .collect::>(); + assert_eq!( + codes, + vec![ + "repo.snapshot.symlink_skipped".to_owned(), + "repo.snapshot.file_too_large".to_owned(), + "repo.snapshot.file_too_large".to_owned() + ] + ); +} diff --git a/crates/lift/tests/repo_git_snapshot.rs b/crates/lift/tests/repo_git_snapshot.rs new file mode 100644 index 000000000..41126062c --- /dev/null +++ b/crates/lift/tests/repo_git_snapshot.rs @@ -0,0 +1,364 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time before unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "substrate-lift-{prefix}-{}-{suffix}", + std::process::id(), + )); + fs::create_dir_all(&path).expect("temp dir should be creatable"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn write_file(path: &Path, contents: &[u8]) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("parent dir should be creatable"); + } + fs::write(path, contents).unwrap_or_else(|error| { + panic!("failed to write {}: {error}", path.display()); + }); +} + +fn snapshot_request( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::SnapshotRequest { + repo::SnapshotRequest { + root: repo::RepoRoot::from_dir(root).expect("repo root should parse"), + source, + options, + } +} + +fn materialize( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::RepoSnapshot { + repo::materialize_snapshot(&snapshot_request(root, source, options)) + .expect("snapshot should build") +} + +fn inventory_paths(snapshot: &repo::RepoSnapshot) -> Vec { + snapshot + .inventory + .iter() + .map(|entry| entry.path.as_str().to_owned()) + .collect() +} + +fn run_git(repo_root: &Path, args: &[&str]) { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +fn git_stdout(repo_root: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8(output.stdout) + .expect("git output should be utf-8") + .trim() + .to_owned() +} + +fn init_git_repo(prefix: &str) -> TempDir { + let temp = TempDir::new(prefix); + run_git(temp.path(), &["init", "--quiet"]); + temp +} + +fn commit_all(repo_root: &Path, message: &str) { + run_git(repo_root, &["add", "-A"]); + run_git( + repo_root, + &[ + "-c", + "user.name=Substrate Lift", + "-c", + "user.email=lift@example.com", + "commit", + "--quiet", + "-m", + message, + ], + ); +} + +#[test] +fn gitrev_snapshot_reads_the_committed_tree_not_dirty_worktree_state() { + let repo_root = init_git_repo("repo-gitrev-dirty-worktree"); + write_file( + &repo_root.path().join("Cargo.toml"), + b"[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ); + write_file( + &repo_root.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"committed\"\n}\n", + ); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + write_file( + &repo_root.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"dirty\"\n}\n", + ); + write_file(&repo_root.path().join("untracked.txt"), b"worktree-only"); + + let snapshot = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head.clone() }, + repo::SnapshotOptions::default(), + ); + let paths = inventory_paths(&snapshot); + + assert_eq!(snapshot.source, repo::SnapshotSource::GitRev { rev: head }); + assert_eq!(paths, vec!["Cargo.toml", "src/lib.rs"]); + assert!( + !paths.iter().any(|path| path == "untracked.txt"), + "git revision snapshots must ignore untracked worktree files" + ); + assert_eq!( + snapshot + .read_bytes(&crate::kernel::RepoPath::parse("src/lib.rs").expect("path should parse")) + .expect("blob bytes should exist"), + b"pub fn fixture_value() -> &'static str {\n \"committed\"\n}\n" + ); +} + +#[test] +fn clean_worktree_and_gitrev_snapshots_have_matching_inventory_and_fingerprint() { + let repo_root = init_git_repo("repo-gitrev-parity"); + write_file( + &repo_root.path().join("Cargo.toml"), + b"[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ); + write_file( + &repo_root.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"same\"\n}\n", + ); + write_file(&repo_root.path().join("docs/guide.md"), b"# guide\n"); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let worktree = materialize( + repo_root.path(), + repo::SnapshotSource::Worktree, + repo::SnapshotOptions::default(), + ); + let gitrev = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head }, + repo::SnapshotOptions::default(), + ); + + assert_eq!(worktree.inventory, gitrev.inventory); + assert_eq!(worktree.blob_store, gitrev.blob_store); + assert_eq!(worktree.fingerprint, gitrev.fingerprint); + assert_eq!(worktree.stats.file_count, gitrev.stats.file_count); + assert_eq!(worktree.stats.total_bytes, gitrev.stats.total_bytes); + assert_eq!(worktree.stats.skipped_by_ignore, 1); + assert_eq!(gitrev.stats.skipped_by_ignore, 0); + assert_eq!( + worktree.stats.skipped_symlinks, + gitrev.stats.skipped_symlinks + ); + assert_eq!( + worktree.stats.skipped_non_utf8_paths, + gitrev.stats.skipped_non_utf8_paths + ); + assert_eq!( + worktree.stats.skipped_large_files, + gitrev.stats.skipped_large_files + ); + assert_eq!( + worktree.stats.skipped_unsupported_file_kinds, + gitrev.stats.skipped_unsupported_file_kinds + ); + assert_eq!(worktree.diagnostics, gitrev.diagnostics); +} + +#[test] +fn gitrev_snapshot_rejects_revisions_that_resolve_to_blobs() { + let repo_root = init_git_repo("repo-gitrev-object-kind"); + write_file( + &repo_root.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"blob\"\n}\n", + ); + commit_all(repo_root.path(), "initial"); + + let error = repo::materialize_snapshot(&snapshot_request( + repo_root.path(), + repo::SnapshotSource::GitRev { + rev: "HEAD:src/lib.rs".to_owned(), + }, + repo::SnapshotOptions::default(), + )) + .expect_err("blob revisions should be rejected"); + + assert_eq!( + error, + repo::RepoError::GitRevisionObjectKind { + rev: "HEAD:src/lib.rs".to_owned(), + actual_kind: "blob".to_owned(), + expected_kind: "tree", + } + ); +} + +#[cfg(unix)] +#[test] +fn gitrev_followed_symlinks_still_compose_with_typed_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = init_git_repo("repo-gitrev-follow-ignore-typed"); + write_file(&repo_root.path().join("target/cache.txt"), b"cached"); + symlink("target/cache.txt", repo_root.path().join("link.txt")).expect("symlink should exist"); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + well_known_excludes: vec![repo::WellKnownExclude::RustTarget], + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head }, + options, + ); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + + assert!(snapshot.entry(&link).is_none()); + assert_eq!(snapshot.stats.skipped_by_ignore, 2); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn gitrev_followed_symlinks_compose_with_directory_glob_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = init_git_repo("repo-gitrev-follow-ignore-dir-glob"); + write_file(&repo_root.path().join("dist/bundle.js"), b"bundle"); + symlink("dist/bundle.js", repo_root.path().join("link.txt")).expect("symlink should exist"); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + exclude_globs: vec!["dist".to_owned()], + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head }, + options, + ); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + + assert!(snapshot.entry(&link).is_none()); + assert!(inventory_paths(&snapshot) + .into_iter() + .all(|path| !path.starts_with("dist/"))); + assert_eq!(snapshot.stats.skipped_by_ignore, 2); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn gitrev_followed_symlinks_compose_with_directory_slash_glob_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = init_git_repo("repo-gitrev-follow-ignore-dir-slash-glob"); + write_file(&repo_root.path().join("dist/bundle.js"), b"bundle"); + symlink("dist/bundle.js", repo_root.path().join("link.txt")).expect("symlink should exist"); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + exclude_globs: vec!["dist/".to_owned()], + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head }, + options, + ); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + + assert!(snapshot.entry(&link).is_none()); + assert!(inventory_paths(&snapshot) + .into_iter() + .all(|path| !path.starts_with("dist/"))); + assert_eq!(snapshot.stats.skipped_by_ignore, 2); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot.diagnostics.is_empty()); +} diff --git a/crates/lift/tests/repo_ignore.rs b/crates/lift/tests/repo_ignore.rs new file mode 100644 index 000000000..3f8899e28 --- /dev/null +++ b/crates/lift/tests/repo_ignore.rs @@ -0,0 +1,193 @@ +#![allow(unused_crate_dependencies)] + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +#[cfg(unix)] +use repo_support::TempDir; +use repo_support::{copy_fixture_tree, default_snapshot_options, inventory_paths, write_file}; + +#[test] +fn invalid_caller_glob_hard_fails() { + let mut options = default_snapshot_options(); + options.exclude_globs = vec!["[".to_owned()]; + let error = repo::CompiledIgnoreSet::compile(&options).expect_err("glob should fail"); + match error { + repo::RepoError::IgnoreGlobCompile { pattern, .. } => assert_eq!(pattern, "["), + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn git_directory_is_intrinsically_excluded_without_caller_input() { + let fixture = copy_fixture_tree("fixtures/repo/trees/basic_worktree", "repo-ignore-git"); + let snapshot = repo_support::materialize(fixture.path(), default_snapshot_options()); + + assert!(inventory_paths(&snapshot) + .into_iter() + .all(|path| !path.starts_with(".git"))); +} + +#[test] +fn explicit_glob_exclusion_still_composes_on_top_of_typed_policy() { + let fixture = copy_fixture_tree( + "fixtures/repo/trees/well_known_excludes", + "repo-ignore-explicit", + ); + write_file(&fixture.path().join("target/cache.txt"), b"cached"); + let mut options = default_snapshot_options(); + options.well_known_excludes = vec![repo::WellKnownExclude::RustTarget]; + options.exclude_globs = vec!["dist".to_owned()]; + + let snapshot = repo_support::materialize(fixture.path(), options); + let paths = inventory_paths(&snapshot); + + assert!(!paths.iter().any(|path| path.starts_with("dist/"))); + assert!(!paths.iter().any(|path| path.starts_with("target/"))); + assert!(paths.contains(&"build/output.txt".to_owned())); + assert!(paths.contains(&"node_modules/pkg/index.js".to_owned())); +} + +#[test] +fn common_cache_and_vendor_dirs_are_not_implicitly_excluded() { + let fixture = copy_fixture_tree( + "fixtures/repo/trees/well_known_excludes", + "repo-ignore-defaults", + ); + write_file(&fixture.path().join("__pycache__/module.pyc"), b"pyc"); + write_file(&fixture.path().join("target/cache.txt"), b"cached"); + let snapshot = repo_support::materialize(fixture.path(), default_snapshot_options()); + let paths = inventory_paths(&snapshot); + + assert!(paths.contains(&".venv/bin/python".to_owned())); + assert!(paths.contains(&"__pycache__/module.pyc".to_owned())); + assert!(paths.contains(&"build/output.txt".to_owned())); + assert!(paths.contains(&"dist/bundle.js".to_owned())); + assert!(paths.contains(&"node_modules/pkg/index.js".to_owned())); + assert!(paths.contains(&"target/cache.txt".to_owned())); + assert!(paths.contains(&"venv/bin/python".to_owned())); + assert!(paths.iter().all(|path| !path.contains('\\'))); +} + +#[test] +fn typed_well_known_excludes_remove_only_the_selected_canonical_directories() { + let fixture = copy_fixture_tree( + "fixtures/repo/trees/well_known_excludes", + "repo-ignore-well-known", + ); + write_file(&fixture.path().join("__pycache__/module.pyc"), b"pyc"); + write_file(&fixture.path().join("target/cache.txt"), b"cached"); + let mut options = default_snapshot_options(); + options.well_known_excludes = vec![ + repo::WellKnownExclude::RustTarget, + repo::WellKnownExclude::NodeModules, + repo::WellKnownExclude::PythonHiddenVenv, + repo::WellKnownExclude::PythonVenv, + repo::WellKnownExclude::PythonPycache, + repo::WellKnownExclude::WebDist, + repo::WellKnownExclude::WebBuild, + ]; + + let snapshot = repo_support::materialize(fixture.path(), options); + let paths = inventory_paths(&snapshot); + + assert!(!paths.iter().any(|path| path.starts_with(".venv/"))); + assert!(!paths.iter().any(|path| path.starts_with("__pycache__/"))); + assert!(!paths.iter().any(|path| path.starts_with("build/"))); + assert!(!paths.iter().any(|path| path.starts_with("dist/"))); + assert!(!paths.iter().any(|path| path.starts_with("node_modules/"))); + assert!(!paths.iter().any(|path| path.starts_with("target/"))); + assert!(!paths.iter().any(|path| path.starts_with("venv/"))); + assert!(paths.contains(&"src/lib.rs".to_owned())); +} + +#[cfg(unix)] +#[test] +fn followed_symlinks_still_compose_with_caller_glob_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-ignore-follow-glob"); + std::fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file(&repo_root.path().join("dist/bundle.js"), b"bundle"); + symlink("dist/bundle.js", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let mut options = default_snapshot_options(); + options.symlink_policy = repo::SymlinkPolicy::Follow; + options.exclude_globs = vec!["dist/**".to_owned()]; + + let snapshot = repo_support::materialize(repo_root.path(), options); + let paths = inventory_paths(&snapshot); + + assert!(!paths.iter().any(|path| path == "link.txt")); + assert!(!paths.iter().any(|path| path.starts_with("dist/"))); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot.stats.skipped_by_ignore >= 2); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn followed_symlinks_compose_with_directory_glob_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-ignore-follow-dir-glob"); + std::fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file(&repo_root.path().join("dist/bundle.js"), b"bundle"); + symlink("dist/bundle.js", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let mut options = default_snapshot_options(); + options.symlink_policy = repo::SymlinkPolicy::Follow; + options.exclude_globs = vec!["dist".to_owned()]; + + let snapshot = repo_support::materialize(repo_root.path(), options); + let paths = inventory_paths(&snapshot); + + assert!(!paths.iter().any(|path| path == "link.txt")); + assert!(!paths.iter().any(|path| path.starts_with("dist/"))); + assert_eq!(snapshot.stats.file_count, 0); + assert_eq!(snapshot.stats.skipped_by_ignore, 3); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn followed_symlinks_compose_with_directory_slash_glob_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-ignore-follow-dir-slash-glob"); + std::fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file(&repo_root.path().join("dist/bundle.js"), b"bundle"); + symlink("dist/bundle.js", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let mut options = default_snapshot_options(); + options.symlink_policy = repo::SymlinkPolicy::Follow; + options.exclude_globs = vec!["dist/".to_owned()]; + + let snapshot = repo_support::materialize(repo_root.path(), options); + let paths = inventory_paths(&snapshot); + + assert!(!paths.iter().any(|path| path == "link.txt")); + assert!(!paths.iter().any(|path| path.starts_with("dist/"))); + assert_eq!(snapshot.stats.file_count, 0); + assert_eq!(snapshot.stats.skipped_by_ignore, 3); + assert!(snapshot.diagnostics.is_empty()); +} diff --git a/crates/lift/tests/repo_purity.rs b/crates/lift/tests/repo_purity.rs new file mode 100644 index 000000000..215de2e1e --- /dev/null +++ b/crates/lift/tests/repo_purity.rs @@ -0,0 +1,223 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time before unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "substrate-lift-{prefix}-{}-{suffix}", + std::process::id(), + )); + fs::create_dir_all(&path).expect("temp dir should be creatable"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn write_file(path: &Path, contents: &[u8]) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("parent dir should be creatable"); + } + fs::write(path, contents).unwrap_or_else(|error| { + panic!("failed to write {}: {error}", path.display()); + }); +} + +fn snapshot_request( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::SnapshotRequest { + repo::SnapshotRequest { + root: repo::RepoRoot::from_dir(root).expect("repo root should parse"), + source, + options, + } +} + +fn materialize( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::RepoSnapshot { + repo::materialize_snapshot(&snapshot_request(root, source, options)) + .expect("snapshot should build") +} + +fn run_git(repo_root: &Path, args: &[&str]) { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +fn git_stdout(repo_root: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8(output.stdout) + .expect("git output should be utf-8") + .trim() + .to_owned() +} + +fn init_git_repo(prefix: &str) -> TempDir { + let temp = TempDir::new(prefix); + run_git(temp.path(), &["init", "--quiet"]); + temp +} + +fn commit_all(repo_root: &Path, message: &str) { + run_git(repo_root, &["add", "-A"]); + run_git( + repo_root, + &[ + "-c", + "user.name=Substrate Lift", + "-c", + "user.email=lift@example.com", + "commit", + "--quiet", + "-m", + message, + ], + ); +} + +#[test] +fn worktree_snapshot_survives_source_mutation_and_deletion() { + let temp = TempDir::new("repo-purity-worktree"); + fs::create_dir_all(temp.path().join(".git")).expect("git dir should exist"); + write_file( + &temp.path().join("Cargo.toml"), + b"[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ); + write_file( + &temp.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"fixture\"\n}\n", + ); + + let snapshot = materialize( + temp.path(), + repo::SnapshotSource::Worktree, + repo::SnapshotOptions::default(), + ); + let path = crate::kernel::RepoPath::parse("src/lib.rs").expect("path should parse"); + let entry_before = snapshot.entry(&path).expect("entry should exist").clone(); + let bytes_before = snapshot + .read_bytes(&path) + .expect("bytes should exist") + .to_vec(); + let fingerprint_before = snapshot.fingerprint.clone(); + + fs::write(temp.path().join("src/lib.rs"), b"changed").expect("source should mutate"); + fs::remove_file(temp.path().join("Cargo.toml")).expect("source should delete"); + + assert_eq!(snapshot.entry(&path), Some(&entry_before)); + assert_eq!( + snapshot.read_bytes(&path).expect("bytes should remain"), + bytes_before + ); + assert_eq!(snapshot.fingerprint, fingerprint_before); +} + +#[test] +fn gitrev_snapshot_survives_head_advance_and_worktree_mutation() { + let repo_root = init_git_repo("repo-purity-gitrev"); + write_file( + &repo_root.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"first\"\n}\n", + ); + commit_all(repo_root.path(), "first"); + let first_rev = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let snapshot = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { + rev: first_rev.clone(), + }, + repo::SnapshotOptions::default(), + ); + let path = crate::kernel::RepoPath::parse("src/lib.rs").expect("path should parse"); + let entry_before = snapshot.entry(&path).expect("entry should exist").clone(); + let bytes_before = snapshot + .read_bytes(&path) + .expect("bytes should exist") + .to_vec(); + let fingerprint_before = snapshot.fingerprint.clone(); + + write_file( + &repo_root.path().join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"second\"\n}\n", + ); + write_file(&repo_root.path().join("notes.txt"), b"new file"); + commit_all(repo_root.path(), "second"); + + assert_eq!( + snapshot.source, + repo::SnapshotSource::GitRev { rev: first_rev } + ); + assert_eq!(snapshot.entry(&path), Some(&entry_before)); + assert_eq!( + snapshot.read_bytes(&path).expect("bytes should remain"), + bytes_before + ); + assert_eq!(snapshot.fingerprint, fingerprint_before); +} diff --git a/crates/lift/tests/repo_root.rs b/crates/lift/tests/repo_root.rs new file mode 100644 index 000000000..c1da4eeda --- /dev/null +++ b/crates/lift/tests/repo_root.rs @@ -0,0 +1,164 @@ +#![allow(unused_crate_dependencies)] + +use std::collections::BTreeSet; +use std::fs; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use repo_support::{write_file, TempDir}; + +#[test] +fn nearest_marker_ancestor_wins_when_starting_from_a_file() { + let temp = TempDir::new("repo-root-file-start"); + let outer = temp.path().join("outer"); + let inner = outer.join("inner"); + fs::create_dir_all(outer.join(".git")).expect("outer marker dir"); + fs::create_dir_all(&inner).expect("inner dir"); + write_file(&inner.join(".git"), b"gitdir: ../.git/worktrees/inner\n"); + let start = inner.join("src/main.rs"); + write_file(&start, b"fn main() {}\n"); + + let root = repo::root::detect_repo_root(&start, &repo::RepoRootDetectionOptions::git_default()) + .expect("root should be detected"); + + assert_eq!( + root.as_path(), + fs::canonicalize(&inner).expect("inner path should canonicalize") + ); +} + +#[test] +fn directory_start_detects_root_with_git_directory_marker() { + let temp = TempDir::new("repo-root-dir-start"); + let repo_dir = temp.path().join("repo"); + fs::create_dir_all(repo_dir.join(".git")).expect("git dir"); + fs::create_dir_all(repo_dir.join("nested/deeper")).expect("nested dir"); + + let root = repo::root::detect_repo_root( + &repo_dir.join("nested/deeper"), + &repo::RepoRootDetectionOptions::git_default(), + ) + .expect("root should be detected"); + + assert_eq!( + root.as_path(), + fs::canonicalize(&repo_dir).expect("repo path should canonicalize") + ); +} + +#[test] +fn ceiling_dir_stops_ascent_before_higher_match() { + let temp = TempDir::new("repo-root-ceiling"); + let outer = temp.path().join("outer"); + let inner = outer.join("inner"); + fs::create_dir_all(outer.join(".git")).expect("git dir"); + fs::create_dir_all(inner.join("nested")).expect("nested dir"); + + let error = repo::root::detect_repo_root( + &inner.join("nested"), + &repo::RepoRootDetectionOptions { + markers: BTreeSet::from([repo::RootMarker::parse(".git").expect("marker")]), + ceiling_dir: Some(inner.clone()), + }, + ) + .expect_err("ceiling dir should stop ascent"); + + assert_eq!( + error, + repo::RepoError::RootNotFound { + start_path: inner.join("nested").display().to_string(), + markers: vec![".git".to_owned()], + } + ); +} + +#[test] +fn marker_order_does_not_change_selected_root() { + let temp = TempDir::new("repo-root-marker-order"); + let repo_dir = temp.path().join("repo"); + fs::create_dir_all(repo_dir.join(".git")).expect("git dir"); + fs::create_dir_all(repo_dir.join("nested")).expect("nested dir"); + write_file( + &repo_dir.join("Cargo.toml"), + b"[package]\nname = \"fixture\"\n", + ); + let start = repo_dir.join("nested/file.txt"); + write_file(&start, b"fixture"); + + let first = repo::root::detect_repo_root( + &start, + &repo::RepoRootDetectionOptions { + markers: BTreeSet::from([ + repo::RootMarker::parse(".git").expect("marker"), + repo::RootMarker::parse("Cargo.toml").expect("marker"), + ]), + ceiling_dir: None, + }, + ) + .expect("root should be detected"); + let second = repo::root::detect_repo_root( + &start, + &repo::RepoRootDetectionOptions { + markers: BTreeSet::from([ + repo::RootMarker::parse("Cargo.toml").expect("marker"), + repo::RootMarker::parse(".git").expect("marker"), + ]), + ceiling_dir: None, + }, + ) + .expect("root should be detected"); + + assert_eq!(first, second); + assert_eq!( + first.as_path(), + fs::canonicalize(&repo_dir).expect("repo path should canonicalize") + ); +} + +#[test] +fn missing_start_path_and_missing_root_emit_typed_failures() { + let temp = TempDir::new("repo-root-errors"); + let missing = temp.path().join("missing"); + let missing_error = + repo::root::detect_repo_root(&missing, &repo::RepoRootDetectionOptions::git_default()) + .expect_err("missing path should fail"); + assert_eq!( + missing_error, + repo::RepoError::StartPathNotFound { + path: missing.display().to_string(), + } + ); + + let existing = temp.path().join("plain"); + fs::create_dir_all(&existing).expect("plain dir"); + let root_error = + repo::root::detect_repo_root(&existing, &repo::RepoRootDetectionOptions::git_default()) + .expect_err("marker-less dir should fail"); + assert_eq!( + root_error, + repo::RepoError::RootNotFound { + start_path: existing.display().to_string(), + markers: vec![".git".to_owned()], + } + ); +} diff --git a/crates/lift/tests/repo_schema.rs b/crates/lift/tests/repo_schema.rs new file mode 100644 index 000000000..a99e4fa5a --- /dev/null +++ b/crates/lift/tests/repo_schema.rs @@ -0,0 +1,463 @@ +#![allow(unused_crate_dependencies)] + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema::Validator; +use predicates as _; +use serde::{Deserialize, Serialize}; +use serde_jcs as _; +use serde_json::Value; +use sha2 as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use repo_support::{ + copy_fixture_tree, default_snapshot_options, load_json, load_text, manifest_from_diff, + manifest_from_snapshot, manifest_from_snapshot_with_source, +}; + +#[derive(Debug, Deserialize, Serialize)] +struct FixtureManifest { + version: u32, + case: String, + source_kind: String, + source_rev: Option, + options: FixtureOptions, + files: Vec, + snapshot_fingerprint: String, + stats: FixtureStats, +} + +#[derive(Debug, Deserialize, Serialize)] +struct FixtureOptions { + symlink_policy: String, + well_known_excludes: Vec, + exclude_globs: Vec, + non_utf8_path_policy: String, + max_file_bytes: Option, + large_file_policy: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct FixtureFile { + path: String, + file_id: String, + blob_fingerprint: String, + size_bytes: u64, +} + +#[derive(Debug, Deserialize, Serialize)] +struct FixtureStats { + file_count: u64, + total_bytes: u64, + skipped_by_ignore: u64, + skipped_symlinks: u64, + skipped_non_utf8_paths: u64, + skipped_large_files: u64, + skipped_unsupported_file_kinds: u64, +} + +#[derive(Debug, Deserialize, Serialize)] +struct DiffFixtureManifest { + version: u32, + case: String, + base_fingerprint: String, + head_fingerprint: String, + entries: Vec, + diff_fingerprint: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct DiffFixtureEntry { + path: String, + kind: String, + before_blob_fingerprint: Option, + after_blob_fingerprint: Option, +} + +fn assert_diff_entry_shape_invariant(entry: &DiffFixtureEntry, context: &str) { + match entry.kind.as_str() { + "added" => { + assert!( + entry.before_blob_fingerprint.is_none(), + "{context} added entries must not include before_blob_fingerprint" + ); + assert!( + entry.after_blob_fingerprint.is_some(), + "{context} added entries must include after_blob_fingerprint" + ); + } + "modified" => { + assert!( + entry.before_blob_fingerprint.is_some(), + "{context} modified entries must include before_blob_fingerprint" + ); + assert!( + entry.after_blob_fingerprint.is_some(), + "{context} modified entries must include after_blob_fingerprint" + ); + } + "removed" => { + assert!( + entry.before_blob_fingerprint.is_some(), + "{context} removed entries must include before_blob_fingerprint" + ); + assert!( + entry.after_blob_fingerprint.is_none(), + "{context} removed entries must not include after_blob_fingerprint" + ); + } + other => panic!("{context} should use lowercase diff kinds, got {other}"), + } +} + +fn assert_diff_manifest_shape_invariants(manifest: &DiffFixtureManifest, context: &str) { + for entry in &manifest.entries { + assert_diff_entry_shape_invariant(entry, context); + } +} + +fn assert_snapshot_manifest_shape_invariants(manifest: &FixtureManifest, context: &str) { + match manifest.source_kind.as_str() { + "worktree" => { + assert!( + manifest.source_rev.is_none(), + "{context} worktree manifests must omit source_rev" + ); + } + "git_rev" => { + let source_rev = manifest + .source_rev + .as_deref() + .expect("git_rev manifests must include source_rev"); + assert!( + !source_rev.is_empty(), + "{context} git_rev manifests must use a non-empty source_rev" + ); + } + other => panic!("{context} should use a supported source_kind, got {other}"), + } + + let unique_excludes = manifest + .options + .well_known_excludes + .iter() + .copied() + .collect::>(); + assert_eq!( + unique_excludes.len(), + manifest.options.well_known_excludes.len(), + "{context} should not repeat well_known_excludes" + ); +} + +#[test] +fn embedded_snapshot_schema_matches_disk() { + assert_eq!( + repo::schema::SNAPSHOT_MANIFEST_V1_SCHEMA_JSON, + load_text("schemas/repo/snapshot_manifest.v1.json") + ); + assert_eq!( + repo::schema::SNAPSHOT_MANIFEST_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/repo/snapshot_manifest.v1.json" + ); + assert_eq!( + repo::schema::SNAPSHOT_MANIFEST_V1_SCHEMA_FILE, + "snapshot_manifest.v1.json" + ); + assert_eq!(repo::schema::SNAPSHOT_MANIFEST_V1_SCHEMA_VERSION, 1); +} + +#[test] +fn embedded_diff_schema_matches_disk() { + assert_eq!( + repo::schema::DIFF_MANIFEST_V1_SCHEMA_JSON, + load_text("schemas/repo/diff_manifest.v1.json") + ); + assert_eq!( + repo::schema::DIFF_MANIFEST_V1_SCHEMA_ID, + "https://schemas.substrate.dev/lift/repo/diff_manifest.v1.json" + ); + assert_eq!( + repo::schema::DIFF_MANIFEST_V1_SCHEMA_FILE, + "diff_manifest.v1.json" + ); + assert_eq!(repo::schema::DIFF_MANIFEST_V1_SCHEMA_VERSION, 1); +} + +#[test] +fn valid_fixture_manifests_validate_and_deserialize() { + let validator = repo_support::snapshot_validator(); + + for relative in [ + "fixtures/repo/valid/manifest_follow_policy.json", + "fixtures/repo/valid/manifest_git_rev.json", + "fixtures/repo/valid/manifest_minimal.json", + ] { + let instance = load_json(relative); + validator + .validate(&instance) + .unwrap_or_else(|_| panic!("{relative} should validate")); + let parsed: FixtureManifest = serde_json::from_value(instance) + .unwrap_or_else(|_| panic!("{relative} should deserialize")); + assert_eq!(parsed.version, 1, "{relative} should be v1"); + assert_snapshot_manifest_shape_invariants(&parsed, relative); + } +} + +#[test] +fn valid_diff_fixture_manifests_validate_and_deserialize() { + let validator = repo_support::diff_validator(); + + for relative in [ + "fixtures/repo/diff/valid/added_file.json", + "fixtures/repo/diff/valid/empty_diff.json", + "fixtures/repo/diff/valid/modified_file.json", + "fixtures/repo/diff/valid/removed_file.json", + "fixtures/repo/diff/valid/rename_as_add_remove.json", + ] { + let instance = load_json(relative); + validator + .validate(&instance) + .unwrap_or_else(|_| panic!("{relative} should validate")); + let parsed: DiffFixtureManifest = serde_json::from_value(instance) + .unwrap_or_else(|_| panic!("{relative} should deserialize")); + assert_eq!(parsed.version, 1, "{relative} should be v1"); + assert_diff_manifest_shape_invariants(&parsed, relative); + } +} + +#[test] +fn invalid_fixture_manifests_fail_validation() { + let validator = repo_support::snapshot_validator(); + for relative in [ + "fixtures/repo/invalid/manifest_bad_repo_path.json", + "fixtures/repo/invalid/manifest_bad_well_known_exclude.json", + "fixtures/repo/invalid/manifest_git_rev_missing_source_rev.json", + "fixtures/repo/invalid/manifest_missing_stats.json", + "fixtures/repo/invalid/manifest_worktree_with_source_rev.json", + ] { + let instance = load_json(relative); + assert!( + validator.validate(&instance).is_err(), + "{relative} should fail validation" + ); + } +} + +#[test] +fn invalid_diff_fixture_manifests_fail_deterministically() { + let validator = repo_support::diff_validator(); + + for relative in [ + "fixtures/repo/diff/invalid/manifest_added_with_before_blob.json", + "fixtures/repo/diff/invalid/manifest_bad_kind.json", + "fixtures/repo/diff/invalid/manifest_bad_repo_path.json", + "fixtures/repo/diff/invalid/manifest_before_after_shape_invalid.json", + "fixtures/repo/diff/invalid/manifest_modified_missing_after_blob.json", + "fixtures/repo/diff/invalid/manifest_modified_missing_before_blob.json", + "fixtures/repo/diff/invalid/manifest_missing_diff_fingerprint.json", + "fixtures/repo/diff/invalid/manifest_removed_with_after_blob.json", + ] { + let instance = load_json(relative); + assert!( + validator.validate(&instance).is_err(), + "{relative} should fail validation" + ); + assert!( + validator.validate(&instance).is_err(), + "{relative} should keep failing on repeated validation" + ); + } +} + +#[test] +fn generated_snapshot_manifest_validates_and_preserves_runtime_invariants() { + let validator: Validator = repo_support::snapshot_validator(); + let fixture = copy_fixture_tree("fixtures/repo/trees/basic_worktree", "repo-schema"); + let mut options = default_snapshot_options(); + options.symlink_policy = repo::SymlinkPolicy::Follow; + options.well_known_excludes = vec![ + repo::WellKnownExclude::RustTarget, + repo::WellKnownExclude::WebDist, + ]; + let snapshot = repo_support::materialize(fixture.path(), options.clone()); + let manifest = manifest_from_snapshot("basic-worktree", &options, &snapshot); + + validator + .validate(&manifest) + .expect("generated manifest should validate"); + + let parsed: FixtureManifest = + serde_json::from_value(manifest.clone()).expect("generated manifest should deserialize"); + assert_snapshot_manifest_shape_invariants(&parsed, "generated snapshot manifest"); + let manifest_paths = parsed + .files + .iter() + .map(|file| file.path.clone()) + .collect::>(); + let mut sorted_paths = manifest_paths.clone(); + sorted_paths.sort(); + assert_eq!(manifest_paths, sorted_paths); + assert_eq!(parsed.stats.file_count as usize, parsed.files.len()); + assert_eq!( + parsed.stats.total_bytes, + parsed.files.iter().map(|file| file.size_bytes).sum::() + ); + assert_eq!( + parsed.snapshot_fingerprint, + snapshot.fingerprint.as_str().to_owned() + ); + assert_eq!( + parsed.options.well_known_excludes, + vec![ + repo::WellKnownExclude::RustTarget, + repo::WellKnownExclude::WebDist, + ] + ); + + let manifest_value: Value = manifest; + assert_eq!( + manifest_value["snapshot_fingerprint"].as_str(), + Some(snapshot.fingerprint.as_str()) + ); + assert_eq!(manifest_value["source_kind"].as_str(), Some("worktree")); + assert!(manifest_value.get("source_rev").is_none()); +} + +#[test] +fn generated_git_rev_manifest_shape_validates() { + let validator: Validator = repo_support::snapshot_validator(); + let fixture = copy_fixture_tree("fixtures/repo/trees/basic_worktree", "repo-schema-git-rev"); + let mut options = default_snapshot_options(); + options.well_known_excludes = vec![repo::WellKnownExclude::NodeModules]; + let snapshot = repo_support::materialize(fixture.path(), options.clone()); + let source = repo::SnapshotSource::GitRev { + rev: "HEAD~1".to_owned(), + }; + let manifest = + manifest_from_snapshot_with_source("basic-git-rev", &source, &options, &snapshot); + + validator + .validate(&manifest) + .expect("generated git_rev manifest should validate"); + + let parsed: FixtureManifest = + serde_json::from_value(manifest.clone()).expect("generated manifest should deserialize"); + assert_snapshot_manifest_shape_invariants(&parsed, "generated git_rev manifest"); + assert_eq!(parsed.source_kind, "git_rev"); + assert_eq!(parsed.source_rev.as_deref(), Some("HEAD~1")); + assert_eq!( + parsed.options.well_known_excludes, + vec![repo::WellKnownExclude::NodeModules] + ); + assert_eq!(manifest["source_kind"].as_str(), Some("git_rev")); + assert_eq!(manifest["source_rev"].as_str(), Some("HEAD~1")); +} + +#[test] +fn generated_diff_manifest_validates() { + let validator: Validator = repo_support::diff_validator(); + let (_base_root, _head_root, base_snapshot, head_snapshot) = + repo_support::materialize_basic_worktree_pair( + "repo-diff-schema-generated", + |_| {}, + |head| { + repo_support::write_file(&head.join("docs/new.txt"), b"new"); + repo_support::write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"schema\"\n}\n", + ); + }, + ); + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + let manifest = manifest_from_diff("generated-diff", &diff); + + validator + .validate(&manifest) + .expect("generated diff manifest should validate"); + + let parsed: DiffFixtureManifest = + serde_json::from_value(manifest).expect("generated diff manifest should deserialize"); + assert_eq!(parsed.version, 1); + assert_diff_manifest_shape_invariants(&parsed, "generated diff manifest"); +} + +#[test] +fn manifest_ordering_matches_runtime_ordering() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = + repo_support::materialize_basic_worktree_pair( + "repo-diff-schema-ordering", + |base| { + repo_support::write_file(&base.join("target/cache.txt"), b"cached"); + }, + |head| { + repo_support::write_file(&head.join("docs/new.txt"), b"new"); + repo_support::write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"ordered\"\n}\n", + ); + }, + ); + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + let manifest = manifest_from_diff("generated-ordering", &diff); + + let runtime_paths = diff + .entries + .iter() + .map(|entry| entry.path.as_str().to_owned()) + .collect::>(); + let manifest_paths = manifest["entries"] + .as_array() + .expect("entries should be an array") + .iter() + .map(|entry| { + entry["path"] + .as_str() + .expect("path should be a string") + .to_owned() + }) + .collect::>(); + + assert_eq!(manifest_paths, runtime_paths); +} + +#[test] +fn manifest_fingerprint_matches_runtime_fingerprint() { + let (_base_root, _head_root, base_snapshot, head_snapshot) = + repo_support::materialize_basic_worktree_pair( + "repo-diff-schema-fingerprint", + |_| {}, + |head| { + repo_support::write_file( + &head.join("src/lib.rs"), + b"pub fn fixture_value() -> &'static str {\n \"fingerprint\"\n}\n", + ); + }, + ); + let diff = repo::build_diff(&base_snapshot, &head_snapshot); + let manifest = manifest_from_diff("generated-fingerprint", &diff); + + assert_eq!( + manifest["base_fingerprint"].as_str(), + Some(diff.base_fingerprint.as_str()) + ); + assert_eq!( + manifest["head_fingerprint"].as_str(), + Some(diff.head_fingerprint.as_str()) + ); + assert_eq!( + manifest["diff_fingerprint"].as_str(), + Some(diff.fingerprint.as_str()) + ); +} diff --git a/crates/lift/tests/repo_snapshot.rs b/crates/lift/tests/repo_snapshot.rs new file mode 100644 index 000000000..e46bcff0c --- /dev/null +++ b/crates/lift/tests/repo_snapshot.rs @@ -0,0 +1,172 @@ +#![allow(unused_crate_dependencies)] + +use std::fs; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; +#[path = "support/repo_support.rs"] +mod repo_support; + +use repo_support::{ + copy_fixture_tree, default_snapshot_options, inventory_paths, write_file, TempDir, +}; + +#[test] +fn snapshot_materializes_inventory_and_blob_store_for_regular_files() { + let fixture = copy_fixture_tree("fixtures/repo/trees/basic_worktree", "repo-snapshot-basic"); + let snapshot = repo_support::materialize(fixture.path(), default_snapshot_options()); + let paths = inventory_paths(&snapshot); + let expected_bytes = fs::read(fixture.path().join("src/lib.rs")).expect("fixture bytes"); + + assert_eq!(snapshot.inventory.len(), snapshot.blob_store.len()); + assert_eq!(snapshot.inventory.len() as u64, snapshot.stats.file_count); + assert!(paths.contains(&"Cargo.toml".to_owned())); + assert!(paths.contains(&"src/lib.rs".to_owned())); + assert_eq!( + snapshot + .read_bytes(&crate::kernel::RepoPath::parse("src/lib.rs").expect("path")) + .expect("blob bytes should exist"), + expected_bytes + ); +} + +#[test] +fn missing_blob_lookup_returns_typed_failure() { + let fixture = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + "repo-snapshot-missing", + ); + let snapshot = repo_support::materialize(fixture.path(), default_snapshot_options()); + let missing = crate::kernel::RepoPath::parse("missing.txt").expect("path"); + + assert_eq!( + snapshot + .read_bytes(&missing) + .expect_err("missing blob should fail"), + repo::RepoError::MissingBlob { path: missing } + ); +} + +#[test] +fn large_file_policy_behaves_as_configured() { + let temp = TempDir::new("repo-snapshot-large-file"); + fs::create_dir_all(temp.path().join(".git")).expect("git dir"); + write_file(&temp.path().join("big.bin"), b"1234567890"); + + let mut error_options = default_snapshot_options(); + error_options.max_file_bytes = Some(3); + error_options.large_file_policy = repo::LargeFilePolicy::Error; + assert_eq!( + repo::materialize_snapshot(&repo_support::snapshot_request(temp.path(), error_options)) + .expect_err("oversized file should error"), + repo::RepoError::FileTooLarge { + display_path: fs::canonicalize(temp.path().join("big.bin")) + .expect("large file path should canonicalize") + .display() + .to_string(), + size_bytes: 10, + max_file_bytes: 3, + } + ); + + let mut skip_options = default_snapshot_options(); + skip_options.max_file_bytes = Some(3); + skip_options.large_file_policy = repo::LargeFilePolicy::Skip; + let snapshot = repo_support::materialize(temp.path(), skip_options); + assert!(inventory_paths(&snapshot).is_empty()); + assert_eq!(snapshot.stats.skipped_large_files, 1); + assert_eq!( + snapshot.diagnostics[0].code.as_str(), + "repo.snapshot.file_too_large" + ); +} + +#[cfg(unix)] +#[test] +fn non_utf8_path_policy_errors_or_skips_deterministically() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let temp = TempDir::new("repo-snapshot-non-utf8"); + fs::create_dir_all(temp.path().join(".git")).expect("git dir"); + let bad_name = OsString::from_vec(vec![0xff, b'.', b't', b'x', b't']); + let bad_path = temp.path().join(bad_name); + if fs::write(&bad_path, b"bad").is_err() { + return; + } + + let error = repo::materialize_snapshot(&repo_support::snapshot_request( + temp.path(), + default_snapshot_options(), + )) + .expect_err("non-utf8 path should error by default"); + match error { + repo::RepoError::NonUtf8Path { .. } => {} + other => panic!("unexpected error: {other:?}"), + } + + let mut skip_options = default_snapshot_options(); + skip_options.non_utf8_path_policy = repo::NonUtf8PathPolicy::Skip; + let snapshot = repo_support::materialize(temp.path(), skip_options); + assert_eq!(snapshot.stats.skipped_non_utf8_paths, 1); + assert!(snapshot.inventory.is_empty()); + assert_eq!( + snapshot.diagnostics[0].code.as_str(), + "repo.snapshot.non_utf8_path" + ); +} + +#[cfg(unix)] +#[test] +fn symlinks_are_skipped_and_never_enter_inventory() { + use std::os::unix::fs::symlink; + + let temp = TempDir::new("repo-snapshot-symlink"); + fs::create_dir_all(temp.path().join(".git")).expect("git dir"); + write_file(&temp.path().join("target.txt"), b"target"); + symlink("target.txt", temp.path().join("link.txt")).expect("symlink should be creatable"); + + let snapshot = repo_support::materialize(temp.path(), default_snapshot_options()); + assert!(!inventory_paths(&snapshot).contains(&"link.txt".to_owned())); + assert_eq!(snapshot.stats.skipped_symlinks, 1); + assert_eq!( + snapshot.diagnostics[0].code.as_str(), + "repo.snapshot.symlink_skipped" + ); +} + +#[cfg(unix)] +#[test] +fn unsupported_file_kinds_are_skipped_with_diagnostics() { + use std::os::unix::net::UnixListener; + + let temp = TempDir::new("repo-snapshot-unsupported"); + fs::create_dir_all(temp.path().join(".git")).expect("git dir"); + let socket_path = temp.path().join("s.sock"); + let _listener = UnixListener::bind(&socket_path).expect("socket should bind"); + + let snapshot = repo_support::materialize(temp.path(), default_snapshot_options()); + assert!(inventory_paths(&snapshot).is_empty()); + assert_eq!(snapshot.stats.skipped_unsupported_file_kinds, 1); + assert_eq!( + snapshot.diagnostics[0].code.as_str(), + "repo.snapshot.unsupported_file_kind" + ); +} diff --git a/crates/lift/tests/repo_symlinks.rs b/crates/lift/tests/repo_symlinks.rs new file mode 100644 index 000000000..9de2808e9 --- /dev/null +++ b/crates/lift/tests/repo_symlinks.rs @@ -0,0 +1,407 @@ +#![allow(unused_crate_dependencies)] +#![cfg(unix)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use assert_cmd as _; +use gix as _; +use globset as _; +use jsonschema as _; +use predicates as _; +use serde as _; +use serde_jcs as _; +use serde_json as _; +use sha2 as _; +use substrate_lift as _; +use thiserror as _; +use toml as _; +use walkdir as _; + +mod kernel { + pub(crate) use substrate_lift::kernel::*; +} +#[path = "../src/repo/mod.rs"] +mod repo; + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time before unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "substrate-lift-{prefix}-{}-{suffix}", + std::process::id(), + )); + fs::create_dir_all(&path).expect("temp dir should be creatable"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn write_file(path: &Path, contents: &[u8]) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("parent dir should be creatable"); + } + fs::write(path, contents).unwrap_or_else(|error| { + panic!("failed to write {}: {error}", path.display()); + }); +} + +fn snapshot_request( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::SnapshotRequest { + repo::SnapshotRequest { + root: repo::RepoRoot::from_dir(root).expect("repo root should parse"), + source, + options, + } +} + +fn materialize( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::RepoSnapshot { + repo::materialize_snapshot(&snapshot_request(root, source, options)) + .expect("snapshot should build") +} + +fn run_git(repo_root: &Path, args: &[&str]) { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +fn git_stdout(repo_root: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8(output.stdout) + .expect("git output should be utf-8") + .trim() + .to_owned() +} + +fn init_git_repo(prefix: &str) -> TempDir { + let temp = TempDir::new(prefix); + run_git(temp.path(), &["init", "--quiet"]); + temp +} + +fn commit_all(repo_root: &Path, message: &str) { + run_git(repo_root, &["add", "-A"]); + run_git( + repo_root, + &[ + "-c", + "user.name=Substrate Lift", + "-c", + "user.email=lift@example.com", + "commit", + "--quiet", + "-m", + message, + ], + ); +} + +#[cfg(unix)] +#[test] +fn worktree_follow_materializes_target_bytes_at_the_link_path() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-symlink-worktree-follow"); + fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file(&repo_root.path().join("dir/target.txt"), b"target-bytes"); + symlink("dir/target.txt", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize(repo_root.path(), repo::SnapshotSource::Worktree, options); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + let target = crate::kernel::RepoPath::parse("dir/target.txt").expect("path should parse"); + + assert_eq!( + snapshot.read_bytes(&link).expect("link bytes should exist"), + b"target-bytes" + ); + assert_eq!( + snapshot + .entry(&link) + .expect("link entry should exist") + .blob_fingerprint, + snapshot + .entry(&target) + .expect("target entry should exist") + .blob_fingerprint + ); + assert_eq!(snapshot.stats.skipped_symlinks, 0); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn worktree_follow_rejects_symlink_escapes_outside_the_repo_root() { + use std::os::unix::fs::symlink; + + let temp = TempDir::new("repo-symlink-worktree-escape"); + let repo_root = temp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("git dir should exist"); + write_file(&temp.path().join("outside.txt"), b"outside"); + symlink("../outside.txt", repo_root.join("link.txt")).expect("symlink should exist"); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + ..repo::SnapshotOptions::default() + }; + let error = repo::materialize_snapshot(&snapshot_request( + &repo_root, + repo::SnapshotSource::Worktree, + options, + )) + .expect_err("escaping symlinks should fail"); + + assert_eq!( + error, + repo::RepoError::SymlinkTargetEscape { + path: crate::kernel::RepoPath::parse("link.txt").expect("path should parse"), + target: "../outside.txt".to_owned(), + } + ); +} + +#[cfg(unix)] +#[test] +fn worktree_follow_detects_symlink_loops() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-symlink-worktree-loop"); + fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + symlink("b.txt", repo_root.path().join("a.txt")).expect("symlink should exist"); + symlink("a.txt", repo_root.path().join("b.txt")).expect("symlink should exist"); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + ..repo::SnapshotOptions::default() + }; + let error = repo::materialize_snapshot(&snapshot_request( + repo_root.path(), + repo::SnapshotSource::Worktree, + options, + )) + .expect_err("symlink loops should fail"); + + assert_eq!( + error, + repo::RepoError::SymlinkTargetLoop { + path: crate::kernel::RepoPath::parse("a.txt").expect("path should parse"), + target: "a.txt".to_owned(), + } + ); +} + +#[cfg(unix)] +#[test] +fn worktree_follow_skips_links_that_resolve_into_intrinsic_git_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-symlink-worktree-ignore-git"); + fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file( + &repo_root.path().join(".git/config"), + b"[core]\nrepositoryformatversion = 0\n", + ); + symlink(".git/config", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize(repo_root.path(), repo::SnapshotSource::Worktree, options); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + + assert!(snapshot.entry(&link).is_none()); + assert_eq!(snapshot.stats.skipped_by_ignore, 2); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn worktree_follow_skips_links_that_resolve_into_typed_excludes() { + use std::os::unix::fs::symlink; + + let repo_root = TempDir::new("repo-symlink-worktree-ignore-typed"); + fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file(&repo_root.path().join("target/cache.txt"), b"cached"); + symlink("target/cache.txt", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + well_known_excludes: vec![repo::WellKnownExclude::RustTarget], + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize(repo_root.path(), repo::SnapshotSource::Worktree, options); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + + assert!(snapshot.entry(&link).is_none()); + assert_eq!(snapshot.stats.skipped_by_ignore, 3); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn worktree_follow_applies_large_file_skip_before_reading_target_bytes() { + use std::os::unix::fs::{symlink, PermissionsExt}; + + let repo_root = TempDir::new("repo-symlink-worktree-large-file"); + fs::create_dir_all(repo_root.path().join(".git")).expect("git dir should exist"); + write_file(&repo_root.path().join("big.bin"), b"1234567890"); + fs::set_permissions( + repo_root.path().join("big.bin"), + fs::Permissions::from_mode(0o000), + ) + .expect("target file permissions should update"); + symlink("big.bin", repo_root.path().join("link.txt")).expect("symlink should exist"); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + max_file_bytes: Some(3), + large_file_policy: repo::LargeFilePolicy::Skip, + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize(repo_root.path(), repo::SnapshotSource::Worktree, options); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + + assert!(snapshot.entry(&link).is_none()); + assert_eq!(snapshot.stats.skipped_large_files, 2); + assert_eq!(snapshot.stats.file_count, 0); + assert!(snapshot + .diagnostics + .iter() + .all(|diagnostic| diagnostic.code.as_str() == "repo.snapshot.file_too_large")); + assert!(snapshot.diagnostics.iter().any(|diagnostic| { + diagnostic + .subject + .as_ref() + .and_then(|subject| subject.repo_path.as_ref()) + == Some(&link) + })); + assert!(snapshot.diagnostics.iter().any(|diagnostic| { + diagnostic + .subject + .as_ref() + .is_some_and(|subject| subject.display_path.ends_with("big.bin")) + })); +} + +#[cfg(unix)] +#[test] +fn gitrev_follow_materializes_committed_symlink_targets() { + use std::os::unix::fs::symlink; + + let repo_root = init_git_repo("repo-symlink-gitrev-follow"); + write_file(&repo_root.path().join("dir/target.txt"), b"git-target"); + symlink("dir/target.txt", repo_root.path().join("link.txt")).expect("symlink should exist"); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + ..repo::SnapshotOptions::default() + }; + let snapshot = materialize( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head }, + options, + ); + let link = crate::kernel::RepoPath::parse("link.txt").expect("path should parse"); + let target = crate::kernel::RepoPath::parse("dir/target.txt").expect("path should parse"); + + assert_eq!( + snapshot.read_bytes(&link).expect("link bytes should exist"), + b"git-target" + ); + assert_eq!( + snapshot + .entry(&link) + .expect("link entry should exist") + .blob_fingerprint, + snapshot + .entry(&target) + .expect("target entry should exist") + .blob_fingerprint + ); + assert_eq!(snapshot.stats.skipped_symlinks, 0); + assert!(snapshot.diagnostics.is_empty()); +} + +#[cfg(unix)] +#[test] +fn gitrev_follow_rejects_dangling_symlink_targets() { + use std::os::unix::fs::symlink; + + let repo_root = init_git_repo("repo-symlink-gitrev-dangling"); + symlink("missing.txt", repo_root.path().join("link.txt")).expect("symlink should exist"); + commit_all(repo_root.path(), "initial"); + let head = git_stdout(repo_root.path(), &["rev-parse", "HEAD"]); + + let options = repo::SnapshotOptions { + symlink_policy: repo::SymlinkPolicy::Follow, + ..repo::SnapshotOptions::default() + }; + let error = repo::materialize_snapshot(&snapshot_request( + repo_root.path(), + repo::SnapshotSource::GitRev { rev: head }, + options, + )) + .expect_err("dangling symlinks should fail"); + + assert_eq!( + error, + repo::RepoError::SymlinkTargetDangling { + path: crate::kernel::RepoPath::parse("link.txt").expect("path should parse"), + target: "missing.txt".to_owned(), + } + ); +} diff --git a/crates/lift/tests/support/lang_support.rs b/crates/lift/tests/support/lang_support.rs new file mode 100644 index 000000000..99d256a40 --- /dev/null +++ b/crates/lift/tests/support/lang_support.rs @@ -0,0 +1,351 @@ +use std::collections::BTreeMap; + +use toml::Value; + +#[allow(dead_code)] +mod lang { + pub(crate) use crate::pack::{LanguageId, QueryEngineKind}; + + pub(crate) mod adapter { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lang/adapter.rs")); + } + pub(crate) mod cache { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lang/cache.rs")); + } + pub(crate) mod capabilities { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/lang/capabilities.rs" + )); + } + pub(crate) mod driver { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lang/driver.rs")); + } + pub(crate) mod error { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lang/error.rs")); + } + pub(crate) mod model { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lang/model.rs")); + } + pub(crate) mod registry { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lang/registry.rs")); + } + + pub(crate) use adapter::{ + AdapterDescriptor, AdapterName, AdapterParseOutput, AdapterParseResult, LanguageAdapter, + ParseInput, + }; + pub(crate) use cache::{ + CachedParseOutcome, NoopParseCache, ParseCache, ParseCacheKey, ParseCacheLookup, + }; + pub(crate) use capabilities::AdapterCapabilities; + pub(crate) use driver::ParseDriver; + pub(crate) use error::{LangError, LangResult}; + pub(crate) use model::{ + compare_symbol_drafts, sort_local_edges, sort_local_symbols, sort_surface_markers, + symbol_identity_lemma, EdgeEndpoint, EdgeEndpointDraft, FailedParse, LocalEdge, + LocalEdgeDraft, LocalEdgeKind, LocalSymbol, LocalSymbolDraft, MissingRequestedLanguage, + ParseRequest, ParseScope, ParseSet, ParseStats, ParsedUnit, ReferenceTarget, + ReferenceTargetDraft, SkippedParse, SkippedReason, SurfaceMarker, SurfaceMarkerDraft, + SymbolKind, SymbolVisibility, + }; + pub(crate) use registry::{LanguageRegistry, LanguageRegistryBuilder}; +} + +#[derive(Clone, Debug)] +pub(crate) struct TomlTestAdapter { + descriptor: lang::AdapterDescriptor, +} + +impl TomlTestAdapter { + pub(crate) fn new() -> Self { + Self { + descriptor: lang::AdapterDescriptor { + name: lang::AdapterName::parse("test.toml_proof").expect("adapter name"), + language: pack::LanguageId::Toml, + version: "proof-v1".to_owned(), + }, + } + } +} + +impl lang::LanguageAdapter for TomlTestAdapter { + fn descriptor(&self) -> lang::AdapterDescriptor { + self.descriptor.clone() + } + + fn recognizes(&self, input: &lang::ParseInput<'_>) -> bool { + input.path.as_str().ends_with(".toml") + } + + fn parse(&self, input: &lang::ParseInput<'_>) -> lang::AdapterParseResult { + let text = match std::str::from_utf8(input.bytes) { + Ok(text) => text, + Err(error) => { + return lang::AdapterParseResult::Failed { + diagnostics: vec![file_diagnostic( + "lang.toml.invalid_utf8", + format!("file is not valid UTF-8: {error}"), + input.path, + )], + }; + } + }; + + let document: Value = match toml::from_str(text) { + Ok(document) => document, + Err(error) => { + return lang::AdapterParseResult::Failed { + diagnostics: vec![file_diagnostic( + "lang.toml.parse_failed", + format!("failed to parse TOML: {error}"), + input.path, + )], + }; + } + }; + + let mut collector = TomlCollector::new(); + match document { + Value::Table(table) => collector.walk_table(&[], &table), + other => collector.walk_value(&[], &other), + } + + let (symbols, edges) = collector.finish(); + + lang::AdapterParseResult::Parsed(lang::AdapterParseOutput { + symbols, + edges, + surface_markers: Vec::new(), + diagnostics: Vec::new(), + }) + } +} + +struct TomlCollector { + symbols: BTreeMap, + edges: Vec, + edge_keys: BTreeSet, +} + +impl TomlCollector { + fn new() -> Self { + Self { + symbols: BTreeMap::new(), + edges: Vec::new(), + edge_keys: BTreeSet::new(), + } + } + + fn walk_table(&mut self, prefix: &[String], table: &toml::map::Map) { + let mut entries = table.iter().collect::>(); + entries.sort_by_key(|(left, _)| *left); + + for (key, value) in entries { + let mut path = prefix.to_vec(); + path.push(key.clone()); + self.ensure_symbol(&path); + self.push_contains(prefix, &path); + self.walk_value(&path, value); + } + } + + fn walk_value(&mut self, current: &[String], value: &Value) { + match value { + Value::Table(table) => self.walk_table(current, table), + Value::Array(items) => { + for item in items { + if let Value::String(text) = item { + self.push_config_ref(current, text); + } + } + } + Value::String(text) => self.push_config_ref(current, text), + Value::Boolean(_) + | Value::Datetime(_) + | Value::Float(_) + | Value::Integer(_) => {} + } + } + + fn ensure_symbol(&mut self, path: &[String]) -> String { + let local_key = symbol_local_key(path); + self.symbols + .entry(local_key.clone()) + .or_insert_with(|| lang::LocalSymbolDraft { + local_key: local_key.clone(), + kind: lang::SymbolKind::ConfigKey, + name: path.last().cloned(), + path: path.to_vec(), + span: crate::kernel::ByteSpan::new(0, 0).expect("zero span"), + visibility: lang::SymbolVisibility::Unknown, + }); + local_key + } + + fn push_contains(&mut self, parent: &[String], child: &[String]) { + let child_local_key = self.ensure_symbol(child); + let source = if parent.is_empty() { + lang::EdgeEndpointDraft::FileRoot + } else { + lang::EdgeEndpointDraft::Symbol { + local_key: self.ensure_symbol(parent), + } + }; + let target = lang::ReferenceTargetDraft::LocalSymbol { + local_key: child_local_key, + }; + self.push_edge(lang::LocalEdgeKind::Contains, source, target); + } + + fn push_config_ref(&mut self, source_path: &[String], value: &str) { + let Some(parts) = parse_config_ref(value) else { + return; + }; + let source = lang::EdgeEndpointDraft::Symbol { + local_key: self.ensure_symbol(source_path), + }; + let target = lang::ReferenceTargetDraft::QualifiedName { parts }; + self.push_edge(lang::LocalEdgeKind::ConfigRef, source, target); + } + + fn push_edge( + &mut self, + kind: lang::LocalEdgeKind, + source: lang::EdgeEndpointDraft, + target: lang::ReferenceTargetDraft, + ) { + let edge_key = format!( + "{}|{}|{}", + edge_kind_key(kind), + endpoint_key(&source), + target_key(&target) + ); + if self.edge_keys.insert(edge_key) { + self.edges.push(lang::LocalEdgeDraft { + kind, + source, + target, + span: None, + }); + } + } + + fn finish(self) -> (Vec, Vec) { + (self.symbols.into_values().collect(), self.edges) + } +} + +fn symbol_local_key(path: &[String]) -> String { + format!("config\0{}", path.join("\0")) +} + +fn parse_config_ref(value: &str) -> Option> { + let tail = value.strip_prefix("config://")?; + let parts = tail + .split('.') + .map(str::trim) + .filter(|segment| !segment.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + (!parts.is_empty()).then_some(parts) +} + +fn file_diagnostic(code: &str, message: String, path: &crate::kernel::RepoPath) -> crate::kernel::Diagnostic { + crate::kernel::Diagnostic { + code: crate::kernel::DiagnosticCode::parse(code).expect("diagnostic code"), + severity: crate::kernel::Severity::Error, + message, + subject: Some(crate::kernel::Locator { + path: path.clone(), + span: None, + json_pointer: None, + }), + related: Vec::new(), + help: None, + } +} + +fn edge_kind_key(kind: lang::LocalEdgeKind) -> &'static str { + match kind { + lang::LocalEdgeKind::Contains => "contains", + lang::LocalEdgeKind::Import => "import", + lang::LocalEdgeKind::Export => "export", + lang::LocalEdgeKind::Call => "call", + lang::LocalEdgeKind::TypeRef => "type_ref", + lang::LocalEdgeKind::Inherit => "inherit", + lang::LocalEdgeKind::Implement => "implement", + lang::LocalEdgeKind::TestRef => "test_ref", + lang::LocalEdgeKind::ConfigRef => "config_ref", + lang::LocalEdgeKind::SchemaRef => "schema_ref", + lang::LocalEdgeKind::Unknown => "unknown", + } +} + +fn endpoint_key(endpoint: &lang::EdgeEndpointDraft) -> String { + match endpoint { + lang::EdgeEndpointDraft::FileRoot => "file_root".to_owned(), + lang::EdgeEndpointDraft::Symbol { local_key } => format!("symbol:{local_key}"), + } +} + +fn target_key(target: &lang::ReferenceTargetDraft) -> String { + match target { + lang::ReferenceTargetDraft::LocalSymbol { local_key } => { + format!("local_symbol:{local_key}") + } + lang::ReferenceTargetDraft::QualifiedName { parts } => { + format!("qualified:{}", parts.join(".")) + } + lang::ReferenceTargetDraft::FilePath { path } => format!("file:{}", path.as_str()), + lang::ReferenceTargetDraft::ExternalPackage { package, symbol } => { + format!("package:{package}:{}", symbol.clone().unwrap_or_default()) + } + lang::ReferenceTargetDraft::Opaque { value } => format!("opaque:{value}"), + } +} + +pub(crate) fn fixture_case_path(case: &str) -> String { + format!("fixtures/lang/proof/toml_consumer_repo/{case}") +} + +pub(crate) fn materialize_fixture_case(case: &str) -> (repo_support::TempDir, repo::RepoSnapshot) { + let temp = repo_support::copy_fixture_tree( + &fixture_case_path(case), + &format!("lang-proof-{case}"), + ); + let snapshot = repo_support::materialize(temp.path(), repo_support::default_snapshot_options()); + (temp, snapshot) +} + +pub(crate) fn parse_snapshot_with_toml( + snapshot: &repo::RepoSnapshot, + request: lang::ParseRequest, +) -> lang::ParseSet { + let registry = lang::LanguageRegistryBuilder::new() + .register(TomlTestAdapter::new()) + .expect("register TOML adapter") + .build() + .expect("build registry"); + + lang::ParseDriver::new(registry) + .parse_snapshot(snapshot, &request) + .expect("parse snapshot") +} + +pub(crate) fn parse_fixture_case( + case: &str, +) -> (repo_support::TempDir, repo::RepoSnapshot, lang::ParseSet) { + let (temp, snapshot) = materialize_fixture_case(case); + let parse_set = parse_snapshot_with_toml(&snapshot, toml_snapshot_request()); + (temp, snapshot, parse_set) +} + +pub(crate) fn toml_snapshot_request() -> lang::ParseRequest { + let mut languages = BTreeSet::new(); + languages.insert(pack::LanguageId::Toml); + lang::ParseRequest { + languages, + scope: lang::ParseScope::Snapshot, + } +} diff --git a/crates/lift/tests/support/repo_support.rs b/crates/lift/tests/support/repo_support.rs new file mode 100644 index 000000000..080f8625c --- /dev/null +++ b/crates/lift/tests/support/repo_support.rs @@ -0,0 +1,318 @@ +#![allow(dead_code)] + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use jsonschema::{Retrieve, Uri, Validator}; +use serde_json::{json, Value}; + +use crate::repo; + +pub(crate) struct TempDir { + path: PathBuf, +} + +impl TempDir { + pub(crate) fn new(prefix: &str) -> Self { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time before unix epoch") + .as_nanos(); + let base = Path::new("/tmp"); + let path = if base.exists() { + base.to_path_buf() + } else { + std::env::temp_dir() + } + .join(format!( + "substrate-lift-{prefix}-{}-{suffix}", + std::process::id(), + )); + fs::create_dir_all(&path).expect("temp dir should be creatable"); + Self { path } + } + + pub(crate) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +pub(crate) fn crate_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} + +pub(crate) fn fixture_path(relative: &str) -> PathBuf { + crate_root().join(relative) +} + +pub(crate) fn load_json(relative: &str) -> Value { + let path = fixture_path(relative); + let contents = fs::read_to_string(&path).unwrap_or_else(|error| { + panic!("failed to read {}: {error}", path.display()); + }); + + serde_json::from_str(&contents).unwrap_or_else(|error| { + panic!("failed to parse {}: {error}", path.display()); + }) +} + +pub(crate) fn load_text(relative: &str) -> String { + let path = fixture_path(relative); + fs::read_to_string(&path).unwrap_or_else(|error| { + panic!("failed to read {}: {error}", path.display()); + }) +} + +pub(crate) fn copy_fixture_tree(relative: &str, prefix: &str) -> TempDir { + let src = fixture_path(relative); + let temp = TempDir::new(prefix); + copy_dir_all(&src, temp.path()); + temp +} + +pub(crate) fn copy_dir_all(src: &Path, dst: &Path) { + fs::create_dir_all(dst).expect("destination should be creatable"); + for entry in fs::read_dir(src).expect("source should be readable") { + let entry = entry.expect("dir entry should be readable"); + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + let file_type = entry.file_type().expect("file type should be readable"); + if file_type.is_dir() { + copy_dir_all(&src_path, &dst_path); + } else { + fs::copy(&src_path, &dst_path).unwrap_or_else(|error| { + panic!( + "failed to copy fixture {} -> {}: {error}", + src_path.display(), + dst_path.display() + ); + }); + } + } +} + +pub(crate) fn write_file(path: &Path, contents: &[u8]) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("parent dir should be creatable"); + } + fs::write(path, contents).unwrap_or_else(|error| { + panic!("failed to write {}: {error}", path.display()); + }); +} + +pub(crate) fn default_snapshot_options() -> repo::SnapshotOptions { + repo::SnapshotOptions::default() +} + +pub(crate) fn snapshot_request( + root: &Path, + options: repo::SnapshotOptions, +) -> repo::SnapshotRequest { + snapshot_request_with_source(root, repo::SnapshotSource::Worktree, options) +} + +pub(crate) fn snapshot_request_with_source( + root: &Path, + source: repo::SnapshotSource, + options: repo::SnapshotOptions, +) -> repo::SnapshotRequest { + repo::SnapshotRequest { + root: repo::root::RepoRoot::from_dir(root).expect("repo root should parse"), + source, + options, + } +} + +pub(crate) fn materialize(root: &Path, options: repo::SnapshotOptions) -> repo::RepoSnapshot { + repo::materialize_snapshot(&snapshot_request(root, options)).expect("snapshot should build") +} + +pub(crate) fn materialize_basic_worktree_pair( + prefix: &str, + mutate_base: FBase, + mutate_head: FHead, +) -> (TempDir, TempDir, repo::RepoSnapshot, repo::RepoSnapshot) +where + FBase: FnOnce(&Path), + FHead: FnOnce(&Path), +{ + let base = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + &format!("{prefix}-base"), + ); + let head = copy_fixture_tree( + "fixtures/repo/trees/basic_worktree", + &format!("{prefix}-head"), + ); + + mutate_base(base.path()); + mutate_head(head.path()); + + let options = default_snapshot_options(); + let base_snapshot = materialize(base.path(), options.clone()); + let head_snapshot = materialize(head.path(), options); + + (base, head, base_snapshot, head_snapshot) +} + +pub(crate) fn inventory_paths(snapshot: &repo::RepoSnapshot) -> Vec { + snapshot + .inventory + .iter() + .map(|entry| entry.path.as_str().to_owned()) + .collect() +} + +pub(crate) fn diff_validator() -> Validator { + let root_schema: Value = serde_json::from_str(repo::schema::DIFF_MANIFEST_V1_SCHEMA_JSON) + .expect("embedded diff schema should parse"); + let kernel_schema: Value = + serde_json::from_str(crate::kernel::PRIMITIVES_V1_SCHEMA_JSON).expect("kernel schema"); + let retriever = InMemoryRetriever { + schemas: HashMap::from([ + ( + crate::kernel::PRIMITIVES_V1_SCHEMA_ID.to_owned(), + kernel_schema.clone(), + ), + ("../kernel/primitives.v1.json".to_owned(), kernel_schema), + ]), + }; + + jsonschema::draft202012::options() + .with_retriever(retriever) + .build(&root_schema) + .expect("diff schema should compile") +} + +pub(crate) fn snapshot_validator() -> Validator { + let root_schema: Value = serde_json::from_str(repo::schema::SNAPSHOT_MANIFEST_V1_SCHEMA_JSON) + .expect("embedded repo schema should parse"); + let kernel_schema: Value = + serde_json::from_str(crate::kernel::PRIMITIVES_V1_SCHEMA_JSON).expect("kernel schema"); + let retriever = InMemoryRetriever { + schemas: HashMap::from([ + ( + crate::kernel::PRIMITIVES_V1_SCHEMA_ID.to_owned(), + kernel_schema.clone(), + ), + ("../kernel/primitives.v1.json".to_owned(), kernel_schema), + ]), + }; + + jsonschema::draft202012::options() + .with_retriever(retriever) + .build(&root_schema) + .expect("repo schema should compile") +} + +pub(crate) fn manifest_from_snapshot( + case: &str, + options: &repo::SnapshotOptions, + snapshot: &repo::RepoSnapshot, +) -> Value { + manifest_from_snapshot_with_source(case, &snapshot.source, options, snapshot) +} + +pub(crate) fn manifest_from_snapshot_with_source( + case: &str, + source: &repo::SnapshotSource, + options: &repo::SnapshotOptions, + snapshot: &repo::RepoSnapshot, +) -> Value { + let mut manifest = json!({ + "version": 1, + "case": case, + "source_kind": match source { + repo::SnapshotSource::Worktree => "worktree", + repo::SnapshotSource::GitRev { .. } => "git_rev", + }, + "options": { + "symlink_policy": match options.symlink_policy { + repo::SymlinkPolicy::Skip => "skip", + repo::SymlinkPolicy::Follow => "follow", + }, + "well_known_excludes": options.well_known_excludes, + "exclude_globs": options.exclude_globs, + "non_utf8_path_policy": match options.non_utf8_path_policy { + repo::NonUtf8PathPolicy::Error => "error", + repo::NonUtf8PathPolicy::Skip => "skip", + }, + "max_file_bytes": options.max_file_bytes, + "large_file_policy": match options.large_file_policy { + repo::LargeFilePolicy::Error => "error", + repo::LargeFilePolicy::Skip => "skip", + }, + }, + "files": snapshot.inventory.iter().map(|entry| { + json!({ + "path": entry.path.as_str(), + "file_id": entry.file_id.as_str(), + "blob_fingerprint": entry.blob_fingerprint.as_str(), + "size_bytes": entry.size_bytes, + }) + }).collect::>(), + "snapshot_fingerprint": snapshot.fingerprint.as_str(), + "stats": { + "file_count": snapshot.stats.file_count, + "total_bytes": snapshot.stats.total_bytes, + "skipped_by_ignore": snapshot.stats.skipped_by_ignore, + "skipped_symlinks": snapshot.stats.skipped_symlinks, + "skipped_non_utf8_paths": snapshot.stats.skipped_non_utf8_paths, + "skipped_large_files": snapshot.stats.skipped_large_files, + "skipped_unsupported_file_kinds": snapshot.stats.skipped_unsupported_file_kinds, + } + }); + + if let repo::SnapshotSource::GitRev { rev } = source { + manifest["source_rev"] = Value::String(rev.clone()); + } + + manifest +} + +pub(crate) fn manifest_from_diff(case: &str, diff: &repo::RepoDiff) -> Value { + json!({ + "version": 1, + "case": case, + "base_fingerprint": diff.base_fingerprint.as_str(), + "head_fingerprint": diff.head_fingerprint.as_str(), + "entries": diff.entries.iter().map(|entry| { + json!({ + "path": entry.path.as_str(), + "kind": match entry.kind { + repo::DiffKind::Added => "added", + repo::DiffKind::Modified => "modified", + repo::DiffKind::Removed => "removed", + }, + "before_blob_fingerprint": entry.before.as_ref().map(|before| before.blob_fingerprint.as_str()), + "after_blob_fingerprint": entry.after.as_ref().map(|after| after.blob_fingerprint.as_str()), + }) + }).collect::>(), + "diff_fingerprint": diff.fingerprint.as_str(), + }) +} + +#[derive(Clone, Debug)] +struct InMemoryRetriever { + schemas: HashMap, +} + +impl Retrieve for InMemoryRetriever { + fn retrieve( + &self, + uri: &Uri, + ) -> Result> { + self.schemas + .get(uri.as_str()) + .cloned() + .ok_or_else(|| format!("schema not found: {uri}").into()) + } +} diff --git a/crates/shell/src/execution/agent_runtime/state_store.rs b/crates/shell/src/execution/agent_runtime/state_store.rs index 92756a366..ba99b089f 100644 --- a/crates/shell/src/execution/agent_runtime/state_store.rs +++ b/crates/shell/src/execution/agent_runtime/state_store.rs @@ -563,7 +563,6 @@ impl AgentRuntimeStateStore { || participant.handle.role != MEMBER_ROLE || participant.handle.execution.scope != AgentExecutionScope::World || !participant.is_authoritative_live() - || !owner_process_is_alive(&participant) { continue; } @@ -3211,6 +3210,42 @@ mod tests { }); } + #[test] + #[serial_test::serial] + fn invalidate_stale_world_members_for_session_does_not_require_live_owner_pid() { + with_store(|store| { + let orchestrator = live_orchestrator("claude_code", "sess_live", "ash_orchestrator"); + let mut stale_member = + live_member("codex", "sess_live", "ash_member_old", "ash_orchestrator"); + stale_member.internal.shell_owner_pid = 999_999_999; + + store + .persist_participant(&orchestrator) + .expect("persist orchestrator"); + store + .persist_participant(&stale_member) + .expect("persist stale member"); + + let invalidated = store + .invalidate_stale_world_members_for_session("sess_live", 3) + .expect("invalidate stale members"); + + assert_eq!(invalidated, vec!["ash_member_old"]); + let stale_member = store + .load_participant("ash_member_old") + .expect("load stale member") + .expect("stale member exists"); + assert_eq!( + stale_member.handle.state, + AgentRuntimeSessionState::Invalidated + ); + assert_eq!( + stale_member.internal.termination_reason.as_deref(), + Some("world generation invalidated by replacement binding") + ); + }); + } + #[test] #[serial_test::serial] fn list_invalidated_participants_reads_authoritative_tombstones_only() {