diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 347be42f9..ba2a2cba2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: targets: wasm32-wasip2 # Component loading is host behaviour, so it is tested once, natively, # rather than on every target the matrix above cross-compiles for. - - run: cargo test --features wasm-plugin-tests --test wasm + - run: cargo xtask test-plugins test_mime_db: name: Test with MIME database diff --git a/.gitignore b/.gitignore index 4ee83031f..12465665a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ __pycache__/ # WASM plugin builds: scripts/build-wasm-plugins.sh plugin.wasm +# The bundled WASM summarizer ships ready to embed without a WASM toolchain. +!/plugins/summarize/plugin.wasm /crates/*/target # Generated data left behind by the retired review web viewer. diff --git a/Cargo.lock b/Cargo.lock index 665d3680b..e978649b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -730,10 +730,9 @@ name = "diffr-plugin-summarize" version = "0.1.0" dependencies = [ "diffr-plugin-sdk", - "reqwest", "serde", "serde_json", - "tokio", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -862,6 +861,7 @@ dependencies = [ "version_check", "wasmtime", "wasmtime-wasi", + "wasmtime-wasi-http", ] [[package]] @@ -1218,7 +1218,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1371,6 +1371,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -1400,6 +1406,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1416,9 +1423,9 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls", + "rustls 0.23.44", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.5", "tower-service", ] @@ -1931,7 +1938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -2269,7 +2276,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.44", "socket2", "thiserror 2.0.20", "tokio", @@ -2291,7 +2298,7 @@ dependencies = [ "rand_pcg", "ring", "rustc-hash", - "rustls", + "rustls 0.23.44", "rustls-pki-types", "slab", "thiserror 2.0.20", @@ -2538,14 +2545,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.44", "rustls-pki-types", "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.5", "tower", "tower-http", "tower-service", @@ -2620,6 +2627,20 @@ dependencies = [ "rustix 1.1.4", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + [[package]] name = "rustls" version = "0.23.44" @@ -2629,7 +2650,7 @@ dependencies = [ "aws-lc-rs", "once_cell", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.15", "subtle", "zeroize", ] @@ -2667,10 +2688,10 @@ dependencies = [ "jni", "log", "once_cell", - "rustls", + "rustls 0.23.44", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki", + "rustls-webpki 0.103.15", "security-framework", "security-framework-sys", "webpki-root-certs", @@ -2683,6 +2704,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.15" @@ -3181,13 +3213,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ - "rustls", + "rustls 0.23.44", "tokio", ] @@ -4115,6 +4158,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -4482,6 +4534,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "wasmtime-wasi-http" +version = "34.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17748938e2c98db4b7710efafb4751836e440d2b2ea0036b92a1000c8b5f37f2" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "futures", + "http", + "http-body", + "http-body-util", + "hyper", + "rustls 0.22.4", + "tokio", + "tokio-rustls 0.25.0", + "tracing", + "wasmtime", + "wasmtime-wasi", + "wasmtime-wasi-io", + "webpki-roots 0.26.11", +] + [[package]] name = "wasmtime-wasi-io" version = "34.0.2" @@ -4553,6 +4629,24 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4875,6 +4969,15 @@ dependencies = [ "wit-parser 0.254.0", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + [[package]] name = "wit-bindgen-rust" version = "0.60.0" diff --git a/Cargo.toml b/Cargo.toml index fad84f4e1..b1f934be5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,6 +153,7 @@ diffr-plugin-hide-files = { path = "plugins/hide-files" } diffr-plugin-removed-runs = { path = "plugins/removed-runs" } diffr-plugin-summarize = { path = "plugins/summarize" } diffr-plugin-test-bodies = { path = "plugins/test-bodies" } +wasmtime-wasi-http = { version = "34.0.2", default-features = false } [target.'cfg(not(any(windows, target_os = "illumos", target_os = "freebsd")))'.dependencies] @@ -199,9 +200,8 @@ default-members = [ "examples/plugins/fixtures", ] -# Test-only exception: the summarizer's HTTP client is native-only today. [workspace.metadata.diffr] -wasm-test-exclude = ["diffr-plugin-summarize"] +wasm-test-exclude = [] [profile.release] lto = "thin" diff --git a/docs/config.md b/docs/config.md index 7f43d12b0..216f43c99 100644 --- a/docs/config.md +++ b/docs/config.md @@ -70,7 +70,7 @@ name = "default-dark" # a bundled terminal theme path = "/path/to/theme.toml" # or a Helix-style theme file [plugins] # see "Plugins" below -order = ["bundled.context", "bundled.hide-files", "bundled.deleted-bodies", "bundled.test-bodies", "bundled.removed-runs", "bundled.summarize", "bundled.group"] +order = ["bundled.context", "bundled.hide-files", "bundled.deleted-bodies", "bundled.summarize", "bundled.test-bodies", "bundled.removed-runs", "bundled.group"] ``` When a file exceeds a `[diff]` limit it falls back to a line diff: the file's @@ -92,7 +92,7 @@ produce on the wire. ```toml [plugins] -order = ["bundled.context", "bundled.hide-files", "bundled.deleted-bodies", "bundled.test-bodies", "bundled.removed-runs", "bundled.summarize", "bundled.group"] +order = ["bundled.context", "bundled.hide-files", "bundled.deleted-bodies", "bundled.summarize", "bundled.test-bodies", "bundled.removed-runs", "bundled.group"] [plugins.bundled.context] # unchanged lines far from any change collapse enabled = true @@ -115,15 +115,17 @@ min_lines = 3 enabled = true min_lines = 5 -[plugins.bundled.summarize] # pseudocode for large new function bodies +[plugins.bundled.summarize] # pseudocode for large new function bodies and right-side tests enabled = false # off by default: it needs an API key provider = "gemini" model = "gemini-3.8-flash" min_lines = 20 +tests = true # include right-side tests even when not newly added +test_min_lines = 20 api_key = "…" # or GEMINI_API_KEY / GOOGLE_API_KEY endpoint = "https://…" # optional base URL override request_timeout_ms = 60000 -max_concurrency = 16 +max_concurrency = 16 # legacy option; ignored by the WASM summarizer retries = 3 system_prompt = """…""" # the model's system instruction; defaults to diffr's own @@ -156,7 +158,7 @@ key. ```toml [plugins] -order = ["bundled.context", "bundled.hide-files", "bundled.deleted-bodies", "bundled.test-bodies", "bundled.removed-runs", "bundled.summarize", "bundled.group", "external.fixtures"] +order = ["bundled.context", "bundled.hide-files", "bundled.deleted-bodies", "bundled.summarize", "bundled.test-bodies", "bundled.removed-runs", "bundled.group", "external.fixtures"] [plugins.external.fixtures] path = "plugins/fixtures" diff --git a/docs/plugins.md b/docs/plugins.md index 7d0d37f5c..b24bd3dc8 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -186,7 +186,7 @@ A component gets full access; diffr does not sandbox it: - WASI with the working directory preopened read-write as `.`. - The environment inherited. -- The network open, with name lookup. +- The network open, with name lookup and standard WASI HTTP/HTTPS outgoing requests. - Its stdout and stderr captured, and written to diffr's stderr a line at a time, each line prefixed with `[] `. Neither may reach diffr's stdout, which is the stream. A native plugin writes to diffr's stderr @@ -270,8 +270,11 @@ cp target/wasm32-wasip2/release/hide_all.wasm plugin.wasm ``` `cargo xtask build-plugins` builds every plugin crate in this repository -that runs as a component into its folder's `plugin.wasm` (the outputs are not -committed). diffr compiles each component once per run, with wasmtime's +that runs as a component into its folder's `plugin.wasm`. The bundled +summarizer's component is committed because normal builds embed it without +requiring a WASM toolchain; rebuild and commit it whenever its source, manifest, +queries or SDK changes. Other plugin build outputs are not committed. +diffr compiles each component once per run, with wasmtime's compilation cache on disk, so an unchanged component is not recompiled on the next run. @@ -298,8 +301,29 @@ next run. path = "/path/to/diffr/plugins/deleted-bodies" ``` - The summarizer's HTTP client does not build for `wasm32-wasip2`, so it runs - natively only. +- `plugins/summarize`: a network-backed component example that runs as WASM + even when bundled. Its transport + uses `wasi:http/outgoing-handler` for HTTP/HTTPS, including request headers, + streaming bodies and timeouts. Build it with `cargo xtask build-plugins`, + then load it without rebuilding diffr: + + ```toml + [plugins] + order = ["external.summarize", "bundled.test-bodies", "bundled.group"] + + [plugins.external.summarize] + path = "/path/to/diffr/plugins/summarize" + enabled = true + # Reads GEMINI_API_KEY or GOOGLE_API_KEY from the inherited environment. + ``` + + `plugins/summarize/src/http.rs` is a complete outgoing-request example for + plugin authors. The host provides TLS; components do not need to embed a + TLS implementation. WASM instances process files serially; the legacy + `max_concurrency` option is accepted but has no effect. Timeouts apply to + connection, first-byte and between-byte waits. + `cargo test --features wasm-plugin-tests --bin diffr external_component_summarizes_over_http` + checks the external component against a local model endpoint, including retries. Native plugin crates register their name and constructor through the same `export!("name", Type)` macro used for WASM exports. The linker gathers the @@ -312,10 +336,25 @@ Repository tooling discovers plugin crates through `cargo metadata` and bundled implementation into diffr; otherwise the plugin uses WASM. This does not add a WASM requirement to `cargo install`. `cargo xtask test-plugins` builds WASM variants and runs parity tests. The workspace's -`wasm-test-exclude` list records test-only exceptions such as the summarizer. +`wasm-test-exclude` list can record test-only exceptions; currently all plugins build as WASM. The host also discovers bundled manifests and query assets from its plugin dependencies, rather than maintaining a second file list. A dependency marked `native = true` contributes its registration; otherwise its prebuilt `plugin.wasm` is embedded alongside its manifest. The default configuration chooses which bundled plugins run; discovering an asset does not enable it. + +### Test summaries and folding order + +Enable `summarize` to get pseudocode for new functions and right-side test +bodies. `tests = false` disables test summaries independently; `test_min_lines` +(default 20) controls their size threshold. Modified and unchanged tests in a +diffed file qualify too; deleted tests do not. A summary must fit in half the +body's nonblank lines or the ordinary test fold remains. + +The default order runs `summarize` before `test-bodies`, then `group`. +`test-bodies` preserves existing labels and still collapses modules and bodies. +Opening a module or aggregate fold reveals each test's pseudocode; opening an +individual body reveals its source. JS/TS `describe` suites remain containers, +with summaries on their individual `it`/`test` callbacks. An explicit custom +`plugins.order` is respected; update it to this order to match the defaults. diff --git a/docs/streaming.md b/docs/streaming.md index c2e174d9c..94de5c62f 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -105,7 +105,7 @@ present. EOF without `complete` means the output was cut off. One shape everywhere: `{"code": "", "message": ""}`. - On a `file` record, `error` replaces `diff` and the run continues. Codes: - `binary`, `not_utf8`, `unsupported_file_type` (symlinks, submodules), + `not_utf8`, `unsupported_file_type` (symlinks, submodules), `unmerged`, `read_failed`, `query_conflict` (two query files capture one syntax node with different fold ranges; the message names the line and both files), or `internal` for a failure diffr did not classify. @@ -310,9 +310,9 @@ Closing stdout stops production once the files in flight finish. Terminate the process to cancel immediately. The CLI also retains its normal SIGPIPE behavior on Unix. -Regular UTF-8 text files are supported. Binary and non-UTF-8 files, symlinks, -submodules and unmerged index entries produce per-file errors. Non-UTF-8 paths -fail discovery. +Regular UTF-8 text files are supported. Binary files containing NUL bytes +produce size-only binary diffs. Non-UTF-8 text files, symlinks, submodules and +unmerged index entries produce per-file errors. Non-UTF-8 paths fail discovery. ## Plugins diff --git a/plugins/summarize/Cargo.toml b/plugins/summarize/Cargo.toml index 09fc44123..845148d86 100644 --- a/plugins/summarize/Cargo.toml +++ b/plugins/summarize/Cargo.toml @@ -11,8 +11,7 @@ publish = false diffr-plugin-sdk = { path = "../../crates/diffr-plugin-sdk" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -reqwest = { version = "0.13", default-features = false, features = ["rustls", "json", "http2"] } -tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "time", "sync"] } +wasi = "=0.14.2" [package.metadata.diffr] -native = true +native = false diff --git a/plugins/summarize/plugin.toml b/plugins/summarize/plugin.toml index 011dda573..50a073898 100644 --- a/plugins/summarize/plugin.toml +++ b/plugins/summarize/plugin.toml @@ -1,12 +1,12 @@ name = "summarize" title = "Summaries" -description = "Pseudocode summaries for large new function bodies." +description = "Pseudocode summaries for large new function bodies and tests." # Off by default: the summarizer needs an API key, and diffr stops at # startup when a plugin that is on cannot be made. [enabled] -title = "Summarize new functions" -description = "Summarize large new function bodies as pseudocode. Needs an API key: `api_key`, or `GEMINI_API_KEY` or `GOOGLE_API_KEY` in the environment." +title = "Summarize functions and tests" +description = "Summarize large new function bodies and right-side tests as pseudocode. Needs an API key: `api_key`, or `GEMINI_API_KEY` or `GOOGLE_API_KEY` in the environment." default = false [options.provider] @@ -42,14 +42,15 @@ description = "Override the provider's base URL, for proxies and tests." type = "integer" minimum = 1 title = "Request timeout (ms)" -description = "Per-request limit in milliseconds." +description = "Connect, first-byte and between-byte timeout, in milliseconds." default = 60000 [options.max_concurrency] type = "integer" minimum = 1 title = "Parallel requests" -description = "Requests in flight at once across files." +description = "Legacy option accepted for compatibility; WASM runs one file at a time. This setting has no effect." +"x-settings" = false default = 16 [options.retries] @@ -71,3 +72,16 @@ possible: about one pseudocode line per five source lines, and never more than a of the body's lines. When a fold lists a doc, also set "summary" to one sentence copied \ verbatim from that doc; otherwise leave it empty. Answer with a JSON array of \ {"id", "summary", "pseudocode"} objects, one per fold.""" + +[options.tests] +type = "boolean" +title = "Summarize tests" +description = "Summarize right-side test bodies, including modified and unchanged tests in diffed files." +default = true + +[options.test_min_lines] +type = "integer" +minimum = 0 +title = "Shortest test body to summarize (lines)" +description = "Test bodies shorter than this keep their ordinary fold labels." +default = 20 diff --git a/plugins/summarize/plugin.wasm b/plugins/summarize/plugin.wasm new file mode 100644 index 000000000..542201db1 Binary files /dev/null and b/plugins/summarize/plugin.wasm differ diff --git a/plugins/summarize/queries/javascript.scm b/plugins/summarize/queries/javascript.scm index 3f29fbeb6..b582737aa 100644 --- a/plugins/summarize/queries/javascript.scm +++ b/plugins/summarize/queries/javascript.scm @@ -18,5 +18,5 @@ (arrow_function body: (statement_block "{" @fold.open "}" @fold.close) @fold) (function_expression body: (statement_block "{" @fold.open "}" @fold.close) @fold) ])) - (#match? @_name "^(it|test|describe)$") + (#match? @_name "^(it|test)$") (#set! tag "summarize:test")) diff --git a/plugins/summarize/src/http.rs b/plugins/summarize/src/http.rs new file mode 100644 index 000000000..cb30eba8b --- /dev/null +++ b/plugins/summarize/src/http.rs @@ -0,0 +1,105 @@ +//! Outgoing HTTP through the standard WASI interface, available to external +//! components without a diffr-specific network import. +use diffr_plugin_sdk::anyhow::{self, anyhow}; +use wasi::http::{outgoing_handler, types::*}; +use wasi::io::streams::StreamError; + +pub fn post(url: &str, key: &str, body: &str, timeout_ms: u64) -> anyhow::Result<(u16, Vec)> { + let (scheme, rest) = url + .split_once("://") + .ok_or_else(|| anyhow!("invalid endpoint URL"))?; + let scheme = match scheme { + "http" => Scheme::Http, + "https" => Scheme::Https, + _ => anyhow::bail!("endpoint must use http or https"), + }; + let (authority, path) = rest.split_once('/').unwrap_or((rest, "")); + let headers = Fields::from_list(&[ + ("content-type".into(), b"application/json".to_vec()), + ("content-length".into(), body.len().to_string().into_bytes()), + ("x-goog-api-key".into(), key.as_bytes().to_vec()), + ]) + .map_err(|e| anyhow!("HTTP headers: {e:?}"))?; + let request = OutgoingRequest::new(headers); + request + .set_method(&Method::Post) + .map_err(|_| anyhow!("HTTP method"))?; + request + .set_scheme(Some(&scheme)) + .map_err(|_| anyhow!("HTTP scheme"))?; + request + .set_authority(Some(authority)) + .map_err(|_| anyhow!("HTTP authority"))?; + request + .set_path_with_query(Some(&format!("/{path}"))) + .map_err(|_| anyhow!("HTTP path"))?; + let outgoing = request.body().map_err(|_| anyhow!("HTTP body"))?; + let options = RequestOptions::new(); + let timeout = Some(timeout_ms.saturating_mul(1_000_000)); + options + .set_connect_timeout(timeout) + .map_err(|_| anyhow!("connect timeout"))?; + options + .set_first_byte_timeout(timeout) + .map_err(|_| anyhow!("response timeout"))?; + options + .set_between_bytes_timeout(timeout) + .map_err(|_| anyhow!("read timeout"))?; + let response = outgoing_handler::handle(request, Some(options)) + .map_err(|e| anyhow!("HTTP request: {e:?}"))?; + { + let stream = outgoing + .write() + .map_err(|_| anyhow!("HTTP output stream"))?; + let mut remaining = body.as_bytes(); + while !remaining.is_empty() { + let capacity = stream + .check_write() + .map_err(|e| anyhow!("HTTP write: {e:?}"))? as usize; + if capacity == 0 { + stream.subscribe().block(); + continue; + } + let length = capacity.min(remaining.len()); + stream + .write(&remaining[..length]) + .map_err(|e| anyhow!("HTTP write: {e:?}"))?; + remaining = &remaining[length..]; + } + // Finish the body below without a separate flush: a peer can respond + // as soon as Content-Length bytes arrive, closing the write stream + // before a post-write flush even though the request succeeded. + } + OutgoingBody::finish(outgoing, None).map_err(|e| anyhow!("HTTP finish: {e:?}"))?; + response.subscribe().block(); + let response = response + .get() + .ok_or_else(|| anyhow!("HTTP response not ready"))? + .map_err(|_| anyhow!("HTTP response already consumed"))? + .map_err(|e| anyhow!("HTTP response: {e:?}"))?; + let status = response.status(); + let incoming = response + .consume() + .map_err(|_| anyhow!("HTTP response body"))?; + let mut bytes = Vec::new(); + { + let stream = incoming + .stream() + .map_err(|_| anyhow!("HTTP input stream"))?; + loop { + match stream.blocking_read(64 * 1024) { + Ok(chunk) => bytes.extend_from_slice(&chunk), + Err(StreamError::Closed) => break, + Err(error) => anyhow::bail!("HTTP read: {error:?}"), + } + } + } + let trailers = IncomingBody::finish(incoming); + trailers.subscribe().block(); + trailers + .get() + .ok_or_else(|| anyhow!("HTTP trailers not ready"))? + .map_err(|_| anyhow!("HTTP trailers already consumed"))? + .map_err(|e| anyhow!("HTTP trailers: {e:?}"))?; + Ok((status, bytes)) +} diff --git a/plugins/summarize/src/lib.rs b/plugins/summarize/src/lib.rs index 65e265239..18aafbf89 100644 --- a/plugins/summarize/src/lib.rs +++ b/plugins/summarize/src/lib.rs @@ -3,9 +3,7 @@ //! //! It needs an API key: `new` fails without one, naming how to set it or //! turn the plugin off, which is why the bundled configuration ships it off. -//! The plugin holds its HTTP client and the runtime every request runs on, -//! and at most `max_concurrency` requests are in flight at once across -//! files. +//! Requests use WASI HTTP. The host calls one file at a time per instance. use diffr_plugin_sdk::anyhow::{self, anyhow, Context as _}; use diffr_plugin_sdk::{ docstring_of, export, has_tag, is_fold, line_count, one_sided, walk, Draft, FileEntry, Move, @@ -15,11 +13,10 @@ use serde::Deserialize; use serde_json::json; use std::collections::BTreeMap; use std::time::Duration; -use tokio::runtime::Runtime; -use tokio::sync::Semaphore; +mod http; /// The plugin's name, and the tags its queries set: a function body, and a -/// test body, which is never summarized. +/// test body, which can be summarized independently of whether it is new. const PLUGIN: &str = "summarize"; const FUNCTION: &str = "summarize:function"; const TEST: &str = "summarize:test"; @@ -31,9 +28,12 @@ pub struct Options { pub provider: Provider, pub model: String, pub min_lines: usize, + pub tests: bool, + pub test_min_lines: usize, pub api_key: Option, pub endpoint: Option, pub request_timeout_ms: u64, + /// Accepted for compatibility with existing configs; WASM calls are serial. pub max_concurrency: usize, pub retries: u32, /// The system instruction sent with every request. @@ -48,15 +48,11 @@ pub enum Provider { const DEFAULT_ENDPOINT: &str = "https://generativelanguage.googleapis.com"; -/// The summarizer: its options, its API key and endpoint, its HTTP client, -/// the runtime its requests run on, and the requests in flight across files. +/// The summarizer's options, API key and endpoint. pub struct Summarize { options: Options, api_key: String, endpoint: String, - client: reqwest::Client, - runtime: Runtime, - limit: Semaphore, } /// One fold to summarize: its region id, 1-based inclusive line range, and @@ -154,56 +150,39 @@ impl Summarize { }); let url = self.url(); let failed = |message: String| anyhow!("{}: {message}", self.options.model); - let text = self.runtime.block_on(async { - let _permit = self - .limit - .acquire() - .await - .expect("semaphore is never closed"); + let text: serde_json::Value = { let mut attempt = 0; loop { - let response = self - .client - .post(&url) - .header("x-goog-api-key", &self.api_key) - .json(&body) - .send() - .await; - let retry = match response { - Ok(response) if response.status().is_success() => { - return response - .json::() - .await - .map_err(|error| failed(error.to_string())); - } - Ok(response) - if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS - || response.status().is_server_error() => - { - format!("HTTP {}", response.status()) + let result = http::post( + &url, + &self.api_key, + &body.to_string(), + self.options.request_timeout_ms, + ); + let retry = match result { + Ok((status, body)) if (200..300).contains(&status) => { + break serde_json::from_slice(&body) + .map_err(|error| failed(error.to_string()))?; } - Ok(response) => { - let status = response.status(); - let detail = response.text().await.unwrap_or_default(); + Ok((status, _)) if status == 429 || status >= 500 => format!("HTTP {status}"), + Ok((status, body)) => { return Err(failed(format!( "HTTP {status} {}", - detail.chars().take(200).collect::() - ))); - } - Err(error) - if error.is_timeout() || error.is_connect() || error.is_request() => - { - error.to_string() + String::from_utf8_lossy(&body) + .chars() + .take(200) + .collect::() + ))) } - Err(error) => return Err(failed(error.to_string())), + Err(error) => error.to_string(), }; if attempt >= self.options.retries { return Err(failed(format!("{retry} after {} attempts", attempt + 1))); } attempt += 1; - tokio::time::sleep(Duration::from_millis(250 * (1 << attempt.min(6)))).await; + std::thread::sleep(Duration::from_millis(250 * (1 << attempt.min(6)))); } - })?; + }; let content = text["candidates"][0]["content"]["parts"] .as_array() .and_then(|parts| parts.last()) @@ -234,16 +213,15 @@ impl Summarize { } } -/// New function bodies on the after side of at least `min_lines` lines: -/// function folds with no line inside them paired with the before side (see -/// `one_sided`), so a body that only moved, or a file diffed by line whose -/// bodies still align, is not new. Only the outermost qualifying body is -/// taken, never one nested inside it; test bodies and folds that already -/// start collapsed are skipped. Each is its id, its 1-based inclusive line -/// range, and its docstring's id. Selection runs before this plugin links -/// any docstring, so a docstring matched across sides never makes the new -/// body under it look paired. -pub fn select(sides: &Pairing, min_lines: usize) -> Vec<(u32, u32, u32, Option)> { +/// Select new right-side function bodies and, when a threshold is supplied, +/// right-side tests regardless of newness or initial collapsed state. Descend +/// through suites/modules so each test gets its own summary. Only the outermost +/// eligible body is selected; docstrings are linked after selection. +pub fn select( + sides: &Pairing, + min_lines: usize, + test_min_lines: Option, +) -> Vec<(u32, u32, u32, Option)> { let (lhs, rhs) = match sides { Pairing::Both { lhs, rhs } => (OtherSide::of(&lhs.regions), rhs), Pairing::RightOnly { rhs } => (OtherSide::default(), rhs), @@ -255,16 +233,19 @@ pub fn select(sides: &Pairing, min_lines: usize) -> Vec<(u32, u32, u32, regions: &[Region], lhs: &OtherSide, min_lines: usize, + test_min_lines: Option, selected: &mut Vec<(u32, u32, u32, Option)>, ) { for region in regions { - if is_fold(region) - && has_tag(region, FUNCTION) - && !has_tag(region, TEST) - && !region.visibility.collapsed - && one_sided(region, lhs) - && line_count(region) >= min_lines - { + let eligible = if has_tag(region, TEST) { + test_min_lines.is_some_and(|minimum| line_count(region) >= minimum) + } else { + has_tag(region, FUNCTION) + && !region.visibility.collapsed + && one_sided(region, lhs) + && line_count(region) >= min_lines + }; + if is_fold(region) && eligible { let lines = region.range.lines(); selected.push(( region.id, @@ -275,11 +256,18 @@ pub fn select(sides: &Pairing, min_lines: usize) -> Vec<(u32, u32, u32, continue; } if let Node::Fold { children } = ®ion.node { - visit(rhs, children, lhs, min_lines, selected); + visit(rhs, children, lhs, min_lines, test_min_lines, selected); } } } - visit(rhs, &rhs.regions, &lhs, min_lines, &mut selected); + visit( + rhs, + &rhs.regions, + &lhs, + min_lines, + test_min_lines, + &mut selected, + ); selected } @@ -369,23 +357,12 @@ impl Plugin for Summarize { fn new(options: Options) -> anyhow::Result { let api_key = resolve_key(&options)?; - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(options.request_timeout_ms)) - .build()?; - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .thread_name("diffr-summarizer") - .enable_all() - .build()?; Ok(Self { api_key, endpoint: options .endpoint .clone() .unwrap_or_else(|| DEFAULT_ENDPOINT.to_owned()), - client, - runtime, - limit: Semaphore::new(options.max_concurrency), options, }) } @@ -435,7 +412,11 @@ impl Plugin for Summarize { } fn mutate(&self, file: &FileEntry, sides: &Pairing) -> anyhow::Result> { - let selected = select(sides, self.options.min_lines); + let selected = select( + sides, + self.options.min_lines, + self.options.tests.then_some(self.options.test_min_lines), + ); let (Pairing::Both { rhs, .. } | Pairing::RightOnly { rhs }) = &sides else { return Ok(Vec::new()); }; diff --git a/plugins/test-bodies/src/lib.rs b/plugins/test-bodies/src/lib.rs index 8ee4c531b..059b58c86 100644 --- a/plugins/test-bodies/src/lib.rs +++ b/plugins/test-bodies/src/lib.rs @@ -84,18 +84,33 @@ impl Plugin for TestBodies { // Every fold has its own id, so each side's fold is its own target // and takes its own label. A test body's docstring, when it has one, // is linked after it collapses. - let mut labels: BTreeMap)> = BTreeMap::new(); + let mut labels: BTreeMap)> = BTreeMap::new(); for source in sides.sides() { walk(&source.regions, &mut |region| { if !is_fold(region) || line_count(region) < options.min_lines { return; } if has_tag(region, MODULE) { - labels.insert(region.id, ("test module", None)); + labels.insert( + region.id, + ( + if region.visibility.label.is_empty() { + "test module".to_owned() + } else { + region.visibility.label.clone() + }, + None, + ), + ); } else if has_tag(region, TEST) { - labels - .entry(region.id) - .or_insert(("test body", docstring_of(source, region, PLUGIN))); + labels.entry(region.id).or_insert(( + if region.visibility.label.is_empty() { + "test body".to_owned() + } else { + region.visibility.label.clone() + }, + docstring_of(source, region, PLUGIN), + )); } }); } diff --git a/src/config/default.toml b/src/config/default.toml index 3456af28d..6056aa8a7 100644 --- a/src/config/default.toml +++ b/src/config/default.toml @@ -6,9 +6,9 @@ order = [ "bundled.context", "bundled.hide-files", "bundled.deleted-bodies", + "bundled.summarize", "bundled.test-bodies", "bundled.removed-runs", - "bundled.summarize", "bundled.group", ] @@ -41,6 +41,8 @@ enabled = false provider = "gemini" model = "gemini-3.8-flash" min_lines = 20 +tests = true +test_min_lines = 20 request_timeout_ms = 60000 max_concurrency = 16 retries = 3 diff --git a/src/git.rs b/src/git.rs index 041b67e64..2f72e19c7 100644 --- a/src/git.rs +++ b/src/git.rs @@ -3,7 +3,7 @@ use crate::config::Params; use crate::pairing::Pairing; use crate::plugin::Pipeline; use crate::protocol; -use crate::summary::DiffResult; +use crate::summary::{DiffResult, FileContent, FileFormat}; use crate::tags::{self, Attributes, Prefix, PREFIX_BYTES}; use anyhow::Context as _; use git2::{Delta, Diff, DiffFindOptions, DiffOptions, Oid, Repository}; @@ -207,7 +207,6 @@ impl FileChange { pub(crate) enum FileError { UnsupportedFileType, ReadFailed, - Binary, NotUtf8, Unmerged, } @@ -217,7 +216,6 @@ impl FileError { match self { Self::UnsupportedFileType => "unsupported_file_type", Self::ReadFailed => "read_failed", - Self::Binary => "binary", Self::NotUtf8 => "not_utf8", Self::Unmerged => "unmerged", } @@ -231,7 +229,6 @@ impl fmt::Display for FileError { "structural diffs currently require regular text files (not symlinks or submodules)" } Self::ReadFailed => "could not read the source", - Self::Binary => "structural diffs currently support text files only", Self::NotUtf8 => "the source is not valid UTF-8", Self::Unmerged => { "unmerged index entry: resolve the conflict before requesting a structural diff" @@ -306,7 +303,7 @@ impl Source { /// The start of the source for the content rules, or `None` when they /// cannot apply: not a regular file, binary, or not UTF-8. Loading the - /// whole source reports those. + /// whole source handles those cases. fn prefix(&self, repo: &Repository) -> anyhow::Result, bool)>> { let Some(mut bytes) = self.head(repo, PREFIX_BYTES + 1)? else { return Ok(None); @@ -346,9 +343,9 @@ impl Source { } } - fn read(&self, repo: &Repository) -> anyhow::Result { + fn read(&self, repo: &Repository) -> anyhow::Result> { let mode = match self { - Self::Absent => return Ok(String::new()), + Self::Absent => return Ok(Vec::new()), Self::Blob { mode, .. } | Self::WorkingFile { mode, .. } => mode, }; if !matches!(mode, git2::FileMode::Blob | git2::FileMode::BlobExecutable) { @@ -363,10 +360,7 @@ impl Source { Self::WorkingFile { path, .. } => std::fs::read(path).context(FileError::ReadFailed)?, Self::Absent => unreachable!(), }; - if bytes.contains(&0) { - return Err(FileError::Binary.into()); - } - String::from_utf8(bytes).context(FileError::NotUtf8) + Ok(bytes) } } @@ -555,8 +549,8 @@ impl DiffSession { /// Sources read on the session thread; diffing needs no repository access. pub(crate) struct LoadedFile { pub(crate) file: FileChange, - before: String, - after: String, + before: Vec, + after: Vec, pub(crate) params: Arc, diff_options: crate::options::DiffOptions, } @@ -568,14 +562,29 @@ impl LoadedFile { /// A fold query conflict fails this file alone. pub(crate) fn diff(&self) -> anyhow::Result { + // Preserve binary files as successful, size-only records. Rejecting + // them while loading bypasses the protocol and TUI's binary support. + if self.before.contains(&0) || self.after.contains(&0) { + return Ok(DiffResult { + file_format: FileFormat::Binary, + lhs_src: FileContent::Binary, + rhs_src: FileContent::Binary, + lhs_positions: vec![], + rhs_positions: vec![], + lhs_folds: vec![], + rhs_folds: vec![], + }); + } + let before = std::str::from_utf8(&self.before).context(FileError::NotUtf8)?; + let after = std::str::from_utf8(&self.after).context(FileError::NotUtf8)?; let options = crate::options::DiffOptions { generated: self.file.tags.iter().any(|tag| tag == tags::GENERATED), ..self.diff_options.clone() }; Ok(DiffResult::from_sources_with_options( self.file.path(), - &self.before, - &self.after, + before, + after, &self.params, &options, )?) diff --git a/src/plugin/tests/mod.rs b/src/plugin/tests/mod.rs index 0274df267..a29e288b8 100644 --- a/src/plugin/tests/mod.rs +++ b/src/plugin/tests/mod.rs @@ -150,7 +150,16 @@ pub(crate) fn bundled(name: &str, overrides: serde_json::Value) -> Pipeline { name, serde_json::Value::Object(options), &|host, options| { - native::registered(native::lookup(name)?.expect("native code"), host, options) + if let Some(bytes) = builtin::component(name) { + let engine = super::wasm::engine()?; + super::wasm::WasmPlugin::load( + &engine, + &super::config::ComponentSource::Bundled(bytes), + )? + .create(host, options) + } else { + native::registered(native::lookup(name)?.expect("native code"), host, options) + } }, ) .unwrap(); diff --git a/src/plugin/tests/summarize.rs b/src/plugin/tests/summarize.rs index 67584de1f..83d369202 100644 --- a/src/plugin/tests/summarize.rs +++ b/src/plugin/tests/summarize.rs @@ -102,11 +102,11 @@ fn summarizer_with(overrides: serde_json::Value) -> Pipeline { #[test] fn selection_takes_new_bodies_of_at_least_min_lines() { let (_, sides) = project("a.py", "", LARGE); - let selected = select(&trees(&sides), 3); + let selected = select(&trees(&sides), 3, None); assert_eq!(selected.len(), 1); assert_eq!((selected[0].1, selected[0].2), (2, 4)); let (_, sides) = project("a.py", LARGE, LARGE); - assert!(select(&trees(&sides), 3).is_empty()); + assert!(select(&trees(&sides), 3, None).is_empty()); } #[test] @@ -124,7 +124,7 @@ fn selection_reads_newness_from_the_lines_when_the_match_fell_back() { ); // Nothing matched, so no fold is paired; the lines still are. Only the // added body, whose lines pair with nothing, is new. - let selected = select(&trees(&sides), 3); + let selected = select(&trees(&sides), 3, None); assert_eq!(selected.len(), 1, "{selected:?}"); assert_eq!((selected[0].1, selected[0].2), (7, 9)); } @@ -135,18 +135,18 @@ fn selection_takes_outermost_function_bodies_only() { // not a function, so the method is the outermost selection. let after = "impl A {\n fn m(&self) {\n a();\n b();\n c();\n let f = || {\n d();\n e();\n g();\n };\n f();\n }\n}\n"; let (_, sides) = project("a.rs", "", after); - let selected = select(&trees(&sides), 3); + let selected = select(&trees(&sides), 3, None); assert_eq!(selected.len(), 1, "{selected:?}"); assert_eq!((selected[0].1, selected[0].2), (3, 11)); // Below the threshold, nothing. - assert!(select(&trees(&sides), 30).is_empty()); + assert!(select(&trees(&sides), 30, None).is_empty()); } #[test] fn selection_skips_test_bodies_and_collapsed_folds() { let after = "#[test]\nfn t() {\n a();\n b();\n c();\n}\n\nfn f() {\n a();\n b();\n c();\n}\n"; let (file, mut sides) = project("a.rs", "", after); - let selected = select(&trees(&sides), 3); + let selected = select(&trees(&sides), 3, None); assert_eq!(selected.len(), 1, "{selected:?}"); assert_eq!((selected[0].1, selected[0].2), (9, 11)); run("test-bodies", json!({"min_lines": 3}), &file, &mut sides); @@ -159,13 +159,13 @@ fn selection_skips_test_bodies_and_collapsed_folds() { region.visibility.collapsed = true; } }); - assert!(select(&sides, 3).is_empty()); + assert!(select(&sides, 3, None).is_empty()); } #[test] fn long_summaries_are_discarded_and_the_body_stays_open() { let (file, mut sides) = project("a.py", "", LARGE); - let id = select(&trees(&sides), 3)[0].0; + let id = select(&trees(&sides), 3, None)[0].0; let (endpoint, server) = serve(vec![(200, gemini_answer(&[(id, "a()\nb()\nc()")]))]); summarizer(&endpoint, 0).run(&file, &mut sides).unwrap(); let sides = trees(&sides); @@ -183,7 +183,7 @@ fn long_summaries_are_discarded_and_the_body_stays_open() { #[test] fn summaries_collapse_selected_folds_behind_pseudocode() { let (file, mut sides) = project("a.py", "", LARGE); - let id = select(&trees(&sides), 3)[0].0; + let id = select(&trees(&sides), 3, None)[0].0; let (endpoint, server) = serve(vec![(200, gemini_answer(&[(id, "call a, b, c")]))]); summarizer(&endpoint, 0).run(&file, &mut sides).unwrap(); let sides = trees(&sides); @@ -220,7 +220,7 @@ fn a_docstring_is_sent_and_only_a_verbatim_sentence_from_it_is_kept() { labels.remove(0) }; let (file, mut sides) = project("a.rs", "", after); - let id = select(&trees(&sides), 3)[0].0; + let id = select(&trees(&sides), 3, None)[0].0; let (endpoint, server) = serve(vec![(200, answer(id, "Sums three numbers."))]); summarizer(&endpoint, 0).run(&file, &mut sides).unwrap(); let sides = trees(&sides); @@ -307,17 +307,17 @@ fn newness_is_the_lines_inside_the_body() { lhs_states.push(region.fold_state_id) }); assert!(!lhs_states.contains(&body.expect("a function body on the after side"))); - assert_eq!(select(&projected, 3).len(), 1); + assert_eq!(select(&projected, 3, None).len(), 1); // The same body grown from one that already had lines: `let x = a;` // still pairs, so this is a rewrite rather than a new body. let (_, sides) = project("a.rs", &grown, &after); - assert!(select(&trees(&sides), 3).is_empty()); + assert!(select(&trees(&sides), 3, None).is_empty()); } #[test] fn the_system_prompt_is_the_configured_one() { let (file, mut sides) = project("a.py", "", LARGE); - let id = select(&trees(&sides), 3)[0].0; + let id = select(&trees(&sides), 3, None)[0].0; let request = |overrides: serde_json::Value, sides: &mut Pairing| { let (endpoint, server) = serve(vec![(200, gemini_answer(&[(id, "call a, b, c")]))]); let mut overrides = overrides; @@ -354,7 +354,7 @@ fn the_system_prompt_is_the_configured_one() { #[test] fn transient_failures_are_retried_then_succeed() { let (file, mut sides) = project("a.py", "", LARGE); - let id = select(&trees(&sides), 3)[0].0; + let id = select(&trees(&sides), 3, None)[0].0; let (endpoint, server) = serve(vec![ (503, "{}".to_owned()), (429, "{}".to_owned()), @@ -400,3 +400,139 @@ fn small_files_never_call_the_model() { let moves = moves(&pipeline, &file, &sides).unwrap(); assert!(moves.is_empty()); } + +/// Exercise the same component a user loads from an external plugin folder. +#[cfg(feature = "wasm-plugin-tests")] +#[test] +fn external_component_summarizes_over_http() { + let (file, mut sides) = project("a.py", "", LARGE); + let id = select(&trees(&sides), 3, None)[0].0; + let (endpoint, server) = serve(vec![ + (429, "{}".into()), + (200, gemini_answer(&[(id, "call a, b, c")])), + ]); + let engine = super::super::wasm::engine().unwrap(); + let plugin = super::super::wasm::WasmPlugin::load( + &engine, + &super::super::config::ComponentSource::File( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("plugins/summarize/plugin.wasm"), + ), + ) + .unwrap(); + let mut options = builtin::manifest("summarize").unwrap().defaults(); + options.extend( + json!({"api_key": "test-key", "endpoint": endpoint, "min_lines": 3, "retries": 1}) + .as_object() + .unwrap() + .clone(), + ); + let mut pipeline = Pipeline::default(); + pipeline + .push( + "summarize", + serde_json::Value::Object(options), + &|host, options| plugin.create(host, options), + ) + .unwrap(); + pipeline.run(&file, &mut sides).unwrap(); + assert_eq!(fold_label(&trees(&sides)), "call a, b, c"); + assert_eq!(server.join().unwrap().len(), 2); +} + +#[test] +fn tests_are_selected_when_added_modified_unchanged_or_already_collapsed() { + for (path, before, after) in [ + ( + "a.py", + "def test_it():\n setup()\n act()\n check()\n", + "def test_it():\n setup()\n act()\n check_new()\n", + ), + ( + "a.rs", + "#[test]\nfn it_works() {\n setup();\n act();\n check();\n}\n", + "#[test]\nfn it_works() {\n setup();\n act();\n check_new();\n}\n", + ), + ( + "a.go", + "package a\nfunc TestIt(t *testing.T) {\n setup()\n act()\n check()\n}\n", + "package a\nfunc TestIt(t *testing.T) {\n setup()\n act()\n checkNew()\n}\n", + ), + ( + "a.ts", + "test('it', () => {\n setup();\n act();\n check();\n});\n", + "test('it', () => {\n setup();\n act();\n checkNew();\n});\n", + ), + ] { + for old in ["", before, after] { + // Entirely identical files bypass parsing. Keep a change outside + // the test to exercise an unchanged body in a diffed file. + let comment = if path.ends_with(".py") { "#" } else { "//" }; + let after = format!("{after}\n{comment} changed elsewhere\n"); + let (file, mut sides) = project(path, old, &after); + assert_eq!(select(&trees(&sides), 3, Some(3)).len(), 1, "{path}: {old}"); + assert!(select(&trees(&sides), 3, None).is_empty()); + assert!(select(&trees(&sides), 3, Some(30)).is_empty()); + run("test-bodies", json!({"min_lines": 3}), &file, &mut sides); + assert_eq!(select(&trees(&sides), 3, Some(3)).len(), 1); + } + } +} + +#[test] +fn suites_select_individual_tests_and_preserve_nested_summary_folds() { + for (path, after, outer_tag) in [ + ("a.rs", "#[cfg(test)]\nmod tests {\n #[test]\n fn one() {\n setup();\n act();\n check();\n }\n #[test]\n fn two() {\n setup();\n act();\n check();\n }\n}\n", "test-bodies:module"), + ("a.ts", "describe('suite', () => {\n it('one', () => {\n setup();\n act();\n check();\n });\n test('two', () => {\n setup();\n act();\n check();\n });\n});\n", "test-bodies:test"), + ] { + let (file, mut sides) = project(path, "", after); + let selected = select(&trees(&sides), 3, Some(3)); + assert_eq!(selected.len(), 2, "{path}: {selected:?}"); + let (endpoint, server) = serve(vec![(200, gemini_answer(&[(selected[0].0, "setup; act; check one"), (selected[1].0, "setup; act; check two")]))]); + summarizer_with(json!({"api_key": "test", "endpoint": endpoint, "test_min_lines": 3})).run(&file, &mut sides).unwrap(); + run("test-bodies", json!({"min_lines": 3}), &file, &mut sides); + server.join().unwrap(); + run("group", json!({}), &file, &mut sides); + let sides = trees(&sides); + let mut found = 0; + let mut outer_state = None; + walk(&rhs(&sides).regions, &mut |region| { + if has_tag(region, outer_tag) && !selected.iter().any(|s| s.0 == region.id) { + assert!(region.visibility.collapsed); + outer_state = Some(region.fold_state_id); + } + if selected.iter().any(|s| s.0 == region.id) { + assert!(region.visibility.collapsed); + assert!(region.visibility.label.starts_with("setup; act; check")); + assert_ne!(Some(region.fold_state_id), outer_state); + found += 1; + } + }); + assert!(outer_state.is_some()); + assert_eq!(found, 2); + } +} + +#[test] +fn bundled_wasm_summarizer_streams_large_prompts() { + // Exceed the host's outgoing body buffer and close the server immediately + // after replying, exercising backpressure and the response/finish race. + let after = format!( + "def test_it():\n setup()\n act()\n check()\n# {}\n", + "context ".repeat(16_384) + ); + let (file, mut sides) = project("a.py", "", &after); + let id = select(&trees(&sides), 3, Some(3))[0].0; + let (endpoint, server) = serve(vec![(200, gemini_answer(&[(id, "setup; act; check")]))]); + let wasm = summarizer_with(json!({ + "api_key": "test", "endpoint": endpoint, "test_min_lines": 3, "retries": 0, + })); + assert!(builtin::component("summarize").is_some()); + wasm.run(&file, &mut sides).unwrap(); + assert_eq!(fold_label(&trees(&sides)), "setup; act; check"); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + let request: serde_json::Value = serde_json::from_str(&requests[0]).unwrap(); + let prompt = request["contents"][0]["parts"][0]["text"].as_str().unwrap(); + assert!(prompt.contains(&"context ".repeat(16_384))); + assert!(prompt.contains(&format!("fold {id}: lines 2-4"))); +} diff --git a/src/plugin/wasm.rs b/src/plugin/wasm.rs index 03366ecce..90530d043 100644 --- a/src/plugin/wasm.rs +++ b/src/plugin/wasm.rs @@ -57,6 +57,7 @@ pub(crate) fn engine() -> anyhow::Result { /// progress, set before each call. struct State { wasi: WasiCtx, + http: wasmtime_wasi_http::WasiHttpCtx, table: ResourceTable, host: Host, } @@ -73,6 +74,12 @@ impl WasiView for State { } } +impl wasmtime_wasi_http::WasiHttpView for State { + fn ctx(&mut self) -> &mut wasmtime_wasi_http::WasiHttpCtx { + &mut self.http + } +} + impl types::Host for State {} impl host::Host for State { @@ -192,6 +199,7 @@ impl WasmPlugin { let component = component.with_context(|| format!("compiling {label}"))?; let mut linker = Linker::::new(engine); wasmtime_wasi::p2::add_to_linker_sync(&mut linker)?; + wasmtime_wasi_http::add_only_http_to_linker_sync(&mut linker)?; bindings::Plugin::add_to_linker::>(&mut linker, |state| state)?; let pre = bindings::PluginPre::new( linker @@ -222,6 +230,7 @@ impl WasmPlugin { &self.engine, State { wasi: wasi.build(), + http: wasmtime_wasi_http::WasiHttpCtx::new(), table: ResourceTable::new(), host, }, diff --git a/tests/view.rs b/tests/view.rs index adc97b08a..2ff7e94ef 100644 --- a/tests/view.rs +++ b/tests/view.rs @@ -226,3 +226,55 @@ fn syntax_spans_come_only_with_the_flag() { "{syntax:?}" ); } + +#[test] +fn git_binary_files_stream_as_diff_records_with_side_sizes() { + let fixture = Fixture::new(); + fixture.write("keep.txt", "unchanged\n"); + let empty = fixture.commit(); + fixture.write("plugin.wasm", "\0asm\x01\0\0\0"); + let added = fixture.commit(); + fixture.write("plugin.wasm", "\0asm\x01\0\0\0extra"); + let modified = fixture.commit(); + fixture.remove("plugin.wasm"); + let deleted = fixture.commit(); + for (base, head, expected) in [ + ( + &empty, + &added, + serde_json::json!({"type": "binary", "rhs": {"size": 8}}), + ), + ( + &added, + &modified, + serde_json::json!({"type": "binary", "lhs": {"size": 8}, "rhs": {"size": 13}}), + ), + ( + &modified, + &deleted, + serde_json::json!({"type": "binary", "lhs": {"size": 13}}), + ), + ] { + let output = fixture.run(base, head); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let events = records(&output); + assert_eq!(events[1]["diff"], expected); + assert!(events[1].get("error").is_none()); + assert_eq!(events.last().unwrap()["succeeded"], 1); + assert_eq!(events.last().unwrap()["failed"], 0); + } + // The working-tree source uses the same binary path as a committed blob. + fixture.write("plugin.wasm", "\0asm"); + let mut index = fixture.repo.index().unwrap(); + index.add_path(std::path::Path::new("plugin.wasm")).unwrap(); + index.write().unwrap(); + let events = records(&fixture.diffr(&[&added, "--format", "ndjson"])); + assert_eq!( + events[1]["diff"], + serde_json::json!({"type": "binary", "lhs": {"size": 8}, "rhs": {"size": 4}}) + ); +} diff --git a/tui/themes/README.md b/tui/themes/README.md index 26f0c0bda..5c36228bf 100644 --- a/tui/themes/README.md +++ b/tui/themes/README.md @@ -4,7 +4,8 @@ These are Helix editor themes, vendored from https://github.com/helix-editor/helix/tree/master/runtime/themes and licensed under the Mozilla Public License 2.0 (see LICENSE in this directory). -One Dark uses a lighter comment foreground for readability on diff highlights. +One Dark uses lighter comment and muted foregrounds for readability on diff +highlights and collapsed summaries. | name | file | | ----------------- | --------------------- | diff --git a/tui/themes/onedark.toml b/tui/themes/onedark.toml index d3a0fb66e..8470c5dac 100644 --- a/tui/themes/onedark.toml +++ b/tui/themes/onedark.toml @@ -115,5 +115,5 @@ light-black = "#2C323C" gray = "#3E4452" faint-gray = "#3B4048" light-gray = "#5C6370" -comment-gray = "#C0C6D0" -linenr = "#4B5263" +comment-gray = "#C8CED8" +linenr = "#687184" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c4242da59..572b87ac3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -198,6 +198,8 @@ fn main() -> Result<()> { "--locked", "--features", "wasm-plugin-tests", + "--bin", + "diffr", "--test", "wasm", ])