diff --git a/.github/workflows/languages.yml b/.github/workflows/languages.yml new file mode 100644 index 000000000..b7baa7bac --- /dev/null +++ b/.github/workflows/languages.yml @@ -0,0 +1,145 @@ +name: Extra languages + +on: + pull_request: + paths: + - 'languages/**' + - 'sample_files/**' + - 'tests/cli.rs' + - 'src/**' + - 'Cargo.*' + - 'build.rs' + - 'xtask/**' + - '.github/workflows/languages.yml' + workflow_dispatch: + inputs: + publish: + description: Publish the exact signed archives and produce a verified catalog + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: languages-${{ github.ref }} + cancel-in-progress: false + +jobs: + guard: + runs-on: ubuntu-22.04 + permissions: + actions: read + steps: + - name: Require protected publication environment + if: inputs.publish + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + run: | + test "$GITHUB_REF" = refs/heads/main + gh api "repos/$REPOSITORY/environments/languages-release" > environment.json + jq -e '.protection_rules | any(.type == "required_reviewers" and (.reviewers | length > 0))' environment.json + + build: + needs: guard + environment: ${{ inputs.publish && 'languages-release' || 'languages-ci' }} + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-14 + - target: x86_64-apple-darwin + os: macos-15-intel + - target: x86_64-unknown-linux-gnu + os: ubuntu-22.04 + - target: aarch64-unknown-linux-gnu + os: ubuntu-22.04-arm + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: dtolnay/rust-toolchain@stable + - name: Check language definitions and publication gate + run: | + cargo xtask check-languages + python3 languages/tests/test_definitions.py + python3 languages/tests/test_licenses.py + python3 languages/tests/test_publish.py + - name: Build locked grammars + run: cargo xtask build-languages --target '${{ matrix.target }}' + - name: Import Whiteboard signing identity + if: inputs.publish && runner.os == 'macOS' + env: + CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE_BASE64 }} + CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + run: | + python3 -c 'import os,base64; open(os.environ["RUNNER_TEMP"]+"/certificate.p12","wb").write(base64.b64decode(os.environ["CERTIFICATE"]))' + password=$(openssl rand -hex 24) + security create-keychain -p "$password" "$RUNNER_TEMP/languages.keychain-db" + security set-keychain-settings -lut 21600 "$RUNNER_TEMP/languages.keychain-db" + security unlock-keychain -p "$password" "$RUNNER_TEMP/languages.keychain-db" + security import "$RUNNER_TEMP/certificate.p12" -k "$RUNNER_TEMP/languages.keychain-db" -P "$CERTIFICATE_PASSWORD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple: -k "$password" "$RUNNER_TEMP/languages.keychain-db" + security list-keychains -d user -s "$RUNNER_TEMP/languages.keychain-db" + echo 'DIFFR_SIGN_IDENTITY=Developer ID Application: Workable Solutions Inc. (PWXY59YDAY)' >> "$GITHUB_ENV" + - name: Native parse and hardened runtime proof + run: python3 languages/tests/probe.py 'target/languages/${{ matrix.target }}/package' + - name: Package final bytes + run: cargo xtask package-languages --target '${{ matrix.target }}' + - name: Installer and static parser parity + run: python3 languages/tests/integration.py 'target/languages/${{ matrix.target }}/catalog-entry.json' --release + - name: Fresh Linux install without development tools or network + if: runner.os == 'Linux' + env: + TARGET: ${{ matrix.target }} + run: | + filename=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["url"].rsplit("/", 1)[1])' "target/languages/$TARGET/catalog-entry.json") + pack="$PWD/target/languages/$TARGET/$filename" + docker run --rm --network none --read-only --user 65534:65534 \ + --tmpfs /tmp:rw,exec,nosuid,size=256m \ + --env DIFFR_PARSER_DIR=/tmp/diffr-parsers \ + --volume "$PWD/target/languages/$TARGET/verification/diffr-native:/usr/local/bin/diffr:ro" \ + --volume "$pack:/opt/extra.tgz:ro" \ + --volume "$PWD/target/languages/$TARGET/test-fixtures.txt:/opt/fixtures.txt:ro" \ + --volume "$PWD/sample_files:/fixtures:ro" \ + --volume "$PWD/languages/tests/container-smoke.sh:/opt/container-smoke.sh:ro" \ + debian:bookworm-slim /bin/sh /opt/container-smoke.sh + - name: Clean signing material + if: always() && runner.os == 'macOS' + run: | + security delete-keychain "$RUNNER_TEMP/languages.keychain-db" || true + rm -f "$RUNNER_TEMP/certificate.p12" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: languages-${{ matrix.target }} + path: | + target/languages/${{ matrix.target }}/*.tgz + target/languages/${{ matrix.target }}/catalog-entry.json + if-no-files-found: error + + publish: + if: inputs.publish + needs: build + environment: languages-release + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: languages-* + path: target/languages/publish + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24' + registry-url: https://registry.npmjs.org + - name: Publish and verify exact archives + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: python3 languages/publish.py target/languages/publish --publish + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: verified-language-catalog + path: target/languages/publish/catalog.json + if-no-files-found: error diff --git a/Cargo.lock b/Cargo.lock index 032aa4f9e..c733a085b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -695,6 +701,8 @@ dependencies = [ "diffr-plugin-test-bodies", "dirs", "encoding_rs", + "flate2", + "fs2", "git2", "glob", "hashbrown 0.17.1", @@ -703,6 +711,7 @@ dependencies = [ "jsonschema", "lazy_static", "libc", + "libloading", "line-numbers", "log", "predicates", @@ -718,10 +727,12 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_yaml_ng", + "sha2", "smallvec", "streaming-iterator", "strsim", "strum", + "tar", "tempfile", "terminal_size", "tikv-jemallocator", @@ -1044,6 +1055,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.8" @@ -1064,6 +1085,17 @@ dependencies = [ "serde", ] +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -1132,6 +1164,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1812,6 +1854,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.8" @@ -1931,6 +1983,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -2926,6 +2988,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -3070,6 +3138,17 @@ dependencies = [ "winx", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -5072,6 +5151,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xtask" version = "0.1.0" @@ -5189,6 +5278,12 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index a8f37b0b0..e29af57a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,8 @@ rust-version = "1.85.0" include = [ "/build.rs", "/src/", + "/languages/catalog.json", + "/languages/extra.json", "/wit/", "/plugins/", "/vendored_parsers/highlights/*.scm", @@ -33,6 +35,12 @@ include = [ ] [dependencies] +libloading = "0.8" +sha2 = "0.10" +flate2 = "1" +tar = "0.4" +fs2 = "0.4" +tempfile = "3.27.0" git2 = { version = "0.20", default-features = false } regex = "1.10.4" clap = { version = "4.0.0", features = ["cargo", "env", "wrap_help", "string"] } @@ -94,24 +102,24 @@ tree-sitter-fortran = { version = "0.6.0", optional = true } tree-sitter-fsharp = { version = "0.3.0", optional = true } tree-sitter-gleam = "1.0.0" tree-sitter-go = "0.25.0" -tree-sitter-haskell = "0.23.1" +tree-sitter-haskell = { version = "0.23.1", optional = true } tree-sitter-hcl = "1.1.0" tree-sitter-html = "0.23.2" tree-sitter-java-orchard = "0.5.9" tree-sitter-javascript = "0.25.0" tree-sitter-json = "0.24.8" -tree-sitter-julia = "0.23.1" +tree-sitter-julia = { version = "0.23.1", optional = true } tree-sitter-lua = "0.5" tree-sitter-make = "1.1.1" tree-sitter-newick = "1.1.0" tree-sitter-nix = "0.3.0" tree-sitter-objc = "3.0.2" -tree-sitter-ocaml = "0.25.0" +tree-sitter-ocaml = { version = "0.25.0", optional = true } tree-sitter-pascal = "0.10.0" tree-sitter-php = "0.24.0" tree-sitter-proto = "0.4.0" tree-sitter-python = "0.25.0" -tree-sitter-qmljs = "0.3.0" +tree-sitter-qmljs = { version = "0.3.0", optional = true } tree-sitter-r = "1.2.0" tree-sitter-racket = "0.24.7" tree-sitter-ruby = "0.23.1" @@ -119,13 +127,13 @@ tree-sitter-rust-orchard = "0.16.6" tree-sitter-scala = "0.26.2" tree-sitter-scheme = "0.24.7" tree-sitter-sequel = "0.3.11" -tree-sitter-sfapex = "2.4.0" +tree-sitter-sfapex = { version = "2.4.0", optional = true } tree-sitter-solidity = "1.2.13" tree-sitter-swift = "0.7.1" tree-sitter-toml-ng = "0.7.0" tree-sitter-typescript = "0.23.2" tree-sitter-verilog = { version = "1.0.3", optional = true } -tree-sitter-vhdl = "1.4.0" +tree-sitter-vhdl = { version = "1.4.0", optional = true } tree-sitter-xml = "0.7.0" tree-sitter-yaml = "0.7.0" tree-sitter-zig = "1.1.2" @@ -165,9 +173,9 @@ assert_cmd = "2.0.17" predicates = "3.1.3" pretty_assertions = "1.3.0" -tempfile = "3.27.0" [build-dependencies] +serde_json = "1" toml = "0.8" # TODO: enable parallel mode, see discussion in # https://github.com/rust-lang/cc-rs/pull/849 @@ -235,8 +243,14 @@ name = "wasm" required-features = ["wasm-plugin-tests"] [features] -all-languages = ["lang-fortran", "lang-fsharp", "lang-verilog"] +all-languages = ["lang-fortran", "lang-fsharp", "lang-verilog", "lang-ocaml", "lang-julia", "lang-haskell", "lang-vhdl", "lang-apex", "lang-qml"] lang-fortran = ["dep:tree-sitter-fortran"] lang-fsharp = ["dep:tree-sitter-fsharp"] lang-verilog = ["dep:tree-sitter-verilog"] +lang-ocaml = ["dep:tree-sitter-ocaml"] +lang-julia = ["dep:tree-sitter-julia"] +lang-haskell = ["dep:tree-sitter-haskell"] +lang-vhdl = ["dep:tree-sitter-vhdl"] +lang-apex = ["dep:tree-sitter-sfapex"] +lang-qml = ["dep:tree-sitter-qmljs"] wasm-plugin-tests = [] diff --git a/README.md b/README.md index 0c73fb4b7..6932e207f 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,13 @@ We hope in the future that we can find some way to upstream some / all of these `diffr` has a powerful, wasm-based plugin API which customizes how it presents changed files. For more details, read the [docs](./docs/plugin.md). +## Optional languages + +Optional parsers can be compiled in with `--features all-languages`. +The native pack installer and its publication status are documented in +[languages/README.md](languages/README.md). Use `diffr languages list --json` +to inspect availability for your executable. + ## License MIT. See `LICENSE` for the terms and `NOTICE` for the third-party work diff --git a/build.rs b/build.rs index 9682ce1c9..adb05edc5 100644 --- a/build.rs +++ b/build.rs @@ -41,7 +41,12 @@ impl TreeSitterParser { } fn main() { + println!( + "cargo:rustc-env=DIFFR_TARGET={}", + std::env::var("TARGET").unwrap() + ); native_plugins(); + extra_languages(); let parsers = vec![ TreeSitterParser { name: "tree-sitter-janet-simple", @@ -86,6 +91,70 @@ fn main() { println!("cargo:rustc-env=JEMALLOC_SYS_WITH_LG_PAGE=16"); } +fn extra_languages() { + use serde_json::Value; + println!("cargo:rerun-if-changed=languages/extra.json"); + let definitions: Vec = + serde_json::from_str(&std::fs::read_to_string("languages/extra.json").unwrap()).unwrap(); + let strings = |values: &Value| { + values + .as_array() + .unwrap() + .iter() + .map(|value| format!("{:?}", value.as_str().unwrap())) + .collect::>() + .join(",") + }; + let pairs = |values: &Value| { + values + .as_array() + .unwrap() + .iter() + .map(|pair| { + format!( + "({:?},{:?})", + pair[0].as_str().unwrap(), + pair[1].as_str().unwrap() + ) + }) + .collect::>() + .join(",") + }; + let mut code = String::from("extra_languages! {\n"); + for definition in definitions { + let text = |key| definition[key].as_str().unwrap(); + let highlights = definition["highlights"] + .as_array() + .unwrap() + .iter() + .map(|query| { + if let Some(local) = query["local"].as_str() { + println!("cargo:rerun-if-changed={local}"); + format!( + "include_str!(concat!(env!(\"CARGO_MANIFEST_DIR\"), {:?}))", + format!("/{local}") + ) + } else { + format!( + "{}::{}", + query["crate"].as_str().unwrap().replace('-', "_"), + query["constant"].as_str().unwrap() + ) + } + }) + .collect::>() + .join(","); + code.push_str(&format!( + "{} => {{ id: {:?}, feature: {:?}, parser: {}, highlights: [{}], atoms: [{}], delimiters: [{}], ignore_trailing: [{}] }},\n", + text("variant"), text("id"), text("feature"), text("parser"), highlights, + strings(&definition["atoms"]), pairs(&definition["delimiters"]), pairs(&definition["ignore_trailing"]), + )); + } + code.push_str("}\n"); + let output = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("extra_languages.rs"); + std::fs::write(output, code).unwrap(); +} + fn commit_info() { if !PathBuf::from(".git").exists() { return; diff --git a/languages/.gitignore b/languages/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/languages/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/languages/ADDING.md b/languages/ADDING.md new file mode 100644 index 000000000..5831f5152 --- /dev/null +++ b/languages/ADDING.md @@ -0,0 +1,52 @@ +# Moving another language into `extra` + +`languages/extra.json` is the single extra-language registry. It drives package +building, generated Rust parser configuration, availability reporting, and tests. +Generated files stay under `target/`; no generated parser or upstream highlight +query or license needs to be copied into this repository. + +1. Move the language's configuration from `tree_sitter_parser.rs` into an entry + in `extra.json`. Preserve its atom nodes, delimiters, and trailing-token rules. + Copy an existing entry with similar rules as a template. +2. Declare the grammar dependency optional in Cargo, retain a `lang-*` feature, + and include it in `all-languages`. Both OCaml grammars share one feature. +3. Supply the pinned crate version/revision, source directory, exported C symbol, + Rust parser constant, license path/hash, and a pair of representative fixtures. +4. Describe highlight inputs by upstream crate, Rust constant, and relative file + path. The package builder reads the files from Cargo's locked sources; built-in + builds use the corresponding constants. QML demonstrates combined queries. + Use `local` only for an existing diffr customization, such as Julia or Verilog. + An empty list preserves OCaml interfaces' existing behavior. Shared query + dependencies include their licenses. Set each `license` to its upstream path + and SHA-256. Packaging reads Cargo sources first; when a crate omits its + license, it fetches from the GitHub repository at Cargo's exact VCS revision. + Verified licenses are cached under `target/languages/licenses`; subsequent + packaging can reuse them offline. Published archives always include licenses. +5. Run the checks below and advance the independent pack version when its contents + change. Keep language names, globs, and detection unconditional. + +Do not add another Rust or Python language list. Cargo generates the Rust registry +from `extra.json`. `check-languages` rejects duplicate built-in configurations, +missing features, license metadata, query inputs, and fixtures. Recognition, annotation +rules, and structural diff rules remain in diffr; the pack carries compiled +parser/scanner libraries and integrity-checked highlight query files. + +```sh +cargo xtask check-languages +python3 languages/tests/test_definitions.py +cargo test --locked +cargo test --locked --features all-languages --bin diffr --test cli +cargo xtask build-languages --target aarch64-apple-darwin +# Set DIFFR_SIGN_IDENTITY to Whiteboard's identity for signed macOS checks. +python3 languages/tests/probe.py target/languages/aarch64-apple-darwin/package +cargo xtask package-languages --target aarch64-apple-darwin +python3 languages/tests/integration.py target/languages/aarch64-apple-darwin/catalog-entry.json --release +``` + +Every registry entry automatically participates in native/static structural, +syntax, tree, and installation tests. Include scanner-heavy fixtures when needed. +The tests also cover concurrent loads/installs, corrupt libraries and queries, +repair, offline use, and the platform's default store. + +Publish and verify all four target archives before updating `catalog.json`, as +explained in [README.md](README.md). Existing CLIs retain their pinned revisions. diff --git a/languages/README.md b/languages/README.md new file mode 100644 index 000000000..2bb92992b --- /dev/null +++ b/languages/README.md @@ -0,0 +1,102 @@ +# Extra native languages + +`diffr languages list --json` inspects the optional language +pack without networking or loading native code. Install the revision pinned by +your executable with `diffr languages install extra`. For offline installation, +pass `--archive /path/to/package.tgz`; the same catalog checks apply. + +**Initial publication is pending.** The checked-in catalog intentionally has no +artifacts until all four signed/final packages have been published and verified. +Until that catalog is incorporated, default builds report `parser_unavailable`. +The per-language `lang-*` features and `all-languages` continue to provide +built-in parsers and take priority over packages. The pack contains Fortran, F#, +Verilog, OCaml (implementation and interface), Julia, Haskell, VHDL, Salesforce +Apex, and QML. LaTeX and general SQL remain bundled. + +The single registry is [extra.json](extra.json); see [ADDING.md](ADDING.md) for +the pathway to move another parser out of the built-in configuration. + +The store is `/diffr/parsers//extra/`. +`DIFFR_PARSER_DIR` replaces the `parsers` root. Revisions coexist, and ordinary +diffs never download or install anything. A missing, incompatible, or corrupt +parser produces a text diff with a `parser_not_installed`, `parser_unavailable`, +or `parser_load_failed` problem. The wire shape and diff exit codes are unchanged. + +## Build and verify + +Run on each native target runner (macOS ARM64/x64 or Linux glibc ARM64/x64): + +```sh +cargo xtask build-languages --target aarch64-apple-darwin +python3 languages/tests/probe.py target/languages/aarch64-apple-darwin/package +cargo xtask package-languages --target aarch64-apple-darwin +python3 languages/tests/integration.py target/languages/aarch64-apple-darwin/catalog-entry.json --release +``` + +Python 3 and a C compiler are build tools only. Cargo's locked metadata locates +registry sources; definitions additionally pin versions and source revisions. +Highlight queries travel with the optional pack and are hash-verified before use. +Built-in feature builds continue to use the upstream crate constants. F# uses +only `fsharp/src`, including its scanner, and Fortran includes its scanner. +Licenses come from the pinned crate/source revisions. The builder reads upstream +queries directly from Cargo sources and preserves diffr's existing Julia and +Verilog custom queries. No grammar package contains JavaScript or install scripts. + +To exercise x64 execution under Rosetta on an ARM64 Mac, first install the Rust +target with `rustup target add x86_64-apple-darwin`, then run the commands above +with `x86_64-apple-darwin`. The builder and probe explicitly select the target's +C architecture, and the integration test builds and runs the x64 CLI. This also +checks that package selection follows the executable architecture. + +On macOS, set `DIFFR_SIGN_IDENTITY` to Whiteboard's Developer ID identity before +running the probe. It signs the libraries and a hardened native host, retaining +library validation. The integration script signs the test CLIs with that identity +as well. Package **after** signing. Packaging never modifies the final libraries. +The integration script temporarily replaces the source catalog to build test +CLIs, then restores it; run it in a dedicated checkout, without concurrent Cargo +builds. There is deliberately no runtime catalog override. + +## Publication + +Before using the manual `Extra languages` workflow with `publish: true`: + +1. Configure the `languages-release` GitHub environment with required reviewers. + The workflow checks that protection exists and permits publication only from + `main`. +2. Supply environment secrets `APPLE_CERTIFICATE_BASE64`, + `APPLE_CERTIFICATE_PASSWORD`, and `NPM_TOKEN`. The certificate must belong to + Whiteboard's Team ID `PWXY59YDAY`; npm access must cover all four package names. +3. Set the independent revision in `package.template.json`. Do not reuse a + revision whose bytes have already been published. +4. Run the workflow. All four runners must build, strip, sign where applicable, + parse with native libraries, and pass installed/static parity before publishing. +5. Download the `verified-language-catalog` artifact, review it, and replace + `languages/catalog.json` in a normal code change. Release the CLI afterward. + +The publisher uploads the exact prepared tarballs and downloads each back for +size/hash verification. An existing immutable revision is accepted only if its +bytes match. It produces no catalog if any target fails. Partial publication can +be retried only with byte-identical artifacts, or with a new pack revision. +Re-signing can change bytes even when source inputs are identical. + +Local signed macOS ARM64/Rosetta x64 and Docker Linux ARM64/x64 checks are +recorded in [VALIDATION.md](VALIDATION.md). Gatekeeper behavior for separately +quarantined libraries and public npm installation remain release evidence to +collect; no validation exemption is added to Whiteboard. + +## Clean Linux installation test + +From the repository root: + +```sh +docker build --platform linux/arm64 -f languages/tests/Dockerfile -t diffr-extra-install-test . +docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,nosuid,size=256m diffr-extra-install-test +``` + +The build stage compiles native Linux artifacts and compares installed parsers +with `all-languages`. The separate Debian runtime contains only diffr, the pack, +fixtures, and base OS utilities. As an unprivileged user, with networking disabled, +it verifies a fresh offline install, every optional grammar, idempotence, corrupt-pack +fallback, and atomic repair. It asserts that Rust, C compilers, Node, npm, and +Python are absent. Switch to `--platform linux/amd64` to exercise the other Linux +target (emulation on ARM hosts is substantially slower). diff --git a/languages/VALIDATION.md b/languages/VALIDATION.md new file mode 100644 index 000000000..cc66a2398 --- /dev/null +++ b/languages/VALIDATION.md @@ -0,0 +1,174 @@ +# Local validation — 2026-09-29 + +Base: diffr `58fa3b2` (includes PR #49). Host: macOS ARM64. No packages were +published, and `catalog.json` remains empty pending verified publication. + +## Registry/query refactor — pack 0.3.0 + +`extra.json` now generates the Rust configuration and drives packaging. Upstream +highlight files are no longer copied into the repository or embedded in default +builds. The pack contains catalog-hashed queries alongside its libraries; +`all-languages` still reads upstream crate constants. + +- Full default workspace tests: 290 passed, one ignored. +- Default and all-languages bin/CLI tests: 271 passed, one ignored, in each build. +- Five registry checks and both publication-gate tests passed. +- Signed macOS ARM64 native/static parity passed for all ten grammars, including + query corruption/fallback/repair, scanners, parallel installs/diffs, offline + installation, the default store, quarantine-marked archives, and old CLI reuse. +- Linux ARM64 Docker passed the same parity suite and the minimal unprivileged, + read-only-root, no-network/no-development-tools runtime installation test. +- The source package includes `extra.json` and the catalog, with no copied upstream + highlight files. Formatting, actionlint, typos, and whitespace checks passed. +- Clippy reports only the six pre-existing warnings listed below. + +Compressed pack sizes: 3,947,947 bytes on macOS ARM64 and 3,748,052 bytes on Linux +ARM64. The four-target workflow will rerun on the updated PR; the x64 results +below describe the previous 0.2.0 pack, before query relocation. + +## Earlier 0.2.0 validation + +## Passed + +- `cargo test --locked`: 290 tests passed, one ignored. +- `cargo test --locked --features all-languages --bin diffr --test cli`: + 271 passed, one ignored. +- Archive/download tests cover interrupted and oversized transfers, wrong + hashes/targets/sizes, missing files, duplicates, symlinks, hardlinks, nested + paths, and traversal. +- `python3 languages/tests/test_publish.py`: both publication-gate tests passed. +- `actionlint .github/workflows/languages.yml`, formatting, and `git diff --check`. +- Cargo's package file list includes the embedded catalog, all ten highlight + queries, and their licenses. +- All ten C grammars built and parsed in a signed hardened native probe. +- Signed release CLIs passed installed/static NDJSON and syntax parity, tree + comparison, scanner-heavy Fortran/F# fixtures, added/deleted/unchanged files, + 120 mixed-language files with eight workers, concurrent installers, corruption, + repair, offline reuse, and revision coexistence. +- A fresh offline install and diff succeeded with an empty `PATH`. +- The signed macOS CLI passed the default `Library/Application Support` store, + an archive marked with `com.apple.quarantine`, truncated-archive rejection, + and actual coexistence with the previously built 0.1.0 CLI and pack. +- Definition checks (three tests), shared OCaml feature coverage, and the + metadata-driven packaging and fixture discovery checks passed. + +Both the probe and full CLI used Team ID `PWXY59YDAY`. The full CLI used the +existing Whiteboard diffr entitlement `allow-unsigned-executable-memory`. +Neither host disabled library validation. Loading was tested from the parser +store outside the application bundle. The original Whiteboard app was not changed. + +Clippy completes with six pre-existing warnings in `src/diff/graph.rs`, +`src/parse/syntax.rs`, and `src/protocol/mod.rs`; `-D warnings` fails on those. + +## Measurements + +One local signed ARM64 pack, revision `0.2.0`, containing ten grammars: + +| Measurement | Bytes | +|---|---:| +| Compressed pack | 3,932,587 | +| Native-pack release CLI | 93,029,568 | +| All-languages release CLI | 148,755,536 | + +The executable is 55,725,968 bytes smaller than the equivalent all-languages +build. Both measurements include Developer ID signatures. These are local release +executables, not publicly downloaded CLI artifacts. + +Installer/loading overhead was separately measured before the pack expansion, +with the original three-parser feature set held constant: 107,839,232 bytes for +the baseline and 112,555,376 for the installer/loader build, an increase of +4,716,144 bytes (4.37%). That comparison isolates installer work; it is not the +expanded pack's net size change. + +Ten new-process Apex diffs had a median of 196 ms; the first measured sample +was 345 ms. A 120-file mixed batch with eight workers took 777 ms. Each process +starts with an empty parser cache, while the batch reuses loaded grammars. +The OS page cache was warm; these are not cold-disk measurements. Timing is +indicative, not a performance gate. + +## Outstanding release evidence + +Public npm downloads and behavior for separately quarantined libraries have +not been exercised locally. The +four-runner workflow must pass before publication. The repository currently +has no configured signing/npm secrets or protected publication environment; +see README.md for the exact setup. Only a catalog emitted after all four +registry downloads match may replace the empty checked-in catalog. + +## Linux ARM64 Docker verification + +A native `aarch64-unknown-linux-gnu` build passed `languages/tests/probe.py` and +the complete installed/static integration suite in Docker. The Linux pack was +3,737,483 bytes compressed (revision `0.2.0`). The separate Debian Bookworm runtime passed +`container-smoke.sh` with networking disabled, a read-only root filesystem, +a temporary writable parser store, and UID/GID 65534. + +The Linux native CLI was 98,161,256 bytes; the all-languages CLI was +153,820,688 bytes. Ten new-process Apex diffs had a median of 228 ms, +and the 120-file mixed batch took 1,554 ms. + +The runtime verifies absence of Cargo, Rust, C compilers, Node, npm, and Python; +then tests fresh offline installation, all ten structural parsers, idempotence, +corrupt-library text fallback, and atomic repair. This test is also wired into +both Linux jobs in `languages.yml`. The local run covers ARM64 only. + +Commands: + +```sh +docker build --platform linux/arm64 -f languages/tests/Dockerfile -t diffr-extra-install-test . +docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,nosuid,size=256m diffr-extra-install-test +``` + +Result: `PASS: fresh offline install, all optional parsers, idempotence, corrupt +fallback, atomic repair; unprivileged and without development tools.` + +The first attempted build used a manually assembled context that omitted tracked +grammar-directory symlinks. Rebuilding with the ordinary Docker context and the +provided Dockerfile-specific ignore file fixed the test setup. No parser or +installer fix was needed. Public npm installation remains unverified until +publication and catalog activation. + +## macOS x64 / Rosetta verification + +The signed `x86_64-apple-darwin` probe and both signed x64 CLIs ran under Rosetta +on the ARM64 host. All ten grammars passed native/static diff, syntax, and tree +parity, scanner-heavy fixtures, added/deleted/unchanged files, the 120-file batch, +concurrent installation, corruption/repair, truncated archive rejection, the +default Application Support store, quarantined-archive installation, and an +empty-PATH fresh install. The executable selected the x64 package. + +Both x64 hosts and libraries use Team ID `PWXY59YDAY` with hardened runtime; +the full CLI retains only Whiteboard's existing executable-memory entitlement. +An additional shared-store check installed both ARM64 and x64 revisions into +one root and confirmed that each executable selected its own target and parsed +QML without fallback. + +| x64 measurement | Bytes | +|---|---:| +| Compressed pack | 4,219,033 | +| Native-pack release CLI | 96,633,808 | +| All-languages release CLI | 152,322,048 | + +The new-process Apex median was 395 ms; the 120-file mixed batch took 1,800 ms. +These are Rosetta timings with warm OS caches, not native Intel benchmarks. + +All four versioned npm tarball URLs returned HTTP 404 on 2026-09-29. Public +network installation remains blocked on publication. The repository has no +signing/npm repository secrets and no protected `languages-release` environment; +only the unprotected test environment `languages-ci` currently exists. + +## Linux x64 Docker verification + +The `linux/amd64` Docker build passed all ten native probes and the complete +installed/static integration suite under emulation on the ARM64 host. Its separate +minimal runtime passed fresh offline installation, all parsers, idempotence, +corrupt-library fallback, and atomic repair with UID/GID 65534, networking disabled, +a read-only root, and no Rust, C compiler, Node, npm, or Python. + +The compressed pack was 3,759,226 bytes; native and all-languages executables +were 100,672,240 and 156,206,248 bytes. The Apex median was 366 ms and the +120-file mixed batch took 2,434 ms; these are warm-cache emulation measurements. + +The native Linux x64 GitHub runner also passed installation/parity and its +minimal Docker runtime check in run `36609243307`, alongside both ARM64 runners. +The macOS Intel job was still running when these local results were recorded. diff --git a/languages/build.py b/languages/build.py new file mode 100644 index 000000000..dc059b43e --- /dev/null +++ b/languages/build.py @@ -0,0 +1,117 @@ +"""Build locked grammar sources; package final (already stripped/signed) bytes.""" +import argparse +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tarfile +from definitions import check +from licenses import package_licenses + +ROOT = Path(__file__).resolve().parent.parent +TARGETS = { + 'aarch64-apple-darwin': ('darwin-arm64', 'darwin', 'arm64'), + 'x86_64-apple-darwin': ('darwin-x64', 'darwin', 'x64'), + 'aarch64-unknown-linux-gnu': ('linux-arm64-gnu', 'linux', 'arm64'), + 'x86_64-unknown-linux-gnu': ('linux-x64-gnu', 'linux', 'x64'), +} + + +def read(path): + return json.loads(path.read_text()) + + +def write(path, value): + path.write_text(json.dumps(value, indent=2) + '\n') + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument('action', choices=['build', 'package', 'check']) + cli.add_argument('--target', choices=TARGETS) + args = cli.parse_args() + definitions = check() + if args.action == 'check': + print(f'{len(definitions)} optional grammar definitions validated') + return + if args.target is None: + cli.error('--target is required for build/package') + suffix, system, cpu = TARGETS[args.target] + host = subprocess.check_output(['rustc', '-vV'], text=True).split('host: ')[1].splitlines()[0] + if args.target != host and not (system == 'darwin' and host.endswith('-apple-darwin')): + cli.error('build/package on the native target runner (macOS also supports cross-architecture builds)') + architecture = ['-arch', 'arm64' if cpu == 'arm64' else 'x86_64'] if system == 'darwin' else [] + output = ROOT / 'target/languages' / args.target + package = output / 'package' + template = read(ROOT / 'languages/package.template.json') + name = '@dev.fast/diffr-languages-extra-' + suffix + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--all-features', '--format-version', '1'], cwd=ROOT)) + if args.action == 'build': + if package.exists(): + shutil.rmtree(package) + package.mkdir(parents=True) + (output / 'test-fixtures.txt').write_text(''.join( + f"{d['fixture']['prefix']}:{d['fixture']['extension']}\n" for d in definitions)) + libraries = [] + for definition in definitions: + crate = next(p for p in metadata['packages'] if p['name'] == definition['crate'] and p['version'] == definition['version']) + crate_root = Path(crate['manifest_path']).parent + assert read(crate_root / '.cargo_vcs_info.json')['git']['sha1'] == definition['revision'] + source = crate_root / definition['source'] + filename = definition['id'] + ('.dylib' if system == 'darwin' else '.so') + sources = [source / 'parser.c'] + if (source / 'scanner.c').exists(): + sources.append(source / 'scanner.c') + subprocess.run([os.environ.get('CC', 'cc'), *architecture, '-O2', '-std=c11', '-fPIC', '-I' + str(source), '-dynamiclib' if system == 'darwin' else '-shared', *map(str, sources), '-o', str(package / filename)], check=True) + subprocess.run(['strip', '-x' if system == 'darwin' else '--strip-unneeded', str(package / filename)], check=True) + abi = int(re.search(r'#define LANGUAGE_VERSION (\d+)', (source / 'parser.c').read_text())[1]) + libraries.append(dict(id=definition['id'], file=filename, symbol=definition['symbol'], abi=abi)) + highlights = [] + for query in definition['highlights']: + if 'local' in query: + path = ROOT / query['local'] + else: + dependency = next(p for p in metadata['packages'] if p['name'] == query['crate']) + dependency_root = Path(dependency['manifest_path']).parent + path = dependency_root / query['path'] + highlights.append(path.read_text()) + (package / (definition['id'] + '.highlights.scm')).write_text(''.join(highlights)) + write(package / 'manifest.json', dict(schema=1, pack='extra', version=template['version'], target=args.target, libraries=libraries)) + write(package / 'package.json', dict(template, name=name, os=[system], cpu=[cpu], **({'libc': ['glibc']} if system == 'linux' else {}))) + else: + manifest = read(package / 'manifest.json') + assert manifest['target'] == args.target and manifest['version'] == template['version'] + expected = {(d['id'], d['symbol']) for d in definitions} + assert {(d['id'], d['symbol']) for d in manifest['libraries']} == expected, 'stale package: rebuild the language libraries' + package_licenses(definitions, metadata, package, ROOT / 'target/languages/licenses') + expected_files = {'manifest.json', 'package.json'} + for library in manifest['libraries']: + expected_files.update([library['file'], library['id'] + '.LICENSE', library['id'] + '.highlights.scm']) + expected_files.update(q['crate'] + '.LICENSE' for d in definitions for q in d['highlights'] if 'crate' in q and q['crate'] != d['crate']) + assert {p.name for p in package.iterdir()} == expected_files, 'unexpected or missing package files' + files = {p.name: dict(size=p.stat().st_size, sha256=digest(p.read_bytes())) for p in sorted(package.iterdir())} + archive = output / (name.split('/')[1] + '-' + template['version'] + '.tgz') + with archive.open('wb') as raw: + with gzip.GzipFile(fileobj=raw, mode='wb', mtime=0, filename='') as compressed: + with tarfile.open(fileobj=compressed, mode='w', format=tarfile.USTAR_FORMAT) as tar: + for filename in files: + data = (package / filename).read_bytes() + entry = tarfile.TarInfo('package/' + filename) + entry.size, entry.mode = len(data), 0o644 + tar.addfile(entry, io.BytesIO(data)) + entry = dict(manifest, name=name, url=f'https://registry.npmjs.org/{name}/-/{archive.name}', size=archive.stat().st_size, sha256=digest(archive.read_bytes()), files=files) + write(output / 'catalog-entry.json', entry) + print(archive) + + +if __name__ == '__main__': + main() diff --git a/languages/catalog.json b/languages/catalog.json new file mode 100644 index 000000000..2d0b35ebb --- /dev/null +++ b/languages/catalog.json @@ -0,0 +1 @@ +{"schema":1,"packages":[]} diff --git a/languages/definitions.py b/languages/definitions.py new file mode 100644 index 000000000..b9fe855f9 --- /dev/null +++ b/languages/definitions.py @@ -0,0 +1,61 @@ +"""Shared definitions for packaging, smoke tests, and optional-parser checks.""" +import json +from pathlib import Path +import re +import subprocess + +ROOT = Path(__file__).resolve().parent.parent + + +def definitions(root=ROOT): + return json.loads((root / 'languages/extra.json').read_text()) + + +def validate(items, metadata, root=ROOT): + package = next(p for p in metadata['packages'] if p['name'] == 'diffr-cli') + dependencies = {d['name']: d for d in package['dependencies']} + builtins = set(re.findall(r'^ (\w+) => \{', (root / 'src/parse/tree_sitter_parser.rs').read_text(), re.MULTILINE)) + assert len({d['id'] for d in items}) == len(items), 'duplicate language ID' + assert len({d['variant'] for d in items}) == len(items), 'duplicate language variant' + assert {d['crate'] for d in items} == {name for name, dep in dependencies.items() if dep['optional'] and name.startswith(('tree-sitter-', 'ts-parser-'))}, 'optional Cargo dependencies and definitions differ' + assert {d['feature'] for d in items} == {name for name in package['features'] if name.startswith('lang-')}, 'language Cargo features and definitions differ' + for definition in items: + id, feature, crate = (definition[k] for k in ('id', 'feature', 'crate')) + assert re.fullmatch(r'[a-z][a-z0-9-]*', id), f'{id}: invalid ID' + assert definition['variant'].isidentifier(), f'{id}: invalid enum variant' + assert definition['variant'] not in builtins, f'{id}: remove duplicate built-in config' + assert package['features'][feature] == ['dep:' + crate], f'{id}: incorrect Cargo feature' + assert feature in package['features']['all-languages'], f'{id}: missing from all-languages' + assert dependencies[crate]['optional'], f'{id}: dependency must be optional' + source = Path(definition['source']) + assert not source.is_absolute() and '..' not in source.parts, f'{id}: invalid source path' + assert re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9]*', definition['symbol']), f'{id}: invalid symbol' + assert definition['smoke'].strip(), f'{id}: missing parse smoke input' + assert re.fullmatch(r'\w+(::\w+)+', definition['parser']), f'{id}: invalid parser constant' + licenses = [definition['license']] + [q['license'] for q in definition['highlights'] if 'crate' in q and q['crate'] != crate] + for license in licenses: + path = Path(license['path']) + assert not path.is_absolute() and '..' not in path.parts, f'{id}: invalid license path' + assert re.fullmatch(r'[0-9a-f]{64}', license['sha256']), f'{id}: invalid license hash' + for query in definition['highlights']: + if 'local' in query: + relative = Path(query['local']) + assert not relative.is_absolute() and '..' not in relative.parts, f'{id}: invalid local query path' + path = root / relative + assert path.is_file(), f'{id}: missing local query' + else: + assert query['crate'] in dependencies, f'{id}: unknown query crate' + assert re.fullmatch(r'\w+(::\w+)*', query['constant']), f'{id}: invalid query constant' + path = Path(query['path']) + assert not path.is_absolute() and '..' not in path.parts, f'{id}: invalid query path' + fixture = definition['fixture'] + for side in (1, 2): + path = root / 'sample_files' / f"{fixture['prefix']}_{side}.{fixture['extension']}" + assert path.is_file(), f'{id}: missing fixture {path}' + + +def check(root=ROOT): + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--no-deps', '--format-version', '1'], cwd=root)) + items = definitions(root) + validate(items, metadata, root) + return items diff --git a/languages/extra.json b/languages/extra.json new file mode 100644 index 000000000..a7abe7eac --- /dev/null +++ b/languages/extra.json @@ -0,0 +1,483 @@ +[ + { + "id": "apex", + "crate": "tree-sitter-sfapex", + "version": "2.4.0", + "source": "apex/src", + "symbol": "tree_sitter_apex", + "revision": "9247e2cd12490f1aac71cb212f726ef1a2b5f7f9", + "repository": "https://github.com/aheber/tree-sitter-sfapex", + "variant": "Apex", + "feature": "lang-apex", + "fixture": { + "prefix": "apex", + "extension": "trigger" + }, + "smoke": "public class Example { public Integer f(Integer x) { return x + 1; } }\n", + "parser": "tree_sitter_sfapex::apex::LANGUAGE", + "atoms": [ + "string_literal", + "null_literal", + "boolean", + "int", + "decimal_floating_point_literal", + "date_literal", + "currency_literal" + ], + "delimiters": [ + [ + "[", + "]" + ], + [ + "(", + ")" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-sfapex", + "constant": "apex::HIGHLIGHTS_QUERY", + "path": "apex/queries/highlights.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "c3d045abbc0a8dec1b00737c0668f6dbb028d248521b74e79778c726671925ed" + } + }, + { + "id": "fortran", + "crate": "tree-sitter-fortran", + "version": "0.6.0", + "source": "src", + "symbol": "tree_sitter_fortran", + "revision": "48593829df8a929bfe58528eedcf405171921005", + "repository": "https://github.com/stadelmanma/tree-sitter-fortran", + "variant": "Fortran", + "feature": "lang-fortran", + "fixture": { + "prefix": "fortran", + "extension": "f90" + }, + "smoke": "program hello\n implicit none\n print *, \"hello\"\nend program hello\n", + "parser": "tree_sitter_fortran::LANGUAGE", + "atoms": [ + "string_literal", + "number_literal" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "(/", + "/)" + ], + [ + "[", + "]" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-fortran", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "c0fc02a86ae2b179c4729694c523bbf42da674e5136ed8e12e3cb2f26556ea9f" + } + }, + { + "id": "fsharp", + "crate": "tree-sitter-fsharp", + "version": "0.3.0", + "source": "fsharp/src", + "symbol": "tree_sitter_fsharp", + "revision": "5247c1197cb290fcaea0e0a793d32829c1396831", + "repository": "https://github.com/ionide/tree-sitter-fsharp", + "variant": "FSharp", + "feature": "lang-fsharp", + "fixture": { + "prefix": "f_sharp", + "extension": "fs" + }, + "smoke": "module Example\nlet f x =\n let y = x + 1\n y * 2\n", + "parser": "tree_sitter_fsharp::LANGUAGE_FSHARP", + "atoms": [ + "string", + "triple_quoted_string" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-fsharp", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "f9352b63e87252b282b252bf27530027c8ca69829e0023f8b8b682a736f271fd" + } + }, + { + "id": "haskell", + "crate": "tree-sitter-haskell", + "version": "0.23.1", + "source": "src", + "symbol": "tree_sitter_haskell", + "revision": "c30d812bc90827f1a54106a25bc9a6307f5cdcec", + "repository": "https://github.com/tree-sitter/tree-sitter-haskell", + "variant": "Haskell", + "feature": "lang-haskell", + "fixture": { + "prefix": "haskell", + "extension": "hs" + }, + "smoke": "module Example where\nf x =\n let y = x + 1\n in y * 2\n", + "parser": "tree_sitter_haskell::LANGUAGE", + "atoms": [ + "qualified_variable" + ], + "delimiters": [ + [ + "[", + "]" + ], + [ + "(", + ")" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-haskell", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "2e0110e07abef7c2548b26ec9d6969775617ca539a0dc8dbeeb14d6452c711d1" + } + }, + { + "id": "julia", + "crate": "tree-sitter-julia", + "version": "0.23.1", + "source": "src", + "symbol": "tree_sitter_julia", + "revision": "a8e1262997d5a45520a06cbe1b86c0737d507054", + "repository": "https://github.com/tree-sitter/tree-sitter-julia", + "variant": "Julia", + "feature": "lang-julia", + "fixture": { + "prefix": "julia", + "extension": "jl" + }, + "smoke": "function f(x)\n x + 1\nend\n", + "parser": "tree_sitter_julia::LANGUAGE", + "atoms": [ + "string_literal", + "prefixed_string_literal", + "command_literal", + "character_literal" + ], + "delimiters": [ + [ + "{", + "}" + ], + [ + "[", + "]" + ], + [ + "(", + ")" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "local": "vendored_parsers/highlights/julia.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "74f800107f200f4e28da33e9c3f86230d37d8f4c0a4916f7111e800c395ee3a2" + } + }, + { + "id": "ocaml", + "crate": "tree-sitter-ocaml", + "version": "0.25.0", + "source": "grammars/ocaml/src", + "symbol": "tree_sitter_ocaml", + "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", + "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", + "variant": "OCaml", + "feature": "lang-ocaml", + "fixture": { + "prefix": "ocaml", + "extension": "ml" + }, + "smoke": "module M = struct\n let f x = x + 1\nend\n", + "parser": "tree_sitter_ocaml::LANGUAGE_OCAML", + "atoms": [ + "character", + "string", + "quoted_string", + "tag", + "type_variable", + "attribute_id" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-ocaml", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "f977a20805dd6e275f14bc090a5f8e638401dad4df366841c1060f12c75b74ae" + } + }, + { + "id": "ocaml-interface", + "crate": "tree-sitter-ocaml", + "version": "0.25.0", + "source": "grammars/interface/src", + "symbol": "tree_sitter_ocaml_interface", + "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", + "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", + "variant": "OCamlInterface", + "feature": "lang-ocaml", + "fixture": { + "prefix": "ocaml_interface", + "extension": "mli" + }, + "smoke": "type t = { value : int }\nval f : int -> t\n", + "parser": "tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE", + "atoms": [ + "character", + "string", + "quoted_string", + "tag", + "type_variable", + "attribute_id" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [], + "license": { + "path": "LICENSE", + "sha256": "f977a20805dd6e275f14bc090a5f8e638401dad4df366841c1060f12c75b74ae" + } + }, + { + "id": "qml", + "crate": "tree-sitter-qmljs", + "version": "0.3.0", + "source": "src", + "symbol": "tree_sitter_qmljs", + "revision": "0bec4359a7eb2f6c9220cd57372d87d236f66d59", + "repository": "https://github.com/yuja/tree-sitter-qmljs", + "variant": "Qml", + "feature": "lang-qml", + "fixture": { + "prefix": "qml", + "extension": "qml" + }, + "smoke": "import QtQuick\nItem { property int count: 1; function f(x) { return x + count; } }\n", + "parser": "tree_sitter_qmljs::LANGUAGE", + "atoms": [ + "string", + "template_string", + "regex" + ], + "delimiters": [ + [ + "{", + "}" + ], + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "<", + ">" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-javascript", + "constant": "HIGHLIGHT_QUERY", + "path": "queries/highlights.scm", + "license": { + "path": "LICENSE", + "sha256": "2e0110e07abef7c2548b26ec9d6969775617ca539a0dc8dbeeb14d6452c711d1" + } + }, + { + "crate": "tree-sitter-typescript", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm", + "license": { + "path": "LICENSE", + "sha256": "49bf33cf78ef5897e4e161ce1517df7de1ae5042a65b6bcfd44401e0fc606559" + } + }, + { + "crate": "tree-sitter-qmljs", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "a8fb3a7df5ba9ef3b6511a1d65ad69869de66828fe2d690b7b65913678739738" + } + }, + { + "id": "verilog", + "crate": "tree-sitter-verilog", + "version": "1.0.3", + "source": "src", + "symbol": "tree_sitter_verilog", + "revision": "521b535e41a5acd2c6539a922d4649bbe8275110", + "repository": "https://github.com/tree-sitter/tree-sitter-verilog", + "variant": "Verilog", + "feature": "lang-verilog", + "fixture": { + "prefix": "verilog", + "extension": "sv" + }, + "smoke": "module example(input a, output b);\nassign b = a;\nendmodule\n", + "parser": "tree_sitter_verilog::LANGUAGE", + "atoms": [ + "integral_number" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "begin", + "end" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "local": "vendored_parsers/highlights/verilog.scm" + } + ], + "license": { + "path": "LICENSE", + "sha256": "8d2ff8d1e308828099436c43dc8bfbb8b6f368ad2a012ab9802a80ca22b03b53" + } + }, + { + "id": "vhdl", + "crate": "tree-sitter-vhdl", + "version": "1.4.0", + "source": "src", + "symbol": "tree_sitter_vhdl", + "revision": "889c521757471093ce5ac83502ca8a89e516404e", + "repository": "https://github.com/jpt13653903/tree-sitter-vhdl", + "variant": "Vhdl", + "feature": "lang-vhdl", + "fixture": { + "prefix": "vhdl", + "extension": "vhd" + }, + "smoke": "entity example is\nend entity example;\narchitecture rtl of example is\nbegin\nend architecture rtl;\n", + "parser": "tree_sitter_vhdl::LANGUAGE", + "atoms": [], + "delimiters": [ + [ + "(", + ")" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-vhdl", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/Neovim/highlights.scm" + } + ], + "license": { + "path": "License.md", + "sha256": "a9323218abe9aabd3429b76634470b40c3be012600e1f96bd16ca254724fa954" + } + } +] diff --git a/languages/licenses.py b/languages/licenses.py new file mode 100644 index 000000000..8bf64ce85 --- /dev/null +++ b/languages/licenses.py @@ -0,0 +1,43 @@ +"""Collect licenses from locked Cargo sources or their exact upstream revision.""" +import hashlib +import json +from pathlib import Path +import re +from urllib.request import urlopen + + +def collect(crate, spec, cache): + root = Path(crate['manifest_path']).parent + path = Path(spec['path']) + assert not path.is_absolute() and '..' not in path.parts, 'invalid license path' + fingerprint = spec['sha256'] + assert re.fullmatch(r'[0-9a-f]{64}', fingerprint), 'invalid license hash' + cached = cache / fingerprint + if (root / path).is_file(): + data = (root / path).read_bytes() + elif cached.is_file(): + data = cached.read_bytes() + else: + revision = json.loads((root / '.cargo_vcs_info.json').read_text())['git']['sha1'] + assert re.fullmatch(r'[0-9a-f]{40}', revision), 'invalid upstream revision' + repository = crate['repository'].removesuffix('.git').removesuffix('/') + assert re.fullmatch(r'https://github.com/[\w.-]+/[\w.-]+', repository), 'unsupported license repository' + url = repository.replace('https://github.com/', 'https://raw.githubusercontent.com/') + '/' + revision + '/' + path.as_posix() + with urlopen(url, timeout=30) as response: + data = response.read(1024 * 1024 + 1) + assert len(data) <= 1024 * 1024, 'license exceeds size limit' + assert hashlib.sha256(data).hexdigest() == fingerprint, f"{crate['name']}: license hash mismatch" + cache.mkdir(parents=True, exist_ok=True) + cached.write_bytes(data) + return data + + +def package_licenses(definitions, metadata, package, cache): + crates = {p['name']: p for p in metadata['packages']} + for definition in definitions: + (package / (definition['id'] + '.LICENSE')).write_bytes( + collect(crates[definition['crate']], definition['license'], cache)) + for query in definition['highlights']: + if 'crate' in query and query['crate'] != definition['crate']: + (package / (query['crate'] + '.LICENSE')).write_bytes( + collect(crates[query['crate']], query['license'], cache)) diff --git a/languages/package.template.json b/languages/package.template.json new file mode 100644 index 000000000..b3aa45794 --- /dev/null +++ b/languages/package.template.json @@ -0,0 +1,6 @@ +{ + "version": "0.3.0", + "description": "Optional native language parsers for diffr", + "license": "MIT", + "repository": "https://github.com/devdotfast/diffr" +} diff --git a/languages/publish.py b/languages/publish.py new file mode 100644 index 000000000..3b8ecd6a3 --- /dev/null +++ b/languages/publish.py @@ -0,0 +1,58 @@ +"""Publish exact prepared archives, verify registry bytes, then emit a catalog. + +This command is used only by the protected publication workflow. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import time +import urllib.error +import urllib.request +from build import TARGETS + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory', type=Path) + parser.add_argument('--publish', action='store_true') + args = parser.parse_args() + (args.directory / "catalog.json").unlink(missing_ok=True) + entries = [json.loads(p.read_text()) for p in sorted(args.directory.rglob('catalog-entry.json'))] + assert len(entries) == 4 and {e['target'] for e in entries} == set(TARGETS), 'need exactly four targets' + assert len({e['version'] for e in entries}) == 1, 'mixed revisions' + for entry in entries: + archives = list(args.directory.rglob(entry['url'].rsplit('/', 1)[1])) + assert len(archives) == 1, 'missing or duplicate archive' + data = archives[0].read_bytes() + assert len(data) == entry['size'] and sha(data) == entry['sha256'] + if args.publish: + try: + with urllib.request.urlopen(entry['url'], timeout=60) as response: + existing = response.read(entry['size'] + 1) + assert len(existing) == entry['size'] and sha(existing) == entry['sha256'], 'immutable revision already has different bytes' + except urllib.error.HTTPError as error: + if error.code != 404: + raise + subprocess.run(['npm', 'publish', str(archives[0]), '--access', 'public', '--ignore-scripts'], check=True) + # Never advance the catalog based on local artifacts alone. + for attempt in range(12): + try: + with urllib.request.urlopen(entry['url'], timeout=60) as response: + downloaded = response.read(entry['size'] + 1) + assert len(downloaded) == entry['size'] and sha(downloaded) == entry['sha256'], 'registry byte mismatch' + break + except urllib.error.HTTPError as error: + if error.code != 404 or attempt == 11: + raise + time.sleep(5) + (args.directory / 'catalog.json').write_text(json.dumps({'schema': 1, 'packages': entries}, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/languages/tests/Dockerfile b/languages/tests/Dockerfile new file mode 100644 index 000000000..fa2a9fcf9 --- /dev/null +++ b/languages/tests/Dockerfile @@ -0,0 +1,27 @@ +FROM rust:1.85-bookworm AS build +RUN apt-get update && apt-get install -y --no-install-recommends python3 cmake clang pkg-config && rm -rf /var/lib/apt/lists/* +ENV RUSTUP_TOOLCHAIN=1.85.0 CARGO_BUILD_JOBS=4 +WORKDIR /work +COPY . . +RUN --mount=type=cache,target=/work/target,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + target=$(rustc -vV | sed -n 's/^host: //p') && \ + cargo xtask build-languages --target "$target" && \ + python3 languages/tests/probe.py "target/languages/$target/package" && \ + cargo xtask package-languages --target "$target" && \ + python3 languages/tests/integration.py "target/languages/$target/catalog-entry.json" --release && \ + mkdir /artifacts && \ + cp "target/languages/$target/verification/diffr-native" /artifacts/diffr && \ + archive=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["url"].rsplit("/", 1)[1])' "target/languages/$target/catalog-entry.json") && \ + cp "target/languages/$target/$archive" /artifacts/extra.tgz && \ + cp "target/languages/$target/test-fixtures.txt" /artifacts/fixtures.txt + +FROM debian:bookworm-slim +COPY --from=build /artifacts/diffr /usr/local/bin/diffr +COPY --from=build /artifacts/extra.tgz /opt/extra.tgz +COPY --from=build /artifacts/fixtures.txt /opt/fixtures.txt +COPY sample_files /fixtures +COPY languages/tests/container-smoke.sh /opt/container-smoke.sh +ENV DIFFR_PARSER_DIR=/tmp/diffr-parsers +USER 65534:65534 +ENTRYPOINT ["/bin/sh", "/opt/container-smoke.sh"] diff --git a/languages/tests/Dockerfile.dockerignore b/languages/tests/Dockerfile.dockerignore new file mode 100644 index 000000000..6e6351b45 --- /dev/null +++ b/languages/tests/Dockerfile.dockerignore @@ -0,0 +1,5 @@ +.git +target +**/target +**/node_modules +**/__pycache__ diff --git a/languages/tests/container-smoke.sh b/languages/tests/container-smoke.sh new file mode 100644 index 000000000..3d6489ae2 --- /dev/null +++ b/languages/tests/container-smoke.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu +for tool in cargo rustc cc gcc node npm python3; do + if command -v "$tool" >/dev/null 2>&1; then + echo "Unexpected development tool: $tool" >&2 + exit 1 + fi +done +test ! -e "$DIFFR_PARSER_DIR" +diffr languages list --json > /tmp/before.json +grep -q 'not_installed' /tmp/before.json +diffr languages install extra --json --archive /opt/extra.tgz +diffr languages list --json > /tmp/after.json +test "$(grep -o '"availability":"installed"' /tmp/after.json | wc -l)" -eq "$(wc -l < /opt/fixtures.txt)" +while IFS=: read -r fixture extension; do + diffr --format ndjson --syntax --no-index "/fixtures/${fixture}_1.$extension" "/fixtures/${fixture}_2.$extension" > "/tmp/$fixture.ndjson" + if grep -q '"fallback":{' "/tmp/$fixture.ndjson"; then + echo "Unexpected fallback for $fixture" >&2 + exit 1 + fi +done < /opt/fixtures.txt +# An unchanged repeat is idempotent and needs no network or development tools. +diffr languages install extra --json --archive /opt/extra.tgz +for library in "$DIFFR_PARSER_DIR"/*/extra/*/fortran.so; do + printf 'broken' > "$library" +done +diffr --format ndjson --no-index /fixtures/fortran_1.f90 /fixtures/fortran_2.f90 > /tmp/corrupt.ndjson +grep -q 'parser_load_failed' /tmp/corrupt.ndjson +diffr languages install extra --json --archive /opt/extra.tgz +diffr --format ndjson --syntax --no-index /fixtures/fortran_1.f90 /fixtures/fortran_2.f90 > /tmp/repaired.ndjson +cmp /tmp/fortran.ndjson /tmp/repaired.ndjson || { + # Run summary timing is allowed to differ; compare deterministic file records. + grep '"type":"file"' /tmp/fortran.ndjson > /tmp/expected-file + grep '"type":"file"' /tmp/repaired.ndjson > /tmp/repaired-file + cmp /tmp/expected-file /tmp/repaired-file +} +printf 'PASS: fresh offline install, all optional parsers, idempotence, corrupt fallback, atomic repair; unprivileged and without development tools.\n' diff --git a/languages/tests/entitlements.plist b/languages/tests/entitlements.plist new file mode 100644 index 000000000..d7757761a --- /dev/null +++ b/languages/tests/entitlements.plist @@ -0,0 +1,5 @@ + + + +com.apple.security.cs.allow-unsigned-executable-memory + diff --git a/languages/tests/fixtures/scanner.f90 b/languages/tests/fixtures/scanner.f90 new file mode 100644 index 000000000..7bba6c05d --- /dev/null +++ b/languages/tests/fixtures/scanner.f90 @@ -0,0 +1,15 @@ +program scanner + implicit none + integer :: i, j + real :: total + character(len=32) :: message + total = 1.25e+2 + & + & 2.5e-1 + message = 'hello & + &world' + do 100 i = 1, 3 + do 100 j = 1, 2 + total = total + real(i * j) +100 continue + print *, message, total +end program scanner diff --git a/languages/tests/fixtures/scanner.fs b/languages/tests/fixtures/scanner.fs new file mode 100644 index 000000000..dc5ed5a72 --- /dev/null +++ b/languages/tests/fixtures/scanner.fs @@ -0,0 +1,14 @@ +module Scanner + +(* outer comment (* nested comment *) ends here *) +let message = """first +second""" + +let compute values = + values + |> List.map (fun value -> + match value with + | Some x -> + let doubled = x * 2 + doubled + 1 + | None -> 0) diff --git a/languages/tests/integration.py b/languages/tests/integration.py new file mode 100644 index 000000000..092597ed3 --- /dev/null +++ b/languages/tests/integration.py @@ -0,0 +1,196 @@ +"""Build test-only pinned CLIs, compare native/static output, exercise offline repair. + +The tracked catalog is restored even on failure; no runtime trust override exists. +""" +import concurrent.futures +import json +import os +from pathlib import Path +import shutil +import statistics +import subprocess +import sys +import tempfile +import time + +root = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(root / 'languages')) +from definitions import definitions +items = definitions() +entry_path = Path(sys.argv[1]).resolve() +entry = json.loads(entry_path.read_text()) +archive = entry_path.parent / entry['url'].rsplit('/', 1)[1] +release = '--release' in sys.argv +profile = 'release' if release else 'debug' +catalog = root / 'languages/catalog.json' +saved = catalog.read_bytes() +output = root / 'target/languages' / entry['target'] / 'verification' +output.mkdir(exist_ok=True) +host = subprocess.check_output(['rustc', '-vV'], text=True).split('host: ')[1].splitlines()[0] + + +def build(features, name): + args = ['cargo', 'build', '--locked', '--bin', 'diffr'] + build_directory = root / 'target' + if entry['target'] != host: + args += ['--target', entry['target']] + build_directory /= entry['target'] + if release: + args += ['--release'] + if features: + args += ['--features', features] + subprocess.run(args, cwd=root, check=True) + destination = output / name + destination.unlink(missing_ok=True) + shutil.copyfile(build_directory / profile / 'diffr', destination) + destination.chmod(0o755) + identity = os.environ.get('DIFFR_SIGN_IDENTITY') + if identity: + subprocess.run(['codesign', '--force', '--timestamp', '--options', 'runtime', '--sign', identity, '--entitlements', str(root / 'languages/tests/entitlements.plist'), str(destination)], check=True) + return destination + + +try: + catalog.write_text(json.dumps({'schema': 1, 'packages': [entry]}) + '\n') + native = build(None, 'diffr-native') + static = build('all-languages', 'diffr-static') +finally: + catalog.write_bytes(saved) + +with tempfile.TemporaryDirectory() as temporary: + store = Path(temporary) / 'store' + env = dict(os.environ, DIFFR_PARSER_DIR=str(store)) + + def run(binary, *args, code=0): + process = subprocess.run([str(binary), *map(str, args)], env=env, cwd=root, capture_output=True, text=True) + assert process.returncode == code, (args, process.returncode, process.stdout, process.stderr) + return [json.loads(line) for line in process.stdout.splitlines()] + + def compare_args(fixture, extension): + return ['--format', 'ndjson', '--syntax', '--no-index', f'sample_files/{fixture}_1.{extension}', f'sample_files/{fixture}_2.{extension}'] + + fixtures = [(d['fixture']['prefix'], d['fixture']['extension']) for d in items] + first_id = items[0]['id'] + args = compare_args(*fixtures[0]) + missing = run(native, *args) + assert next(e for e in missing if e['type'] == 'file')['diff']['stats']['fallback']['code'] == 'parser_not_installed' + assert not store.exists(), 'ordinary diffs must not create/download packs' + damaged = Path(temporary) / 'damaged.tgz' + damaged.write_bytes(archive.read_bytes()[:-32]) + rejected = run(native, 'languages', 'install', 'extra', '--json', '--archive', damaged, code=2) + assert rejected[0]['error']['code'] == 'language_install_failed' + assert not (store / entry['target'] / 'extra' / entry['version']).exists() + previous = entry_path.parent / 'previous' + previous_tested = (previous / 'diffr').exists() + if previous_tested: + previous_entry = json.loads((previous / 'catalog-entry.json').read_text()) + previous_archive = previous / previous_entry['url'].rsplit('/', 1)[1] + run(previous / 'diffr', 'languages', 'install', 'extra', '--json', '--archive', previous_archive) + older = store / entry['target'] / 'extra' / '0.0.0' + older.mkdir(parents=True) + (older / 'sentinel').write_text('older CLI') + installed = lambda: run(native, 'languages', 'install', 'extra', '--json', '--archive', archive) + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(lambda _: installed(), range(4))) + status = run(native, 'languages', 'list', '--json')[0] + assert status['target'] == entry['target'], 'select packages using executable architecture' + assert {l['id'] for l in status['languages']} == {d['id'] for d in items} + assert all(l['availability'] == 'installed' for l in status['languages']) + for fixture, extension in fixtures: + args = compare_args(fixture, extension) + expected = run(static, *args) + actual = run(native, *args) + # Run summaries include elapsed time; file and syntax events are deterministic. + relevant = lambda events: [e for e in events if e['type'] in ('file', 'syntax')] + assert relevant(actual) == relevant(expected), fixture + assert not next(e for e in actual if e['type'] == 'file')['diff']['stats'].get('fallback'), fixture + for flag in ['--dump-ts', '--dump-syntax']: + path = f'sample_files/{fixture}_1.{extension}' + a = subprocess.check_output([native, 'debug', flag, path], env=env, cwd=root) + b = subprocess.check_output([static, 'debug', flag, path], env=env, cwd=root) + assert a == b, (fixture, flag) + for extension in ['f90', 'fs']: + original = root / f'languages/tests/fixtures/scanner.{extension}' + changed = Path(temporary) / f'changed.{extension}' + changed.write_text(original.read_text().replace('2', '4')) + args = ['--format', 'ndjson', '--syntax', '--no-index', str(original), str(changed)] + assert relevant(run(native, *args)) == relevant(run(static, *args)) + tree = subprocess.check_output([native, 'debug', '--dump-ts', original], env=env, cwd=root) + assert b'ERROR' not in tree and b'MISSING' not in tree, extension + # Exercise synchronized first loads and repeated grammars in a single process. + repository = Path(temporary) / 'repository' + repository.mkdir() + def git(*args): + subprocess.run(['git', '-C', str(repository), *args], check=True, capture_output=True) + git('init') + for fixture, extension in fixtures: + for index in range(12): + (repository / f'{fixture}-{index}.{extension}').write_bytes((root / f'sample_files/{fixture}_1.{extension}').read_bytes()) + git('add', '.') + git('-c', 'user.name=Pack test', '-c', 'user.email=pack@example.invalid', 'commit', '-m', 'fixtures') + for fixture, extension in fixtures: + for index in range(12): + (repository / f'{fixture}-{index}.{extension}').write_bytes((root / f'sample_files/{fixture}_2.{extension}').read_bytes()) + before = str(root / f'sample_files/{fixture}_1.{extension}') + for left, right in [(before, before), ('/dev/null', before), (before, '/dev/null')]: + args = ['--format', 'ndjson', '--syntax', '--no-index', left, right] + assert relevant(run(native, *args)) == relevant(run(static, *args)) + args = ['--repo', repository, '--format', 'ndjson', '--syntax', '--jobs', '8', 'HEAD'] + ordered = lambda events: sorted((json.dumps(e, sort_keys=True) for e in relevant(events))) + start = time.perf_counter() + batch = run(native, *args) + batch_ms = (time.perf_counter() - start) * 1000 + assert ordered(batch) == ordered(run(static, *args)) + revision = store / entry['target'] / 'extra' / entry['version'] + library = revision / next(l['file'] for l in entry['libraries'] if l['id'] == first_id) + library.write_bytes(b'broken') + broken = run(native, *compare_args(*fixtures[0])) + assert next(e for e in broken if e['type'] == 'file')['diff']['stats']['fallback']['code'] == 'parser_load_failed' + assert all(l['availability'] == 'built_in' for l in run(static, 'languages', 'list', '--json')[0]['languages']) + installed() + assert library.stat().st_size == entry['files'][library.name]['size'] + query = revision / (first_id + '.highlights.scm') + query.write_text('(invalid query') + broken_query = run(native, *compare_args(*fixtures[0])) + assert next(e for e in broken_query if e['type'] == 'file')['diff']['stats']['fallback']['code'] == 'parser_load_failed' + installed() + assert relevant(run(native, *compare_args(*fixtures[0]))) == relevant(run(static, *compare_args(*fixtures[0]))) + installed() # idempotent, offline + timings = [] + for _ in range(10): + start = time.perf_counter() + run(native, *compare_args(*fixtures[0])) + timings.append((time.perf_counter() - start) * 1000) + print(json.dumps({'archive_bytes': archive.stat().st_size, 'native_executable_bytes': native.stat().st_size, 'static_executable_bytes': static.stat().st_size, 'first_diff_ms': timings[0], 'median_process_diff_ms': statistics.median(timings), 'mixed_file_count': len(fixtures) * 12, 'mixed_batch_diff_ms': batch_ms}, indent=2)) + assert (older / 'sentinel').read_text() == 'older CLI' + if previous_tested: + previous_status = run(previous / 'diffr', 'languages', 'list', '--json')[0] + assert previous_status['revision'] == previous_entry['version'] != entry['version'] + assert all(l['availability'] == 'installed' for l in previous_status['languages']) + run(previous / 'diffr', *compare_args('fortran', 'f90')) + # Exercise the platform's default data directory, including macOS's spaced path. + home = Path(temporary) / 'home' + home.mkdir() + env.pop('DIFFR_PARSER_DIR') + env['HOME'] = str(home) + env['XDG_DATA_HOME'] = str(home / 'data') + env['XDG_CONFIG_HOME'] = str(home / 'config') + installed() + default_root = home / ('Library/Application Support' if sys.platform == 'darwin' else 'data') / 'diffr/parsers' + assert (default_root / entry['target'] / 'extra' / entry['version'] / 'manifest.json').exists() + run(native, *compare_args(*fixtures[0])) + if sys.platform == 'darwin': + # Match a browser-downloaded archive's quarantine attribute without weakening library validation. + quarantined = Path(temporary) / 'quarantined.tgz' + shutil.copyfile(archive, quarantined) + subprocess.run(['/usr/bin/xattr', '-w', 'com.apple.quarantine', '0081;00000000;diffr-test;', str(quarantined)], check=True) + env['DIFFR_PARSER_DIR'] = str(Path(temporary) / 'quarantine-store') + run(native, 'languages', 'install', 'extra', '--json', '--archive', quarantined) + run(native, *compare_args(*fixtures[0])) + print(json.dumps({'previous_cli_coexistence': previous_tested, 'default_store': str(default_root), 'quarantined_archive': sys.platform == 'darwin'})) + env['PATH'] = '' + env['DIFFR_PARSER_DIR'] = str(Path(temporary) / 'fresh-store') + run(native, 'languages', 'list', '--json') + installed() + run(native, *compare_args(*fixtures[0])) + print('Native/static NDJSON, trees, syntax, concurrent install, corruption, repair and offline reuse passed.') diff --git a/languages/tests/probe.c b/languages/tests/probe.c new file mode 100644 index 000000000..f93634db4 --- /dev/null +++ b/languages/tests/probe.c @@ -0,0 +1,19 @@ +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 4) return 2; + void *library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!library) { fprintf(stderr, "%s\n", dlerror()); return 2; } + const TSLanguage *(*language)(void) = dlsym(library, argv[2]); + TSParser *parser = ts_parser_new(); + if (!language || !ts_parser_set_language(parser, language())) return 2; + TSTree *tree = ts_parser_parse_string(parser, NULL, argv[3], strlen(argv[3])); + if (!tree || ts_node_has_error(ts_tree_root_node(tree))) return 1; + ts_tree_delete(tree); + ts_parser_delete(parser); + dlclose(library); + return 0; +} diff --git a/languages/tests/probe.py b/languages/tests/probe.py new file mode 100644 index 000000000..fde9e4951 --- /dev/null +++ b/languages/tests/probe.py @@ -0,0 +1,29 @@ +"""Compile a small native host and parse with every pack library. + +Set DIFFR_SIGN_IDENTITY on macOS to test hardened runtime library validation. +""" +import json +import os +from pathlib import Path +import subprocess +import sys + +root = Path(__file__).resolve().parents[2] +package = Path(sys.argv[1]).resolve() +metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--format-version', '1'], cwd=root)) +runtime = Path(next(p for p in metadata['packages'] if p['name'] == 'tree-sitter')['manifest_path']).parent +probe = package.parent / 'probe' +manifest = json.loads((package / 'manifest.json').read_text()) +architecture = ['-arch', 'arm64' if manifest['target'].startswith('aarch64-') else 'x86_64'] if sys.platform == 'darwin' else [] +subprocess.run(['cc', *architecture, '-O2', '-D_DEFAULT_SOURCE', '-I' + str(runtime / 'include'), '-I' + str(runtime / 'src'), str(runtime / 'src/lib.c'), str(root / 'languages/tests/probe.c'), '-o', str(probe), *(['-ldl'] if sys.platform != 'darwin' else [])], check=True) +identity = os.environ.get('DIFFR_SIGN_IDENTITY') +if identity: + for path in [probe, *[package / entry['file'] for entry in manifest['libraries']]]: + subprocess.run(['codesign', '--force', '--timestamp', '--options', 'runtime', '--sign', identity, str(path)], check=True) + subprocess.run(['codesign', '--verify', '--strict', str(path)], check=True) +sys.path.insert(0, str(root / 'languages')) +from definitions import definitions +fixtures = {definition['id']: definition['smoke'] for definition in definitions()} +for entry in manifest['libraries']: + subprocess.run([str(probe), str(package / entry['file']), entry['symbol'], fixtures[entry['id']]], check=True) + print(entry['id'] + ': loaded and parsed') diff --git a/languages/tests/test_definitions.py b/languages/tests/test_definitions.py new file mode 100644 index 000000000..e850a6ad6 --- /dev/null +++ b/languages/tests/test_definitions.py @@ -0,0 +1,52 @@ +import copy +import json +from pathlib import Path +import subprocess +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from definitions import ROOT, definitions, validate + + +class DefinitionChecks(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--no-deps', '--format-version', '1'], cwd=ROOT)) + cls.items = definitions() + + def test_current_definitions_and_shared_ocaml_feature(self): + validate(self.items, self.metadata) + ocaml = [d for d in self.items if d['feature'] == 'lang-ocaml'] + self.assertEqual({d['id'] for d in ocaml}, {'ocaml', 'ocaml-interface'}) + + def test_missing_definitions_and_feature_entries_are_rejected(self): + with self.assertRaisesRegex(AssertionError, 'dependencies'): + validate(self.items[1:], self.metadata) + metadata = copy.deepcopy(self.metadata) + package = next(p for p in metadata['packages'] if p['name'] == 'diffr-cli') + package['features']['all-languages'].remove(self.items[0]['feature']) + with self.assertRaisesRegex(AssertionError, 'all-languages'): + validate(self.items, metadata) + + def test_missing_fixtures_are_rejected(self): + items = copy.deepcopy(self.items) + items[0]['fixture']['prefix'] = 'missing-extra-language-fixture' + with self.assertRaisesRegex(AssertionError, 'missing fixture'): + validate(items, self.metadata) + + def test_duplicate_builtin_config_is_rejected(self): + items = copy.deepcopy(self.items) + items[0]['variant'] = 'Ada' + with self.assertRaisesRegex(AssertionError, 'duplicate built-in'): + validate(items, self.metadata) + + def test_query_path_traversal_is_rejected(self): + items = copy.deepcopy(self.items) + items[0]['highlights'][0]['path'] = '../highlights.scm' + with self.assertRaisesRegex(AssertionError, 'invalid query path'): + validate(items, self.metadata) + + +if __name__ == '__main__': + unittest.main() diff --git a/languages/tests/test_licenses.py b/languages/tests/test_licenses.py new file mode 100644 index 000000000..d0b5b0437 --- /dev/null +++ b/languages/tests/test_licenses.py @@ -0,0 +1,49 @@ +import hashlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from licenses import collect + + +class LicenseChecks(unittest.TestCase): + def test_locked_download_cache_and_integrity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + revision = 'a' * 40 + (root / '.cargo_vcs_info.json').write_text(json.dumps({'git': {'sha1': revision}})) + crate = dict(name='test', manifest_path=str(root / 'Cargo.toml'), repository='https://github.com/example/grammar') + data = b'Upstream license\n' + spec = dict(path='LICENSE', sha256=hashlib.sha256(data).hexdigest()) + cache = root / 'cache' + with patch('licenses.urlopen', return_value=io.BytesIO(data)) as fetch: + self.assertEqual(collect(crate, spec, cache), data) + fetch.assert_called_once_with(f'https://raw.githubusercontent.com/example/grammar/{revision}/LICENSE', timeout=30) + with patch('licenses.urlopen', side_effect=AssertionError('unexpected network')): + self.assertEqual(collect(crate, spec, cache), data) + (cache / spec['sha256']).write_bytes(b'corrupt') + with self.assertRaisesRegex(AssertionError, 'hash mismatch'): + collect(crate, spec, cache) + (root / 'LICENSE').write_bytes(data) + self.assertEqual(collect(crate, spec, cache), data) + with self.assertRaisesRegex(AssertionError, 'invalid license path'): + collect(crate, dict(spec, path='../LICENSE'), cache) + + def test_failed_download_does_not_cache(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / '.cargo_vcs_info.json').write_text(json.dumps({'git': {'sha1': 'b' * 40}})) + crate = dict(name='test', manifest_path=str(root / 'Cargo.toml'), repository='https://github.com/example/grammar') + with patch('licenses.urlopen', return_value=io.BytesIO(b'wrong license')): + with self.assertRaisesRegex(AssertionError, 'hash mismatch'): + collect(crate, dict(path='LICENSE', sha256='0' * 64), root / 'cache') + self.assertFalse((root / 'cache').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/languages/tests/test_publish.py b/languages/tests/test_publish.py new file mode 100644 index 000000000..969fde8cd --- /dev/null +++ b/languages/tests/test_publish.py @@ -0,0 +1,52 @@ +import hashlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import publish +from build import TARGETS + + +class CatalogPublication(unittest.TestCase): + def test_all_registry_bytes_must_match_before_catalog_exists(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + payloads = {} + for target in TARGETS: + folder = directory / target + folder.mkdir() + payload = target.encode() + filename = target + '.tgz' + url = 'https://registry.npmjs.org/' + filename + payloads[url] = payload + (folder / filename).write_bytes(payload) + (folder / 'catalog-entry.json').write_text(json.dumps({ + 'target': target, 'version': '0.1.0', 'url': url, + 'size': len(payload), 'sha256': hashlib.sha256(payload).hexdigest(), + })) + with patch.object(sys, 'argv', ['publish.py', temporary]), patch.object( + publish.urllib.request, 'urlopen', side_effect=lambda url, **_: io.BytesIO(payloads[url]) + ): + publish.main() + self.assertEqual(len(json.loads((directory / 'catalog.json').read_text())['packages']), 4) + payloads[next(iter(payloads))] = b'wrong registry bytes' + with patch.object(sys, 'argv', ['publish.py', temporary]), patch.object( + publish.urllib.request, 'urlopen', side_effect=lambda url, **_: io.BytesIO(payloads[url]) + ), self.assertRaises(AssertionError): + publish.main() + self.assertFalse((directory / 'catalog.json').exists()) + + def test_partial_target_set_cannot_produce_a_catalog(self): + with tempfile.TemporaryDirectory() as temporary: + with patch.object(sys, 'argv', ['publish.py', temporary]), self.assertRaises(AssertionError): + publish.main() + self.assertFalse((Path(temporary) / 'catalog.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/sample_files/ocaml_interface_1.mli b/sample_files/ocaml_interface_1.mli new file mode 100644 index 000000000..3e647cea8 --- /dev/null +++ b/sample_files/ocaml_interface_1.mli @@ -0,0 +1,3 @@ +(** Public interface *) +type result = { value : int } +val compute : int -> result diff --git a/sample_files/ocaml_interface_2.mli b/sample_files/ocaml_interface_2.mli new file mode 100644 index 000000000..b0da7420d --- /dev/null +++ b/sample_files/ocaml_interface_2.mli @@ -0,0 +1,3 @@ +(** Public interface *) +type result = { value : int; label : string } +val compute : int -> string -> result diff --git a/src/cli.rs b/src/cli.rs index cf1e08637..70e8d6d90 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -87,7 +87,7 @@ pub(crate) fn run() -> Result { .arg(Arg::new("value").required(true)), ), ) - .after_help("Examples:\n diffr\n diffr --cached\n diffr main...HEAD -- src/\n diffr --no-index -- before.rs after.rs\n diffr main HEAD --format ndjson\n\nUnsupported Git flags are rejected; this is not a complete git diff implementation.") + .after_help("Examples:\n diffr\n diffr --cached\n diffr main...HEAD -- src/\n diffr --no-index -- before.rs after.rs\n diffr main HEAD --format ndjson\n diffr languages list\n diffr languages install extra\n\nUnsupported Git flags are rejected; this is not a complete git diff implementation.") .get_matches_from(argv); if let Some(("config", sub)) = args.subcommand() { return run_config(&args, sub); diff --git a/src/config.rs b/src/config.rs index c1ad8bf4d..7fe1f345d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -137,7 +137,10 @@ impl std::fmt::Display for ConfigError { impl std::error::Error for ConfigError {} pub(crate) struct Params { - languages: DftHashMap>>, + languages: + DftHashMap, crate::languages::ParserError>>>, + deferred: DftHashMap>, + order: Vec, pub(crate) diff: DiffConfig, } @@ -275,24 +278,32 @@ impl Config { let mut languages: DftHashMap<_, _> = Language::iter() .map(|language| (language, OnceLock::new())) .collect(); + let mut deferred = DftHashMap::default(); for (name, sources) in queries::assemble(&queries)? { let language = Language::iter() .find(|language| format!("{language:?}").to_lowercase() == name) .ok_or_else(|| ConfigError(format!("unknown language: {name}")))?; - let parser = tree_sitter_parser::from_language(language); + if !crate::languages::builtin(language) { + deferred.insert(language, sources); + continue; + } + let parser = tree_sitter_parser::from_language(language) + .map_err(|e| ConfigError(e.to_string()))?; let query = AnnotationQuery::compile(&parser.language, &sources)?; check_tags(&query, &self.plugins.order)?; languages.insert( language, - OnceLock::from(Arc::new(LanguageParams { + OnceLock::from(Ok(Arc::new(LanguageParams { parser, query, sub_languages: OnceLock::new(), - })), + }))), ); } Ok(Params { languages, + deferred, + order: self.plugins.order, diff: self.diff, }) } @@ -322,26 +333,50 @@ fn check_tags(query: &AnnotationQuery, order: &[String]) -> Result<(), ConfigErr } impl Params { - pub(crate) fn language(&self, language: Language) -> &Arc { - let config = self.languages[&language].get_or_init(|| { - // Languages without annotation rules still support structural diffing. - // Keep their grammars lazy, as in the existing parser registry. - let parser = tree_sitter_parser::from_language(language); - Arc::new(LanguageParams { - parser, - query: AnnotationQuery::compile(&parser.language, &[]).expect("an empty query"), - sub_languages: OnceLock::new(), + pub(crate) fn language( + &self, + language: Language, + ) -> Result<&Arc, crate::languages::ParserError> { + let config = self.languages[&language] + .get_or_init(|| { + let parser = tree_sitter_parser::from_language(language)?; + let sources = self + .deferred + .get(&language) + .map(Vec::as_slice) + .unwrap_or(&[]); + let query = AnnotationQuery::compile(&parser.language, sources) + .and_then(|query| { + check_tags(&query, &self.order)?; + Ok(query) + }) + .map_err(|e| { + crate::languages::ParserError::new( + crate::summary::FallbackCause::ParserLoadFailed, + e, + ) + })?; + Ok(Arc::new(LanguageParams { + parser, + query, + sub_languages: OnceLock::new(), + })) }) - }); + .as_ref() + .map_err(Clone::clone)?; config.sub_languages.get_or_init(|| { config .parser .sub_languages .iter() - .map(|sub| (sub, Arc::clone(self.language(sub.parse_as)))) + .filter_map(|sub| { + self.language(sub.parse_as) + .ok() + .map(|params| (sub, Arc::clone(params))) + }) .collect() }); - config + Ok(config) } } diff --git a/src/diff/sliders.rs b/src/diff/sliders.rs index 03b56c99d..6c8df8aaf 100644 --- a/src/diff/sliders.rs +++ b/src/diff/sliders.rs @@ -800,7 +800,9 @@ mod tests { fn test_slider_two_steps() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs = parse(&arena, "A B", config, false).unwrap(); let rhs = parse(&arena, "A B X\n A B", config, false).unwrap(); @@ -826,7 +828,9 @@ mod tests { fn test_slider_partially_unchanged() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs = parse(&arena, "(A B) X \n (A B)", config, false).unwrap(); let rhs = parse(&arena, "((novel) A B)", config, false).unwrap(); diff --git a/src/diff/unchanged.rs b/src/diff/unchanged.rs index 33b85fb52..f09a8260a 100644 --- a/src/diff/unchanged.rs +++ b/src/diff/unchanged.rs @@ -497,7 +497,9 @@ mod tests { fn test_shrink_unchanged_at_start() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "unchanged A B", config, false).unwrap(); let rhs_nodes = parse(&arena, "unchanged X", config, false).unwrap(); @@ -524,7 +526,9 @@ mod tests { fn test_shrink_unchanged_at_end() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "A B unchanged", config, false).unwrap(); let rhs_nodes = parse(&arena, "X unchanged", config, false).unwrap(); @@ -551,7 +555,9 @@ mod tests { fn test_shrink_unchanged_nested() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -589,7 +595,9 @@ mod tests { fn test_split_unchanged_toplevel_at_start() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); // Make sure that the initial unchanged node exceeds TINY_TREE_THRESHOLD. let lhs_nodes = parse( @@ -631,7 +639,9 @@ mod tests { fn test_split_unchanged_toplevel_at_end() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -672,7 +682,9 @@ mod tests { fn test_split_preserves_outer_delimiters() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "(A)", config, false).unwrap(); let rhs_nodes = parse(&arena, "(B)", config, false).unwrap(); @@ -700,7 +712,9 @@ mod tests { fn test_split_unchanged_middle() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -742,7 +756,9 @@ mod tests { fn test_split_unchanged_multiple() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -771,7 +787,9 @@ mod tests { fn test_split_unchanged_outer_delimiter() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -803,7 +821,9 @@ mod tests { fn test_split_mostly_unchanged_toplevel() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -829,7 +849,9 @@ mod tests { fn test_count_common_unique() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); // There are two subtrees that are unique on both sides and // shared between the two sides here: @@ -859,7 +881,9 @@ mod tests { fn test_similar_with_common_grandchildren() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "((novel-lhs 1 2 3 4 5)) x", config, false).unwrap(); let rhs_nodes = parse(&arena, "((novel-rhs 1 2 3 4 5)) y", config, false).unwrap(); @@ -874,7 +898,9 @@ mod tests { fn test_similar_ignore_delimiter() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "(novel-lhs 1 2 3 4 5) x", config, false).unwrap(); let rhs_nodes = parse(&arena, "[novel-rhs 1 2 3 4 5] y", config, false).unwrap(); diff --git a/src/engine.rs b/src/engine.rs index ef4d8eb02..a6a68f228 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -108,13 +108,23 @@ pub(crate) fn diff_file_content( }; let language = guess(Path::new(display_path), guess_src, overrides); - let lang_config = language.map(|lang| (lang, params.language(lang))); + let resolved = language + .map(|lang| params.language(lang).map(|config| (lang, config))) + .transpose(); + let fallback = resolved + .as_ref() + .err() + .map(|error| FileFormat::TextFallback { + cause: error.cause, + reason: error.message.clone(), + }); + let lang_config = resolved.ok().flatten(); if lhs_src == rhs_src { - let file_format = match language { + let file_format = fallback.clone().unwrap_or(match language { Some(language) => FileFormat::SupportedLanguage(language), None => FileFormat::PlainText, - }; + }); // If the two files are byte-for-byte identical, return early // rather than doing any more work. @@ -141,7 +151,7 @@ pub(crate) fn diff_file_content( (file_format, lhs_positions, rhs_positions) } None => { - let file_format = FileFormat::PlainText; + let file_format = fallback.unwrap_or(FileFormat::PlainText); let (lhs_positions, rhs_positions) = line_parser::change_positions(lhs_src, rhs_src); (file_format, lhs_positions, rhs_positions) } diff --git a/src/languages.rs b/src/languages.rs new file mode 100644 index 000000000..7c2d378aa --- /dev/null +++ b/src/languages.rs @@ -0,0 +1,589 @@ +//! Only the embedded catalog grants trust; installation never discovers versions. +use anyhow::{ensure, Context, Result}; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; +use strum::IntoEnumIterator; + +use crate::parse::guess_language::Language; +use crate::summary::FallbackCause; + +pub(crate) const TARGET: &str = env!("DIFFR_TARGET"); +const MAX_ARCHIVE: u64 = 64 * 1024 * 1024; +const MAX_FILE: u64 = 128 * 1024 * 1024; +const MAX_UNPACKED: u64 = 256 * 1024 * 1024; + +#[derive(Deserialize)] +struct Catalog { + schema: u32, + packages: Vec, +} +#[derive(Deserialize, Serialize, Clone)] +pub(crate) struct Library { + pub id: String, + pub file: String, + pub symbol: String, + pub abi: usize, +} +#[derive(Deserialize, Serialize, Clone)] +struct Fingerprint { + size: u64, + sha256: String, +} +#[derive(Deserialize, Serialize, Clone)] +pub(crate) struct Package { + schema: u32, + pack: String, + pub version: String, + target: String, + name: String, + url: String, + size: u64, + sha256: String, + pub libraries: Vec, + files: BTreeMap, +} +static CATALOG: LazyLock = LazyLock::new(|| { + let catalog: Catalog = serde_json::from_str(include_str!("../languages/catalog.json")) + .expect("invalid embedded language catalog"); + assert_eq!(catalog.schema, 1); + catalog +}); + +pub(crate) use crate::parse::optional::{builtin, id as optional_id}; +pub(crate) fn package() -> Option<&'static Package> { + CATALOG.packages.iter().find(|p| p.target == TARGET) +} +fn root() -> Result { + let path = std::env::var_os("DIFFR_PARSER_DIR") + .map(PathBuf::from) + .or_else(|| dirs::data_local_dir().map(|p| p.join("diffr/parsers"))) + .context("cannot determine parser store")?; + Ok(std::path::absolute(path)?) +} +pub(crate) fn directory(package: &Package) -> Result { + Ok(root()? + .join(&package.target) + .join("extra") + .join(&package.version)) +} +fn hash(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn read_verified(path: &Path, fingerprint: &Fingerprint) -> Result> { + let metadata = fs::symlink_metadata(path)?; + ensure!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "not a regular file: {}", + path.display() + ); + ensure!( + fingerprint.size <= MAX_FILE && metadata.len() == fingerprint.size, + "wrong size: {}", + path.display() + ); + let mut bytes = Vec::new(); + File::open(path)? + .take(fingerprint.size + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() as u64 == fingerprint.size && hash(&bytes) == fingerprint.sha256, + "integrity failure: {}", + path.display() + ); + Ok(bytes) +} +pub(crate) fn verify_library( + package: &Package, + library: &Library, + directory: &Path, +) -> Result { + let expected = fs::canonicalize(root()?)? + .join(&package.target) + .join("extra") + .join(&package.version); + ensure!( + fs::canonicalize(directory)? == expected, + "linked pack directory outside parser store" + ); + let path = directory.join(&library.file); + read_verified( + &path, + package + .files + .get(&library.file) + .context("uncataloged library")?, + )?; + let canonical = fs::canonicalize(&path)?; + ensure!( + canonical.parent() == Some(fs::canonicalize(directory)?.as_path()), + "library outside parser store" + ); + Ok(canonical) +} + +pub(crate) fn highlights(package: &Package, id: &str, directory: &Path) -> Result { + let name = format!("{id}.highlights.scm"); + let bytes = read_verified( + &directory.join(&name), + package + .files + .get(&name) + .context("missing catalog highlights")?, + )?; + Ok(String::from_utf8(bytes)?) +} +fn verify_install(package: &Package, directory: &Path) -> Result<()> { + ensure!( + !fs::symlink_metadata(directory)?.file_type().is_symlink(), + "linked pack directory" + ); + for (file, fingerprint) in &package.files { + read_verified(&directory.join(file), fingerprint)?; + } + Ok(()) +} +pub(crate) fn lock(package: &Package) -> Result { + let directory = directory(package)?; + let parent = directory.parent().unwrap(); + fs::create_dir_all(parent)?; + let expected = fs::canonicalize(root()?)? + .join(&package.target) + .join("extra"); + ensure!( + fs::canonicalize(parent)? == expected, + "linked directory outside parser store" + ); + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(parent.join(format!(".{}.lock", package.version)))?; + lock.lock_exclusive()?; + Ok(lock) +} + +#[derive(Debug, Clone)] +pub(crate) struct ParserError { + pub cause: FallbackCause, + pub message: String, +} +impl ParserError { + pub(crate) fn new(cause: FallbackCause, detail: impl std::fmt::Display) -> Self { + Self { cause, message: format!("{detail}; run `diffr languages install extra` to install or repair additional language support") } + } +} +impl std::fmt::Display for ParserError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} +impl std::error::Error for ParserError {} + +fn unpack(package: &Package, archive: &Path, destination: &Path) -> Result<()> { + ensure!(package.size <= MAX_ARCHIVE, "archive exceeds limit"); + let bytes = read_verified( + archive, + &Fingerprint { + size: package.size, + sha256: package.sha256.clone(), + }, + )?; + ensure!(package.size <= MAX_ARCHIVE, "archive exceeds limit"); + let decoder = flate2::read::GzDecoder::new(bytes.as_slice()); + let mut tar = tar::Archive::new(decoder.take(MAX_UNPACKED + 1)); + let mut seen = BTreeSet::new(); + for entry in tar.entries()? { + let mut entry = entry?; + ensure!( + entry.header().entry_type().is_file(), + "archive contains non-file entry" + ); + let path = entry.path_bytes(); + let path = std::str::from_utf8(&path)?; + let name = path + .strip_prefix("package/") + .context("invalid archive prefix")? + .to_owned(); + ensure!( + !name.contains('/') && !name.contains('\\') && name != "." && name != "..", + "invalid archive path" + ); + let expected = package + .files + .get(&name) + .context("unexpected archive file")?; + ensure!(seen.insert(name.clone()), "duplicate archive file"); + ensure!( + entry.size() == expected.size && expected.size <= MAX_FILE, + "archive file exceeds limit" + ); + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + ensure!( + bytes.len() as u64 == expected.size && hash(&bytes) == expected.sha256, + "archive file integrity failure" + ); + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(destination.join(name))?; + file.write_all(&bytes)?; + file.sync_all()?; + } + ensure!(seen.len() == package.files.len(), "incomplete archive"); + let manifest: serde_json::Value = + serde_json::from_slice(&fs::read(destination.join("manifest.json"))?)?; + ensure!( + manifest["schema"] == 1 + && manifest["pack"] == "extra" + && manifest["target"] == package.target + && manifest["version"] == package.version + && manifest["libraries"] == serde_json::to_value(&package.libraries)?, + "incompatible manifest" + ); + Ok(()) +} +fn install(package: &Package, archive: Option<&Path>) -> Result<()> { + ensure!( + package.schema == 1 && package.pack == "extra" && package.target == TARGET, + "incompatible package" + ); + ensure!(package.size <= MAX_ARCHIVE, "archive exceeds limit"); + let _lock = lock(package)?; + let destination = directory(package)?; + if verify_install(package, &destination).is_ok() { + return Ok(()); + } + let parent = destination.parent().unwrap(); + let staging = tempfile::tempdir_in(parent)?; + let download = staging.path().join("archive.tgz"); + let archive = match archive { + Some(path) => path, + None => { + ensure!( + package + .url + .starts_with("https://registry.npmjs.org/@dev.fast/diffr-languages-extra-"), + "invalid catalog URL" + ); + eprintln!( + "Downloading extra {} ({} bytes)", + package.version, package.size + ); + tokio::runtime::Runtime::new()?.block_on(download_archive( + &package.url, + package.size, + &download, + ))?; + &download + } + }; + let prepared = staging.path().join("prepared"); + fs::create_dir(&prepared)?; + unpack(package, archive, &prepared)?; + File::open(&prepared)?.sync_all()?; + publish_directory(&prepared, &destination)?; + File::open(parent)?.sync_all()?; + Ok(()) +} + +async fn download_archive(url: &str, expected_size: u64, destination: &Path) -> Result<()> { + ensure!(expected_size <= MAX_ARCHIVE, "archive exceeds limit"); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(180)) + .build()?; + let mut response = client.get(url).send().await?.error_for_status()?; + let mut file = File::create(destination)?; + let mut size = 0; + while let Some(chunk) = response.chunk().await? { + size += chunk.len() as u64; + ensure!(size <= expected_size, "download exceeds pinned size"); + file.write_all(&chunk)?; + } + ensure!(size == expected_size, "incomplete download"); + file.sync_all()?; + Ok(()) +} + +fn publish_directory(prepared: &Path, destination: &Path) -> Result<()> { + match fs::symlink_metadata(destination) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::rename(prepared, destination)?; + return Ok(()); + } + Err(error) => return Err(error.into()), + Ok(_) => {} + } + // Exchange complete directories so interruption cannot expose a partial repair. + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))] + { + use std::os::unix::ffi::OsStrExt; + let from = std::ffi::CString::new(prepared.as_os_str().as_bytes())?; + let to = std::ffi::CString::new(destination.as_os_str().as_bytes())?; + #[cfg(target_os = "macos")] + let result = unsafe { libc::renamex_np(from.as_ptr(), to.as_ptr(), libc::RENAME_SWAP) }; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + let result = unsafe { + libc::renameat2( + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_EXCHANGE, + ) + }; + if result != 0 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) + } + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))] + anyhow::bail!("atomic pack repair is unsupported on this target") +} + +pub(crate) fn run() -> Result { + let command = clap::Command::new("diffr languages") + .subcommand_required(true) + .subcommand( + clap::Command::new("list").arg( + clap::Arg::new("json") + .long("json") + .action(clap::ArgAction::SetTrue), + ), + ) + .subcommand( + clap::Command::new("install") + .arg( + clap::Arg::new("pack") + .required(true) + .value_parser(["extra"]), + ) + .arg( + clap::Arg::new("json") + .long("json") + .action(clap::ArgAction::SetTrue), + ) + .arg( + clap::Arg::new("archive") + .long("archive") + .value_parser(clap::value_parser!(PathBuf)), + ), + ); + let args = command.try_get_matches_from( + std::iter::once(std::ffi::OsString::from("diffr languages")) + .chain(std::env::args_os().skip(2)), + ); + let args = match args { + Ok(args) => args, + Err(error) => { + let code = error.exit_code(); + error.print()?; + return Ok(code); + } + }; + let (action, args) = args.subcommand().unwrap(); + let json = args.get_flag("json"); + if action == "list" { + let package = package(); + let state = package + .map(|p| directory(p).and_then(|dir| verify_install(p, &dir))) + .transpose(); + let languages: Vec<_> = Language::iter() + .filter(|language| optional_id(*language).is_some()) + .map(|language| { + let availability = if builtin(language) { + "built_in" + } else if package.is_none() { + "unavailable" + } else if state.as_ref().is_ok_and(|s| s.is_some()) { + "installed" + } else if package + .and_then(|p| directory(p).ok()) + .is_some_and(|p| p.exists()) + { + "corrupt" + } else { + "not_installed" + }; + serde_json::json!({"id": optional_id(language), "availability": availability}) + }) + .collect(); + let result = serde_json::json!({"target": TARGET, "pack": "extra", "revision": package.map(|p| &p.version), "download_size": package.map(|p| p.size), "languages": languages}); + if json { + println!("{result}"); + } else { + match package { + Some(pack) => println!("extra {}: {} ({} bytes)", pack.version, TARGET, pack.size), + None => println!("extra: no published revision for {TARGET}"), + } + for language in languages { + println!( + "{}: {}", + language["id"].as_str().unwrap(), + language["availability"].as_str().unwrap() + ); + } + } + return Ok(0); + } + let result = package() + .context("no published extra pack is pinned for this executable target") + .and_then(|p| install(p, args.get_one::("archive").map(PathBuf::as_path))); + match result { + Ok(()) => { + if json { + println!( + "{}", + serde_json::json!({"ok": true, "pack": "extra", "target": TARGET, "revision": package().unwrap().version}) + ); + } else { + println!("Installed extra {}", package().unwrap().version); + } + Ok(0) + } + Err(error) => { + if json { + println!( + "{}", + serde_json::json!({"ok": false, "error": {"code": "language_install_failed", "message": format!("{error:#}")}}) + ); + } else { + eprintln!("{error:#}"); + } + Ok(2) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::{write::GzEncoder, Compression}; + + fn fixture(entries: &[(&str, &[u8], tar::EntryType)]) -> (tempfile::TempDir, PathBuf, Package) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pack.tgz"); + let encoder = GzEncoder::new(File::create(&path).unwrap(), Compression::default()); + let mut archive = tar::Builder::new(encoder); + let manifest = serde_json::json!({"schema": 1, "pack": "extra", "version": "0.0.0", "target": TARGET, "libraries": []}).to_string(); + let mut files = BTreeMap::new(); + for (name, bytes, kind) in std::iter::once(( + "manifest.json", + manifest.as_bytes(), + tar::EntryType::Regular, + )) + .chain(entries.iter().copied()) + { + let mut header = tar::Header::new_ustar(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_entry_type(kind); + if kind.is_symlink() { + header.set_link_name("/tmp/outside").unwrap(); + } + let path = format!("package/{name}"); + header.as_mut_bytes()[..path.len()].copy_from_slice(path.as_bytes()); + header.set_cksum(); + archive.append(&header, bytes).unwrap(); + files.insert( + name.to_owned(), + Fingerprint { + size: bytes.len() as u64, + sha256: hash(bytes), + }, + ); + } + archive.into_inner().unwrap().finish().unwrap(); + let bytes = fs::read(&path).unwrap(); + let package = Package { + schema: 1, + pack: "extra".into(), + version: "0.0.0".into(), + target: TARGET.into(), + name: "test".into(), + url: "unused".into(), + size: bytes.len() as u64, + sha256: hash(&bytes), + libraries: vec![], + files, + }; + (directory, path, package) + } + + #[test] + fn streamed_download_is_bounded_and_rejects_interruption() { + for (body, content_length, expected_size, succeeds) in [ + ("abcdefgh", 8, 8, true), + ("abcdefgh", 8, 4, false), + ("abc", 8, 8, false), + ] { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/pack.tgz", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0; 4096]; + stream.read(&mut request).unwrap(); + write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n{body}").unwrap(); + }); + let directory = tempfile::tempdir().unwrap(); + let destination = directory.path().join("download"); + let result = tokio::runtime::Runtime::new() + .unwrap() + .block_on(download_archive(&url, expected_size, &destination)); + server.join().unwrap(); + assert_eq!(result.is_ok(), succeeds, "{result:?}"); + assert!(destination.metadata().unwrap().len() <= expected_size); + } + } + + #[test] + fn archive_requires_exact_bytes_files_and_target() { + let (_dir, archive, package) = + fixture(&[("parser.so", b"parser", tar::EntryType::Regular)]); + let destination = tempfile::tempdir().unwrap(); + unpack(&package, &archive, destination.path()).unwrap(); + verify_install(&package, destination.path()).unwrap(); + fs::write(destination.path().join("parser.so"), b"broken").unwrap(); + assert!(verify_install(&package, destination.path()).is_err()); + for mutation in 0..4 { + let mut bad = package.clone(); + match mutation { + 0 => bad.sha256 = "00".repeat(32), + 1 => bad.target = "wrong-target".into(), + 2 => { + bad.files.remove("parser.so"); + } + _ => { + bad.files.get_mut("parser.so").unwrap().size = MAX_FILE + 1; + } + } + assert!(unpack(&bad, &archive, tempfile::tempdir().unwrap().path()).is_err()); + } + let mut bytes = fs::read(&archive).unwrap(); + bytes.truncate(bytes.len() / 2); + fs::write(&archive, bytes).unwrap(); + assert!(unpack(&package, &archive, tempfile::tempdir().unwrap().path()).is_err()); + } + + #[test] + fn archives_reject_links_nested_paths_and_duplicates() { + for entries in [ + vec![("linked", b"".as_slice(), tar::EntryType::Symlink)], + vec![("hardlink", b"".as_slice(), tar::EntryType::Link)], + vec![("../escape", b"x".as_slice(), tar::EntryType::Regular)], + vec![("nested/file", b"x".as_slice(), tar::EntryType::Regular)], + vec![("duplicate", b"x".as_slice(), tar::EntryType::Regular); 2], + ] { + let (_dir, archive, package) = fixture(&entries); + assert!(unpack(&package, &archive, tempfile::tempdir().unwrap().path()).is_err()); + } + } +} diff --git a/src/main.rs b/src/main.rs index 6dd75d718..0450c16c4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,7 @@ mod constants; mod diff; mod engine; mod exit_codes; +mod languages; use engine::diff_file_content; mod files; mod git; @@ -135,6 +136,7 @@ fn main() { reset_sigpipe(); let result = match std::env::args_os().nth(1).as_deref() { + Some(arg) if arg == "languages" => languages::run().map_err(Into::into), Some(arg) if arg == "debug" => { run_debug(); return; @@ -165,7 +167,10 @@ fn run_debug() { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = tsp::from_language(lang); + let ts_lang = tsp::from_language(lang).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(2) + }); let tree = tsp::to_tree(&src, ts_lang); tsp::print_tree(&src, &tree); } @@ -186,7 +191,10 @@ fn run_debug() { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = params.language(lang); + let ts_lang = params.language(lang).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(2) + }); let arena = Arena::new(); let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); init_all_info(&ast, &[]); @@ -209,7 +217,10 @@ fn run_debug() { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = params.language(lang); + let ts_lang = params.language(lang).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(2) + }); let arena = Arena::new(); let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); init_all_info(&ast, &[]); diff --git a/src/parse/guess_language.rs b/src/parse/guess_language.rs index c36ca5fc6..82cbb8b1e 100644 --- a/src/parse/guess_language.rs +++ b/src/parse/guess_language.rs @@ -38,9 +38,7 @@ pub(crate) enum Language { EmacsLisp, Erlang, Fish, - #[cfg(feature = "lang-fsharp")] FSharp, - #[cfg(feature = "lang-fortran")] Fortran, Gleam, Go, @@ -81,7 +79,6 @@ pub(crate) enum Language { Toml, TypeScript, TypeScriptTsx, - #[cfg(feature = "lang-verilog")] Verilog, Vhdl, Xml, @@ -142,9 +139,7 @@ pub(crate) fn language_name(language: Language) -> &'static str { EmacsLisp => "Emacs Lisp", Erlang => "Erlang", Fish => "Fish", - #[cfg(feature = "lang-fsharp")] FSharp => "F#", - #[cfg(feature = "lang-fortran")] Fortran => "Fortran", Gleam => "Gleam", Go => "Go", @@ -185,7 +180,6 @@ pub(crate) fn language_name(language: Language) -> &'static str { Toml => "TOML", TypeScript => "TypeScript", TypeScriptTsx => "TypeScript TSX", - #[cfg(feature = "lang-verilog")] Verilog => "Verilog", Vhdl => "VHDL", Xml => "XML", @@ -295,9 +289,7 @@ pub(crate) fn language_globs(language: Language) -> Vec { "rebar.lock", ], Fish => &["*.fish"], - #[cfg(feature = "lang-fsharp")] FSharp => &["*.fs", "*.fsx", "*.fsi"], - #[cfg(feature = "lang-fortran")] Fortran => &["*.f", "*.for", "*.f90", "*.F", "*.FOR", "*.F90"], Gleam => &["*.gleam"], Go => &["*.go"], @@ -404,7 +396,6 @@ pub(crate) fn language_globs(language: Language) -> Vec { ], TypeScript => &["*.ts", "*.cts", "*.mts"], TypeScriptTsx => &["*.tsx"], - #[cfg(feature = "lang-verilog")] Verilog => &["*.v", "*.sv", "*.vh"], Vhdl => &["*.vhdl", "*.vhd"], Xml => &[ @@ -554,9 +545,7 @@ fn from_emacs_mode_header(src: &str) -> Option { "elm" => Elm, "emacs-lisp" => EmacsLisp, "fish" => Fish, - #[cfg(feature = "lang-fsharp")] "fsharp" => FSharp, - #[cfg(feature = "lang-fortran")] "fortran" => Fortran, "gleam" => Gleam, "go" => Go, @@ -583,7 +572,6 @@ fn from_emacs_mode_header(src: &str) -> Option { "toml" => Toml, "tuareg" => OCaml, "typescript" => TypeScript, - #[cfg(feature = "lang-verilog")] "verilog" => Verilog, "vhdl" => Vhdl, "yaml" => Yaml, diff --git a/src/parse/mod.rs b/src/parse/mod.rs index d698599c9..e4ab624bf 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -5,3 +5,7 @@ pub(crate) mod guess_language; pub(crate) mod query; pub(crate) mod syntax; pub(crate) mod tree_sitter_parser; + +pub(crate) mod native; + +pub(crate) mod optional; diff --git a/src/parse/native.rs b/src/parse/native.rs new file mode 100644 index 000000000..9d2d17c0e --- /dev/null +++ b/src/parse/native.rs @@ -0,0 +1,75 @@ +//! A grammar's code must outlive every Tree-sitter language, parser and tree. +use crate::hash::DftHashMap; +use crate::languages::{self, ParserError}; +use crate::summary::FallbackCause; +use anyhow::{ensure, Context}; +use libloading::Library; +use std::sync::{LazyLock, Mutex}; +use tree_sitter_language::LanguageFn; + +struct Loaded { + language: tree_sitter::Language, + highlights: String, + _library: Library, +} + +pub(crate) fn load(id: &'static str) -> Result<(tree_sitter::Language, String), ParserError> { + static LOADED: LazyLock>> = + LazyLock::new(|| Mutex::new(DftHashMap::default())); + let mut loaded = LOADED.lock().unwrap(); + if let Some(grammar) = loaded.get(id) { + return Ok((grammar.language.clone(), grammar.highlights.clone())); + } + let package = languages::package().ok_or_else(|| { + ParserError::new( + FallbackCause::ParserUnavailable, + format!( + "{id}: no compatible published pack for {}", + languages::TARGET + ), + ) + })?; + let directory = languages::directory(package) + .map_err(|e| ParserError::new(FallbackCause::ParserLoadFailed, e))?; + if !directory.exists() { + return Err(ParserError::new( + FallbackCause::ParserNotInstalled, + format!("{id} parser is not installed"), + )); + } + let result = (|| -> anyhow::Result { + let _lock = languages::lock(package)?; + let entry = package + .libraries + .iter() + .find(|l| l.id == id) + .context("missing catalog language")?; + ensure!( + (tree_sitter::MIN_COMPATIBLE_LANGUAGE_VERSION..=tree_sitter::LANGUAGE_VERSION) + .contains(&entry.abi), + "incompatible catalog ABI" + ); + let path = languages::verify_library(package, entry, &directory)?; + let highlights = languages::highlights(package, id, &directory)?; + // Only exact, catalog-hashed code is loaded. The handle stays in LOADED forever. + let library = unsafe { Library::new(path)? }; + let function = + unsafe { library.get:: *const ()>(entry.symbol.as_bytes())? }; + ensure!(!unsafe { function() }.is_null(), "null grammar pointer"); + let language = tree_sitter::Language::new(unsafe { LanguageFn::from_raw(*function) }); + ensure!( + language.abi_version() == entry.abi, + "grammar ABI differs from catalog" + ); + tree_sitter::Parser::new().set_language(&language)?; + Ok(Loaded { + language, + highlights, + _library: library, + }) + })() + .map_err(|e| ParserError::new(FallbackCause::ParserLoadFailed, format!("{id}: {e:#}")))?; + let parser = (result.language.clone(), result.highlights.clone()); + loaded.insert(id, result); + Ok(parser) +} diff --git a/src/parse/optional.rs b/src/parse/optional.rs new file mode 100644 index 000000000..702d01a58 --- /dev/null +++ b/src/parse/optional.rs @@ -0,0 +1,51 @@ +//! The pack builder and runtime registry share languages/extra.json. +use super::guess_language::Language; +use super::tree_sitter_parser::TreeSitterConfig; +use crate::languages::ParserError; +use crate::summary::FallbackCause; + +macro_rules! extra_languages { + ($( $variant:ident => { + id: $id:literal, feature: $feature:literal, parser: $parser:path, + highlights: [$($query:expr),*], atoms: [$($atom:literal),*], + delimiters: [$($delimiter:expr),*], ignore_trailing: [$($trailing:expr),*] + }, )*) => { + pub(crate) fn id(language: Language) -> Option<&'static str> { + match language { $(Language::$variant => Some($id),)* _ => None } + } + #[allow(clippy::match_like_matches_macro)] + pub(crate) fn builtin(language: Language) -> bool { + match language { $(Language::$variant => cfg!(feature = $feature),)* _ => true } + } + pub(crate) fn config(language: Language) -> Result, ParserError> { + match language { + $(Language::$variant => { + #[cfg(feature = $feature)] + let compiled = { + let queries: &[&str] = &[$($query),*]; + Some((tree_sitter::Language::new($parser), queries.concat())) + }; + #[cfg(not(feature = $feature))] + let compiled = None; + let (language, highlights) = match compiled { + Some(parser) => parser, + None => super::native::load($id)?, + }; + let highlight_query = tree_sitter::Query::new(&language, &highlights) + .map_err(|error| ParserError::new(FallbackCause::ParserLoadFailed, error))?; + Ok(Some(TreeSitterConfig { + language, + highlight_query, + atom_nodes: [$($atom),*].into_iter().collect(), + delimiter_tokens: vec![$($delimiter),*], + ignore_trailing_tokens: vec![$($trailing),*], + sub_languages: vec![], + })) + },)* + _ => Ok(None), + } + } + }; +} + +include!(concat!(env!("OUT_DIR"), "/extra_languages.rs")); diff --git a/src/parse/tree_sitter_parser.rs b/src/parse/tree_sitter_parser.rs index 1aef0c877..9e9aef31d 100644 --- a/src/parse/tree_sitter_parser.rs +++ b/src/parse/tree_sitter_parser.rs @@ -57,12 +57,12 @@ pub(crate) struct TreeSitterConfig { /// By forcing the tree-sitter subtree to be a difftastic atom, we /// guarantee a correct diff, at the cost of losing some structure /// in the tree-sitter AST. - atom_nodes: DftHashSet<&'static str>, + pub(super) atom_nodes: DftHashSet<&'static str>, /// We want to consider delimiter tokens as part of lists, not /// standalone atoms. Tree-sitter includes delimiter tokens, so /// mark which token pairs we consider to be delimiters. - delimiter_tokens: Vec<(&'static str, &'static str)>, + pub(super) delimiter_tokens: Vec<(&'static str, &'static str)>, /// Tokens that we may ignore if they occur in a trailing /// position. This ensures that `[1, 2]` and `[1, 2,]` don't show @@ -74,11 +74,11 @@ pub(crate) struct TreeSitterConfig { /// /// Note that core diffing still sees these tokens, and we only /// handle them specially in post-processing. - ignore_trailing_tokens: Vec<(&'static str, &'static str)>, + pub(super) ignore_trailing_tokens: Vec<(&'static str, &'static str)>, /// The tree-sitter query used for syntax highlighting this /// language. - highlight_query: ts::Query, + pub(super) highlight_query: ts::Query, /// Sub-languages in use, if any. pub(crate) sub_languages: Vec, @@ -91,17 +91,9 @@ extern "C" { fn tree_sitter_smali() -> ts::Language; } -// TODO: begin/end and object/end. -const OCAML_ATOM_NODES: [&str; 6] = [ - "character", - "string", - "quoted_string", - "tag", - "type_variable", - "attribute_id", -]; - -pub(crate) fn from_language(language: guess::Language) -> &'static TreeSitterConfig { +pub(crate) fn from_language( + language: guess::Language, +) -> Result<&'static TreeSitterConfig, crate::languages::ParserError> { // Constructing a tree sitter query is relatively expensive: it // can take tens of milliseconds. // @@ -112,9 +104,16 @@ pub(crate) fn from_language(language: guess::Language) -> &'static TreeSitterCon LazyLock::new(|| Mutex::new(DftHashMap::default())); let mut cache = CONFIG_CACHE.lock().unwrap(); - cache - .entry(language) - .or_insert_with(|| Box::leak(Box::new(build_config(language)))) + if let Some(config) = cache.get(&language) { + return Ok(config); + } + let config = match super::optional::config(language)? { + Some(config) => config, + None => build_config(language), + }; + let config = Box::leak(Box::new(config)); + cache.insert(language, config); + Ok(config) } fn build_config(language: guess::Language) -> TreeSitterConfig { @@ -138,33 +137,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - Apex => { - let language_fn = tree_sitter_sfapex::apex::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: [ - "string_literal", - "null_literal", - "boolean", - "int", - "decimal_floating_point_literal", - "date_literal", - "currency_literal", - ] - .into_iter() - .collect(), - delimiter_tokens: vec![("[", "]"), ("(", ")"), ("{", "}")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - tree_sitter_sfapex::apex::HIGHLIGHTS_QUERY, - ) - .unwrap(), - sub_languages: vec![], - } - } Asm => { let language_fn = tree_sitter_asm::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -456,36 +428,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - #[cfg(feature = "lang-fsharp")] - FSharp => { - let language_fn = tree_sitter_fsharp::LANGUAGE_FSHARP; - let language = tree_sitter::Language::new(language_fn); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["string", "triple_quoted_string"].into_iter().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_fsharp::HIGHLIGHTS_QUERY) - .unwrap(), - - sub_languages: vec![], - } - } - #[cfg(feature = "lang-fortran")] - Fortran => { - let language_fn = tree_sitter_fortran::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["string_literal", "number_literal"].into_iter().collect(), - delimiter_tokens: vec![("(", ")"), ("(/", "/)"), ("[", "]")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_fortran::HIGHLIGHTS_QUERY) - .unwrap(), - sub_languages: vec![], - } - } Gleam => { let language_fn = tree_sitter_gleam::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -517,19 +459,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - Haskell => { - let language_fn = tree_sitter_haskell::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["qualified_variable"].into_iter().collect(), - delimiter_tokens: vec![("[", "]"), ("(", ")")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_haskell::HIGHLIGHTS_QUERY) - .unwrap(), - sub_languages: vec![], - } - } Hcl => { let language_fn = tree_sitter_hcl::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -689,30 +618,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - Julia => { - let language_fn = tree_sitter_julia::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: [ - "string_literal", - "prefixed_string_literal", - "command_literal", - "character_literal", - ] - .into_iter() - .collect(), - delimiter_tokens: vec![("{", "}"), ("[", "]"), ("(", ")")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../vendored_parsers/highlights/julia.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Kotlin => { let language = unsafe { tree_sitter_kotlin() }; TreeSitterConfig { @@ -843,32 +748,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - OCaml => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML; - let language = tree_sitter::Language::new(language_fn); - TreeSitterConfig { - language: language.clone(), - atom_nodes: OCAML_ATOM_NODES.iter().copied().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_ocaml::HIGHLIGHTS_QUERY) - .unwrap(), - sub_languages: vec![], - } - } - OCamlInterface => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE; - let language = tree_sitter::Language::new(language_fn); - TreeSitterConfig { - language: language.clone(), - atom_nodes: OCAML_ATOM_NODES.iter().copied().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], - ignore_trailing_tokens: vec![], - // TODO: why doesn't tree_sitter_ocaml::HIGHLIGHTS_QUERY work here? - highlight_query: ts::Query::new(&language, "").unwrap(), - sub_languages: vec![], - } - } Pascal => { let language_fn = tree_sitter_pascal::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -961,23 +840,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - Qml => { - let language_fn = tree_sitter_qmljs::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - - let mut highlight_query = tree_sitter_javascript::HIGHLIGHT_QUERY.to_owned(); - highlight_query.push_str(tree_sitter_typescript::HIGHLIGHTS_QUERY); - highlight_query.push_str(tree_sitter_qmljs::HIGHLIGHTS_QUERY); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["string", "template_string", "regex"].into_iter().collect(), - delimiter_tokens: vec![("{", "}"), ("(", ")"), ("[", "]"), ("<", ">")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, &highlight_query).unwrap(), - sub_languages: vec![], - } - } R => { let language_fn = tree_sitter_r::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -1248,37 +1110,6 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - #[cfg(feature = "lang-verilog")] - Verilog => { - let language_fn = tree_sitter_verilog::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - TreeSitterConfig { - ignore_trailing_tokens: vec![], - language: language.clone(), - atom_nodes: ["integral_number"].into_iter().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("begin", "end")], - highlight_query: ts::Query::new( - &language, - include_str!("../../vendored_parsers/highlights/verilog.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } - Vhdl => { - let language_fn = tree_sitter_vhdl::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: [].into_iter().collect(), - delimiter_tokens: vec![("(", ")")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_vhdl::HIGHLIGHTS_QUERY) - .unwrap(), - sub_languages: vec![], - } - } Zig => { let language_fn = tree_sitter_zig::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -1295,6 +1126,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } + _ => unreachable!("language must be registered in built-in or extra config"), } } @@ -2160,7 +1992,7 @@ mod tests { fn test_parse() { let arena = Arena::new(); let params = Params::default(); - let css_config = params.language(guess::Language::Css); + let css_config = params.language(guess::Language::Css).unwrap(); parse(&arena, ".foo {}", css_config, false).unwrap(); } @@ -2168,7 +2000,7 @@ mod tests { fn test_parse_empty_file() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess::Language::EmacsLisp); + let config = params.language(guess::Language::EmacsLisp).unwrap(); let res = parse(&arena, "", config, false).unwrap(); let expected: Vec<&Syntax> = vec![]; @@ -2181,7 +2013,7 @@ mod tests { fn test_subtrees() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess::Language::Html); + let config = params.language(guess::Language::Html).unwrap(); let res = parse(&arena, "", config, false).unwrap(); match res[0] { @@ -2204,7 +2036,9 @@ mod tests { #[test] fn test_configs_valid() { for language in guess::Language::iter() { - from_language(language); + if crate::languages::builtin(language) { + from_language(language).unwrap(); + } } } } diff --git a/src/protocol/project.rs b/src/protocol/project.rs index 9ae85a034..eee74c541 100644 --- a/src/protocol/project.rs +++ b/src/protocol/project.rs @@ -129,6 +129,9 @@ fn stats(result: &DiffResult, lhs_src: &str, rhs_src: &str) -> Stats { /// is the engine's own prose, with the numbers. fn fallback_code(cause: FallbackCause) -> &'static str { match cause { + FallbackCause::ParserNotInstalled => "parser_not_installed", + FallbackCause::ParserUnavailable => "parser_unavailable", + FallbackCause::ParserLoadFailed => "parser_load_failed", FallbackCause::Generated => "generated", FallbackCause::ByteLimit => "too_large", FallbackCause::GraphLimit => "too_complex", @@ -1320,7 +1323,8 @@ mod tests { fn syntax_spans_are_per_line_sorted_and_innermost() { let parser = crate::parse::tree_sitter_parser::from_language( crate::parse::guess_language::Language::Python, - ); + ) + .unwrap(); let spans = syntax_spans("def f(x):\n return \"a\"\n", parser); for pair in spans.windows(2) { assert!( diff --git a/src/protocol/stream.rs b/src/protocol/stream.rs index 6dcdb8d86..29ba1dcef 100644 --- a/src/protocol/stream.rs +++ b/src/protocol/stream.rs @@ -270,7 +270,10 @@ fn syntax_spans( let FileFormat::SupportedLanguage(language) = &diff.file_format else { return (Vec::new(), Vec::new()); }; - let parser = params.language(*language).parser; + let Ok(config) = params.language(*language) else { + return (Vec::new(), Vec::new()); + }; + let parser = config.parser; let spans = |content: &FileContent| match content { FileContent::Text(src) => project::syntax_spans(src, parser), FileContent::Binary => Vec::new(), diff --git a/src/summary.rs b/src/summary.rs index c961536f1..429a1d0e2 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -30,6 +30,9 @@ pub(crate) enum FileFormat { pub(crate) enum FallbackCause { /// A side is larger than the byte limit. ByteLimit, + ParserNotInstalled, + ParserUnavailable, + ParserLoadFailed, /// The AST matching graph grew past the graph limit. GraphLimit, /// A side has more parse errors than the parse error limit. diff --git a/tests/cli.rs b/tests/cli.rs index 7eff6f7ac..8d0d893b4 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -26,7 +26,7 @@ fn list_languages() { } #[test] -fn optional_languages_follow_build_features() { +fn optional_languages_are_recognized_without_compiled_parsers() { let listed = debug_command() .arg("--list-languages") .assert() @@ -39,13 +39,33 @@ fn optional_languages_follow_build_features() { let config = dir.path().join("config.toml"); std::fs::write(&config, "[plugins]\norder = []\n").unwrap(); - for (name, fixture, extension, enabled) in [ - ("Fortran", "fortran", "f90", cfg!(feature = "lang-fortran")), - ("Verilog", "verilog", "sv", cfg!(feature = "lang-verilog")), - ("F#", "f_sharp", "fs", cfg!(feature = "lang-fsharp")), - ] { - assert_eq!(listed.contains(name), enabled, "{name}"); + let status = get_base_command() + .env("DIFFR_PARSER_DIR", dir.path().join("parsers")) + .args(["languages", "list", "--json"]) + .output() + .unwrap(); + let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); + let missing_code = if status["revision"].is_null() { + "parser_unavailable" + } else { + "parser_not_installed" + }; + let definitions: Vec = + serde_json::from_str(include_str!("../languages/extra.json")).unwrap(); + for definition in definitions { + let name = definition["id"].as_str().unwrap(); + let fixture = definition["fixture"]["prefix"].as_str().unwrap(); + let extension = definition["fixture"]["extension"].as_str().unwrap(); + let language = status["languages"] + .as_array() + .unwrap() + .iter() + .find(|l| l["id"] == name) + .unwrap(); + let enabled = language["availability"] == "built_in"; + assert!(listed.contains(&format!("*.{extension}")), "{name}"); let output = get_base_command() + .env("DIFFR_PARSER_DIR", dir.path().join("parsers")) .arg("--config") .arg(&config) .args([ @@ -69,11 +89,7 @@ fn optional_languages_follow_build_features() { assert_eq!(file["diff"]["type"], "text", "{name}: {file}"); assert_eq!( file["diff"]["stats"]["fallback"]["code"].as_str(), - if enabled { - None - } else { - Some("unsupported_language") - }, + if enabled { None } else { Some(missing_code) }, "{name}", ); } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 572b87ac3..f39e148a5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -186,6 +186,20 @@ fn main() -> Result<()> { let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); let task = std::env::args().nth(1).unwrap_or_default(); match task.as_str() { + "build-languages" | "package-languages" | "check-languages" => { + let action = match task.as_str() { + "build-languages" => "build", + "package-languages" => "package", + _ => "check", + }; + run( + Command::new("python3") + .arg(root.join("languages/build.py")) + .arg(action) + .args(std::env::args_os().skip(2)), + "Building language pack", + ) + } "install" => install(root, true), "install-tui" => install(root, false), "build-plugins" => build_plugins(root),