From fa11cbe72db2a6115f85550fdee7588260ff52c7 Mon Sep 17 00:00:00 2001 From: thesiti92 Date: Tue, 29 Sep 2026 14:02:15 -0400 Subject: [PATCH 1/4] Add installable native extra-language pack Preserve compile-time parser features and add pinned native loading, atomic installation, metadata-driven packaging, and protected release validation. AI-assisted implementation and validation with Codex. --- .github/workflows/languages.yml | 142 ++++++ Cargo.lock | 95 ++++ Cargo.toml | 31 +- README.md | 7 + build.rs | 4 + languages/.gitignore | 1 + languages/ADDING.md | 82 ++++ languages/README.md | 92 ++++ languages/VALIDATION.md | 104 ++++ languages/apex/LICENSE | 21 + languages/apex/highlights.scm | 299 ++++++++++++ languages/apex/language.json | 16 + languages/build.py | 105 +++++ languages/catalog.json | 1 + languages/definitions.py | 46 ++ languages/fortran/LICENSE | 19 + languages/fortran/highlights.scm | 204 ++++++++ languages/fortran/language.json | 16 + languages/fsharp/LICENSE | 21 + languages/fsharp/highlights.scm | 421 +++++++++++++++++ languages/fsharp/language.json | 16 + languages/haskell/LICENSE | 21 + languages/haskell/highlights.scm | 443 +++++++++++++++++ languages/haskell/language.json | 16 + languages/julia/LICENSE | 21 + languages/julia/highlights.scm | 330 +++++++++++++ languages/julia/language.json | 16 + languages/ocaml-interface/LICENSE | 21 + languages/ocaml-interface/highlights.scm | 0 languages/ocaml-interface/language.json | 16 + languages/ocaml/LICENSE | 21 + languages/ocaml/highlights.scm | 148 ++++++ languages/ocaml/language.json | 16 + languages/package.template.json | 6 + languages/publish.py | 58 +++ languages/qml/LICENSE | 20 + languages/qml/highlights.scm | 61 +++ languages/qml/language.json | 16 + languages/tests/Dockerfile | 27 ++ languages/tests/Dockerfile.dockerignore | 5 + languages/tests/container-smoke.sh | 37 ++ languages/tests/entitlements.plist | 5 + languages/tests/fixtures/scanner.f90 | 15 + languages/tests/fixtures/scanner.fs | 14 + languages/tests/integration.py | 190 ++++++++ languages/tests/probe.c | 19 + languages/tests/probe.py | 29 ++ languages/tests/test_definitions.py | 40 ++ languages/tests/test_publish.py | 52 ++ languages/verilog/LICENSE | 21 + languages/verilog/highlights.scm | 322 +++++++++++++ languages/verilog/language.json | 16 + languages/vhdl/LICENSE | 21 + languages/vhdl/highlights.scm | 435 +++++++++++++++++ languages/vhdl/language.json | 16 + sample_files/ocaml_interface_1.mli | 3 + sample_files/ocaml_interface_2.mli | 3 + src/cli.rs | 2 +- src/config.rs | 67 ++- src/diff/sliders.rs | 8 +- src/diff/unchanged.rs | 52 +- src/engine.rs | 18 +- src/languages.rs | 577 +++++++++++++++++++++++ src/main.rs | 17 +- src/parse/guess_language.rs | 12 - src/parse/mod.rs | 4 + src/parse/native.rs | 72 +++ src/parse/optional.rs | 27 ++ src/parse/tree_sitter_parser.rs | 146 ++++-- src/protocol/project.rs | 6 +- src/protocol/stream.rs | 5 +- src/summary.rs | 3 + tests/cli.rs | 44 +- xtask/src/main.rs | 14 + 74 files changed, 5196 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/languages.yml create mode 100644 languages/.gitignore create mode 100644 languages/ADDING.md create mode 100644 languages/README.md create mode 100644 languages/VALIDATION.md create mode 100644 languages/apex/LICENSE create mode 100644 languages/apex/highlights.scm create mode 100644 languages/apex/language.json create mode 100644 languages/build.py create mode 100644 languages/catalog.json create mode 100644 languages/definitions.py create mode 100644 languages/fortran/LICENSE create mode 100644 languages/fortran/highlights.scm create mode 100644 languages/fortran/language.json create mode 100644 languages/fsharp/LICENSE create mode 100644 languages/fsharp/highlights.scm create mode 100644 languages/fsharp/language.json create mode 100644 languages/haskell/LICENSE create mode 100644 languages/haskell/highlights.scm create mode 100644 languages/haskell/language.json create mode 100644 languages/julia/LICENSE create mode 100644 languages/julia/highlights.scm create mode 100644 languages/julia/language.json create mode 100644 languages/ocaml-interface/LICENSE create mode 100644 languages/ocaml-interface/highlights.scm create mode 100644 languages/ocaml-interface/language.json create mode 100644 languages/ocaml/LICENSE create mode 100644 languages/ocaml/highlights.scm create mode 100644 languages/ocaml/language.json create mode 100644 languages/package.template.json create mode 100644 languages/publish.py create mode 100644 languages/qml/LICENSE create mode 100644 languages/qml/highlights.scm create mode 100644 languages/qml/language.json create mode 100644 languages/tests/Dockerfile create mode 100644 languages/tests/Dockerfile.dockerignore create mode 100644 languages/tests/container-smoke.sh create mode 100644 languages/tests/entitlements.plist create mode 100644 languages/tests/fixtures/scanner.f90 create mode 100644 languages/tests/fixtures/scanner.fs create mode 100644 languages/tests/integration.py create mode 100644 languages/tests/probe.c create mode 100644 languages/tests/probe.py create mode 100644 languages/tests/test_definitions.py create mode 100644 languages/tests/test_publish.py create mode 100644 languages/verilog/LICENSE create mode 100644 languages/verilog/highlights.scm create mode 100644 languages/verilog/language.json create mode 100644 languages/vhdl/LICENSE create mode 100644 languages/vhdl/highlights.scm create mode 100644 languages/vhdl/language.json create mode 100644 sample_files/ocaml_interface_1.mli create mode 100644 sample_files/ocaml_interface_2.mli create mode 100644 src/languages.rs create mode 100644 src/parse/native.rs create mode 100644 src/parse/optional.rs diff --git a/.github/workflows/languages.yml b/.github/workflows/languages.yml new file mode 100644 index 000000000..4c4c4e51f --- /dev/null +++ b/.github/workflows/languages.yml @@ -0,0 +1,142 @@ +name: Extra languages + +on: + pull_request: + paths: + - 'languages/**' + - 'sample_files/**' + - 'tests/cli.rs' + - 'src/**' + - 'Cargo.*' + - 'build.rs' + - 'xtask/**' + - '.github/workflows/languages.yml' + workflow_dispatch: + inputs: + publish: + description: Publish the exact signed archives and produce a verified catalog + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: languages-${{ github.ref }} + cancel-in-progress: false + +jobs: + guard: + runs-on: ubuntu-22.04 + steps: + - name: Require protected publication environment + if: inputs.publish + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + run: | + test "$GITHUB_REF" = refs/heads/main + gh api "repos/$REPOSITORY/environments/languages-release" > environment.json + jq -e '.protection_rules | any(.type == "required_reviewers" and (.reviewers | length > 0))' environment.json + + build: + needs: guard + environment: ${{ inputs.publish && 'languages-release' || 'languages-ci' }} + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-14 + - target: x86_64-apple-darwin + os: macos-15-intel + - target: x86_64-unknown-linux-gnu + os: ubuntu-22.04 + - target: aarch64-unknown-linux-gnu + os: ubuntu-22.04-arm + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: dtolnay/rust-toolchain@stable + - name: Check language definitions and publication gate + run: | + cargo xtask check-languages + python3 languages/tests/test_definitions.py + python3 languages/tests/test_publish.py + - name: Build locked grammars + run: cargo xtask build-languages --target '${{ matrix.target }}' + - name: Import Whiteboard signing identity + if: inputs.publish && runner.os == 'macOS' + env: + CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE_BASE64 }} + CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + run: | + python3 -c 'import os,base64; open(os.environ["RUNNER_TEMP"]+"/certificate.p12","wb").write(base64.b64decode(os.environ["CERTIFICATE"]))' + password=$(openssl rand -hex 24) + security create-keychain -p "$password" "$RUNNER_TEMP/languages.keychain-db" + security set-keychain-settings -lut 21600 "$RUNNER_TEMP/languages.keychain-db" + security unlock-keychain -p "$password" "$RUNNER_TEMP/languages.keychain-db" + security import "$RUNNER_TEMP/certificate.p12" -k "$RUNNER_TEMP/languages.keychain-db" -P "$CERTIFICATE_PASSWORD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple: -k "$password" "$RUNNER_TEMP/languages.keychain-db" + security list-keychains -d user -s "$RUNNER_TEMP/languages.keychain-db" + echo 'DIFFR_SIGN_IDENTITY=Developer ID Application: Workable Solutions Inc. (PWXY59YDAY)' >> "$GITHUB_ENV" + - name: Native parse and hardened runtime proof + run: python3 languages/tests/probe.py 'target/languages/${{ matrix.target }}/package' + - name: Package final bytes + run: cargo xtask package-languages --target '${{ matrix.target }}' + - name: Installer and static parser parity + run: python3 languages/tests/integration.py 'target/languages/${{ matrix.target }}/catalog-entry.json' --release + - name: Fresh Linux install without development tools or network + if: runner.os == 'Linux' + env: + TARGET: ${{ matrix.target }} + run: | + filename=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["url"].rsplit("/", 1)[1])' "target/languages/$TARGET/catalog-entry.json") + pack="$PWD/target/languages/$TARGET/$filename" + docker run --rm --network none --read-only --user 65534:65534 \ + --tmpfs /tmp:rw,exec,nosuid,size=256m \ + --env DIFFR_PARSER_DIR=/tmp/diffr-parsers \ + --volume "$PWD/target/languages/$TARGET/verification/diffr-native:/usr/local/bin/diffr:ro" \ + --volume "$pack:/opt/extra.tgz:ro" \ + --volume "$PWD/target/languages/$TARGET/test-fixtures.txt:/opt/fixtures.txt:ro" \ + --volume "$PWD/sample_files:/fixtures:ro" \ + --volume "$PWD/languages/tests/container-smoke.sh:/opt/container-smoke.sh:ro" \ + debian:bookworm-slim /bin/sh /opt/container-smoke.sh + - name: Clean signing material + if: always() && runner.os == 'macOS' + run: | + security delete-keychain "$RUNNER_TEMP/languages.keychain-db" || true + rm -f "$RUNNER_TEMP/certificate.p12" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: languages-${{ matrix.target }} + path: | + target/languages/${{ matrix.target }}/*.tgz + target/languages/${{ matrix.target }}/catalog-entry.json + if-no-files-found: error + + publish: + if: inputs.publish + needs: build + environment: languages-release + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: languages-* + path: target/languages/publish + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24' + registry-url: https://registry.npmjs.org + - name: Publish and verify exact archives + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: python3 languages/publish.py target/languages/publish --publish + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: verified-language-catalog + path: target/languages/publish/catalog.json + if-no-files-found: error diff --git a/Cargo.lock b/Cargo.lock index 032aa4f9e..c733a085b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -695,6 +701,8 @@ dependencies = [ "diffr-plugin-test-bodies", "dirs", "encoding_rs", + "flate2", + "fs2", "git2", "glob", "hashbrown 0.17.1", @@ -703,6 +711,7 @@ dependencies = [ "jsonschema", "lazy_static", "libc", + "libloading", "line-numbers", "log", "predicates", @@ -718,10 +727,12 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_yaml_ng", + "sha2", "smallvec", "streaming-iterator", "strsim", "strum", + "tar", "tempfile", "terminal_size", "tikv-jemallocator", @@ -1044,6 +1055,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.8" @@ -1064,6 +1085,17 @@ dependencies = [ "serde", ] +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -1132,6 +1164,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1812,6 +1854,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.8" @@ -1931,6 +1983,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -2926,6 +2988,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -3070,6 +3138,17 @@ dependencies = [ "winx", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -5072,6 +5151,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xtask" version = "0.1.0" @@ -5189,6 +5278,12 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index a8f37b0b0..cc188043a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,10 @@ rust-version = "1.85.0" include = [ "/build.rs", "/src/", + "/languages/catalog.json", + "/languages/*/language.json", + "/languages/*/highlights.scm", + "/languages/*/LICENSE", "/wit/", "/plugins/", "/vendored_parsers/highlights/*.scm", @@ -33,6 +37,12 @@ include = [ ] [dependencies] +libloading = "0.8" +sha2 = "0.10" +flate2 = "1" +tar = "0.4" +fs2 = "0.4" +tempfile = "3.27.0" git2 = { version = "0.20", default-features = false } regex = "1.10.4" clap = { version = "4.0.0", features = ["cargo", "env", "wrap_help", "string"] } @@ -94,24 +104,24 @@ tree-sitter-fortran = { version = "0.6.0", optional = true } tree-sitter-fsharp = { version = "0.3.0", optional = true } tree-sitter-gleam = "1.0.0" tree-sitter-go = "0.25.0" -tree-sitter-haskell = "0.23.1" +tree-sitter-haskell = { version = "0.23.1", optional = true } tree-sitter-hcl = "1.1.0" tree-sitter-html = "0.23.2" tree-sitter-java-orchard = "0.5.9" tree-sitter-javascript = "0.25.0" tree-sitter-json = "0.24.8" -tree-sitter-julia = "0.23.1" +tree-sitter-julia = { version = "0.23.1", optional = true } tree-sitter-lua = "0.5" tree-sitter-make = "1.1.1" tree-sitter-newick = "1.1.0" tree-sitter-nix = "0.3.0" tree-sitter-objc = "3.0.2" -tree-sitter-ocaml = "0.25.0" +tree-sitter-ocaml = { version = "0.25.0", optional = true } tree-sitter-pascal = "0.10.0" tree-sitter-php = "0.24.0" tree-sitter-proto = "0.4.0" tree-sitter-python = "0.25.0" -tree-sitter-qmljs = "0.3.0" +tree-sitter-qmljs = { version = "0.3.0", optional = true } tree-sitter-r = "1.2.0" tree-sitter-racket = "0.24.7" tree-sitter-ruby = "0.23.1" @@ -119,13 +129,13 @@ tree-sitter-rust-orchard = "0.16.6" tree-sitter-scala = "0.26.2" tree-sitter-scheme = "0.24.7" tree-sitter-sequel = "0.3.11" -tree-sitter-sfapex = "2.4.0" +tree-sitter-sfapex = { version = "2.4.0", optional = true } tree-sitter-solidity = "1.2.13" tree-sitter-swift = "0.7.1" tree-sitter-toml-ng = "0.7.0" tree-sitter-typescript = "0.23.2" tree-sitter-verilog = { version = "1.0.3", optional = true } -tree-sitter-vhdl = "1.4.0" +tree-sitter-vhdl = { version = "1.4.0", optional = true } tree-sitter-xml = "0.7.0" tree-sitter-yaml = "0.7.0" tree-sitter-zig = "1.1.2" @@ -165,7 +175,6 @@ assert_cmd = "2.0.17" predicates = "3.1.3" pretty_assertions = "1.3.0" -tempfile = "3.27.0" [build-dependencies] toml = "0.8" @@ -235,8 +244,14 @@ name = "wasm" required-features = ["wasm-plugin-tests"] [features] -all-languages = ["lang-fortran", "lang-fsharp", "lang-verilog"] +all-languages = ["lang-fortran", "lang-fsharp", "lang-verilog", "lang-ocaml", "lang-julia", "lang-haskell", "lang-vhdl", "lang-apex", "lang-qml"] lang-fortran = ["dep:tree-sitter-fortran"] lang-fsharp = ["dep:tree-sitter-fsharp"] lang-verilog = ["dep:tree-sitter-verilog"] +lang-ocaml = ["dep:tree-sitter-ocaml"] +lang-julia = ["dep:tree-sitter-julia"] +lang-haskell = ["dep:tree-sitter-haskell"] +lang-vhdl = ["dep:tree-sitter-vhdl"] +lang-apex = ["dep:tree-sitter-sfapex"] +lang-qml = ["dep:tree-sitter-qmljs"] wasm-plugin-tests = [] diff --git a/README.md b/README.md index 0c73fb4b7..6932e207f 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,13 @@ We hope in the future that we can find some way to upstream some / all of these `diffr` has a powerful, wasm-based plugin API which customizes how it presents changed files. For more details, read the [docs](./docs/plugin.md). +## Optional languages + +Optional parsers can be compiled in with `--features all-languages`. +The native pack installer and its publication status are documented in +[languages/README.md](languages/README.md). Use `diffr languages list --json` +to inspect availability for your executable. + ## License MIT. See `LICENSE` for the terms and `NOTICE` for the third-party work diff --git a/build.rs b/build.rs index 9682ce1c9..b9767eb08 100644 --- a/build.rs +++ b/build.rs @@ -41,6 +41,10 @@ impl TreeSitterParser { } fn main() { + println!( + "cargo:rustc-env=DIFFR_TARGET={}", + std::env::var("TARGET").unwrap() + ); native_plugins(); let parsers = vec![ TreeSitterParser { diff --git a/languages/.gitignore b/languages/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/languages/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/languages/ADDING.md b/languages/ADDING.md new file mode 100644 index 000000000..c7f9e8db8 --- /dev/null +++ b/languages/ADDING.md @@ -0,0 +1,82 @@ +# Moving another parser into `extra` + +An extra parser is still a first-party diffr language. Recognition, highlight +queries, annotation queries, and diff rules remain in diffr. Only generated C +parser/scanner code moves into the native pack. Keep its existing compile-time +feature so `all-languages` remains self-contained. + +## 1. Define the grammar + +Create `languages//language.json`, `highlights.scm`, and `LICENSE`. +Use an existing definition such as `qml/language.json` as the template: + +| Field | Meaning | +|---|---| +| `id` | Stable canonical ID, also the library filename stem | +| `variant` | Existing Rust `Language` enum variant | +| `feature` | Cargo feature enabling the built-in parser, e.g. `lang-qml` | +| `crate`, `version`, `revision`, `repository` | Locked crate and upstream source pin | +| `source` | Relative path to the generated `parser.c` and optional `scanner.c` | +| `symbol` | Exported Tree-sitter function, e.g. `tree_sitter_qmljs` | +| `fixture.prefix`, `fixture.extension` | Existing `sample_files/_{1,2}.` pair | +| `smoke` | Small valid program that the native host must parse without errors | + +Copy queries and the upstream license from the pinned source. Preserve any +existing diffr-specific highlight query. An intentionally empty query is valid; +OCaml interfaces retain their existing empty query. Scanner includes are resolved +relative to the Cargo source tree, so shared scanner headers need not be copied. + +Multiple grammar entries can share a Cargo feature/dependency. OCaml and OCaml +interfaces demonstrate this. Include only grammar variants diffr actually uses; +the pack excludes F# signatures, OCaml standalone types, and Salesforce's +standalone SOQL/SOSL/log grammars. + +## 2. Keep built-in and installed behavior aligned + +- Make the parser dependency optional in `Cargo.toml`. Add `lang-` pointing + to `dep:` and include that feature in `all-languages`. +- Add one entry to `src/parse/optional.rs`. This single table supplies both the + canonical ID and compile-time availability; `languages list` discovers it. +- In the existing `tree_sitter_parser.rs` arm, select the compiled grammar behind + its feature and the already-resolved external grammar otherwise. Follow the + QML arm. Load its own highlight query with `include_str!` from `languages/`. + Preserve shared queries, atom rules, delimiters, and embedded-language behavior. +- Keep the `Language` variant, names, globs, and mode detection unconditional. + +Do not add another language list to packaging or test scripts. They discover all +`languages/*/language.json` files. `check-languages` rejects registry/Cargo-feature +mismatches, missing `all-languages` entries, licenses, queries, and fixtures. + +## 3. Validate the change + +```sh +cargo xtask check-languages +python3 languages/tests/test_definitions.py +cargo test --locked +cargo test --locked --features all-languages --bin diffr --test cli +cargo xtask build-languages --target aarch64-apple-darwin +# Set DIFFR_SIGN_IDENTITY to Whiteboard's identity for the signed macOS checks. +python3 languages/tests/probe.py target/languages/aarch64-apple-darwin/package +cargo xtask package-languages --target aarch64-apple-darwin +python3 languages/tests/integration.py target/languages/aarch64-apple-darwin/catalog-entry.json --release +``` + +Packaging and native smoke tests automatically include every definition. The +integration suite compares every grammar's structural output, syntax spans, +parse trees, and added/deleted/unchanged files with an `all-languages` build; +it also checks parallel diffing, installation, corruption, repair, offline use, +and the platform's default store. Add scanner-heavy fixtures when the existing +pair does not cover the grammar's scanner behavior. + +The generated `target/languages//test-fixtures.txt` drives the minimal +Linux runtime test; no compiler, Rust, Node, npm, or Python is present there. +The four-target workflow remains the release gate. Moving a language is not +complete until native and installed/static checks pass for that language. + +## 4. Advance the pack revision + +Choose a new independent version in `package.template.json` when changing pack +contents. Never reuse published bytes/version pairs. Build, strip, sign, package, +publish, and verify all four targets before updating `catalog.json`, as described +in README.md. Existing CLIs keep their own pinned revisions. Do not hand-author +hashes or temporarily trust arbitrary runtime manifests. diff --git a/languages/README.md b/languages/README.md new file mode 100644 index 000000000..f626c453a --- /dev/null +++ b/languages/README.md @@ -0,0 +1,92 @@ +# Extra native languages + +`diffr languages list --json` inspects the optional language +pack without networking or loading native code. Install the revision pinned by +your executable with `diffr languages install extra`. For offline installation, +pass `--archive /path/to/package.tgz`; the same catalog checks apply. + +**Initial publication is pending.** The checked-in catalog intentionally has no +artifacts until all four signed/final packages have been published and verified. +Until that catalog is incorporated, default builds report `parser_unavailable`. +The per-language `lang-*` features and `all-languages` continue to provide +built-in parsers and take priority over packages. The pack contains Fortran, F#, +Verilog, OCaml (implementation and interface), Julia, Haskell, VHDL, Salesforce +Apex, and QML. LaTeX and general SQL remain bundled. + +See [ADDING.md](ADDING.md) for the validated pathway to move another parser. + +The store is `/diffr/parsers//extra/`. +`DIFFR_PARSER_DIR` replaces the `parsers` root. Revisions coexist, and ordinary +diffs never download or install anything. A missing, incompatible, or corrupt +parser produces a text diff with a `parser_not_installed`, `parser_unavailable`, +or `parser_load_failed` problem. The wire shape and diff exit codes are unchanged. + +## Build and verify + +Run on each native target runner (macOS ARM64/x64 or Linux glibc ARM64/x64): + +```sh +cargo xtask build-languages --target aarch64-apple-darwin +python3 languages/tests/probe.py target/languages/aarch64-apple-darwin/package +cargo xtask package-languages --target aarch64-apple-darwin +python3 languages/tests/integration.py target/languages/aarch64-apple-darwin/catalog-entry.json --release +``` + +Python 3 and a C compiler are build tools only. Cargo's locked metadata locates +registry sources; definitions additionally pin versions and source revisions. +Highlights are compiled into diffr. F# uses only `fsharp/src`, including its +scanner, and Fortran includes its scanner. Licenses and highlights were copied +from the pinned crate/source revisions, except Verilog highlights, which retain +diffr's existing query. No grammar package contains JavaScript or install scripts. + +On macOS, set `DIFFR_SIGN_IDENTITY` to Whiteboard's Developer ID identity before +running the probe. It signs the libraries and a hardened native host, retaining +library validation. The integration script signs the test CLIs with that identity +as well. Package **after** signing. Packaging never modifies the final libraries. +The integration script temporarily replaces the source catalog to build test +CLIs, then restores it; run it in a dedicated checkout, without concurrent Cargo +builds. There is deliberately no runtime catalog override. + +## Publication + +Before using the manual `Extra languages` workflow with `publish: true`: + +1. Configure the `languages-release` GitHub environment with required reviewers. + The workflow checks that protection exists and permits publication only from + `main`. +2. Supply environment secrets `APPLE_CERTIFICATE_BASE64`, + `APPLE_CERTIFICATE_PASSWORD`, and `NPM_TOKEN`. The certificate must belong to + Whiteboard's Team ID `PWXY59YDAY`; npm access must cover all four package names. +3. Set the independent revision in `package.template.json`. Do not reuse a + revision whose bytes have already been published. +4. Run the workflow. All four runners must build, strip, sign where applicable, + parse with native libraries, and pass installed/static parity before publishing. +5. Download the `verified-language-catalog` artifact, review it, and replace + `languages/catalog.json` in a normal code change. Release the CLI afterward. + +The publisher uploads the exact prepared tarballs and downloads each back for +size/hash verification. An existing immutable revision is accepted only if its +bytes match. It produces no catalog if any target fails. Partial publication can +be retried only with byte-identical artifacts, or with a new pack revision. +Re-signing can change bytes even when source inputs are identical. + +Local macOS ARM64 and Docker Linux ARM64 proof do not establish the x64 targets, +Gatekeeper behavior for separately quarantined artifacts, or a public npm install. Those remain +release evidence to collect; no validation exemption is added to Whiteboard. + +## Clean Linux installation test + +From the repository root: + +```sh +docker build --platform linux/arm64 -f languages/tests/Dockerfile -t diffr-extra-install-test . +docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,nosuid,size=256m diffr-extra-install-test +``` + +The build stage compiles native Linux artifacts and compares installed parsers +with `all-languages`. The separate Debian runtime contains only diffr, the pack, +fixtures, and base OS utilities. As an unprivileged user, with networking disabled, +it verifies a fresh offline install, every optional grammar, idempotence, corrupt-pack +fallback, and atomic repair. It asserts that Rust, C compilers, Node, npm, and +Python are absent. Switch to `--platform linux/amd64` to exercise the other Linux +target (emulation on ARM hosts is substantially slower). diff --git a/languages/VALIDATION.md b/languages/VALIDATION.md new file mode 100644 index 000000000..1f10f3867 --- /dev/null +++ b/languages/VALIDATION.md @@ -0,0 +1,104 @@ +# Local validation — 2026-09-29 + +Base: diffr `58fa3b2` (includes PR #49). Host: macOS ARM64. No packages were +published, and `catalog.json` remains empty pending verified publication. + +## Passed + +- `cargo test --locked`: 290 tests passed, one ignored. +- `cargo test --locked --features all-languages --bin diffr --test cli`: + 271 passed, one ignored. +- Archive/download tests cover interrupted and oversized transfers, wrong + hashes/targets/sizes, missing files, duplicates, symlinks, hardlinks, nested + paths, and traversal. +- `python3 languages/tests/test_publish.py`: both publication-gate tests passed. +- `actionlint .github/workflows/languages.yml`, formatting, and `git diff --check`. +- Cargo's package file list includes the embedded catalog, all ten highlight + queries, and their licenses. +- All ten C grammars built and parsed in a signed hardened native probe. +- Signed release CLIs passed installed/static NDJSON and syntax parity, tree + comparison, scanner-heavy Fortran/F# fixtures, added/deleted/unchanged files, + 120 mixed-language files with eight workers, concurrent installers, corruption, + repair, offline reuse, and revision coexistence. +- A fresh offline install and diff succeeded with an empty `PATH`. +- The signed macOS CLI passed the default `Library/Application Support` store, + an archive marked with `com.apple.quarantine`, truncated-archive rejection, + and actual coexistence with the previously built 0.1.0 CLI and pack. +- Definition checks (three tests), shared OCaml feature coverage, and the + metadata-driven packaging and fixture discovery checks passed. + +Both the probe and full CLI used Team ID `PWXY59YDAY`. The full CLI used the +existing Whiteboard diffr entitlement `allow-unsigned-executable-memory`. +Neither host disabled library validation. Loading was tested from the parser +store outside the application bundle. The original Whiteboard app was not changed. + +Clippy completes with six pre-existing warnings in `src/diff/graph.rs`, +`src/parse/syntax.rs`, and `src/protocol/mod.rs`; `-D warnings` fails on those. + +## Measurements + +One local signed ARM64 pack, revision `0.2.0`, containing ten grammars: + +| Measurement | Bytes | +|---|---:| +| Compressed pack | 3,932,587 | +| Native-pack release CLI | 93,029,568 | +| All-languages release CLI | 148,755,536 | + +The executable is 55,725,968 bytes smaller than the equivalent all-languages +build. Both measurements include Developer ID signatures. These are local release +executables, not publicly downloaded CLI artifacts. + +Installer/loading overhead was separately measured before the pack expansion, +with the original three-parser feature set held constant: 107,839,232 bytes for +the baseline and 112,555,376 for the installer/loader build, an increase of +4,716,144 bytes (4.37%). That comparison isolates installer work; it is not the +expanded pack's net size change. + +Ten new-process Apex diffs had a median of 196 ms; the first measured sample +was 345 ms. A 120-file mixed batch with eight workers took 777 ms. Each process +starts with an empty parser cache, while the batch reuses loaded grammars. +The OS page cache was warm; these are not cold-disk measurements. Timing is +indicative, not a performance gate. + +## Outstanding release evidence + +macOS x64, Linux x64, Rosetta execution, public npm downloads, and behavior +for separately quarantined artifacts have not been exercised locally. The +four-runner workflow must pass before publication. The repository currently +has no configured signing/npm secrets or protected publication environment; +see README.md for the exact setup. Only a catalog emitted after all four +registry downloads match may replace the empty checked-in catalog. + +## Linux ARM64 Docker verification + +A native `aarch64-unknown-linux-gnu` build passed `languages/tests/probe.py` and +the complete installed/static integration suite in Docker. The Linux pack was +3,737,483 bytes compressed (revision `0.2.0`). The separate Debian Bookworm runtime passed +`container-smoke.sh` with networking disabled, a read-only root filesystem, +a temporary writable parser store, and UID/GID 65534. + +The Linux native CLI was 98,161,256 bytes; the all-languages CLI was +153,820,688 bytes. Ten new-process Apex diffs had a median of 228 ms, +and the 120-file mixed batch took 1,554 ms. + +The runtime verifies absence of Cargo, Rust, C compilers, Node, npm, and Python; +then tests fresh offline installation, all ten structural parsers, idempotence, +corrupt-library text fallback, and atomic repair. This test is also wired into +both Linux jobs in `languages.yml`. The local run covers ARM64 only. + +Commands: + +```sh +docker build --platform linux/arm64 -f languages/tests/Dockerfile -t diffr-extra-install-test . +docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,nosuid,size=256m diffr-extra-install-test +``` + +Result: `PASS: fresh offline install, all optional parsers, idempotence, corrupt +fallback, atomic repair; unprivileged and without development tools.` + +The first attempted build used a manually assembled context that omitted tracked +grammar-directory symlinks. Rebuilding with the ordinary Docker context and the +provided Dockerfile-specific ignore file fixed the test setup. No parser or +installer fix was needed. Public npm installation remains unverified until +publication and catalog activation. diff --git a/languages/apex/LICENSE b/languages/apex/LICENSE new file mode 100644 index 000000000..4d78f763e --- /dev/null +++ b/languages/apex/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2012 Anthony Heber + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/languages/apex/highlights.scm b/languages/apex/highlights.scm new file mode 100644 index 000000000..c130edf39 --- /dev/null +++ b/languages/apex/highlights.scm @@ -0,0 +1,299 @@ +;; attempting to match concepts represented here: +;; https://code.visualstudio.com/api/language-extensions/semantic-highlight-guide + +[ + "[" + "]" + "{" + "}" + "?" + ";" +] @punctuation + +;; Methods + +(method_declaration + name: (identifier) @method) +(method_declaration + type: (type_identifier) @type) + +(method_invocation + name: (identifier) @method) +(argument_list + (identifier) @variable) +(super) @function.defaultLibrary + +(explicit_constructor_invocation + arguments: (argument_list + (identifier) @variable )) + +;; Annotations + +(annotation + name: (identifier) @decorator) + +"@" @operator + +(annotation_key_value + (identifier) @variable) + + +;; Types + +;; because itendifying it when declared doesn't carry to use +;; leans on the convention that "screaming snake case" is a const +((identifier) @variable.readonly + (#match? @variable.readonly "^_*[A-Z][A-Z\\d_]+$")) + +(interface_declaration + name: (identifier) @interface) +(class_declaration + name: (identifier) @class) +(class_declaration + (superclass) @class) +(enum_declaration + name: (identifier) @enum) +(enum_constant + name: (identifier) @enumMember) + +(interfaces + (type_list + (type_identifier) @interface )) + +(local_variable_declaration + (type_identifier) @type ) + +( expression_statement (_ (identifier)) @variable) + +(type_arguments "<" @punctuation) +(type_arguments ">" @punctuation) + +; (identifier) @variable + +((field_access + object: (identifier) @type)) ;; don't know what type of thing it is + +(generic_type + (type_identifier) @type) +(type_arguments (type_identifier) @type) + +(field_access + field: (identifier) @property) + +((scoped_identifier + scope: (identifier) @type) + (#match? @type "^[A-Z]")) +((method_invocation + object: (identifier) @type) + (#match? @type "^[A-Z]")) + + +(field_declaration + type: (type_identifier) @type) + +(formal_parameter + type: (type_identifier) @type + (identifier) @variable) + +(method_declaration + (formal_parameters + (formal_parameter + name: (identifier) @parameter))) + +(enhanced_for_statement + type: (type_identifier) @type + name: (identifier) @variable ) + +(enhanced_for_statement + value: (identifier) @variable) + +(enhanced_for_statement + name: (identifier) @variable) + +(object_creation_expression + type: (type_identifier) @type) + +(array_creation_expression + type: (type_identifier) @type) + +(array_type + element: (type_identifier) @type) + +(return_statement + (identifier) @variable) + +(local_variable_declaration + (variable_declarator + name: (identifier) @variable )) + +(for_statement + condition: (binary_expression + (identifier) @variable)) + +(for_statement + update: (update_expression + (identifier) @variable)) + +(constructor_declaration + name: (identifier) @class) + +(dml_type) @function.defaultLibrary + +(bound_apex_expression + (identifier) @variable) + +(assignment_operator) @operator + +(update_operator) @operator + +(instanceof_expression + left: (identifier) @variable + right: (type_identifier) @type ) + +(cast_expression + type: (type_identifier) @type + value: (identifier) @variable) + +(switch_expression + condition: (identifier) @variable) + +(switch_rule + (switch_label + (identifier) @enumMember )) + +(when_sobject_type + (type_identifier) @type + (identifier) @variable ) + +(trigger_declaration + name: (identifier) @type + object: (identifier) @type + (trigger_event) @keyword + ("," (trigger_event) @keyword)*) + +(binary_expression + operator: [ + ">" + "<" + ">=" + "<=" + "==" + "===" + "!=" + "!==" + "&&" + "||" + "+" + "-" + "*" + "/" + "&" + "|" + "^" + "%" + "<<" + ">>" + ">>>"] @operator) + +(binary_expression + (identifier) @variable) + +(unary_expression + operator: [ + "+" + "-" + "!" + "~" + ]) @operator + +("=>" @operator) + +[ + (boolean_type) + (void_type) +] @type.defaultLibrary + +; Variables + +(field_declaration (variable_declarator + (identifier) @property)) + +(field_declaration + (modifiers (modifier [(final) (static)])(modifier [(final) (static)])) + (variable_declarator + name: (identifier) @variable.readonly)) + +(this) @variable.defaultLibrary + +; Literals + +[ + (int) +] @number + +[ + (string_literal) +] @string + +[ + (line_comment) + (block_comment) +] @comment + +;; Keywords + +[ + (abstract) + (all_rows_clause) + "break" + "catch" + "class" + "continue" + "do" + "else" + "enum" + "extends" + (final) + "finally" + "for" + "get" + (global) + "if" + "implements" + "instanceof" + "interface" + "new" + "on" + (override) + (private) + (protected) + (public) + "return" + "set" + (static) + "switch" + (testMethod) + (webservice) + "throw" + (transient) + "try" + "trigger" + (virtual) + "when" + "while" + (with_sharing) + (without_sharing) + (inherited_sharing) +] @keyword + +(assignment_expression + left: (identifier) @variable) + +; (type_identifier) @type ;; not respecting precedence... +;; I don't love this but couldn't break them up right now +;; can't figure out how to let that be special without conflicting +;; in the grammar +"System.runAs" @method.defaultLibrary + +(scoped_type_identifier + (type_identifier) @type) diff --git a/languages/apex/language.json b/languages/apex/language.json new file mode 100644 index 000000000..26f01c75b --- /dev/null +++ b/languages/apex/language.json @@ -0,0 +1,16 @@ +{ + "id": "apex", + "crate": "tree-sitter-sfapex", + "version": "2.4.0", + "source": "apex/src", + "symbol": "tree_sitter_apex", + "revision": "9247e2cd12490f1aac71cb212f726ef1a2b5f7f9", + "repository": "https://github.com/aheber/tree-sitter-sfapex", + "variant": "Apex", + "feature": "lang-apex", + "fixture": { + "prefix": "apex", + "extension": "trigger" + }, + "smoke": "public class Example { public Integer f(Integer x) { return x + 1; } }\n" +} diff --git a/languages/build.py b/languages/build.py new file mode 100644 index 000000000..e5f092d7a --- /dev/null +++ b/languages/build.py @@ -0,0 +1,105 @@ +"""Build locked grammar sources; package final (already stripped/signed) bytes.""" +import argparse +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tarfile +from definitions import check + +ROOT = Path(__file__).resolve().parent.parent +TARGETS = { + 'aarch64-apple-darwin': ('darwin-arm64', 'darwin', 'arm64'), + 'x86_64-apple-darwin': ('darwin-x64', 'darwin', 'x64'), + 'aarch64-unknown-linux-gnu': ('linux-arm64-gnu', 'linux', 'arm64'), + 'x86_64-unknown-linux-gnu': ('linux-x64-gnu', 'linux', 'x64'), +} + + +def read(path): + return json.loads(path.read_text()) + + +def write(path, value): + path.write_text(json.dumps(value, indent=2) + '\n') + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument('action', choices=['build', 'package', 'check']) + cli.add_argument('--target', choices=TARGETS) + args = cli.parse_args() + definitions = check() + if args.action == 'check': + print(f'{len(definitions)} optional grammar definitions validated') + return + if args.target is None: + cli.error('--target is required for build/package') + suffix, system, cpu = TARGETS[args.target] + host = subprocess.check_output(['rustc', '-vV'], text=True).split('host: ')[1].splitlines()[0] + if args.target != host and not (system == 'darwin' and host.endswith('-apple-darwin')): + cli.error('build/package on the native target runner (macOS also supports cross-architecture builds)') + architecture = ['-arch', 'arm64' if cpu == 'arm64' else 'x86_64'] if system == 'darwin' else [] + output = ROOT / 'target/languages' / args.target + package = output / 'package' + template = read(ROOT / 'languages/package.template.json') + name = '@dev.fast/diffr-languages-extra-' + suffix + if args.action == 'build': + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--all-features', '--format-version', '1'], cwd=ROOT)) + if package.exists(): + shutil.rmtree(package) + package.mkdir(parents=True) + (output / 'test-fixtures.txt').write_text(''.join( + f"{d['fixture']['prefix']}:{d['fixture']['extension']}\n" for d in definitions)) + libraries = [] + for definition in definitions: + crate = next(p for p in metadata['packages'] if p['name'] == definition['crate'] and p['version'] == definition['version']) + crate_root = Path(crate['manifest_path']).parent + assert read(crate_root / '.cargo_vcs_info.json')['git']['sha1'] == definition['revision'] + source = crate_root / definition['source'] + filename = definition['id'] + ('.dylib' if system == 'darwin' else '.so') + sources = [source / 'parser.c'] + if (source / 'scanner.c').exists(): + sources.append(source / 'scanner.c') + subprocess.run([os.environ.get('CC', 'cc'), *architecture, '-O2', '-std=c11', '-fPIC', '-I' + str(source), '-dynamiclib' if system == 'darwin' else '-shared', *map(str, sources), '-o', str(package / filename)], check=True) + subprocess.run(['strip', '-x' if system == 'darwin' else '--strip-unneeded', str(package / filename)], check=True) + abi = int(re.search(r'#define LANGUAGE_VERSION (\d+)', (source / 'parser.c').read_text())[1]) + libraries.append(dict(id=definition['id'], file=filename, symbol=definition['symbol'], abi=abi)) + shutil.copyfile(ROOT / 'languages' / definition['id'] / 'LICENSE', package / (definition['id'] + '.LICENSE')) + write(package / 'manifest.json', dict(schema=1, pack='extra', version=template['version'], target=args.target, libraries=libraries)) + write(package / 'package.json', dict(template, name=name, os=[system], cpu=[cpu], **({'libc': ['glibc']} if system == 'linux' else {}))) + else: + manifest = read(package / 'manifest.json') + assert manifest['target'] == args.target and manifest['version'] == template['version'] + expected = {(d['id'], d['symbol']) for d in definitions} + assert {(d['id'], d['symbol']) for d in manifest['libraries']} == expected, 'stale package: rebuild the language libraries' + expected_files = {'manifest.json', 'package.json'} + for library in manifest['libraries']: + expected_files.update([library['file'], library['id'] + '.LICENSE']) + assert {p.name for p in package.iterdir()} == expected_files, 'unexpected or missing package files' + files = {p.name: dict(size=p.stat().st_size, sha256=digest(p.read_bytes())) for p in sorted(package.iterdir())} + archive = output / (name.split('/')[1] + '-' + template['version'] + '.tgz') + with archive.open('wb') as raw: + with gzip.GzipFile(fileobj=raw, mode='wb', mtime=0, filename='') as compressed: + with tarfile.open(fileobj=compressed, mode='w', format=tarfile.USTAR_FORMAT) as tar: + for filename in files: + data = (package / filename).read_bytes() + entry = tarfile.TarInfo('package/' + filename) + entry.size, entry.mode = len(data), 0o644 + tar.addfile(entry, io.BytesIO(data)) + entry = dict(manifest, name=name, url=f'https://registry.npmjs.org/{name}/-/{archive.name}', size=archive.stat().st_size, sha256=digest(archive.read_bytes()), files=files) + write(output / 'catalog-entry.json', entry) + print(archive) + + +if __name__ == '__main__': + main() diff --git a/languages/catalog.json b/languages/catalog.json new file mode 100644 index 000000000..2d0b35ebb --- /dev/null +++ b/languages/catalog.json @@ -0,0 +1 @@ +{"schema":1,"packages":[]} diff --git a/languages/definitions.py b/languages/definitions.py new file mode 100644 index 000000000..efc510a01 --- /dev/null +++ b/languages/definitions.py @@ -0,0 +1,46 @@ +"""Shared definitions for packaging, smoke tests, and optional-parser checks.""" +import json +from pathlib import Path +import re +import subprocess + +ROOT = Path(__file__).resolve().parent.parent + + +def definitions(root=ROOT): + return [json.loads(path.read_text()) for path in sorted((root / 'languages').glob('*/language.json'))] + + +def validate(items, metadata, root=ROOT): + package = next(p for p in metadata['packages'] if p['name'] == 'diffr-cli') + dependencies = {d['name']: d for d in package['dependencies']} + registry = set(re.findall(r'(\w+)\s*=>\s*\("([^"]+)",\s*"([^"]+)"\)', (root / 'src/parse/optional.rs').read_text())) + expected = {(d['variant'], d['id'], d['feature']) for d in items} + assert registry == expected, 'optional Rust registry and language definitions differ' + assert len({d['id'] for d in items}) == len(items), 'duplicate language ID' + assert {d['crate'] for d in items} == {name for name, dep in dependencies.items() if dep['optional'] and name.startswith(('tree-sitter-', 'ts-parser-'))}, 'optional Cargo dependencies and definitions differ' + assert {d['feature'] for d in items} == {name for name in package['features'] if name.startswith('lang-')}, 'language Cargo features and definitions differ' + for definition in items: + id, feature, crate = (definition[k] for k in ('id', 'feature', 'crate')) + assert re.fullmatch(r'[a-z][a-z0-9-]*', id), f'{id}: invalid ID' + assert definition['variant'].isidentifier(), f'{id}: invalid enum variant' + assert package['features'][feature] == ['dep:' + crate], f'{id}: incorrect Cargo feature' + assert feature in package['features']['all-languages'], f'{id}: missing from all-languages' + assert dependencies[crate]['optional'], f'{id}: dependency must be optional' + source = Path(definition['source']) + assert not source.is_absolute() and '..' not in source.parts, f'{id}: invalid source path' + assert re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9]*', definition['symbol']), f'{id}: invalid symbol' + assert definition['smoke'].strip(), f'{id}: missing parse smoke input' + for filename in ['LICENSE', 'highlights.scm']: + assert (root / 'languages' / id / filename).is_file(), f'{id}: missing {filename}' + fixture = definition['fixture'] + for side in (1, 2): + path = root / 'sample_files' / f"{fixture['prefix']}_{side}.{fixture['extension']}" + assert path.is_file(), f'{id}: missing fixture {path}' + + +def check(root=ROOT): + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--no-deps', '--format-version', '1'], cwd=root)) + items = definitions(root) + validate(items, metadata, root) + return items diff --git a/languages/fortran/LICENSE b/languages/fortran/LICENSE new file mode 100644 index 000000000..d13cc4b26 --- /dev/null +++ b/languages/fortran/LICENSE @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/fortran/highlights.scm b/languages/fortran/highlights.scm new file mode 100644 index 000000000..a56775ca8 --- /dev/null +++ b/languages/fortran/highlights.scm @@ -0,0 +1,204 @@ +[ + (identifier) + (module_name) +] @variable + +(string_literal) @string +(number_literal) @number +(statement_label) @number +(statement_label_reference) @number +(boolean_literal) @boolean +(comment) @comment +(custom_directive) @custom_directive + +[ + (derived_type) + (derived_type_statement) + (import_statement) + (intrinsic_type) + (type_name) +] @type + +(intrinsic_type) @type.builtin + +(base_type_specifier + (identifier) @type) + +[ + (module_statement) + (submodule_statement) +] @module + +[ + (abstract_specifier) + (access_specifier) + (block_label) + (block_label_start_expression) + (none) + (procedure_attributes) + (procedure_qualifier) + (type_qualifier) +] @attribute + +[ + "#define" + "#elif" + "#endif" + "#if" + "#ifdef" + (base_type_specifier) + (block_construct) + (contains_statement) + (default) + (end_associate_statement) + (end_block_construct_statement) + (end_block_data_statement) + (end_coarray_critical_statement) + (end_coarray_team_statement) + (end_do_loop_statement) + (end_enum_statement) + (end_enumeration_type_statement) + (end_forall_statement) + (end_function_statement) + (end_if_statement) + (end_interface_statement) + (end_module_procedure_statement) + (end_module_statement) + (end_program_statement) + (end_select_statement) + (end_submodule_statement) + (end_subroutine_statement) + (end_type_statement) + (end_where_statement) + (enum_statement) + (enumeration_type_statement) + (enumerator_statement) + (equivalence_statement) + (function_statement) + (implicit_statement) + (interface_statement) + (keyword_statement) + (language_binding) + (namelist_statement) + (print_statement) + (procedure_statement) + (program_statement) + (subroutine_statement) +] @keyword + +(use_statement "use" @keyword) +(use_statement "intrinsic" @keyword) +(included_items "only" @keyword) +(allocate_statement "allocate" @keyword) +(deallocate_statement "deallocate" @keyword) +(subroutine_call "call" @keyword) +(do_statement "do" @keyword) +(while_statement "while" @keyword) +(if_statement ["if" "then"] @keyword) +(elseif_clause ["else" "if" "elseif"] @keyword) +(else_clause "else" @keyword) +(open_statement "open" @keyword) +(write_statement "write" @keyword) +(private_statement "private" @keyword) +(public_statement "public" @keyword) + + +(select_case_statement "select" @keyword "case" @keyword) +(select_type_statement "select" @keyword "type" @keyword) +(select_rank_statement "select" @keyword "rank" @keyword) +(case_statement "case" @keyword) +(type_statement "type" @keyword) +(rank_statement "rank" @keyword) + + +[ + "*" + "+" + "-" + "/" + "=" + "<" + ">" + "<=" + ">=" + "==" + "/=" + ".and." + ".or." + ".lt." + ".gt." + ".ge." + ".le." + ".eq." + ".eqv." + ".neqv." + ".ne." +] @operator + +;; Brackets +[ + "(" + ")" + "[" + "]" + "<<<" + ">>>" +] @punctuation.bracket + +;; Delimiter +[ + "::" + "," + "%" + ":" +] @punctuation.delimiter + +"&" @punctuation.special + +(parameters + (identifier) @variable.parameter) + +(program_statement + (name) @variable) + +(module_statement + (name) @variable) + +(submodule_statement + (module_name) (name) @variable) + +(function_statement + (name) @function) + +(subroutine_statement + (name) @function) + +(module_procedure_statement + (name) @function) + +(end_program_statement + (name) @variable) + +(end_module_statement + (name) @variable) + +(end_submodule_statement + (name) @variable) + +(end_function_statement + (name) @function) + +(end_subroutine_statement + (name) @function) + +(end_module_procedure_statement + (name) @function) + +(subroutine_call + (identifier) @function) + +(keyword_argument + name: (identifier) @keyword) + +(derived_type_member_expression + (type_member) @property) diff --git a/languages/fortran/language.json b/languages/fortran/language.json new file mode 100644 index 000000000..317c6ade3 --- /dev/null +++ b/languages/fortran/language.json @@ -0,0 +1,16 @@ +{ + "id": "fortran", + "crate": "tree-sitter-fortran", + "version": "0.6.0", + "source": "src", + "symbol": "tree_sitter_fortran", + "revision": "48593829df8a929bfe58528eedcf405171921005", + "repository": "https://github.com/stadelmanma/tree-sitter-fortran", + "variant": "Fortran", + "feature": "lang-fortran", + "fixture": { + "prefix": "fortran", + "extension": "f90" + }, + "smoke": "program hello\n implicit none\n print *, \"hello\"\nend program hello\n" +} diff --git a/languages/fsharp/LICENSE b/languages/fsharp/LICENSE new file mode 100644 index 000000000..e80345de4 --- /dev/null +++ b/languages/fsharp/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Nikolaj Sidorenco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/fsharp/highlights.scm b/languages/fsharp/highlights.scm new file mode 100644 index 000000000..be6f2294c --- /dev/null +++ b/languages/fsharp/highlights.scm @@ -0,0 +1,421 @@ +;; ---------------------------------------------------------------------------- +;; Literals and comments + +[ + (line_comment) + (xml_doc) + (block_comment) +] @comment @spell + +(xml_doc) @comment.documentation @spell + +(const + [ + (_) @constant + (unit) @constant.builtin + ]) + +(primary_constr_args (_) @variable.parameter) + +(class_as_reference + (_) @variable.parameter.builtin) + + +((argument_patterns (long_identifier (identifier) @character.special)) + (#match? @character.special "^\_.*")) + +;; ---------------------------------------------------------------------------- +;; Punctuation + +(type_name type_name: (_) @type.definition) +(exception_definition exception_name: (_) @type.definition) + +[ + (_type) + (atomic_type) +] @type + +(member_signature + . + (identifier) @function.member + (curried_spec + (arguments_spec + "*"* @operator + (argument_spec + (argument_name_spec + "?"? @character.special + name: (_) @variable.parameter))))) + +(union_type_case (identifier) @constant) + +(rules + (rule + pattern: (_) @constant + block: (_))) + +(wildcard_pattern) @character.special + +(identifier_pattern + . + (_) @constant + . + (_) @variable) + +(optional_pattern + "?" @character.special) + +(fsi_directive_decl . (string) @module) + +(import_decl . (_) @module) +(named_module + name: (_) @module) +(namespace + name: (_) @module) +(module_defn + . + (_) @module) + +(ce_expression + . + (_) @constant.macro) + +(field_initializer + field: (_) @property) + +(record_fields + (record_field + . + (identifier) @property)) + +(value_declaration_left . (_) @variable) + +(function_declaration_left + . (_) @function) + +(argument_patterns) @variable.parameter +(typed_pattern + (_pattern) @variable.parameter + (_type) @type) + +(member_defn + (method_or_prop_defn + [ + (property_or_ident) @function + (property_or_ident + instance: (identifier) @variable.parameter.builtin + method: (identifier) @function.method) + ] + args: (_)* @variable.parameter)) + + +(dot_expression + . + (_) @variable.member + . + (_)) + +(application_expression + . + (long_identifier_or_op + (identifier) @function.call) + . + (_) @variable) + +(application_expression + . + (dot_expression + field: (long_identifier_or_op + (identifier) @function.call)) + . + (_) @variable) + +(application_expression + . + (typed_expression + (long_identifier_or_op + (identifier) @function.call) + (_)) + . + (_) @variable) + +(application_expression + . + (typed_expression + (dot_expression + field: (long_identifier_or_op + (identifier) @function.call)) + (_)) + . + (_) @variable) + +((infix_expression + . + (_) + . + (infix_op) @operator + . + (_) @function.call + ) + (#eq? @operator "|>") + ) + +((infix_expression + . + (_) @function.call + . + (infix_op) @operator + . + (_) + ) + (#eq? @operator "<|") + ) + +[ + (xint) + (int) + (int16) + (uint16) + (int32) + (uint32) + (int64) + (uint64) + (nativeint) + (unativeint) +] @number + +[ + (ieee32) + (ieee64) + (float) + (decimal) +] @number.float + +(bool) @boolean + +([ + (string) + (triple_quoted_string) + (verbatim_string) + (char) + (format_string) + (format_triple_quoted_string) +] @spell @string) + +(compiler_directive_decl) @keyword.directive + +(preproc_line + "#line" @keyword.directive) + +(attribute + target: (identifier)? @keyword + (_type) @attribute) + +[ + "(" + ")" + "{" + "}" + "[" + "]" + "[|" + "|]" + "{|" + "|}" +] @punctuation.bracket + +[ + "[<" + ">]" +] @punctuation.special + +(format_string_eval + [ + "{" + "}" + ] @punctuation.special) + +[ + "," + ";" + ":" + "." +] @punctuation.delimiter + +[ + "|" + "=" + ">" + "<" + "-" + "~" + "->" + "<-" + "&" + "&&" + "|" + "||" + ":>" + ":?>" + ".." + (infix_op) + (prefix_op) + (op_identifier) +] @operator + +(generic_type + [ + "<" + ">" + ] @punctuation.bracket) + +(typed_expression + ">" @punctuation.bracket) + +[ + "if" + "then" + "else" + "elif" + "when" + "match" + "match!" +] @keyword.conditional + +[ + "and" + "or" + "not" + "upcast" + "downcast" +] @keyword.operator + +[ + "return" + "return!" + "yield" + "yield!" +] @keyword.return + +[ + "for" + "while" + "downto" + "to" +] @keyword.repeat + + +[ + "open" + "#r" + "#load" +] @keyword.import + +[ + "abstract" + "delegate" + "extern" + "static" + "inline" + "mutable" + "override" + "rec" + "global" + (access_modifier) +] @keyword.modifier + +[ + "let" + "let!" + "use" + "use!" + "and!" + "member" +] @keyword.function + +[ + "enum" + "type" + "exception" + "inherit" + "interface" + "and" + "class" + "struct" +] @keyword.type + +((identifier) @keyword.exception + (#any-of? @keyword.exception "failwith" "failwithf" "raise" "reraise")) + +[ + "as" + "assert" + "begin" + "end" + "done" + "default" + "in" + "do" + "do!" + "fun" + "function" + "get" + "set" + "lazy" + "new" + "of" + "struct" + "val" + "module" + "namespace" + "with" +] @keyword + +[ + "null" +] @constant.builtin + +(match_expression "with" @keyword.conditional) + +(try_expression + [ + "try" + "with" + "finally" + ] @keyword.exception) + +((_type + (simple_type + (long_identifier + (identifier) @type.builtin))) + (#any-of? @type.builtin "bool" "byte" "sbyte" "int16" "uint16" "int" "uint" "int64" "uint64" "nativeint" "unativeint" "decimal" "float" "double" "float32" "single" "char" "string" "unit")) + +(preproc_if + [ + "#if" @keyword.directive + "#endif" @keyword.directive + ] + condition: (_)? @keyword.directive) + +(preproc_else + "#else" @keyword.directive) + +((long_identifier + (identifier)+ @variable.member + . + (identifier))) + +((identifier) @module.builtin + (#any-of? @module.builtin "Array" "Async" "Directory" "File" "List" "Option" "Path" "Map" "Set" "Lazy" "Seq" "Task" "String" "Result" )) + +((value_declaration + (attributes + (attribute + (_type + (simple_type + (long_identifier + (identifier) @attribute))))) + (function_or_value_defn + (value_declaration_left + . + (_) @constant))) + (#eq? @attribute "Literal")) diff --git a/languages/fsharp/language.json b/languages/fsharp/language.json new file mode 100644 index 000000000..bfcab294a --- /dev/null +++ b/languages/fsharp/language.json @@ -0,0 +1,16 @@ +{ + "id": "fsharp", + "crate": "tree-sitter-fsharp", + "version": "0.3.0", + "source": "fsharp/src", + "symbol": "tree_sitter_fsharp", + "revision": "5247c1197cb290fcaea0e0a793d32829c1396831", + "repository": "https://github.com/ionide/tree-sitter-fsharp", + "variant": "FSharp", + "feature": "lang-fsharp", + "fixture": { + "prefix": "f_sharp", + "extension": "fs" + }, + "smoke": "module Example\nlet f x =\n let y = x + 1\n y * 2\n" +} diff --git a/languages/haskell/LICENSE b/languages/haskell/LICENSE new file mode 100644 index 000000000..4b52d191c --- /dev/null +++ b/languages/haskell/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/haskell/highlights.scm b/languages/haskell/highlights.scm new file mode 100644 index 000000000..a81cced88 --- /dev/null +++ b/languages/haskell/highlights.scm @@ -0,0 +1,443 @@ +; ---------------------------------------------------------------------------- +; Parameters and variables +; NOTE: These are at the top, so that they have low priority, +; and don't override destructured parameters +(variable) @variable + +(pattern/wildcard) @variable + +(decl/function + patterns: (patterns + (_) @variable.parameter)) + +(expression/lambda + (_)+ @variable.parameter + "->") + +(decl/function + (infix + (pattern) @variable.parameter)) + +; ---------------------------------------------------------------------------- +; Literals and comments +(integer) @number + +(negation) @number + +(expression/literal + (float)) @number.float + +(char) @character + +(string) @string + +(unit) @string.special.symbol ; unit, as in () + +(comment) @comment + +((haddock) @comment.documentation) + +; ---------------------------------------------------------------------------- +; Punctuation +[ + "(" + ")" + "{" + "}" + "[" + "]" +] @punctuation.bracket + +[ + "," + ";" +] @punctuation.delimiter + +; ---------------------------------------------------------------------------- +; Keywords, operators, includes +[ + "forall" + ; "∀" ; utf-8 is not cross-platform safe +] @keyword.repeat + +(pragma) @keyword.directive + +[ + "if" + "then" + "else" + "case" + "of" +] @keyword.conditional + +[ + "import" + "qualified" + "module" +] @keyword.import + +[ + (operator) + (constructor_operator) + (all_names) + (wildcard) + "." + ".." + "=" + "|" + "::" + "=>" + "->" + "<-" + "\\" + "`" + "@" +] @operator + +; TODO broken, also huh? +; ((qualified_module +; (module) @constructor) +; . +; (module)) + +(module + (module_id) @module) + +[ + "where" + "let" + "in" + "class" + "instance" + "pattern" + "data" + "newtype" + "family" + "type" + "as" + "hiding" + "deriving" + "via" + "stock" + "anyclass" + "do" + "mdo" + "rec" + "infix" + "infixl" + "infixr" +] @keyword + +; ---------------------------------------------------------------------------- +; Functions and variables +(decl + [ + name: (variable) @function + names: (binding_list (variable) @function) + ]) + +(decl/bind + name: (variable) @variable) + +; Consider signatures (and accompanying functions) +; with only one value on the rhs as variables +(decl/signature + name: (variable) @variable + type: (type)) + +((decl/signature + name: (variable) @_name + type: (type)) + . + (decl + name: (variable) @variable) + match: (_) + (#eq? @_name @variable)) + +; but consider a type that involves 'IO' a decl/function +(decl/signature + name: (variable) @function + type: (type/apply + constructor: (name) @_type) + (#eq? @_type "IO")) + +((decl/signature + name: (variable) @_name + type: (type/apply + constructor: (name) @_type) + (#eq? @_type "IO")) + . + (decl + name: (variable) @function) + match: (_) + (#eq? @_name @function)) + +((decl/signature) @function + . + (decl/function + name: (variable) @function)) + +(decl/bind + name: (variable) @function + (match + expression: (expression/lambda))) + +; view patterns +(view_pattern + [ + (expression/variable) @function.call + (expression/qualified + (variable) @function.call) + ]) + +; consider infix functions as operators +(infix_id + [ + (variable) @operator + (qualified + (variable) @operator) + ]) + +; decl/function calls with an infix operator +; e.g. func <$> a <*> b +(infix + [ + (variable) @function.call + (qualified + ((module) @module + (variable) @function.call)) + ] + . + (operator)) + +; infix operators applied to variables +((expression/variable) @variable + . + (operator)) + +((operator) + . + [ + (expression/variable) @variable + (expression/qualified + (variable) @variable) + ]) + +; decl/function calls with infix operators +([ + (expression/variable) @function.call + (expression/qualified + (variable) @function.call) + ] + . + (operator) @_op + (#any-of? @_op "$" "<$>" ">>=" "=<<")) + +; right hand side of infix operator +((infix + [ + (operator) + (infix_id (variable)) + ] ; infix or `func` + . + [ + (variable) @function.call + (qualified + (variable) @function.call) + ]) + . + (operator) @_op + (#any-of? @_op "$" "<$>" "=<<")) + +; decl/function composition, arrows, monadic composition (lhs) +( + [ + (expression/variable) @function + (expression/qualified + (variable) @function) + ] + . + (operator) @_op + (#any-of? @_op "." ">>>" "***" ">=>" "<=<")) + +; right hand side of infix operator +((infix + [ + (operator) + (infix_id (variable)) + ] ; infix or `func` + . + [ + (variable) @function + (qualified + (variable) @function) + ]) + . + (operator) @_op + (#any-of? @_op "." ">>>" "***" ">=>" "<=<")) + +; function composition, arrows, monadic composition (rhs) +((operator) @_op + . + [ + (expression/variable) @function + (expression/qualified + (variable) @function) + ] + (#any-of? @_op "." ">>>" "***" ">=>" "<=<")) + +; function defined in terms of a function composition +(decl/function + name: (variable) @function + (match + expression: (infix + operator: (operator) @_op + (#any-of? @_op "." ">>>" "***" ">=>" "<=<")))) + +(apply + [ + (expression/variable) @function.call + (expression/qualified + (variable) @function.call) + ]) + +; function compositions, in parentheses, applied +; lhs +(apply + . + (expression/parens + (infix + [ + (variable) @function.call + (qualified + (variable) @function.call) + ] + . + (operator)))) + +; rhs +(apply + . + (expression/parens + (infix + (operator) + . + [ + (variable) @function.call + (qualified + (variable) @function.call) + ]))) + +; variables being passed to a function call +(apply + (_) + . + [ + (expression/variable) @variable + (expression/qualified + (variable) @variable) + ]) + +; main is always a function +; (this prevents `main = undefined` from being highlighted as a variable) +(decl/bind + name: (variable) @function + (#eq? @function "main")) + +; scoped function types (func :: a -> b) +(signature + pattern: (pattern/variable) @function + type: (quantified_type)) + +; signatures that have a function type +; + binds that follow them +(decl/signature + name: (variable) @function + type: (quantified_type)) + +((decl/signature + name: (variable) @_name + type: (quantified_type)) + . + (decl/bind + (variable) @function) + (#eq? @function @_name)) + +; ---------------------------------------------------------------------------- +; Types +(name) @type + +(type/star) @type + +(variable) @type + +(constructor) @constructor + +; True or False +((constructor) @boolean + (#any-of? @boolean "True" "False")) + +; otherwise (= True) +((variable) @boolean + (#eq? @boolean "otherwise")) + +; ---------------------------------------------------------------------------- +; Quasi-quotes +(quoter) @function.call + +(quasiquote + [ + (quoter) @_name + (_ + (variable) @_name) + ] + (#eq? @_name "qq") + (quasiquote_body) @string) + +(quasiquote + (_ + (variable) @_name) + (#eq? @_name "qq") + (quasiquote_body) @string) + +; namespaced quasi-quoter +(quasiquote + (_ + (module) @module + . + (variable) @function.call)) + +; Highlighting of quasiquote_body for other languages is handled by injections.scm +; ---------------------------------------------------------------------------- +; Exceptions/error handling +((variable) @keyword.exception + (#any-of? @keyword.exception + "error" "undefined" "try" "tryJust" "tryAny" "catch" "catches" "catchJust" "handle" "handleJust" + "throw" "throwIO" "throwTo" "throwError" "ioError" "mask" "mask_" "uninterruptibleMask" + "uninterruptibleMask_" "bracket" "bracket_" "bracketOnErrorSource" "finally" "fail" + "onException" "expectationFailure")) + +; ---------------------------------------------------------------------------- +; Debugging +((variable) @keyword.debug + (#any-of? @keyword.debug + "trace" "traceId" "traceShow" "traceShowId" "traceWith" "traceShowWith" "traceStack" "traceIO" + "traceM" "traceShowM" "traceEvent" "traceEventWith" "traceEventIO" "flushEventLog" "traceMarker" + "traceMarkerIO")) + +; ---------------------------------------------------------------------------- +; Fields + +(field_name + (variable) @variable.member) + +(import_name + (name) + . + (children + (variable) @variable.member)) + + +; ---------------------------------------------------------------------------- +; Spell checking +(comment) @spell diff --git a/languages/haskell/language.json b/languages/haskell/language.json new file mode 100644 index 000000000..700b40745 --- /dev/null +++ b/languages/haskell/language.json @@ -0,0 +1,16 @@ +{ + "id": "haskell", + "crate": "tree-sitter-haskell", + "version": "0.23.1", + "source": "src", + "symbol": "tree_sitter_haskell", + "revision": "c30d812bc90827f1a54106a25bc9a6307f5cdcec", + "repository": "https://github.com/tree-sitter/tree-sitter-haskell", + "variant": "Haskell", + "feature": "lang-haskell", + "fixture": { + "prefix": "haskell", + "extension": "hs" + }, + "smoke": "module Example where\nf x =\n let y = x + 1\n in y * 2\n" +} diff --git a/languages/julia/LICENSE b/languages/julia/LICENSE new file mode 100644 index 000000000..607d94b7f --- /dev/null +++ b/languages/julia/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld, GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/julia/highlights.scm b/languages/julia/highlights.scm new file mode 100644 index 000000000..79f1c1819 --- /dev/null +++ b/languages/julia/highlights.scm @@ -0,0 +1,330 @@ +; https://github.com/tree-sitter/tree-sitter-julia/blob/e01c928d11375513138a175a68485c4d53e55ea9/queries/highlights.scm +; MIT license + +; Identifiers +(identifier) @variable + +(field_expression + (identifier) @variable.member .) + +; Symbols +(quote_expression + ":" @string.special.symbol + [ + (identifier) + (operator) + ] @string.special.symbol) + +; Function calls +(call_expression + (identifier) @function.call) + +(call_expression + (field_expression + (identifier) @function.call .)) + +(broadcast_call_expression + (identifier) @function.call) + +(broadcast_call_expression + (field_expression + (identifier) @function.call .)) + +; Macros +(macro_identifier + "@" @function.macro + (_) @function.macro) + +(macro_definition + (signature + (call_expression + . + (identifier) @function.macro))) + +; Built-in functions +; filter(name -> Base.eval(Core, name) isa Core.Builtin, names(Core)) +((identifier) @function.builtin + (#any-of? @function.builtin + "applicable" "fieldtype" "getfield" "getglobal" "invoke" "isa" "isdefined" "modifyfield!" + "modifyglobal!" "nfields" "replacefield!" "replaceglobal!" "setfield!" "setfieldonce!" + "setglobal!" "setglobalonce!" "swapfield!" "swapglobal!" "throw" "tuple" "typeassert" "typeof")) + +; Type definitions +(type_head (_) @type.definition) + +; Type annotations +(parametrized_type_expression + [ + (identifier) @type + (field_expression + (identifier) @type .) + ] + (curly_expression + (_) @type)) + +(typed_expression + (identifier) @type .) + +(unary_typed_expression + (identifier) @type .) + +(where_expression + (_) @type .) + +(binary_expression + (_) @type + (operator) @operator + (_) @type + (#any-of? @operator "<:" ">:")) + +; Built-in types +; filter(name -> typeof(Base.eval(Core, name)) in [DataType, UnionAll], names(Core)) +((identifier) @type.builtin + (#any-of? @type.builtin + "AbstractArray" "AbstractChar" "AbstractFloat" "AbstractString" "Any" "ArgumentError" "Array" + "AssertionError" "Bool" "BoundsError" "Char" "ConcurrencyViolationError" "Cvoid" "DataType" + "DenseArray" "DivideError" "DomainError" "ErrorException" "Exception" "Expr" "Float16" "Float32" + "Float64" "Function" "GlobalRef" "IO" "InexactError" "InitError" "Int" "Int128" "Int16" "Int32" + "Int64" "Int8" "Integer" "InterruptException" "LineNumberNode" "LoadError" "Method" + "MethodError" "Module" "NTuple" "NamedTuple" "Nothing" "Number" "OutOfMemoryError" + "OverflowError" "Pair" "Ptr" "QuoteNode" "ReadOnlyMemoryError" "Real" "Ref" "SegmentationFault" + "Signed" "StackOverflowError" "String" "Symbol" "Task" "Tuple" "Type" "TypeError" "TypeVar" + "UInt" "UInt128" "UInt16" "UInt32" "UInt64" "UInt8" "UndefInitializer" "UndefKeywordError" + "UndefRefError" "UndefVarError" "Union" "UnionAll" "Unsigned" "VecElement" "WeakRef")) + +; Keywords +[ + "const" + "global" + "local" +] @keyword + +(compound_statement + [ + "begin" + "end" + ] @keyword) + +(quote_statement + [ + "quote" + "end" + ] @keyword) + +(let_statement + [ + "let" + "end" + ] @keyword) + +(if_statement + [ + "if" + "end" + ] @keyword.conditional) + +(elseif_clause + "elseif" @keyword.conditional) + +(else_clause + "else" @keyword.conditional) + +(ternary_expression + [ + "?" + ":" + ] @keyword.conditional.ternary) + +(try_statement + [ + "try" + "end" + ] @keyword.exception) + +(catch_clause + "catch" @keyword.exception) + +(finally_clause + "finally" @keyword.exception) + +(for_statement + [ + "for" + "end" + ] @keyword.repeat) + +(for_binding + "outer" @keyword.repeat) + +; comprehensions +(for_clause + "for" @keyword.repeat) + +(if_clause + "if" @keyword.conditional) + +(while_statement + [ + "while" + "end" + ] @keyword.repeat) + +[ + (break_statement) + (continue_statement) +] @keyword.repeat + +(function_definition + [ + "function" + "end" + ] @keyword.function) + +(do_clause + [ + "do" + "end" + ] @keyword.function) + +(macro_definition + [ + "macro" + "end" + ] @keyword) + +(return_statement + "return" @keyword.return) + +(module_definition + [ + "module" + "baremodule" + "end" + ] @keyword.import) + +(export_statement + "export" @keyword.import) + +(public_statement + "public" @keyword.import) + +(import_statement + "import" @keyword.import) + +(using_statement + "using" @keyword.import) + +(import_alias + "as" @keyword.import) + +(selected_import + ":" @punctuation.delimiter) + +(struct_definition + [ + "mutable" + "struct" + "end" + ] @keyword.type) + +(abstract_definition + [ + "abstract" + "type" + "end" + ] @keyword.type) + +(primitive_definition + [ + "primitive" + "type" + "end" + ] @keyword.type) + +; Operators & Punctuation +(operator) @operator + +(adjoint_expression + "'" @operator) + +(range_expression + ":" @operator) + +(arrow_function_expression + "->" @operator) + +[ + "." + "..." + "::" +] @punctuation + +[ + "," + ";" +] @punctuation.delimiter + +[ + "(" + ")" + "[" + "]" + "{" + "}" +] @punctuation.bracket + +; Keyword operators +((operator) @keyword.operator + (#any-of? @keyword.operator "in" "isa")) + +(where_expression + "where" @keyword.operator) + +; Built-in constants +((identifier) @constant.builtin + (#any-of? @constant.builtin "nothing" "missing")) + +((identifier) @variable.builtin + (#any-of? @variable.builtin "begin" "end") + (#has-ancestor? @variable.builtin index_expression)) + +; Literals +(boolean_literal) @boolean + +(integer_literal) @number + +(float_literal) @number.float + +((identifier) @number.float + (#any-of? @number.float "NaN" "NaN16" "NaN32" "Inf" "Inf16" "Inf32")) + +(character_literal) @character + +(escape_sequence) @string.escape + +(string_literal) @string + +(prefixed_string_literal + prefix: (identifier) @function.macro) @string + +(command_literal) @string.special + +(prefixed_command_literal + prefix: (identifier) @function.macro) @string.special + +((string_literal) @string.documentation + . + [ + (abstract_definition) + (assignment) + (const_statement) + (function_definition) + (macro_definition) + (module_definition) + (struct_definition) + ]) + +[ + (line_comment) + (block_comment) +] @comment diff --git a/languages/julia/language.json b/languages/julia/language.json new file mode 100644 index 000000000..d1e7f1101 --- /dev/null +++ b/languages/julia/language.json @@ -0,0 +1,16 @@ +{ + "id": "julia", + "crate": "tree-sitter-julia", + "version": "0.23.1", + "source": "src", + "symbol": "tree_sitter_julia", + "revision": "a8e1262997d5a45520a06cbe1b86c0737d507054", + "repository": "https://github.com/tree-sitter/tree-sitter-julia", + "variant": "Julia", + "feature": "lang-julia", + "fixture": { + "prefix": "julia", + "extension": "jl" + }, + "smoke": "function f(x)\n x + 1\nend\n" +} diff --git a/languages/ocaml-interface/LICENSE b/languages/ocaml-interface/LICENSE new file mode 100644 index 000000000..82557a14d --- /dev/null +++ b/languages/ocaml-interface/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Max Brunsfeld and Pieter Goetschalckx + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/ocaml-interface/highlights.scm b/languages/ocaml-interface/highlights.scm new file mode 100644 index 000000000..e69de29bb diff --git a/languages/ocaml-interface/language.json b/languages/ocaml-interface/language.json new file mode 100644 index 000000000..78114b6ee --- /dev/null +++ b/languages/ocaml-interface/language.json @@ -0,0 +1,16 @@ +{ + "id": "ocaml-interface", + "crate": "tree-sitter-ocaml", + "version": "0.25.0", + "source": "grammars/interface/src", + "symbol": "tree_sitter_ocaml_interface", + "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", + "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", + "variant": "OCamlInterface", + "feature": "lang-ocaml", + "fixture": { + "prefix": "ocaml_interface", + "extension": "mli" + }, + "smoke": "type t = { value : int }\nval f : int -> t\n" +} diff --git a/languages/ocaml/LICENSE b/languages/ocaml/LICENSE new file mode 100644 index 000000000..82557a14d --- /dev/null +++ b/languages/ocaml/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Max Brunsfeld and Pieter Goetschalckx + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/ocaml/highlights.scm b/languages/ocaml/highlights.scm new file mode 100644 index 000000000..8f73a9fb4 --- /dev/null +++ b/languages/ocaml/highlights.scm @@ -0,0 +1,148 @@ +; Punctuation +;------------ + +[ + "," "." ";" ":" "=" "|" "~" "?" "+" "-" "!" ">" "&" + "->" ";;" ":>" "+=" ":=" ".." +] @punctuation.delimiter + +["(" ")" "[" "]" "{" "}" "[|" "|]" "[<" "[>"] @punctuation.bracket + +(object_type ["<" ">"] @punctuation.bracket) + +"%" @punctuation.special + +(attribute ["[@" "]"] @punctuation.special) +(item_attribute ["[@@" "]"] @punctuation.special) +(floating_attribute ["[@@@" "]"] @punctuation.special) +(extension ["[%" "]"] @punctuation.special) +(item_extension ["[%%" "]"] @punctuation.special) +(quoted_extension ["{%" "}"] @punctuation.special) +(quoted_item_extension ["{%%" "}"] @punctuation.special) + +; Keywords +;--------- + +[ + "and" "as" "assert" "begin" "class" "constraint" "do" "done" "downto" "effect" + "else" "end" "exception" "external" "for" "fun" "function" "functor" "if" "in" + "include" "inherit" "initializer" "lazy" "let" "match" "method" "module" + "mutable" "new" "nonrec" "object" "of" "open" "private" "rec" "sig" "struct" + "then" "to" "try" "type" "val" "virtual" "when" "while" "with" +] @keyword + +; Operators +;---------- + +[ + (prefix_operator) + (sign_operator) + (pow_operator) + (mult_operator) + (add_operator) + (concat_operator) + (rel_operator) + (and_operator) + (or_operator) + (assign_operator) + (hash_operator) + (indexing_operator) + (let_operator) + (let_and_operator) + (match_operator) +] @operator + +(match_expression (match_operator) @keyword) + +(value_definition [(let_operator) (let_and_operator)] @keyword) + +["*" "#" "::" "<-"] @operator + +; Constants +;---------- + +(boolean) @constant + +[(number) (signed_number)] @number + +[(string) (character)] @string + +(quoted_string "{" @string "}" @string) @string + +(escape_sequence) @escape + +(conversion_specification) @string.special + +; Variables +;---------- + +[(value_name) (type_variable)] @variable + +(value_pattern) @variable.parameter + +; Properties +;----------- + +[(label_name) (field_name) (instance_variable_name)] @property + +; Functions +;---------- + +(let_binding + pattern: (value_name) @function + (parameter)) + +(let_binding + pattern: (value_name) @function + body: [(fun_expression) (function_expression)]) + +(value_specification (value_name) @function) + +(external (value_name) @function) + +(method_name) @function.method + +(application_expression + function: (value_path (value_name) @function)) + +(infix_expression + left: (value_path (value_name) @function) + operator: (concat_operator) @operator + (#eq? @operator "@@")) + +(infix_expression + operator: (rel_operator) @operator + right: (value_path (value_name) @function) + (#eq? @operator "|>")) + +( + (value_name) @function.builtin + (#match? @function.builtin "^(raise(_notrace)?|failwith|invalid_arg)$") +) + +; Types +;------ + +[(class_name) (class_type_name) (type_constructor)] @type + +( + (type_constructor) @type.builtin + (#match? @type.builtin "^(int|char|bytes|string|float|bool|unit|exn|array|list|option|int32|int64|nativeint|format6|lazy_t)$") +) + +[(constructor_name) (tag)] @constructor + +; Modules +;-------- + +[(module_name) (module_type_name)] @module + +; Attributes +;----------- + +(attribute_id) @tag + +; Comments +;--------- + +[(comment) (line_number_directive) (directive) (shebang)] @comment diff --git a/languages/ocaml/language.json b/languages/ocaml/language.json new file mode 100644 index 000000000..e35339e28 --- /dev/null +++ b/languages/ocaml/language.json @@ -0,0 +1,16 @@ +{ + "id": "ocaml", + "crate": "tree-sitter-ocaml", + "version": "0.25.0", + "source": "grammars/ocaml/src", + "symbol": "tree_sitter_ocaml", + "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", + "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", + "variant": "OCaml", + "feature": "lang-ocaml", + "fixture": { + "prefix": "ocaml", + "extension": "ml" + }, + "smoke": "module M = struct\n let f x = x + 1\nend\n" +} diff --git a/languages/package.template.json b/languages/package.template.json new file mode 100644 index 000000000..b22de7282 --- /dev/null +++ b/languages/package.template.json @@ -0,0 +1,6 @@ +{ + "version": "0.2.0", + "description": "Optional native language parsers for diffr", + "license": "MIT", + "repository": "https://github.com/devdotfast/diffr" +} diff --git a/languages/publish.py b/languages/publish.py new file mode 100644 index 000000000..3b8ecd6a3 --- /dev/null +++ b/languages/publish.py @@ -0,0 +1,58 @@ +"""Publish exact prepared archives, verify registry bytes, then emit a catalog. + +This command is used only by the protected publication workflow. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import time +import urllib.error +import urllib.request +from build import TARGETS + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory', type=Path) + parser.add_argument('--publish', action='store_true') + args = parser.parse_args() + (args.directory / "catalog.json").unlink(missing_ok=True) + entries = [json.loads(p.read_text()) for p in sorted(args.directory.rglob('catalog-entry.json'))] + assert len(entries) == 4 and {e['target'] for e in entries} == set(TARGETS), 'need exactly four targets' + assert len({e['version'] for e in entries}) == 1, 'mixed revisions' + for entry in entries: + archives = list(args.directory.rglob(entry['url'].rsplit('/', 1)[1])) + assert len(archives) == 1, 'missing or duplicate archive' + data = archives[0].read_bytes() + assert len(data) == entry['size'] and sha(data) == entry['sha256'] + if args.publish: + try: + with urllib.request.urlopen(entry['url'], timeout=60) as response: + existing = response.read(entry['size'] + 1) + assert len(existing) == entry['size'] and sha(existing) == entry['sha256'], 'immutable revision already has different bytes' + except urllib.error.HTTPError as error: + if error.code != 404: + raise + subprocess.run(['npm', 'publish', str(archives[0]), '--access', 'public', '--ignore-scripts'], check=True) + # Never advance the catalog based on local artifacts alone. + for attempt in range(12): + try: + with urllib.request.urlopen(entry['url'], timeout=60) as response: + downloaded = response.read(entry['size'] + 1) + assert len(downloaded) == entry['size'] and sha(downloaded) == entry['sha256'], 'registry byte mismatch' + break + except urllib.error.HTTPError as error: + if error.code != 404 or attempt == 11: + raise + time.sleep(5) + (args.directory / 'catalog.json').write_text(json.dumps({'schema': 1, 'packages': entries}, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/languages/qml/LICENSE b/languages/qml/LICENSE new file mode 100644 index 000000000..0b46abcac --- /dev/null +++ b/languages/qml/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2021 Yuya Nishihara + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/languages/qml/highlights.scm b/languages/qml/highlights.scm new file mode 100644 index 000000000..04a3e52bd --- /dev/null +++ b/languages/qml/highlights.scm @@ -0,0 +1,61 @@ +;;; Annotations + +(ui_annotation + "@" @operator + type_name: [ + (identifier) @attribute + (nested_identifier (identifier) @attribute) + ]) + +;;; Properties + +(ui_object_definition_binding + name: [ + (identifier) @property + (nested_identifier (identifier) @property) + ]) + +(ui_binding + name: [ + (identifier) @property + (nested_identifier (identifier) @property) + ]) + +;; locals query appears not working unless id: isn't a parameter. +(ui_binding + name: (identifier) @property + (#eq? @property "id") + value: (expression_statement (identifier) @variable.parameter)) + +(ui_property + name: (identifier) @property) + +(ui_required + name: (identifier) @property) + +(ui_list_property_type + ["<" ">"] @punctuation.bracket) + +;;; Signals + +(ui_signal + name: (identifier) @function.signal) + +(ui_signal_parameter + (identifier) @variable.parameter) + +;;; Keywords + +[ + "as" + "component" + "default" + "final" + "import" + "on" + "pragma" + "property" + "readonly" + "required" + "signal" +] @keyword diff --git a/languages/qml/language.json b/languages/qml/language.json new file mode 100644 index 000000000..8e0840a19 --- /dev/null +++ b/languages/qml/language.json @@ -0,0 +1,16 @@ +{ + "id": "qml", + "crate": "tree-sitter-qmljs", + "version": "0.3.0", + "source": "src", + "symbol": "tree_sitter_qmljs", + "revision": "0bec4359a7eb2f6c9220cd57372d87d236f66d59", + "repository": "https://github.com/yuja/tree-sitter-qmljs", + "variant": "Qml", + "feature": "lang-qml", + "fixture": { + "prefix": "qml", + "extension": "qml" + }, + "smoke": "import QtQuick\nItem { property int count: 1; function f(x) { return x + count; } }\n" +} diff --git a/languages/tests/Dockerfile b/languages/tests/Dockerfile new file mode 100644 index 000000000..fa2a9fcf9 --- /dev/null +++ b/languages/tests/Dockerfile @@ -0,0 +1,27 @@ +FROM rust:1.85-bookworm AS build +RUN apt-get update && apt-get install -y --no-install-recommends python3 cmake clang pkg-config && rm -rf /var/lib/apt/lists/* +ENV RUSTUP_TOOLCHAIN=1.85.0 CARGO_BUILD_JOBS=4 +WORKDIR /work +COPY . . +RUN --mount=type=cache,target=/work/target,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + target=$(rustc -vV | sed -n 's/^host: //p') && \ + cargo xtask build-languages --target "$target" && \ + python3 languages/tests/probe.py "target/languages/$target/package" && \ + cargo xtask package-languages --target "$target" && \ + python3 languages/tests/integration.py "target/languages/$target/catalog-entry.json" --release && \ + mkdir /artifacts && \ + cp "target/languages/$target/verification/diffr-native" /artifacts/diffr && \ + archive=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["url"].rsplit("/", 1)[1])' "target/languages/$target/catalog-entry.json") && \ + cp "target/languages/$target/$archive" /artifacts/extra.tgz && \ + cp "target/languages/$target/test-fixtures.txt" /artifacts/fixtures.txt + +FROM debian:bookworm-slim +COPY --from=build /artifacts/diffr /usr/local/bin/diffr +COPY --from=build /artifacts/extra.tgz /opt/extra.tgz +COPY --from=build /artifacts/fixtures.txt /opt/fixtures.txt +COPY sample_files /fixtures +COPY languages/tests/container-smoke.sh /opt/container-smoke.sh +ENV DIFFR_PARSER_DIR=/tmp/diffr-parsers +USER 65534:65534 +ENTRYPOINT ["/bin/sh", "/opt/container-smoke.sh"] diff --git a/languages/tests/Dockerfile.dockerignore b/languages/tests/Dockerfile.dockerignore new file mode 100644 index 000000000..6e6351b45 --- /dev/null +++ b/languages/tests/Dockerfile.dockerignore @@ -0,0 +1,5 @@ +.git +target +**/target +**/node_modules +**/__pycache__ diff --git a/languages/tests/container-smoke.sh b/languages/tests/container-smoke.sh new file mode 100644 index 000000000..3d6489ae2 --- /dev/null +++ b/languages/tests/container-smoke.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu +for tool in cargo rustc cc gcc node npm python3; do + if command -v "$tool" >/dev/null 2>&1; then + echo "Unexpected development tool: $tool" >&2 + exit 1 + fi +done +test ! -e "$DIFFR_PARSER_DIR" +diffr languages list --json > /tmp/before.json +grep -q 'not_installed' /tmp/before.json +diffr languages install extra --json --archive /opt/extra.tgz +diffr languages list --json > /tmp/after.json +test "$(grep -o '"availability":"installed"' /tmp/after.json | wc -l)" -eq "$(wc -l < /opt/fixtures.txt)" +while IFS=: read -r fixture extension; do + diffr --format ndjson --syntax --no-index "/fixtures/${fixture}_1.$extension" "/fixtures/${fixture}_2.$extension" > "/tmp/$fixture.ndjson" + if grep -q '"fallback":{' "/tmp/$fixture.ndjson"; then + echo "Unexpected fallback for $fixture" >&2 + exit 1 + fi +done < /opt/fixtures.txt +# An unchanged repeat is idempotent and needs no network or development tools. +diffr languages install extra --json --archive /opt/extra.tgz +for library in "$DIFFR_PARSER_DIR"/*/extra/*/fortran.so; do + printf 'broken' > "$library" +done +diffr --format ndjson --no-index /fixtures/fortran_1.f90 /fixtures/fortran_2.f90 > /tmp/corrupt.ndjson +grep -q 'parser_load_failed' /tmp/corrupt.ndjson +diffr languages install extra --json --archive /opt/extra.tgz +diffr --format ndjson --syntax --no-index /fixtures/fortran_1.f90 /fixtures/fortran_2.f90 > /tmp/repaired.ndjson +cmp /tmp/fortran.ndjson /tmp/repaired.ndjson || { + # Run summary timing is allowed to differ; compare deterministic file records. + grep '"type":"file"' /tmp/fortran.ndjson > /tmp/expected-file + grep '"type":"file"' /tmp/repaired.ndjson > /tmp/repaired-file + cmp /tmp/expected-file /tmp/repaired-file +} +printf 'PASS: fresh offline install, all optional parsers, idempotence, corrupt fallback, atomic repair; unprivileged and without development tools.\n' diff --git a/languages/tests/entitlements.plist b/languages/tests/entitlements.plist new file mode 100644 index 000000000..d7757761a --- /dev/null +++ b/languages/tests/entitlements.plist @@ -0,0 +1,5 @@ + + + +com.apple.security.cs.allow-unsigned-executable-memory + diff --git a/languages/tests/fixtures/scanner.f90 b/languages/tests/fixtures/scanner.f90 new file mode 100644 index 000000000..7bba6c05d --- /dev/null +++ b/languages/tests/fixtures/scanner.f90 @@ -0,0 +1,15 @@ +program scanner + implicit none + integer :: i, j + real :: total + character(len=32) :: message + total = 1.25e+2 + & + & 2.5e-1 + message = 'hello & + &world' + do 100 i = 1, 3 + do 100 j = 1, 2 + total = total + real(i * j) +100 continue + print *, message, total +end program scanner diff --git a/languages/tests/fixtures/scanner.fs b/languages/tests/fixtures/scanner.fs new file mode 100644 index 000000000..dc5ed5a72 --- /dev/null +++ b/languages/tests/fixtures/scanner.fs @@ -0,0 +1,14 @@ +module Scanner + +(* outer comment (* nested comment *) ends here *) +let message = """first +second""" + +let compute values = + values + |> List.map (fun value -> + match value with + | Some x -> + let doubled = x * 2 + doubled + 1 + | None -> 0) diff --git a/languages/tests/integration.py b/languages/tests/integration.py new file mode 100644 index 000000000..5b41475b2 --- /dev/null +++ b/languages/tests/integration.py @@ -0,0 +1,190 @@ +"""Build test-only pinned CLIs, compare native/static output, exercise offline repair. + +The tracked catalog is restored even on failure; no runtime trust override exists. +""" +import concurrent.futures +import json +import os +from pathlib import Path +import shutil +import statistics +import subprocess +import sys +import tempfile +import time + +root = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(root / 'languages')) +from definitions import definitions +items = definitions() +entry_path = Path(sys.argv[1]).resolve() +entry = json.loads(entry_path.read_text()) +archive = entry_path.parent / entry['url'].rsplit('/', 1)[1] +release = '--release' in sys.argv +profile = 'release' if release else 'debug' +catalog = root / 'languages/catalog.json' +saved = catalog.read_bytes() +output = root / 'target/languages' / entry['target'] / 'verification' +output.mkdir(exist_ok=True) +host = subprocess.check_output(['rustc', '-vV'], text=True).split('host: ')[1].splitlines()[0] + + +def build(features, name): + args = ['cargo', 'build', '--locked', '--bin', 'diffr'] + build_directory = root / 'target' + if entry['target'] != host: + args += ['--target', entry['target']] + build_directory /= entry['target'] + if release: + args += ['--release'] + if features: + args += ['--features', features] + subprocess.run(args, cwd=root, check=True) + destination = output / name + destination.unlink(missing_ok=True) + shutil.copyfile(build_directory / profile / 'diffr', destination) + destination.chmod(0o755) + identity = os.environ.get('DIFFR_SIGN_IDENTITY') + if identity: + subprocess.run(['codesign', '--force', '--timestamp', '--options', 'runtime', '--sign', identity, '--entitlements', str(root / 'languages/tests/entitlements.plist'), str(destination)], check=True) + return destination + + +try: + catalog.write_text(json.dumps({'schema': 1, 'packages': [entry]}) + '\n') + native = build(None, 'diffr-native') + static = build('all-languages', 'diffr-static') +finally: + catalog.write_bytes(saved) + +with tempfile.TemporaryDirectory() as temporary: + store = Path(temporary) / 'store' + env = dict(os.environ, DIFFR_PARSER_DIR=str(store)) + + def run(binary, *args, code=0): + process = subprocess.run([str(binary), *map(str, args)], env=env, cwd=root, capture_output=True, text=True) + assert process.returncode == code, (args, process.returncode, process.stdout, process.stderr) + return [json.loads(line) for line in process.stdout.splitlines()] + + def compare_args(fixture, extension): + return ['--format', 'ndjson', '--syntax', '--no-index', f'sample_files/{fixture}_1.{extension}', f'sample_files/{fixture}_2.{extension}'] + + fixtures = [(d['fixture']['prefix'], d['fixture']['extension']) for d in items] + first_id = items[0]['id'] + args = compare_args(*fixtures[0]) + missing = run(native, *args) + assert next(e for e in missing if e['type'] == 'file')['diff']['stats']['fallback']['code'] == 'parser_not_installed' + assert not store.exists(), 'ordinary diffs must not create/download packs' + damaged = Path(temporary) / 'damaged.tgz' + damaged.write_bytes(archive.read_bytes()[:-32]) + rejected = run(native, 'languages', 'install', 'extra', '--json', '--archive', damaged, code=2) + assert rejected[0]['error']['code'] == 'language_install_failed' + assert not (store / entry['target'] / 'extra' / entry['version']).exists() + previous = entry_path.parent / 'previous' + previous_tested = (previous / 'diffr').exists() + if previous_tested: + previous_entry = json.loads((previous / 'catalog-entry.json').read_text()) + previous_archive = previous / previous_entry['url'].rsplit('/', 1)[1] + run(previous / 'diffr', 'languages', 'install', 'extra', '--json', '--archive', previous_archive) + older = store / entry['target'] / 'extra' / '0.0.0' + older.mkdir(parents=True) + (older / 'sentinel').write_text('older CLI') + installed = lambda: run(native, 'languages', 'install', 'extra', '--json', '--archive', archive) + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(lambda _: installed(), range(4))) + status = run(native, 'languages', 'list', '--json')[0] + assert status['target'] == entry['target'], 'select packages using executable architecture' + assert {l['id'] for l in status['languages']} == {d['id'] for d in items} + assert all(l['availability'] == 'installed' for l in status['languages']) + for fixture, extension in fixtures: + args = compare_args(fixture, extension) + expected = run(static, *args) + actual = run(native, *args) + # Run summaries include elapsed time; file and syntax events are deterministic. + relevant = lambda events: [e for e in events if e['type'] in ('file', 'syntax')] + assert relevant(actual) == relevant(expected), fixture + assert not next(e for e in actual if e['type'] == 'file')['diff']['stats'].get('fallback'), fixture + for flag in ['--dump-ts', '--dump-syntax']: + path = f'sample_files/{fixture}_1.{extension}' + a = subprocess.check_output([native, 'debug', flag, path], env=env, cwd=root) + b = subprocess.check_output([static, 'debug', flag, path], env=env, cwd=root) + assert a == b, (fixture, flag) + for extension in ['f90', 'fs']: + original = root / f'languages/tests/fixtures/scanner.{extension}' + changed = Path(temporary) / f'changed.{extension}' + changed.write_text(original.read_text().replace('2', '4')) + args = ['--format', 'ndjson', '--syntax', '--no-index', str(original), str(changed)] + assert relevant(run(native, *args)) == relevant(run(static, *args)) + tree = subprocess.check_output([native, 'debug', '--dump-ts', original], env=env, cwd=root) + assert b'ERROR' not in tree and b'MISSING' not in tree, extension + # Exercise synchronized first loads and repeated grammars in a single process. + repository = Path(temporary) / 'repository' + repository.mkdir() + def git(*args): + subprocess.run(['git', '-C', str(repository), *args], check=True, capture_output=True) + git('init') + for fixture, extension in fixtures: + for index in range(12): + (repository / f'{fixture}-{index}.{extension}').write_bytes((root / f'sample_files/{fixture}_1.{extension}').read_bytes()) + git('add', '.') + git('-c', 'user.name=Pack test', '-c', 'user.email=pack@example.invalid', 'commit', '-m', 'fixtures') + for fixture, extension in fixtures: + for index in range(12): + (repository / f'{fixture}-{index}.{extension}').write_bytes((root / f'sample_files/{fixture}_2.{extension}').read_bytes()) + before = str(root / f'sample_files/{fixture}_1.{extension}') + for left, right in [(before, before), ('/dev/null', before), (before, '/dev/null')]: + args = ['--format', 'ndjson', '--syntax', '--no-index', left, right] + assert relevant(run(native, *args)) == relevant(run(static, *args)) + args = ['--repo', repository, '--format', 'ndjson', '--syntax', '--jobs', '8', 'HEAD'] + ordered = lambda events: sorted((json.dumps(e, sort_keys=True) for e in relevant(events))) + start = time.perf_counter() + batch = run(native, *args) + batch_ms = (time.perf_counter() - start) * 1000 + assert ordered(batch) == ordered(run(static, *args)) + revision = store / entry['target'] / 'extra' / entry['version'] + library = revision / next(l['file'] for l in entry['libraries'] if l['id'] == first_id) + library.write_bytes(b'broken') + broken = run(native, *compare_args(*fixtures[0])) + assert next(e for e in broken if e['type'] == 'file')['diff']['stats']['fallback']['code'] == 'parser_load_failed' + assert all(l['availability'] == 'built_in' for l in run(static, 'languages', 'list', '--json')[0]['languages']) + installed() + assert library.stat().st_size == entry['files'][library.name]['size'] + installed() # idempotent, offline + timings = [] + for _ in range(10): + start = time.perf_counter() + run(native, *compare_args(*fixtures[0])) + timings.append((time.perf_counter() - start) * 1000) + print(json.dumps({'archive_bytes': archive.stat().st_size, 'native_executable_bytes': native.stat().st_size, 'static_executable_bytes': static.stat().st_size, 'first_diff_ms': timings[0], 'median_process_diff_ms': statistics.median(timings), 'mixed_file_count': len(fixtures) * 12, 'mixed_batch_diff_ms': batch_ms}, indent=2)) + assert (older / 'sentinel').read_text() == 'older CLI' + if previous_tested: + previous_status = run(previous / 'diffr', 'languages', 'list', '--json')[0] + assert previous_status['revision'] == previous_entry['version'] != entry['version'] + assert all(l['availability'] == 'installed' for l in previous_status['languages']) + run(previous / 'diffr', *compare_args('fortran', 'f90')) + # Exercise the platform's default data directory, including macOS's spaced path. + home = Path(temporary) / 'home' + home.mkdir() + env.pop('DIFFR_PARSER_DIR') + env['HOME'] = str(home) + env['XDG_DATA_HOME'] = str(home / 'data') + env['XDG_CONFIG_HOME'] = str(home / 'config') + installed() + default_root = home / ('Library/Application Support' if sys.platform == 'darwin' else 'data') / 'diffr/parsers' + assert (default_root / entry['target'] / 'extra' / entry['version'] / 'manifest.json').exists() + run(native, *compare_args(*fixtures[0])) + if sys.platform == 'darwin': + # Match a browser-downloaded archive's quarantine attribute without weakening library validation. + quarantined = Path(temporary) / 'quarantined.tgz' + shutil.copyfile(archive, quarantined) + subprocess.run(['/usr/bin/xattr', '-w', 'com.apple.quarantine', '0081;00000000;diffr-test;', str(quarantined)], check=True) + env['DIFFR_PARSER_DIR'] = str(Path(temporary) / 'quarantine-store') + run(native, 'languages', 'install', 'extra', '--json', '--archive', quarantined) + run(native, *compare_args(*fixtures[0])) + print(json.dumps({'previous_cli_coexistence': previous_tested, 'default_store': str(default_root), 'quarantined_archive': sys.platform == 'darwin'})) + env['PATH'] = '' + env['DIFFR_PARSER_DIR'] = str(Path(temporary) / 'fresh-store') + run(native, 'languages', 'list', '--json') + installed() + run(native, *compare_args(*fixtures[0])) + print('Native/static NDJSON, trees, syntax, concurrent install, corruption, repair and offline reuse passed.') diff --git a/languages/tests/probe.c b/languages/tests/probe.c new file mode 100644 index 000000000..f93634db4 --- /dev/null +++ b/languages/tests/probe.c @@ -0,0 +1,19 @@ +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 4) return 2; + void *library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!library) { fprintf(stderr, "%s\n", dlerror()); return 2; } + const TSLanguage *(*language)(void) = dlsym(library, argv[2]); + TSParser *parser = ts_parser_new(); + if (!language || !ts_parser_set_language(parser, language())) return 2; + TSTree *tree = ts_parser_parse_string(parser, NULL, argv[3], strlen(argv[3])); + if (!tree || ts_node_has_error(ts_tree_root_node(tree))) return 1; + ts_tree_delete(tree); + ts_parser_delete(parser); + dlclose(library); + return 0; +} diff --git a/languages/tests/probe.py b/languages/tests/probe.py new file mode 100644 index 000000000..fde9e4951 --- /dev/null +++ b/languages/tests/probe.py @@ -0,0 +1,29 @@ +"""Compile a small native host and parse with every pack library. + +Set DIFFR_SIGN_IDENTITY on macOS to test hardened runtime library validation. +""" +import json +import os +from pathlib import Path +import subprocess +import sys + +root = Path(__file__).resolve().parents[2] +package = Path(sys.argv[1]).resolve() +metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--format-version', '1'], cwd=root)) +runtime = Path(next(p for p in metadata['packages'] if p['name'] == 'tree-sitter')['manifest_path']).parent +probe = package.parent / 'probe' +manifest = json.loads((package / 'manifest.json').read_text()) +architecture = ['-arch', 'arm64' if manifest['target'].startswith('aarch64-') else 'x86_64'] if sys.platform == 'darwin' else [] +subprocess.run(['cc', *architecture, '-O2', '-D_DEFAULT_SOURCE', '-I' + str(runtime / 'include'), '-I' + str(runtime / 'src'), str(runtime / 'src/lib.c'), str(root / 'languages/tests/probe.c'), '-o', str(probe), *(['-ldl'] if sys.platform != 'darwin' else [])], check=True) +identity = os.environ.get('DIFFR_SIGN_IDENTITY') +if identity: + for path in [probe, *[package / entry['file'] for entry in manifest['libraries']]]: + subprocess.run(['codesign', '--force', '--timestamp', '--options', 'runtime', '--sign', identity, str(path)], check=True) + subprocess.run(['codesign', '--verify', '--strict', str(path)], check=True) +sys.path.insert(0, str(root / 'languages')) +from definitions import definitions +fixtures = {definition['id']: definition['smoke'] for definition in definitions()} +for entry in manifest['libraries']: + subprocess.run([str(probe), str(package / entry['file']), entry['symbol'], fixtures[entry['id']]], check=True) + print(entry['id'] + ': loaded and parsed') diff --git a/languages/tests/test_definitions.py b/languages/tests/test_definitions.py new file mode 100644 index 000000000..ca02002d3 --- /dev/null +++ b/languages/tests/test_definitions.py @@ -0,0 +1,40 @@ +import copy +import json +from pathlib import Path +import subprocess +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from definitions import ROOT, definitions, validate + + +class DefinitionChecks(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--no-deps', '--format-version', '1'], cwd=ROOT)) + cls.items = definitions() + + def test_current_definitions_and_shared_ocaml_feature(self): + validate(self.items, self.metadata) + ocaml = [d for d in self.items if d['feature'] == 'lang-ocaml'] + self.assertEqual({d['id'] for d in ocaml}, {'ocaml', 'ocaml-interface'}) + + def test_missing_registry_and_feature_entries_are_rejected(self): + with self.assertRaisesRegex(AssertionError, 'registry'): + validate(self.items[1:], self.metadata) + metadata = copy.deepcopy(self.metadata) + package = next(p for p in metadata['packages'] if p['name'] == 'diffr-cli') + package['features']['all-languages'].remove(self.items[0]['feature']) + with self.assertRaisesRegex(AssertionError, 'all-languages'): + validate(self.items, metadata) + + def test_missing_fixtures_are_rejected(self): + items = copy.deepcopy(self.items) + items[0]['fixture']['prefix'] = 'missing-extra-language-fixture' + with self.assertRaisesRegex(AssertionError, 'missing fixture'): + validate(items, self.metadata) + + +if __name__ == '__main__': + unittest.main() diff --git a/languages/tests/test_publish.py b/languages/tests/test_publish.py new file mode 100644 index 000000000..969fde8cd --- /dev/null +++ b/languages/tests/test_publish.py @@ -0,0 +1,52 @@ +import hashlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import publish +from build import TARGETS + + +class CatalogPublication(unittest.TestCase): + def test_all_registry_bytes_must_match_before_catalog_exists(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + payloads = {} + for target in TARGETS: + folder = directory / target + folder.mkdir() + payload = target.encode() + filename = target + '.tgz' + url = 'https://registry.npmjs.org/' + filename + payloads[url] = payload + (folder / filename).write_bytes(payload) + (folder / 'catalog-entry.json').write_text(json.dumps({ + 'target': target, 'version': '0.1.0', 'url': url, + 'size': len(payload), 'sha256': hashlib.sha256(payload).hexdigest(), + })) + with patch.object(sys, 'argv', ['publish.py', temporary]), patch.object( + publish.urllib.request, 'urlopen', side_effect=lambda url, **_: io.BytesIO(payloads[url]) + ): + publish.main() + self.assertEqual(len(json.loads((directory / 'catalog.json').read_text())['packages']), 4) + payloads[next(iter(payloads))] = b'wrong registry bytes' + with patch.object(sys, 'argv', ['publish.py', temporary]), patch.object( + publish.urllib.request, 'urlopen', side_effect=lambda url, **_: io.BytesIO(payloads[url]) + ), self.assertRaises(AssertionError): + publish.main() + self.assertFalse((directory / 'catalog.json').exists()) + + def test_partial_target_set_cannot_produce_a_catalog(self): + with tempfile.TemporaryDirectory() as temporary: + with patch.object(sys, 'argv', ['publish.py', temporary]), self.assertRaises(AssertionError): + publish.main() + self.assertFalse((Path(temporary) / 'catalog.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/languages/verilog/LICENSE b/languages/verilog/LICENSE new file mode 100644 index 000000000..d7a5632c8 --- /dev/null +++ b/languages/verilog/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2023 Aliaksei Chapyzhenka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/verilog/highlights.scm b/languages/verilog/highlights.scm new file mode 100644 index 000000000..e5511687a --- /dev/null +++ b/languages/verilog/highlights.scm @@ -0,0 +1,322 @@ +;; https://github.com/helix-editor/helix/blob/f6878f62f74430cff188e7978d06c5ed143179e9/runtime/queries/verilog/highlights.scm + +; Keywords + +[ + ; block delimiters + (module_keyword) + "endmodule" + "program" + "endprogram" + "class" + "endclass" + "interface" + "endinterface" + "package" + "endpackage" + "checker" + "endchecker" + + ; these work in helix, but panic on 1.0.3. Unsure why. + ; "config" + ; "endconfig" + + "pure" + "virtual" + "extends" + "implements" + "super" + (class_item_qualifier) + + "parameter" + "localparam" + "defparam" + "assign" + "typedef" + "modport" + "fork" + "join" + "join_none" + "join_any" + "default" + "break" + "assert" + (unique_priority) + "tagged" + "extern" +] @keyword + +[ + "function" + "endfunction" + + "task" + "endtask" +] @keyword.function + +"return" @keyword.control.return + +[ + "begin" + "end" +] @label + +[ + (always_keyword) + "generate" + "for" + "foreach" + "repeat" + "forever" + "initial" + "while" +] @keyword.control + +[ + "if" + "else" + (case_keyword) + "endcase" +] @keyword.control.conditional + +(comment) @comment + +(include_compiler_directive) @keyword.directive +(package_import_declaration + "import" @keyword.control.import) + +(package_import_declaration + (package_import_item + (package_identifier + (simple_identifier) @constant))) + +(text_macro_identifier + (simple_identifier) @keyword.directive) + +(package_scope + (package_identifier + (simple_identifier) @constant)) + +(package_declaration + (package_identifier + (simple_identifier) @constant)) + +(parameter_port_list + "#" @constructor) + +[ + "=" + "-" + "+" + "/" + "*" + "**" + "^" + "&" + "|" + "&&" + "||" + ":" + (unary_operator) + "{" + "}" + "'{" + "<=" + "@" + "==" + "!=" + "===" + "!==" + "-:" + "<" + ">" + ">=" + "%" + ">>" + "<<" + ">>>" + "<<<" + "|=" + (inc_or_dec_operator) + "?" +] @operator +[ + "or" + "and" +] @keyword.operator + +(cast + ["'" "(" ")"] @keyword.operator) + +(edge_identifier) @constant + +(port_direction) @label +(port_identifier + (simple_identifier) @variable) + +(variable_port_header ("var") @type.builtin) +(data_declaration ("var") @type.builtin) +(tf_port_item1 ("var") @type.builtin) +[ + (net_type) + (integer_vector_type) + (integer_atom_type) +] @type.builtin + +[ + "signed" + "unsigned" +] @label + +(data_type + (simple_identifier) @type) + +(method_call_body + (method_identifier) @variable.other.member) + +(interface_identifier + (simple_identifier) @type) + +(modport_identifier + (modport_identifier + (simple_identifier) @variable.other.member)) + +(net_port_type1 + (simple_identifier) @type) + +[ + (double_quoted_string) + (string_literal) +] @string + +[ + (include_compiler_directive) + (default_nettype_compiler_directive) + (timescale_compiler_directive) +] @keyword.directive + +; begin/end label +(seq_block + (simple_identifier) @comment) + +[ + ";" + "::" + "," + "." +] @punctuation.delimiter + + +(default_nettype_compiler_directive + (default_nettype_value) @string) + +(text_macro_identifier + (simple_identifier) @function.macro) + +(module_declaration + (module_header + (simple_identifier) @constructor)) + +(class_constructor_declaration + "new" @constructor) + +(parameter_identifier + (simple_identifier) @variable.parameter) + +[ + (integral_number) + (unsigned_number) + (unbased_unsized_literal) +] @constant.numeric + +(time_unit) @constant + +(checker_instantiation + (checker_identifier + (simple_identifier) @constructor)) + +(module_instantiation + (simple_identifier) @constructor) + +(name_of_instance + (instance_identifier + (simple_identifier) @variable)) + +(interface_port_declaration + (interface_identifier + (simple_identifier) @type)) + +(net_declaration + (simple_identifier) @type) + +(lifetime) @label + +(function_identifier + (function_identifier + (simple_identifier) @function)) + +(function_subroutine_call + (subroutine_call + (tf_call + (simple_identifier) @function))) + +(function_subroutine_call + (subroutine_call + (system_tf_call + (system_tf_identifier) @function.builtin))) + +(task_identifier + (task_identifier + (simple_identifier) @function.method)) + +;;TODO: fixme +;(assignment_pattern_expression + ;(assignment_pattern + ;(parameter_identifier) @variable.other.member)) + +(type_declaration + (data_type ["packed"] @label)) + +(struct_union) @type + +[ + "enum" +] @type + +(enum_name_declaration + (enum_identifier + (simple_identifier) @constant)) + +(type_declaration + (simple_identifier) @type) + +[ + (integer_atom_type) + (non_integer_type) + "genvar" +] @type.builtin + +(struct_union_member + (list_of_variable_decl_assignments + (variable_decl_assignment + (simple_identifier) @variable.other.member))) + +(member_identifier + (simple_identifier) @variable.other.member) + +(struct_union_member + (data_type_or_void + (data_type + (simple_identifier) @type))) + +(type_declaration + (simple_identifier) @type) + +(generate_block_identifier) @comment + +[ + "[" + "]" + "(" + ")" +] @punctuation.bracket diff --git a/languages/verilog/language.json b/languages/verilog/language.json new file mode 100644 index 000000000..5991efa20 --- /dev/null +++ b/languages/verilog/language.json @@ -0,0 +1,16 @@ +{ + "id": "verilog", + "crate": "tree-sitter-verilog", + "version": "1.0.3", + "source": "src", + "symbol": "tree_sitter_verilog", + "revision": "521b535e41a5acd2c6539a922d4649bbe8275110", + "repository": "https://github.com/tree-sitter/tree-sitter-verilog", + "variant": "Verilog", + "feature": "lang-verilog", + "fixture": { + "prefix": "verilog", + "extension": "sv" + }, + "smoke": "module example(input a, output b);\nassign b = a;\nendmodule\n" +} diff --git a/languages/vhdl/LICENSE b/languages/vhdl/LICENSE new file mode 100644 index 000000000..14dda3100 --- /dev/null +++ b/languages/vhdl/LICENSE @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2024 John-Philip Taylor + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/vhdl/highlights.scm b/languages/vhdl/highlights.scm new file mode 100644 index 000000000..dc005e8e9 --- /dev/null +++ b/languages/vhdl/highlights.scm @@ -0,0 +1,435 @@ +;------------------------------------------------------------------------------- +; +; Maintainer: jpt13653903 +; Capture Reference: https://neovim.io/doc/user/treesitter.html#treesitter-highlight-groups +;------------------------------------------------------------------------------- +; +(line_comment + (comment_content) @spell) @comment + +(block_comment + (comment_content) @spell) @comment.documentation + +(identifier) @variable + +[ + "access" + "after" + "alias" + "architecture" + "array" + ; "assume" + "attribute" + "block" + "body" + "component" + "configuration" + "context" + ; "cover" + "disconnect" + "entity" + ; "fairness" + "file" + "force" + "generate" + "generic" + "group" + "label" + "literal" + "map" + "new" + "package" + "parameter" + "port" + "property" + "range" + "reject" + "release" + ; "restrict" + "sequence" + "transport" + "unaffected" + "view" + ; "vmode" + ; "vpkg" + ; "vprop" + "vunit" +] @keyword + +[ + (ALL) + (OTHERS) + "<>" + (DEFAULT) + (OPEN) +] @constant.builtin + +[ + "is" + "begin" + "end" +] @keyword + +(parameter_specification + "in" @keyword) + +[ + "process" + "wait" + "on" + "until" +] @keyword.coroutine + +(end_process + "end" @keyword.coroutine) + +(timeout_clause + "for" @keyword.coroutine) + +[ + "function" + "procedure" +] @keyword.function + +(subprogram_end + "end" @keyword.function) + +[ + "to" + "downto" + "of" +] @keyword.operator + +[ + "library" + "use" +] @keyword.import + +[ + "subtype" + "type" + "record" + "units" + "constant" + "signal" + "variable" +] @keyword.type + +[ + "protected" + "private" + "pure" + "impure" + "inertial" + "postponed" + ; "strong" + "guarded" + "out" + "inout" + "linkage" + "buffer" + "register" + "bus" + "shared" +] @keyword.modifier + +(mode + "in" @keyword.modifier) + +(force_mode + "in" @keyword.modifier) + +[ + "while" + "loop" + "next" + "exit" +] @keyword.repeat + +(end_loop + "end" @keyword.repeat) + +(for_loop + "for" @keyword.repeat) + +(block_configuration + "for" @keyword) + +(configuration_specification + "for" @keyword) + +(component_configuration + "for" @keyword) + +(end_for + "for" @keyword) + +"return" @keyword.return + +[ + "assert" + "report" + "severity" +] @keyword.debug + +[ + "if" + "then" + "elsif" + "case" +] @keyword.conditional + +(end_if + "end" @keyword.conditional) + +(end_case + "end" @keyword.conditional) + +(when_element + "when" @keyword.conditional) + +(case_generate_alternative + "when" @keyword.conditional) + +(else_statement + "else" @keyword.conditional) + +(else_generate + "else" @keyword.conditional) + +[ + "with" + "select" +] @keyword.conditional.ternary + +(when_expression + "when" @keyword.conditional.ternary) + +(else_expression + "else" @keyword.conditional.ternary) + +(else_waveform + "else" @keyword.conditional.ternary) + +(else_expression_or_unaffected + "else" @keyword.conditional.ternary) + +"null" @constant.builtin + +(user_directive) @keyword.directive + +(protect_directive) @keyword.directive + +(warning_directive) @keyword.directive + +(error_directive) @keyword.directive + +(if_conditional_analysis + "if" @keyword.directive) + +(if_conditional_analysis + "then" @keyword.directive) + +(elsif_conditional_analysis + "elsif" @keyword.directive) + +(else_conditional_analysis + "else" @keyword.directive) + +(end_conditional_analysis + "end" @keyword.directive) + +(end_conditional_analysis + "if" @keyword.directive) + +(directive_body) @keyword.directive + +(directive_constant_builtin) @constant.macro + +(directive_error) @comment.error + +(directive_protect) @keyword.directive + +(directive_warning) @comment.warning + +[ + (condition_conversion) + (relational_operator) + (sign) + (adding_operator) + (exponentiate) + (variable_assignment) + (signal_assignment) + "*" + "/" + ":" + "=>" +] @operator + +[ + (unary_operator) + (logical_operator) + (shift_operator) + "mod" + "not" + "rem" +] @keyword.operator + +[ + "'" + "," + "." + ";" +] @punctuation.delimiter + +[ + "(" + ")" + "[" + "]" + "<<" + ">>" +] @punctuation.bracket + +"@" @punctuation.special + +[ + (decimal_integer) + (string_literal_std_logic) +] @number + +(decimal_float) @number.float + +(bit_string_length) @property + +(bit_string_base) @type.builtin + +(bit_string_value) @number + +(based_literal + (based_base) @type.builtin + (based_integer) @number) + +(based_literal + (based_base) @type.builtin + (based_float) @number.float) + +(string_literal) @string @spell + +(character_literal) @character + +(library_constant_std_logic) @constant.builtin + +(library_constant) @constant.builtin + +(library_function) @function.builtin + +(library_constant_boolean) @boolean + +(library_constant_character) @character + +(library_constant_debug) @keyword.debug + +(unit) @keyword.modifier + +(library_constant_unit) @keyword.modifier + +(label) @label + +(generic_map_aspect + "generic" @constructor + "map" @constructor) + +(port_map_aspect + "port" @constructor + "map" @constructor) + +(selection + (identifier) @variable.member) + +(_ + view: (_) @type) + +(_ + type: (_) @type) + +(_ + library: (_) @module) + +(_ + package: (_) @module) + +(_ + entity: (_) @module) + +(_ + component: (_) @module) + +(_ + configuration: (_) @property) + +(_ + architecture: (_) @property) + +(_ + function: (_) @function) + +(_ + procedure: (_) @function.method) + +(_ + attribute: (_) @attribute) + +(_ + constant: (_) @constant) + +(_ + generic: (_) @variable.parameter) + +(_ + view: (name + (_)) @type) + +(_ + type: (name + (_)) @type) + +(_ + entity: (name + (_)) @module) + +(_ + component: (name + (_)) @module) + +(_ + configuration: (name + (_)) @module) + +(library_type) @type.builtin + +[ + (attribute_function) + (attribute_impure_function) + (attribute_mode_view) + (attribute_pure_function) + (attribute_range) + (attribute_signal) + (attribute_subtype) + (attribute_type) + (attribute_value) + (library_attribute) +] @attribute.builtin + +(library_namespace) @module.builtin + +(subtype_declaration + (identifier) @type.definition) + +(type_declaration + (identifier) @type.definition) + +(mode_view_declaration + (identifier) @type.definition) + +; (ERROR) @error diff --git a/languages/vhdl/language.json b/languages/vhdl/language.json new file mode 100644 index 000000000..22333754c --- /dev/null +++ b/languages/vhdl/language.json @@ -0,0 +1,16 @@ +{ + "id": "vhdl", + "crate": "tree-sitter-vhdl", + "version": "1.4.0", + "source": "src", + "symbol": "tree_sitter_vhdl", + "revision": "889c521757471093ce5ac83502ca8a89e516404e", + "repository": "https://github.com/jpt13653903/tree-sitter-vhdl", + "variant": "Vhdl", + "feature": "lang-vhdl", + "fixture": { + "prefix": "vhdl", + "extension": "vhd" + }, + "smoke": "entity example is\nend entity example;\narchitecture rtl of example is\nbegin\nend architecture rtl;\n" +} diff --git a/sample_files/ocaml_interface_1.mli b/sample_files/ocaml_interface_1.mli new file mode 100644 index 000000000..3e647cea8 --- /dev/null +++ b/sample_files/ocaml_interface_1.mli @@ -0,0 +1,3 @@ +(** Public interface *) +type result = { value : int } +val compute : int -> result diff --git a/sample_files/ocaml_interface_2.mli b/sample_files/ocaml_interface_2.mli new file mode 100644 index 000000000..b0da7420d --- /dev/null +++ b/sample_files/ocaml_interface_2.mli @@ -0,0 +1,3 @@ +(** Public interface *) +type result = { value : int; label : string } +val compute : int -> string -> result diff --git a/src/cli.rs b/src/cli.rs index cf1e08637..70e8d6d90 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -87,7 +87,7 @@ pub(crate) fn run() -> Result { .arg(Arg::new("value").required(true)), ), ) - .after_help("Examples:\n diffr\n diffr --cached\n diffr main...HEAD -- src/\n diffr --no-index -- before.rs after.rs\n diffr main HEAD --format ndjson\n\nUnsupported Git flags are rejected; this is not a complete git diff implementation.") + .after_help("Examples:\n diffr\n diffr --cached\n diffr main...HEAD -- src/\n diffr --no-index -- before.rs after.rs\n diffr main HEAD --format ndjson\n diffr languages list\n diffr languages install extra\n\nUnsupported Git flags are rejected; this is not a complete git diff implementation.") .get_matches_from(argv); if let Some(("config", sub)) = args.subcommand() { return run_config(&args, sub); diff --git a/src/config.rs b/src/config.rs index c1ad8bf4d..7fe1f345d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -137,7 +137,10 @@ impl std::fmt::Display for ConfigError { impl std::error::Error for ConfigError {} pub(crate) struct Params { - languages: DftHashMap>>, + languages: + DftHashMap, crate::languages::ParserError>>>, + deferred: DftHashMap>, + order: Vec, pub(crate) diff: DiffConfig, } @@ -275,24 +278,32 @@ impl Config { let mut languages: DftHashMap<_, _> = Language::iter() .map(|language| (language, OnceLock::new())) .collect(); + let mut deferred = DftHashMap::default(); for (name, sources) in queries::assemble(&queries)? { let language = Language::iter() .find(|language| format!("{language:?}").to_lowercase() == name) .ok_or_else(|| ConfigError(format!("unknown language: {name}")))?; - let parser = tree_sitter_parser::from_language(language); + if !crate::languages::builtin(language) { + deferred.insert(language, sources); + continue; + } + let parser = tree_sitter_parser::from_language(language) + .map_err(|e| ConfigError(e.to_string()))?; let query = AnnotationQuery::compile(&parser.language, &sources)?; check_tags(&query, &self.plugins.order)?; languages.insert( language, - OnceLock::from(Arc::new(LanguageParams { + OnceLock::from(Ok(Arc::new(LanguageParams { parser, query, sub_languages: OnceLock::new(), - })), + }))), ); } Ok(Params { languages, + deferred, + order: self.plugins.order, diff: self.diff, }) } @@ -322,26 +333,50 @@ fn check_tags(query: &AnnotationQuery, order: &[String]) -> Result<(), ConfigErr } impl Params { - pub(crate) fn language(&self, language: Language) -> &Arc { - let config = self.languages[&language].get_or_init(|| { - // Languages without annotation rules still support structural diffing. - // Keep their grammars lazy, as in the existing parser registry. - let parser = tree_sitter_parser::from_language(language); - Arc::new(LanguageParams { - parser, - query: AnnotationQuery::compile(&parser.language, &[]).expect("an empty query"), - sub_languages: OnceLock::new(), + pub(crate) fn language( + &self, + language: Language, + ) -> Result<&Arc, crate::languages::ParserError> { + let config = self.languages[&language] + .get_or_init(|| { + let parser = tree_sitter_parser::from_language(language)?; + let sources = self + .deferred + .get(&language) + .map(Vec::as_slice) + .unwrap_or(&[]); + let query = AnnotationQuery::compile(&parser.language, sources) + .and_then(|query| { + check_tags(&query, &self.order)?; + Ok(query) + }) + .map_err(|e| { + crate::languages::ParserError::new( + crate::summary::FallbackCause::ParserLoadFailed, + e, + ) + })?; + Ok(Arc::new(LanguageParams { + parser, + query, + sub_languages: OnceLock::new(), + })) }) - }); + .as_ref() + .map_err(Clone::clone)?; config.sub_languages.get_or_init(|| { config .parser .sub_languages .iter() - .map(|sub| (sub, Arc::clone(self.language(sub.parse_as)))) + .filter_map(|sub| { + self.language(sub.parse_as) + .ok() + .map(|params| (sub, Arc::clone(params))) + }) .collect() }); - config + Ok(config) } } diff --git a/src/diff/sliders.rs b/src/diff/sliders.rs index 03b56c99d..6c8df8aaf 100644 --- a/src/diff/sliders.rs +++ b/src/diff/sliders.rs @@ -800,7 +800,9 @@ mod tests { fn test_slider_two_steps() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs = parse(&arena, "A B", config, false).unwrap(); let rhs = parse(&arena, "A B X\n A B", config, false).unwrap(); @@ -826,7 +828,9 @@ mod tests { fn test_slider_partially_unchanged() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs = parse(&arena, "(A B) X \n (A B)", config, false).unwrap(); let rhs = parse(&arena, "((novel) A B)", config, false).unwrap(); diff --git a/src/diff/unchanged.rs b/src/diff/unchanged.rs index 33b85fb52..f09a8260a 100644 --- a/src/diff/unchanged.rs +++ b/src/diff/unchanged.rs @@ -497,7 +497,9 @@ mod tests { fn test_shrink_unchanged_at_start() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "unchanged A B", config, false).unwrap(); let rhs_nodes = parse(&arena, "unchanged X", config, false).unwrap(); @@ -524,7 +526,9 @@ mod tests { fn test_shrink_unchanged_at_end() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "A B unchanged", config, false).unwrap(); let rhs_nodes = parse(&arena, "X unchanged", config, false).unwrap(); @@ -551,7 +555,9 @@ mod tests { fn test_shrink_unchanged_nested() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -589,7 +595,9 @@ mod tests { fn test_split_unchanged_toplevel_at_start() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); // Make sure that the initial unchanged node exceeds TINY_TREE_THRESHOLD. let lhs_nodes = parse( @@ -631,7 +639,9 @@ mod tests { fn test_split_unchanged_toplevel_at_end() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -672,7 +682,9 @@ mod tests { fn test_split_preserves_outer_delimiters() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "(A)", config, false).unwrap(); let rhs_nodes = parse(&arena, "(B)", config, false).unwrap(); @@ -700,7 +712,9 @@ mod tests { fn test_split_unchanged_middle() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -742,7 +756,9 @@ mod tests { fn test_split_unchanged_multiple() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -771,7 +787,9 @@ mod tests { fn test_split_unchanged_outer_delimiter() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -803,7 +821,9 @@ mod tests { fn test_split_mostly_unchanged_toplevel() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -829,7 +849,9 @@ mod tests { fn test_count_common_unique() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); // There are two subtrees that are unique on both sides and // shared between the two sides here: @@ -859,7 +881,9 @@ mod tests { fn test_similar_with_common_grandchildren() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "((novel-lhs 1 2 3 4 5)) x", config, false).unwrap(); let rhs_nodes = parse(&arena, "((novel-rhs 1 2 3 4 5)) y", config, false).unwrap(); @@ -874,7 +898,9 @@ mod tests { fn test_similar_ignore_delimiter() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "(novel-lhs 1 2 3 4 5) x", config, false).unwrap(); let rhs_nodes = parse(&arena, "[novel-rhs 1 2 3 4 5] y", config, false).unwrap(); diff --git a/src/engine.rs b/src/engine.rs index ef4d8eb02..a6a68f228 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -108,13 +108,23 @@ pub(crate) fn diff_file_content( }; let language = guess(Path::new(display_path), guess_src, overrides); - let lang_config = language.map(|lang| (lang, params.language(lang))); + let resolved = language + .map(|lang| params.language(lang).map(|config| (lang, config))) + .transpose(); + let fallback = resolved + .as_ref() + .err() + .map(|error| FileFormat::TextFallback { + cause: error.cause, + reason: error.message.clone(), + }); + let lang_config = resolved.ok().flatten(); if lhs_src == rhs_src { - let file_format = match language { + let file_format = fallback.clone().unwrap_or(match language { Some(language) => FileFormat::SupportedLanguage(language), None => FileFormat::PlainText, - }; + }); // If the two files are byte-for-byte identical, return early // rather than doing any more work. @@ -141,7 +151,7 @@ pub(crate) fn diff_file_content( (file_format, lhs_positions, rhs_positions) } None => { - let file_format = FileFormat::PlainText; + let file_format = fallback.unwrap_or(FileFormat::PlainText); let (lhs_positions, rhs_positions) = line_parser::change_positions(lhs_src, rhs_src); (file_format, lhs_positions, rhs_positions) } diff --git a/src/languages.rs b/src/languages.rs new file mode 100644 index 000000000..e21339415 --- /dev/null +++ b/src/languages.rs @@ -0,0 +1,577 @@ +//! Only the embedded catalog grants trust; installation never discovers versions. +use anyhow::{ensure, Context, Result}; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; +use strum::IntoEnumIterator; + +use crate::parse::guess_language::Language; +use crate::summary::FallbackCause; + +pub(crate) const TARGET: &str = env!("DIFFR_TARGET"); +const MAX_ARCHIVE: u64 = 64 * 1024 * 1024; +const MAX_FILE: u64 = 128 * 1024 * 1024; +const MAX_UNPACKED: u64 = 256 * 1024 * 1024; + +#[derive(Deserialize)] +struct Catalog { + schema: u32, + packages: Vec, +} +#[derive(Deserialize, Serialize, Clone)] +pub(crate) struct Library { + pub id: String, + pub file: String, + pub symbol: String, + pub abi: usize, +} +#[derive(Deserialize, Serialize, Clone)] +struct Fingerprint { + size: u64, + sha256: String, +} +#[derive(Deserialize, Serialize, Clone)] +pub(crate) struct Package { + schema: u32, + pack: String, + pub version: String, + target: String, + name: String, + url: String, + size: u64, + sha256: String, + pub libraries: Vec, + files: BTreeMap, +} +static CATALOG: LazyLock = LazyLock::new(|| { + let catalog: Catalog = serde_json::from_str(include_str!("../languages/catalog.json")) + .expect("invalid embedded language catalog"); + assert_eq!(catalog.schema, 1); + catalog +}); + +pub(crate) use crate::parse::optional::{builtin, id as optional_id}; +pub(crate) fn package() -> Option<&'static Package> { + CATALOG.packages.iter().find(|p| p.target == TARGET) +} +fn root() -> Result { + let path = std::env::var_os("DIFFR_PARSER_DIR") + .map(PathBuf::from) + .or_else(|| dirs::data_local_dir().map(|p| p.join("diffr/parsers"))) + .context("cannot determine parser store")?; + Ok(std::path::absolute(path)?) +} +pub(crate) fn directory(package: &Package) -> Result { + Ok(root()? + .join(&package.target) + .join("extra") + .join(&package.version)) +} +fn hash(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn read_verified(path: &Path, fingerprint: &Fingerprint) -> Result> { + let metadata = fs::symlink_metadata(path)?; + ensure!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "not a regular file: {}", + path.display() + ); + ensure!( + fingerprint.size <= MAX_FILE && metadata.len() == fingerprint.size, + "wrong size: {}", + path.display() + ); + let mut bytes = Vec::new(); + File::open(path)? + .take(fingerprint.size + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() as u64 == fingerprint.size && hash(&bytes) == fingerprint.sha256, + "integrity failure: {}", + path.display() + ); + Ok(bytes) +} +pub(crate) fn verify_library( + package: &Package, + library: &Library, + directory: &Path, +) -> Result { + let expected = fs::canonicalize(root()?)? + .join(&package.target) + .join("extra") + .join(&package.version); + ensure!( + fs::canonicalize(directory)? == expected, + "linked pack directory outside parser store" + ); + let path = directory.join(&library.file); + read_verified( + &path, + package + .files + .get(&library.file) + .context("uncatalogued library")?, + )?; + let canonical = fs::canonicalize(&path)?; + ensure!( + canonical.parent() == Some(fs::canonicalize(directory)?.as_path()), + "library outside parser store" + ); + Ok(canonical) +} +fn verify_install(package: &Package, directory: &Path) -> Result<()> { + ensure!( + !fs::symlink_metadata(directory)?.file_type().is_symlink(), + "linked pack directory" + ); + for (file, fingerprint) in &package.files { + read_verified(&directory.join(file), fingerprint)?; + } + Ok(()) +} +pub(crate) fn lock(package: &Package) -> Result { + let directory = directory(package)?; + let parent = directory.parent().unwrap(); + fs::create_dir_all(parent)?; + let expected = fs::canonicalize(root()?)? + .join(&package.target) + .join("extra"); + ensure!( + fs::canonicalize(parent)? == expected, + "linked directory outside parser store" + ); + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(parent.join(format!(".{}.lock", package.version)))?; + lock.lock_exclusive()?; + Ok(lock) +} + +#[derive(Debug, Clone)] +pub(crate) struct ParserError { + pub cause: FallbackCause, + pub message: String, +} +impl ParserError { + pub(crate) fn new(cause: FallbackCause, detail: impl std::fmt::Display) -> Self { + Self { cause, message: format!("{detail}; run `diffr languages install extra` to install or repair additional language support") } + } +} +impl std::fmt::Display for ParserError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} +impl std::error::Error for ParserError {} + +fn unpack(package: &Package, archive: &Path, destination: &Path) -> Result<()> { + ensure!(package.size <= MAX_ARCHIVE, "archive exceeds limit"); + let bytes = read_verified( + archive, + &Fingerprint { + size: package.size, + sha256: package.sha256.clone(), + }, + )?; + ensure!(package.size <= MAX_ARCHIVE, "archive exceeds limit"); + let decoder = flate2::read::GzDecoder::new(bytes.as_slice()); + let mut tar = tar::Archive::new(decoder.take(MAX_UNPACKED + 1)); + let mut seen = BTreeSet::new(); + for entry in tar.entries()? { + let mut entry = entry?; + ensure!( + entry.header().entry_type().is_file(), + "archive contains non-file entry" + ); + let path = entry.path_bytes(); + let path = std::str::from_utf8(&path)?; + let name = path + .strip_prefix("package/") + .context("invalid archive prefix")? + .to_owned(); + ensure!( + !name.contains('/') && !name.contains('\\') && name != "." && name != "..", + "invalid archive path" + ); + let expected = package + .files + .get(&name) + .context("unexpected archive file")?; + ensure!(seen.insert(name.clone()), "duplicate archive file"); + ensure!( + entry.size() == expected.size && expected.size <= MAX_FILE, + "archive file exceeds limit" + ); + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + ensure!( + bytes.len() as u64 == expected.size && hash(&bytes) == expected.sha256, + "archive file integrity failure" + ); + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(destination.join(name))?; + file.write_all(&bytes)?; + file.sync_all()?; + } + ensure!(seen.len() == package.files.len(), "incomplete archive"); + let manifest: serde_json::Value = + serde_json::from_slice(&fs::read(destination.join("manifest.json"))?)?; + ensure!( + manifest["schema"] == 1 + && manifest["pack"] == "extra" + && manifest["target"] == package.target + && manifest["version"] == package.version + && manifest["libraries"] == serde_json::to_value(&package.libraries)?, + "incompatible manifest" + ); + Ok(()) +} +fn install(package: &Package, archive: Option<&Path>) -> Result<()> { + ensure!( + package.schema == 1 && package.pack == "extra" && package.target == TARGET, + "incompatible package" + ); + ensure!(package.size <= MAX_ARCHIVE, "archive exceeds limit"); + let _lock = lock(package)?; + let destination = directory(package)?; + if verify_install(package, &destination).is_ok() { + return Ok(()); + } + let parent = destination.parent().unwrap(); + let staging = tempfile::tempdir_in(parent)?; + let download = staging.path().join("archive.tgz"); + let archive = match archive { + Some(path) => path, + None => { + ensure!( + package + .url + .starts_with("https://registry.npmjs.org/@dev.fast/diffr-languages-extra-"), + "invalid catalog URL" + ); + eprintln!( + "Downloading extra {} ({} bytes)", + package.version, package.size + ); + tokio::runtime::Runtime::new()?.block_on(download_archive( + &package.url, + package.size, + &download, + ))?; + &download + } + }; + let prepared = staging.path().join("prepared"); + fs::create_dir(&prepared)?; + unpack(package, archive, &prepared)?; + File::open(&prepared)?.sync_all()?; + publish_directory(&prepared, &destination)?; + File::open(parent)?.sync_all()?; + Ok(()) +} + +async fn download_archive(url: &str, expected_size: u64, destination: &Path) -> Result<()> { + ensure!(expected_size <= MAX_ARCHIVE, "archive exceeds limit"); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(180)) + .build()?; + let mut response = client.get(url).send().await?.error_for_status()?; + let mut file = File::create(destination)?; + let mut size = 0; + while let Some(chunk) = response.chunk().await? { + size += chunk.len() as u64; + ensure!(size <= expected_size, "download exceeds pinned size"); + file.write_all(&chunk)?; + } + ensure!(size == expected_size, "incomplete download"); + file.sync_all()?; + Ok(()) +} + +fn publish_directory(prepared: &Path, destination: &Path) -> Result<()> { + match fs::symlink_metadata(destination) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::rename(prepared, destination)?; + return Ok(()); + } + Err(error) => return Err(error.into()), + Ok(_) => {} + } + // Exchange complete directories so interruption cannot expose a partial repair. + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))] + { + use std::os::unix::ffi::OsStrExt; + let from = std::ffi::CString::new(prepared.as_os_str().as_bytes())?; + let to = std::ffi::CString::new(destination.as_os_str().as_bytes())?; + #[cfg(target_os = "macos")] + let result = unsafe { libc::renamex_np(from.as_ptr(), to.as_ptr(), libc::RENAME_SWAP) }; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + let result = unsafe { + libc::renameat2( + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_EXCHANGE, + ) + }; + if result != 0 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) + } + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))] + anyhow::bail!("atomic pack repair is unsupported on this target") +} + +pub(crate) fn run() -> Result { + let command = clap::Command::new("diffr languages") + .subcommand_required(true) + .subcommand( + clap::Command::new("list").arg( + clap::Arg::new("json") + .long("json") + .action(clap::ArgAction::SetTrue), + ), + ) + .subcommand( + clap::Command::new("install") + .arg( + clap::Arg::new("pack") + .required(true) + .value_parser(["extra"]), + ) + .arg( + clap::Arg::new("json") + .long("json") + .action(clap::ArgAction::SetTrue), + ) + .arg( + clap::Arg::new("archive") + .long("archive") + .value_parser(clap::value_parser!(PathBuf)), + ), + ); + let args = command.try_get_matches_from( + std::iter::once(std::ffi::OsString::from("diffr languages")) + .chain(std::env::args_os().skip(2)), + ); + let args = match args { + Ok(args) => args, + Err(error) => { + let code = error.exit_code(); + error.print()?; + return Ok(code); + } + }; + let (action, args) = args.subcommand().unwrap(); + let json = args.get_flag("json"); + if action == "list" { + let package = package(); + let state = package + .map(|p| directory(p).and_then(|dir| verify_install(p, &dir))) + .transpose(); + let languages: Vec<_> = Language::iter() + .filter(|language| optional_id(*language).is_some()) + .map(|language| { + let availability = if builtin(language) { + "built_in" + } else if package.is_none() { + "unavailable" + } else if state.as_ref().is_ok_and(|s| s.is_some()) { + "installed" + } else if package + .and_then(|p| directory(p).ok()) + .is_some_and(|p| p.exists()) + { + "corrupt" + } else { + "not_installed" + }; + serde_json::json!({"id": optional_id(language), "availability": availability}) + }) + .collect(); + let result = serde_json::json!({"target": TARGET, "pack": "extra", "revision": package.map(|p| &p.version), "download_size": package.map(|p| p.size), "languages": languages}); + if json { + println!("{result}"); + } else { + match package { + Some(pack) => println!("extra {}: {} ({} bytes)", pack.version, TARGET, pack.size), + None => println!("extra: no published revision for {TARGET}"), + } + for language in languages { + println!( + "{}: {}", + language["id"].as_str().unwrap(), + language["availability"].as_str().unwrap() + ); + } + } + return Ok(0); + } + let result = package() + .context("no published extra pack is pinned for this executable target") + .and_then(|p| install(p, args.get_one::("archive").map(PathBuf::as_path))); + match result { + Ok(()) => { + if json { + println!( + "{}", + serde_json::json!({"ok": true, "pack": "extra", "target": TARGET, "revision": package().unwrap().version}) + ); + } else { + println!("Installed extra {}", package().unwrap().version); + } + Ok(0) + } + Err(error) => { + if json { + println!( + "{}", + serde_json::json!({"ok": false, "error": {"code": "language_install_failed", "message": format!("{error:#}")}}) + ); + } else { + eprintln!("{error:#}"); + } + Ok(2) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::{write::GzEncoder, Compression}; + + fn fixture(entries: &[(&str, &[u8], tar::EntryType)]) -> (tempfile::TempDir, PathBuf, Package) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pack.tgz"); + let encoder = GzEncoder::new(File::create(&path).unwrap(), Compression::default()); + let mut archive = tar::Builder::new(encoder); + let manifest = serde_json::json!({"schema": 1, "pack": "extra", "version": "0.0.0", "target": TARGET, "libraries": []}).to_string(); + let mut files = BTreeMap::new(); + for (name, bytes, kind) in std::iter::once(( + "manifest.json", + manifest.as_bytes(), + tar::EntryType::Regular, + )) + .chain(entries.iter().copied()) + { + let mut header = tar::Header::new_ustar(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_entry_type(kind); + if kind.is_symlink() { + header.set_link_name("/tmp/outside").unwrap(); + } + let path = format!("package/{name}"); + header.as_mut_bytes()[..path.len()].copy_from_slice(path.as_bytes()); + header.set_cksum(); + archive.append(&header, bytes).unwrap(); + files.insert( + name.to_owned(), + Fingerprint { + size: bytes.len() as u64, + sha256: hash(bytes), + }, + ); + } + archive.into_inner().unwrap().finish().unwrap(); + let bytes = fs::read(&path).unwrap(); + let package = Package { + schema: 1, + pack: "extra".into(), + version: "0.0.0".into(), + target: TARGET.into(), + name: "test".into(), + url: "unused".into(), + size: bytes.len() as u64, + sha256: hash(&bytes), + libraries: vec![], + files, + }; + (directory, path, package) + } + + #[test] + fn streamed_download_is_bounded_and_rejects_interruption() { + for (body, content_length, expected_size, succeeds) in [ + ("abcdefgh", 8, 8, true), + ("abcdefgh", 8, 4, false), + ("abc", 8, 8, false), + ] { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/pack.tgz", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0; 4096]; + stream.read(&mut request).unwrap(); + write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n{body}").unwrap(); + }); + let directory = tempfile::tempdir().unwrap(); + let destination = directory.path().join("download"); + let result = tokio::runtime::Runtime::new() + .unwrap() + .block_on(download_archive(&url, expected_size, &destination)); + server.join().unwrap(); + assert_eq!(result.is_ok(), succeeds, "{result:?}"); + assert!(destination.metadata().unwrap().len() <= expected_size); + } + } + + #[test] + fn archive_requires_exact_bytes_files_and_target() { + let (_dir, archive, package) = + fixture(&[("parser.so", b"parser", tar::EntryType::Regular)]); + let destination = tempfile::tempdir().unwrap(); + unpack(&package, &archive, destination.path()).unwrap(); + verify_install(&package, destination.path()).unwrap(); + fs::write(destination.path().join("parser.so"), b"broken").unwrap(); + assert!(verify_install(&package, destination.path()).is_err()); + for mutation in 0..4 { + let mut bad = package.clone(); + match mutation { + 0 => bad.sha256 = "00".repeat(32), + 1 => bad.target = "wrong-target".into(), + 2 => { + bad.files.remove("parser.so"); + } + _ => { + bad.files.get_mut("parser.so").unwrap().size = MAX_FILE + 1; + } + } + assert!(unpack(&bad, &archive, tempfile::tempdir().unwrap().path()).is_err()); + } + let mut bytes = fs::read(&archive).unwrap(); + bytes.truncate(bytes.len() / 2); + fs::write(&archive, bytes).unwrap(); + assert!(unpack(&package, &archive, tempfile::tempdir().unwrap().path()).is_err()); + } + + #[test] + fn archives_reject_links_nested_paths_and_duplicates() { + for entries in [ + vec![("linked", b"".as_slice(), tar::EntryType::Symlink)], + vec![("hardlink", b"".as_slice(), tar::EntryType::Link)], + vec![("../escape", b"x".as_slice(), tar::EntryType::Regular)], + vec![("nested/file", b"x".as_slice(), tar::EntryType::Regular)], + vec![("duplicate", b"x".as_slice(), tar::EntryType::Regular); 2], + ] { + let (_dir, archive, package) = fixture(&entries); + assert!(unpack(&package, &archive, tempfile::tempdir().unwrap().path()).is_err()); + } + } +} diff --git a/src/main.rs b/src/main.rs index 6dd75d718..0450c16c4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,7 @@ mod constants; mod diff; mod engine; mod exit_codes; +mod languages; use engine::diff_file_content; mod files; mod git; @@ -135,6 +136,7 @@ fn main() { reset_sigpipe(); let result = match std::env::args_os().nth(1).as_deref() { + Some(arg) if arg == "languages" => languages::run().map_err(Into::into), Some(arg) if arg == "debug" => { run_debug(); return; @@ -165,7 +167,10 @@ fn run_debug() { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = tsp::from_language(lang); + let ts_lang = tsp::from_language(lang).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(2) + }); let tree = tsp::to_tree(&src, ts_lang); tsp::print_tree(&src, &tree); } @@ -186,7 +191,10 @@ fn run_debug() { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = params.language(lang); + let ts_lang = params.language(lang).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(2) + }); let arena = Arena::new(); let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); init_all_info(&ast, &[]); @@ -209,7 +217,10 @@ fn run_debug() { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = params.language(lang); + let ts_lang = params.language(lang).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(2) + }); let arena = Arena::new(); let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); init_all_info(&ast, &[]); diff --git a/src/parse/guess_language.rs b/src/parse/guess_language.rs index c36ca5fc6..82cbb8b1e 100644 --- a/src/parse/guess_language.rs +++ b/src/parse/guess_language.rs @@ -38,9 +38,7 @@ pub(crate) enum Language { EmacsLisp, Erlang, Fish, - #[cfg(feature = "lang-fsharp")] FSharp, - #[cfg(feature = "lang-fortran")] Fortran, Gleam, Go, @@ -81,7 +79,6 @@ pub(crate) enum Language { Toml, TypeScript, TypeScriptTsx, - #[cfg(feature = "lang-verilog")] Verilog, Vhdl, Xml, @@ -142,9 +139,7 @@ pub(crate) fn language_name(language: Language) -> &'static str { EmacsLisp => "Emacs Lisp", Erlang => "Erlang", Fish => "Fish", - #[cfg(feature = "lang-fsharp")] FSharp => "F#", - #[cfg(feature = "lang-fortran")] Fortran => "Fortran", Gleam => "Gleam", Go => "Go", @@ -185,7 +180,6 @@ pub(crate) fn language_name(language: Language) -> &'static str { Toml => "TOML", TypeScript => "TypeScript", TypeScriptTsx => "TypeScript TSX", - #[cfg(feature = "lang-verilog")] Verilog => "Verilog", Vhdl => "VHDL", Xml => "XML", @@ -295,9 +289,7 @@ pub(crate) fn language_globs(language: Language) -> Vec { "rebar.lock", ], Fish => &["*.fish"], - #[cfg(feature = "lang-fsharp")] FSharp => &["*.fs", "*.fsx", "*.fsi"], - #[cfg(feature = "lang-fortran")] Fortran => &["*.f", "*.for", "*.f90", "*.F", "*.FOR", "*.F90"], Gleam => &["*.gleam"], Go => &["*.go"], @@ -404,7 +396,6 @@ pub(crate) fn language_globs(language: Language) -> Vec { ], TypeScript => &["*.ts", "*.cts", "*.mts"], TypeScriptTsx => &["*.tsx"], - #[cfg(feature = "lang-verilog")] Verilog => &["*.v", "*.sv", "*.vh"], Vhdl => &["*.vhdl", "*.vhd"], Xml => &[ @@ -554,9 +545,7 @@ fn from_emacs_mode_header(src: &str) -> Option { "elm" => Elm, "emacs-lisp" => EmacsLisp, "fish" => Fish, - #[cfg(feature = "lang-fsharp")] "fsharp" => FSharp, - #[cfg(feature = "lang-fortran")] "fortran" => Fortran, "gleam" => Gleam, "go" => Go, @@ -583,7 +572,6 @@ fn from_emacs_mode_header(src: &str) -> Option { "toml" => Toml, "tuareg" => OCaml, "typescript" => TypeScript, - #[cfg(feature = "lang-verilog")] "verilog" => Verilog, "vhdl" => Vhdl, "yaml" => Yaml, diff --git a/src/parse/mod.rs b/src/parse/mod.rs index d698599c9..e4ab624bf 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -5,3 +5,7 @@ pub(crate) mod guess_language; pub(crate) mod query; pub(crate) mod syntax; pub(crate) mod tree_sitter_parser; + +pub(crate) mod native; + +pub(crate) mod optional; diff --git a/src/parse/native.rs b/src/parse/native.rs new file mode 100644 index 000000000..bfc17826a --- /dev/null +++ b/src/parse/native.rs @@ -0,0 +1,72 @@ +//! A grammar's code must outlive every Tree-sitter language, parser and tree. +use crate::hash::DftHashMap; +use crate::languages::{self, ParserError}; +use crate::summary::FallbackCause; +use anyhow::{ensure, Context}; +use libloading::Library; +use std::sync::{LazyLock, Mutex}; +use tree_sitter_language::LanguageFn; + +struct Loaded { + language: tree_sitter::Language, + _library: Library, +} + +pub(crate) fn load(id: &'static str) -> Result { + static LOADED: LazyLock>> = + LazyLock::new(|| Mutex::new(DftHashMap::default())); + let mut loaded = LOADED.lock().unwrap(); + if let Some(grammar) = loaded.get(id) { + return Ok(grammar.language.clone()); + } + let package = languages::package().ok_or_else(|| { + ParserError::new( + FallbackCause::ParserUnavailable, + format!( + "{id}: no compatible published pack for {}", + languages::TARGET + ), + ) + })?; + let directory = languages::directory(package) + .map_err(|e| ParserError::new(FallbackCause::ParserLoadFailed, e))?; + if !directory.exists() { + return Err(ParserError::new( + FallbackCause::ParserNotInstalled, + format!("{id} parser is not installed"), + )); + } + let result = (|| -> anyhow::Result { + let _lock = languages::lock(package)?; + let entry = package + .libraries + .iter() + .find(|l| l.id == id) + .context("missing catalog language")?; + ensure!( + (tree_sitter::MIN_COMPATIBLE_LANGUAGE_VERSION..=tree_sitter::LANGUAGE_VERSION) + .contains(&entry.abi), + "incompatible catalog ABI" + ); + let path = languages::verify_library(package, entry, &directory)?; + // Only exact, catalog-hashed code is loaded. The handle stays in LOADED forever. + let library = unsafe { Library::new(path)? }; + let function = + unsafe { library.get:: *const ()>(entry.symbol.as_bytes())? }; + ensure!(!unsafe { function() }.is_null(), "null grammar pointer"); + let language = tree_sitter::Language::new(unsafe { LanguageFn::from_raw(*function) }); + ensure!( + language.abi_version() == entry.abi, + "grammar ABI differs from catalog" + ); + tree_sitter::Parser::new().set_language(&language)?; + Ok(Loaded { + language, + _library: library, + }) + })() + .map_err(|e| ParserError::new(FallbackCause::ParserLoadFailed, format!("{id}: {e:#}")))?; + let language = result.language.clone(); + loaded.insert(id, result); + Ok(language) +} diff --git a/src/parse/optional.rs b/src/parse/optional.rs new file mode 100644 index 000000000..fb8921eb3 --- /dev/null +++ b/src/parse/optional.rs @@ -0,0 +1,27 @@ +//! Optional parser identity and compile-time availability. +use super::guess_language::Language; + +macro_rules! optional_languages { + ($( $variant:ident => ($id:literal, $feature:literal), )*) => { + pub(crate) fn id(language: Language) -> Option<&'static str> { + match language { $(Language::$variant => Some($id),)* _ => None } + } + #[allow(clippy::match_like_matches_macro)] + pub(crate) fn builtin(language: Language) -> bool { + match language { $(Language::$variant => cfg!(feature = $feature),)* _ => true } + } + }; +} + +optional_languages! { + Apex => ("apex", "lang-apex"), + Fortran => ("fortran", "lang-fortran"), + FSharp => ("fsharp", "lang-fsharp"), + Haskell => ("haskell", "lang-haskell"), + Julia => ("julia", "lang-julia"), + OCaml => ("ocaml", "lang-ocaml"), + OCamlInterface => ("ocaml-interface", "lang-ocaml"), + Qml => ("qml", "lang-qml"), + Verilog => ("verilog", "lang-verilog"), + Vhdl => ("vhdl", "lang-vhdl"), +} diff --git a/src/parse/tree_sitter_parser.rs b/src/parse/tree_sitter_parser.rs index 1aef0c877..f85fec002 100644 --- a/src/parse/tree_sitter_parser.rs +++ b/src/parse/tree_sitter_parser.rs @@ -101,7 +101,9 @@ const OCAML_ATOM_NODES: [&str; 6] = [ "attribute_id", ]; -pub(crate) fn from_language(language: guess::Language) -> &'static TreeSitterConfig { +pub(crate) fn from_language( + language: guess::Language, +) -> Result<&'static TreeSitterConfig, crate::languages::ParserError> { // Constructing a tree sitter query is relatively expensive: it // can take tens of milliseconds. // @@ -112,12 +114,22 @@ pub(crate) fn from_language(language: guess::Language) -> &'static TreeSitterCon LazyLock::new(|| Mutex::new(DftHashMap::default())); let mut cache = CONFIG_CACHE.lock().unwrap(); - cache - .entry(language) - .or_insert_with(|| Box::leak(Box::new(build_config(language)))) + if let Some(config) = cache.get(&language) { + return Ok(config); + } + let external = if crate::languages::builtin(language) { + None + } else { + Some(super::native::load( + crate::languages::optional_id(language).unwrap(), + )?) + }; + let config = Box::leak(Box::new(build_config(language, external))); + cache.insert(language, config); + Ok(config) } -fn build_config(language: guess::Language) -> TreeSitterConfig { +fn build_config(language: guess::Language, _external: Option) -> TreeSitterConfig { use guess::Language::*; match language { Ada => { @@ -139,8 +151,10 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Apex => { - let language_fn = tree_sitter_sfapex::apex::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-apex")] + let language = tree_sitter::Language::new(tree_sitter_sfapex::apex::LANGUAGE); + #[cfg(not(feature = "lang-apex"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), @@ -159,7 +173,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { ignore_trailing_tokens: vec![], highlight_query: ts::Query::new( &language, - tree_sitter_sfapex::apex::HIGHLIGHTS_QUERY, + include_str!("../../languages/apex/highlights.scm"), ) .unwrap(), sub_languages: vec![], @@ -456,33 +470,41 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - #[cfg(feature = "lang-fsharp")] FSharp => { - let language_fn = tree_sitter_fsharp::LANGUAGE_FSHARP; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-fsharp")] + let language = tree_sitter::Language::new(tree_sitter_fsharp::LANGUAGE_FSHARP); + #[cfg(not(feature = "lang-fsharp"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), atom_nodes: ["string", "triple_quoted_string"].into_iter().collect(), delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_fsharp::HIGHLIGHTS_QUERY) - .unwrap(), + highlight_query: ts::Query::new( + &language, + include_str!("../../languages/fsharp/highlights.scm"), + ) + .unwrap(), sub_languages: vec![], } } - #[cfg(feature = "lang-fortran")] Fortran => { - let language_fn = tree_sitter_fortran::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-fortran")] + let language = tree_sitter::Language::new(tree_sitter_fortran::LANGUAGE); + #[cfg(not(feature = "lang-fortran"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), atom_nodes: ["string_literal", "number_literal"].into_iter().collect(), delimiter_tokens: vec![("(", ")"), ("(/", "/)"), ("[", "]")], ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_fortran::HIGHLIGHTS_QUERY) - .unwrap(), + highlight_query: ts::Query::new( + &language, + include_str!("../../languages/fortran/highlights.scm"), + ) + .unwrap(), sub_languages: vec![], } } @@ -518,15 +540,20 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Haskell => { - let language_fn = tree_sitter_haskell::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-haskell")] + let language = tree_sitter::Language::new(tree_sitter_haskell::LANGUAGE); + #[cfg(not(feature = "lang-haskell"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), atom_nodes: ["qualified_variable"].into_iter().collect(), delimiter_tokens: vec![("[", "]"), ("(", ")")], ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_haskell::HIGHLIGHTS_QUERY) - .unwrap(), + highlight_query: ts::Query::new( + &language, + include_str!("../../languages/haskell/highlights.scm"), + ) + .unwrap(), sub_languages: vec![], } } @@ -690,8 +717,10 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Julia => { - let language_fn = tree_sitter_julia::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-julia")] + let language = tree_sitter::Language::new(tree_sitter_julia::LANGUAGE); + #[cfg(not(feature = "lang-julia"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), @@ -707,7 +736,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { ignore_trailing_tokens: vec![], highlight_query: ts::Query::new( &language, - include_str!("../../vendored_parsers/highlights/julia.scm"), + include_str!("../../languages/julia/highlights.scm"), ) .unwrap(), sub_languages: vec![], @@ -844,28 +873,39 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } OCaml => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-ocaml")] + let language = tree_sitter::Language::new(tree_sitter_ocaml::LANGUAGE_OCAML); + #[cfg(not(feature = "lang-ocaml"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), atom_nodes: OCAML_ATOM_NODES.iter().copied().collect(), delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_ocaml::HIGHLIGHTS_QUERY) - .unwrap(), + highlight_query: ts::Query::new( + &language, + include_str!("../../languages/ocaml/highlights.scm"), + ) + .unwrap(), sub_languages: vec![], } } OCamlInterface => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-ocaml")] + let language = tree_sitter::Language::new(tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE); + #[cfg(not(feature = "lang-ocaml"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), atom_nodes: OCAML_ATOM_NODES.iter().copied().collect(), delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], ignore_trailing_tokens: vec![], - // TODO: why doesn't tree_sitter_ocaml::HIGHLIGHTS_QUERY work here? - highlight_query: ts::Query::new(&language, "").unwrap(), + // Preserve the existing empty highlight query for interfaces. + highlight_query: ts::Query::new( + &language, + include_str!("../../languages/ocaml-interface/highlights.scm"), + ) + .unwrap(), sub_languages: vec![], } } @@ -962,12 +1002,14 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Qml => { - let language_fn = tree_sitter_qmljs::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-qml")] + let language = tree_sitter::Language::new(tree_sitter_qmljs::LANGUAGE); + #[cfg(not(feature = "lang-qml"))] + let language = _external.expect("resolved external grammar"); let mut highlight_query = tree_sitter_javascript::HIGHLIGHT_QUERY.to_owned(); highlight_query.push_str(tree_sitter_typescript::HIGHLIGHTS_QUERY); - highlight_query.push_str(tree_sitter_qmljs::HIGHLIGHTS_QUERY); + highlight_query.push_str(include_str!("../../languages/qml/highlights.scm")); TreeSitterConfig { language: language.clone(), @@ -1248,10 +1290,11 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { sub_languages: vec![], } } - #[cfg(feature = "lang-verilog")] Verilog => { - let language_fn = tree_sitter_verilog::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-verilog")] + let language = tree_sitter::Language::new(tree_sitter_verilog::LANGUAGE); + #[cfg(not(feature = "lang-verilog"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { ignore_trailing_tokens: vec![], language: language.clone(), @@ -1259,23 +1302,28 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { delimiter_tokens: vec![("(", ")"), ("[", "]"), ("begin", "end")], highlight_query: ts::Query::new( &language, - include_str!("../../vendored_parsers/highlights/verilog.scm"), + include_str!("../../languages/verilog/highlights.scm"), ) .unwrap(), sub_languages: vec![], } } Vhdl => { - let language_fn = tree_sitter_vhdl::LANGUAGE; - let language = tree_sitter::Language::new(language_fn); + #[cfg(feature = "lang-vhdl")] + let language = tree_sitter::Language::new(tree_sitter_vhdl::LANGUAGE); + #[cfg(not(feature = "lang-vhdl"))] + let language = _external.expect("resolved external grammar"); TreeSitterConfig { language: language.clone(), atom_nodes: [].into_iter().collect(), delimiter_tokens: vec![("(", ")")], ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, tree_sitter_vhdl::HIGHLIGHTS_QUERY) - .unwrap(), + highlight_query: ts::Query::new( + &language, + include_str!("../../languages/vhdl/highlights.scm"), + ) + .unwrap(), sub_languages: vec![], } } @@ -2160,7 +2208,7 @@ mod tests { fn test_parse() { let arena = Arena::new(); let params = Params::default(); - let css_config = params.language(guess::Language::Css); + let css_config = params.language(guess::Language::Css).unwrap(); parse(&arena, ".foo {}", css_config, false).unwrap(); } @@ -2168,7 +2216,7 @@ mod tests { fn test_parse_empty_file() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess::Language::EmacsLisp); + let config = params.language(guess::Language::EmacsLisp).unwrap(); let res = parse(&arena, "", config, false).unwrap(); let expected: Vec<&Syntax> = vec![]; @@ -2181,7 +2229,7 @@ mod tests { fn test_subtrees() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess::Language::Html); + let config = params.language(guess::Language::Html).unwrap(); let res = parse(&arena, "", config, false).unwrap(); match res[0] { @@ -2204,7 +2252,9 @@ mod tests { #[test] fn test_configs_valid() { for language in guess::Language::iter() { - from_language(language); + if crate::languages::builtin(language) { + from_language(language).unwrap(); + } } } } diff --git a/src/protocol/project.rs b/src/protocol/project.rs index 9ae85a034..eee74c541 100644 --- a/src/protocol/project.rs +++ b/src/protocol/project.rs @@ -129,6 +129,9 @@ fn stats(result: &DiffResult, lhs_src: &str, rhs_src: &str) -> Stats { /// is the engine's own prose, with the numbers. fn fallback_code(cause: FallbackCause) -> &'static str { match cause { + FallbackCause::ParserNotInstalled => "parser_not_installed", + FallbackCause::ParserUnavailable => "parser_unavailable", + FallbackCause::ParserLoadFailed => "parser_load_failed", FallbackCause::Generated => "generated", FallbackCause::ByteLimit => "too_large", FallbackCause::GraphLimit => "too_complex", @@ -1320,7 +1323,8 @@ mod tests { fn syntax_spans_are_per_line_sorted_and_innermost() { let parser = crate::parse::tree_sitter_parser::from_language( crate::parse::guess_language::Language::Python, - ); + ) + .unwrap(); let spans = syntax_spans("def f(x):\n return \"a\"\n", parser); for pair in spans.windows(2) { assert!( diff --git a/src/protocol/stream.rs b/src/protocol/stream.rs index 6dcdb8d86..29ba1dcef 100644 --- a/src/protocol/stream.rs +++ b/src/protocol/stream.rs @@ -270,7 +270,10 @@ fn syntax_spans( let FileFormat::SupportedLanguage(language) = &diff.file_format else { return (Vec::new(), Vec::new()); }; - let parser = params.language(*language).parser; + let Ok(config) = params.language(*language) else { + return (Vec::new(), Vec::new()); + }; + let parser = config.parser; let spans = |content: &FileContent| match content { FileContent::Text(src) => project::syntax_spans(src, parser), FileContent::Binary => Vec::new(), diff --git a/src/summary.rs b/src/summary.rs index c961536f1..429a1d0e2 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -30,6 +30,9 @@ pub(crate) enum FileFormat { pub(crate) enum FallbackCause { /// A side is larger than the byte limit. ByteLimit, + ParserNotInstalled, + ParserUnavailable, + ParserLoadFailed, /// The AST matching graph grew past the graph limit. GraphLimit, /// A side has more parse errors than the parse error limit. diff --git a/tests/cli.rs b/tests/cli.rs index 7eff6f7ac..0c9bc9287 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -26,7 +26,7 @@ fn list_languages() { } #[test] -fn optional_languages_follow_build_features() { +fn optional_languages_are_recognized_without_compiled_parsers() { let listed = debug_command() .arg("--list-languages") .assert() @@ -39,13 +39,37 @@ fn optional_languages_follow_build_features() { let config = dir.path().join("config.toml"); std::fs::write(&config, "[plugins]\norder = []\n").unwrap(); - for (name, fixture, extension, enabled) in [ - ("Fortran", "fortran", "f90", cfg!(feature = "lang-fortran")), - ("Verilog", "verilog", "sv", cfg!(feature = "lang-verilog")), - ("F#", "f_sharp", "fs", cfg!(feature = "lang-fsharp")), - ] { - assert_eq!(listed.contains(name), enabled, "{name}"); + let status = get_base_command() + .env("DIFFR_PARSER_DIR", dir.path().join("parsers")) + .args(["languages", "list", "--json"]) + .output() + .unwrap(); + let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); + let missing_code = if status["revision"].is_null() { + "parser_unavailable" + } else { + "parser_not_installed" + }; + for entry in std::fs::read_dir(concat!(env!("CARGO_MANIFEST_DIR"), "/languages")).unwrap() { + let path = entry.unwrap().path().join("language.json"); + if !path.is_file() { + continue; + } + let definition: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + let name = definition["id"].as_str().unwrap(); + let fixture = definition["fixture"]["prefix"].as_str().unwrap(); + let extension = definition["fixture"]["extension"].as_str().unwrap(); + let language = status["languages"] + .as_array() + .unwrap() + .iter() + .find(|l| l["id"] == name) + .unwrap(); + let enabled = language["availability"] == "built_in"; + assert!(listed.contains(&format!("*.{extension}")), "{name}"); let output = get_base_command() + .env("DIFFR_PARSER_DIR", dir.path().join("parsers")) .arg("--config") .arg(&config) .args([ @@ -69,11 +93,7 @@ fn optional_languages_follow_build_features() { assert_eq!(file["diff"]["type"], "text", "{name}: {file}"); assert_eq!( file["diff"]["stats"]["fallback"]["code"].as_str(), - if enabled { - None - } else { - Some("unsupported_language") - }, + if enabled { None } else { Some(missing_code) }, "{name}", ); } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 572b87ac3..f39e148a5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -186,6 +186,20 @@ fn main() -> Result<()> { let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); let task = std::env::args().nth(1).unwrap_or_default(); match task.as_str() { + "build-languages" | "package-languages" | "check-languages" => { + let action = match task.as_str() { + "build-languages" => "build", + "package-languages" => "package", + _ => "check", + }; + run( + Command::new("python3") + .arg(root.join("languages/build.py")) + .arg(action) + .args(std::env::args_os().skip(2)), + "Building language pack", + ) + } "install" => install(root, true), "install-tui" => install(root, false), "build-plugins" => build_plugins(root), From 0d7165caad4b69dcc9915214cff5de5f1631fd1e Mon Sep 17 00:00:00 2001 From: thesiti92 Date: Tue, 29 Sep 2026 14:17:55 -0400 Subject: [PATCH 2/4] Record x64 installation proof and fix CI checks Verify signed macOS x64 loading under Rosetta and Linux x64 installation without development tools. Grant the publication guard read access to environment protection metadata and fix an error-message spelling. AI-assisted implementation and validation with Codex. --- .github/workflows/languages.yml | 2 ++ languages/README.md | 13 +++++++-- languages/VALIDATION.md | 49 +++++++++++++++++++++++++++++++-- src/languages.rs | 2 +- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/.github/workflows/languages.yml b/.github/workflows/languages.yml index 4c4c4e51f..bceacdaea 100644 --- a/.github/workflows/languages.yml +++ b/.github/workflows/languages.yml @@ -28,6 +28,8 @@ concurrency: jobs: guard: runs-on: ubuntu-22.04 + permissions: + actions: read steps: - name: Require protected publication environment if: inputs.publish diff --git a/languages/README.md b/languages/README.md index f626c453a..990cc2ed4 100644 --- a/languages/README.md +++ b/languages/README.md @@ -39,6 +39,12 @@ scanner, and Fortran includes its scanner. Licenses and highlights were copied from the pinned crate/source revisions, except Verilog highlights, which retain diffr's existing query. No grammar package contains JavaScript or install scripts. +To exercise x64 execution under Rosetta on an ARM64 Mac, first install the Rust +target with `rustup target add x86_64-apple-darwin`, then run the commands above +with `x86_64-apple-darwin`. The builder and probe explicitly select the target's +C architecture, and the integration test builds and runs the x64 CLI. This also +checks that package selection follows the executable architecture. + On macOS, set `DIFFR_SIGN_IDENTITY` to Whiteboard's Developer ID identity before running the probe. It signs the libraries and a hardened native host, retaining library validation. The integration script signs the test CLIs with that identity @@ -70,9 +76,10 @@ bytes match. It produces no catalog if any target fails. Partial publication can be retried only with byte-identical artifacts, or with a new pack revision. Re-signing can change bytes even when source inputs are identical. -Local macOS ARM64 and Docker Linux ARM64 proof do not establish the x64 targets, -Gatekeeper behavior for separately quarantined artifacts, or a public npm install. Those remain -release evidence to collect; no validation exemption is added to Whiteboard. +Local signed macOS ARM64/Rosetta x64 and Docker Linux ARM64/x64 checks are +recorded in [VALIDATION.md](VALIDATION.md). Gatekeeper behavior for separately +quarantined libraries and public npm installation remain release evidence to +collect; no validation exemption is added to Whiteboard. ## Clean Linux installation test diff --git a/languages/VALIDATION.md b/languages/VALIDATION.md index 1f10f3867..087df9b9f 100644 --- a/languages/VALIDATION.md +++ b/languages/VALIDATION.md @@ -63,8 +63,8 @@ indicative, not a performance gate. ## Outstanding release evidence -macOS x64, Linux x64, Rosetta execution, public npm downloads, and behavior -for separately quarantined artifacts have not been exercised locally. The +Public npm downloads and behavior for separately quarantined libraries have +not been exercised locally. The four-runner workflow must pass before publication. The repository currently has no configured signing/npm secrets or protected publication environment; see README.md for the exact setup. Only a catalog emitted after all four @@ -102,3 +102,48 @@ grammar-directory symlinks. Rebuilding with the ordinary Docker context and the provided Dockerfile-specific ignore file fixed the test setup. No parser or installer fix was needed. Public npm installation remains unverified until publication and catalog activation. + +## macOS x64 / Rosetta verification + +The signed `x86_64-apple-darwin` probe and both signed x64 CLIs ran under Rosetta +on the ARM64 host. All ten grammars passed native/static diff, syntax, and tree +parity, scanner-heavy fixtures, added/deleted/unchanged files, the 120-file batch, +concurrent installation, corruption/repair, truncated archive rejection, the +default Application Support store, quarantined-archive installation, and an +empty-PATH fresh install. The executable selected the x64 package. + +Both x64 hosts and libraries use Team ID `PWXY59YDAY` with hardened runtime; +the full CLI retains only Whiteboard's existing executable-memory entitlement. +An additional shared-store check installed both ARM64 and x64 revisions into +one root and confirmed that each executable selected its own target and parsed +QML without fallback. + +| x64 measurement | Bytes | +|---|---:| +| Compressed pack | 4,219,033 | +| Native-pack release CLI | 96,633,808 | +| All-languages release CLI | 152,322,048 | + +The new-process Apex median was 395 ms; the 120-file mixed batch took 1,800 ms. +These are Rosetta timings with warm OS caches, not native Intel benchmarks. + +All four versioned npm tarball URLs returned HTTP 404 on 2026-09-29. Public +network installation remains blocked on publication. The repository has no +signing/npm repository secrets and no protected `languages-release` environment; +only the unprotected test environment `languages-ci` currently exists. + +## Linux x64 Docker verification + +The `linux/amd64` Docker build passed all ten native probes and the complete +installed/static integration suite under emulation on the ARM64 host. Its separate +minimal runtime passed fresh offline installation, all parsers, idempotence, +corrupt-library fallback, and atomic repair with UID/GID 65534, networking disabled, +a read-only root, and no Rust, C compiler, Node, npm, or Python. + +The compressed pack was 3,759,226 bytes; native and all-languages executables +were 100,672,240 and 156,206,248 bytes. The Apex median was 366 ms and the +120-file mixed batch took 2,434 ms; these are warm-cache emulation measurements. + +The native Linux x64 GitHub runner also passed installation/parity and its +minimal Docker runtime check in run `36609243307`, alongside both ARM64 runners. +The macOS Intel job was still running when these local results were recorded. diff --git a/src/languages.rs b/src/languages.rs index e21339415..2d54347e3 100644 --- a/src/languages.rs +++ b/src/languages.rs @@ -117,7 +117,7 @@ pub(crate) fn verify_library( package .files .get(&library.file) - .context("uncatalogued library")?, + .context("uncataloged library")?, )?; let canonical = fs::canonicalize(&path)?; ensure!( From 72c7bb9dd0a46ad5b1eaef4b82d444cb1853b8f4 Mon Sep 17 00:00:00 2001 From: thesiti92 Date: Tue, 29 Sep 2026 14:49:50 -0400 Subject: [PATCH 3/4] Drive extra parsers from one registry and package their queries Generate optional Rust configurations from languages/extra.json and remove their arms from the built-in configuration. Build highlight assets from locked upstream sources, verify them before use, and include shared query licenses. Preserve compiled language features and existing Julia/Verilog queries. Validated default/all-language tests, signed macOS installation parity, and clean Linux Docker installation with corrupt-query repair. AI-assisted implementation and validation with Codex. --- Cargo.toml | 4 +- build.rs | 65 ++++ languages/ADDING.md | 105 ++---- languages/README.md | 13 +- languages/VALIDATION.md | 25 ++ languages/apex/highlights.scm | 299 --------------- languages/apex/language.json | 16 - languages/build.py | 16 +- languages/definitions.py | 25 +- languages/extra.json | 436 ++++++++++++++++++++++ languages/fortran/highlights.scm | 204 ----------- languages/fortran/language.json | 16 - languages/fsharp/highlights.scm | 421 --------------------- languages/fsharp/language.json | 16 - languages/haskell/highlights.scm | 443 ----------------------- languages/haskell/language.json | 16 - languages/julia/highlights.scm | 330 ----------------- languages/julia/language.json | 16 - languages/ocaml-interface/highlights.scm | 0 languages/ocaml-interface/language.json | 16 - languages/ocaml/highlights.scm | 148 -------- languages/ocaml/language.json | 16 - languages/package.template.json | 2 +- languages/qml/highlights.scm | 61 ---- languages/qml/language.json | 16 - languages/tests/integration.py | 6 + languages/tests/test_definitions.py | 16 +- languages/typescript/LICENSE | 21 ++ languages/verilog/highlights.scm | 322 ---------------- languages/verilog/language.json | 16 - languages/vhdl/highlights.scm | 435 ---------------------- languages/vhdl/language.json | 16 - src/languages.rs | 12 + src/parse/native.rs | 11 +- src/parse/optional.rs | 54 ++- src/parse/tree_sitter_parser.rs | 236 +----------- tests/cli.rs | 10 +- 37 files changed, 719 insertions(+), 3161 deletions(-) delete mode 100644 languages/apex/highlights.scm delete mode 100644 languages/apex/language.json create mode 100644 languages/extra.json delete mode 100644 languages/fortran/highlights.scm delete mode 100644 languages/fortran/language.json delete mode 100644 languages/fsharp/highlights.scm delete mode 100644 languages/fsharp/language.json delete mode 100644 languages/haskell/highlights.scm delete mode 100644 languages/haskell/language.json delete mode 100644 languages/julia/highlights.scm delete mode 100644 languages/julia/language.json delete mode 100644 languages/ocaml-interface/highlights.scm delete mode 100644 languages/ocaml-interface/language.json delete mode 100644 languages/ocaml/highlights.scm delete mode 100644 languages/ocaml/language.json delete mode 100644 languages/qml/highlights.scm delete mode 100644 languages/qml/language.json create mode 100644 languages/typescript/LICENSE delete mode 100644 languages/verilog/highlights.scm delete mode 100644 languages/verilog/language.json delete mode 100644 languages/vhdl/highlights.scm delete mode 100644 languages/vhdl/language.json diff --git a/Cargo.toml b/Cargo.toml index cc188043a..bc71adafa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,7 @@ include = [ "/build.rs", "/src/", "/languages/catalog.json", - "/languages/*/language.json", - "/languages/*/highlights.scm", + "/languages/extra.json", "/languages/*/LICENSE", "/wit/", "/plugins/", @@ -177,6 +176,7 @@ predicates = "3.1.3" pretty_assertions = "1.3.0" [build-dependencies] +serde_json = "1" toml = "0.8" # TODO: enable parallel mode, see discussion in # https://github.com/rust-lang/cc-rs/pull/849 diff --git a/build.rs b/build.rs index b9767eb08..adb05edc5 100644 --- a/build.rs +++ b/build.rs @@ -46,6 +46,7 @@ fn main() { std::env::var("TARGET").unwrap() ); native_plugins(); + extra_languages(); let parsers = vec![ TreeSitterParser { name: "tree-sitter-janet-simple", @@ -90,6 +91,70 @@ fn main() { println!("cargo:rustc-env=JEMALLOC_SYS_WITH_LG_PAGE=16"); } +fn extra_languages() { + use serde_json::Value; + println!("cargo:rerun-if-changed=languages/extra.json"); + let definitions: Vec = + serde_json::from_str(&std::fs::read_to_string("languages/extra.json").unwrap()).unwrap(); + let strings = |values: &Value| { + values + .as_array() + .unwrap() + .iter() + .map(|value| format!("{:?}", value.as_str().unwrap())) + .collect::>() + .join(",") + }; + let pairs = |values: &Value| { + values + .as_array() + .unwrap() + .iter() + .map(|pair| { + format!( + "({:?},{:?})", + pair[0].as_str().unwrap(), + pair[1].as_str().unwrap() + ) + }) + .collect::>() + .join(",") + }; + let mut code = String::from("extra_languages! {\n"); + for definition in definitions { + let text = |key| definition[key].as_str().unwrap(); + let highlights = definition["highlights"] + .as_array() + .unwrap() + .iter() + .map(|query| { + if let Some(local) = query["local"].as_str() { + println!("cargo:rerun-if-changed={local}"); + format!( + "include_str!(concat!(env!(\"CARGO_MANIFEST_DIR\"), {:?}))", + format!("/{local}") + ) + } else { + format!( + "{}::{}", + query["crate"].as_str().unwrap().replace('-', "_"), + query["constant"].as_str().unwrap() + ) + } + }) + .collect::>() + .join(","); + code.push_str(&format!( + "{} => {{ id: {:?}, feature: {:?}, parser: {}, highlights: [{}], atoms: [{}], delimiters: [{}], ignore_trailing: [{}] }},\n", + text("variant"), text("id"), text("feature"), text("parser"), highlights, + strings(&definition["atoms"]), pairs(&definition["delimiters"]), pairs(&definition["ignore_trailing"]), + )); + } + code.push_str("}\n"); + let output = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("extra_languages.rs"); + std::fs::write(output, code).unwrap(); +} + fn commit_info() { if !PathBuf::from(".git").exists() { return; diff --git a/languages/ADDING.md b/languages/ADDING.md index c7f9e8db8..35b59ea8d 100644 --- a/languages/ADDING.md +++ b/languages/ADDING.md @@ -1,53 +1,32 @@ -# Moving another parser into `extra` - -An extra parser is still a first-party diffr language. Recognition, highlight -queries, annotation queries, and diff rules remain in diffr. Only generated C -parser/scanner code moves into the native pack. Keep its existing compile-time -feature so `all-languages` remains self-contained. - -## 1. Define the grammar - -Create `languages//language.json`, `highlights.scm`, and `LICENSE`. -Use an existing definition such as `qml/language.json` as the template: - -| Field | Meaning | -|---|---| -| `id` | Stable canonical ID, also the library filename stem | -| `variant` | Existing Rust `Language` enum variant | -| `feature` | Cargo feature enabling the built-in parser, e.g. `lang-qml` | -| `crate`, `version`, `revision`, `repository` | Locked crate and upstream source pin | -| `source` | Relative path to the generated `parser.c` and optional `scanner.c` | -| `symbol` | Exported Tree-sitter function, e.g. `tree_sitter_qmljs` | -| `fixture.prefix`, `fixture.extension` | Existing `sample_files/_{1,2}.` pair | -| `smoke` | Small valid program that the native host must parse without errors | - -Copy queries and the upstream license from the pinned source. Preserve any -existing diffr-specific highlight query. An intentionally empty query is valid; -OCaml interfaces retain their existing empty query. Scanner includes are resolved -relative to the Cargo source tree, so shared scanner headers need not be copied. - -Multiple grammar entries can share a Cargo feature/dependency. OCaml and OCaml -interfaces demonstrate this. Include only grammar variants diffr actually uses; -the pack excludes F# signatures, OCaml standalone types, and Salesforce's -standalone SOQL/SOSL/log grammars. - -## 2. Keep built-in and installed behavior aligned - -- Make the parser dependency optional in `Cargo.toml`. Add `lang-` pointing - to `dep:` and include that feature in `all-languages`. -- Add one entry to `src/parse/optional.rs`. This single table supplies both the - canonical ID and compile-time availability; `languages list` discovers it. -- In the existing `tree_sitter_parser.rs` arm, select the compiled grammar behind - its feature and the already-resolved external grammar otherwise. Follow the - QML arm. Load its own highlight query with `include_str!` from `languages/`. - Preserve shared queries, atom rules, delimiters, and embedded-language behavior. -- Keep the `Language` variant, names, globs, and mode detection unconditional. - -Do not add another language list to packaging or test scripts. They discover all -`languages/*/language.json` files. `check-languages` rejects registry/Cargo-feature -mismatches, missing `all-languages` entries, licenses, queries, and fixtures. - -## 3. Validate the change +# Moving another language into `extra` + +`languages/extra.json` is the single extra-language registry. It drives package +building, generated Rust parser configuration, availability reporting, and tests. +Generated files stay under `target/`; no generated parser or upstream highlight +query needs to be copied into this repository. + +1. Move the language's configuration from `tree_sitter_parser.rs` into an entry + in `extra.json`. Preserve its atom nodes, delimiters, and trailing-token rules. + Copy an existing entry with similar rules as a template. +2. Declare the grammar dependency optional in Cargo, retain a `lang-*` feature, + and include it in `all-languages`. Both OCaml grammars share one feature. +3. Supply the pinned crate version/revision, source directory, exported C symbol, + Rust parser constant, license, and a pair of representative fixtures. +4. Describe highlight inputs by upstream crate, Rust constant, and relative file + path. The package builder reads the files from Cargo's locked sources; built-in + builds use the corresponding constants. QML demonstrates combined queries. + Use `local` only for an existing diffr customization, such as Julia or Verilog. + An empty list preserves OCaml interfaces' existing behavior. Shared query + dependencies include their licenses; use `license` for a repository copy if + the upstream crate omits that file. +5. Run the checks below and advance the independent pack version when its contents + change. Keep language names, globs, and detection unconditional. + +Do not add another Rust or Python language list. Cargo generates the Rust registry +from `extra.json`. `check-languages` rejects duplicate built-in configurations, +missing features, licenses, query inputs, and fixtures. Recognition, annotation +rules, and structural diff rules remain in diffr; the pack carries compiled +parser/scanner libraries and integrity-checked highlight query files. ```sh cargo xtask check-languages @@ -55,28 +34,16 @@ python3 languages/tests/test_definitions.py cargo test --locked cargo test --locked --features all-languages --bin diffr --test cli cargo xtask build-languages --target aarch64-apple-darwin -# Set DIFFR_SIGN_IDENTITY to Whiteboard's identity for the signed macOS checks. +# Set DIFFR_SIGN_IDENTITY to Whiteboard's identity for signed macOS checks. python3 languages/tests/probe.py target/languages/aarch64-apple-darwin/package cargo xtask package-languages --target aarch64-apple-darwin python3 languages/tests/integration.py target/languages/aarch64-apple-darwin/catalog-entry.json --release ``` -Packaging and native smoke tests automatically include every definition. The -integration suite compares every grammar's structural output, syntax spans, -parse trees, and added/deleted/unchanged files with an `all-languages` build; -it also checks parallel diffing, installation, corruption, repair, offline use, -and the platform's default store. Add scanner-heavy fixtures when the existing -pair does not cover the grammar's scanner behavior. - -The generated `target/languages//test-fixtures.txt` drives the minimal -Linux runtime test; no compiler, Rust, Node, npm, or Python is present there. -The four-target workflow remains the release gate. Moving a language is not -complete until native and installed/static checks pass for that language. - -## 4. Advance the pack revision +Every registry entry automatically participates in native/static structural, +syntax, tree, and installation tests. Include scanner-heavy fixtures when needed. +The tests also cover concurrent loads/installs, corrupt libraries and queries, +repair, offline use, and the platform's default store. -Choose a new independent version in `package.template.json` when changing pack -contents. Never reuse published bytes/version pairs. Build, strip, sign, package, -publish, and verify all four targets before updating `catalog.json`, as described -in README.md. Existing CLIs keep their own pinned revisions. Do not hand-author -hashes or temporarily trust arbitrary runtime manifests. +Publish and verify all four target archives before updating `catalog.json`, as +explained in [README.md](README.md). Existing CLIs retain their pinned revisions. diff --git a/languages/README.md b/languages/README.md index 990cc2ed4..2bb92992b 100644 --- a/languages/README.md +++ b/languages/README.md @@ -13,7 +13,8 @@ built-in parsers and take priority over packages. The pack contains Fortran, F#, Verilog, OCaml (implementation and interface), Julia, Haskell, VHDL, Salesforce Apex, and QML. LaTeX and general SQL remain bundled. -See [ADDING.md](ADDING.md) for the validated pathway to move another parser. +The single registry is [extra.json](extra.json); see [ADDING.md](ADDING.md) for +the pathway to move another parser out of the built-in configuration. The store is `/diffr/parsers//extra/`. `DIFFR_PARSER_DIR` replaces the `parsers` root. Revisions coexist, and ordinary @@ -34,10 +35,12 @@ python3 languages/tests/integration.py target/languages/aarch64-apple-darwin/cat Python 3 and a C compiler are build tools only. Cargo's locked metadata locates registry sources; definitions additionally pin versions and source revisions. -Highlights are compiled into diffr. F# uses only `fsharp/src`, including its -scanner, and Fortran includes its scanner. Licenses and highlights were copied -from the pinned crate/source revisions, except Verilog highlights, which retain -diffr's existing query. No grammar package contains JavaScript or install scripts. +Highlight queries travel with the optional pack and are hash-verified before use. +Built-in feature builds continue to use the upstream crate constants. F# uses +only `fsharp/src`, including its scanner, and Fortran includes its scanner. +Licenses come from the pinned crate/source revisions. The builder reads upstream +queries directly from Cargo sources and preserves diffr's existing Julia and +Verilog custom queries. No grammar package contains JavaScript or install scripts. To exercise x64 execution under Rosetta on an ARM64 Mac, first install the Rust target with `rustup target add x86_64-apple-darwin`, then run the commands above diff --git a/languages/VALIDATION.md b/languages/VALIDATION.md index 087df9b9f..cc66a2398 100644 --- a/languages/VALIDATION.md +++ b/languages/VALIDATION.md @@ -3,6 +3,31 @@ Base: diffr `58fa3b2` (includes PR #49). Host: macOS ARM64. No packages were published, and `catalog.json` remains empty pending verified publication. +## Registry/query refactor — pack 0.3.0 + +`extra.json` now generates the Rust configuration and drives packaging. Upstream +highlight files are no longer copied into the repository or embedded in default +builds. The pack contains catalog-hashed queries alongside its libraries; +`all-languages` still reads upstream crate constants. + +- Full default workspace tests: 290 passed, one ignored. +- Default and all-languages bin/CLI tests: 271 passed, one ignored, in each build. +- Five registry checks and both publication-gate tests passed. +- Signed macOS ARM64 native/static parity passed for all ten grammars, including + query corruption/fallback/repair, scanners, parallel installs/diffs, offline + installation, the default store, quarantine-marked archives, and old CLI reuse. +- Linux ARM64 Docker passed the same parity suite and the minimal unprivileged, + read-only-root, no-network/no-development-tools runtime installation test. +- The source package includes `extra.json` and the catalog, with no copied upstream + highlight files. Formatting, actionlint, typos, and whitespace checks passed. +- Clippy reports only the six pre-existing warnings listed below. + +Compressed pack sizes: 3,947,947 bytes on macOS ARM64 and 3,748,052 bytes on Linux +ARM64. The four-target workflow will rerun on the updated PR; the x64 results +below describe the previous 0.2.0 pack, before query relocation. + +## Earlier 0.2.0 validation + ## Passed - `cargo test --locked`: 290 tests passed, one ignored. diff --git a/languages/apex/highlights.scm b/languages/apex/highlights.scm deleted file mode 100644 index c130edf39..000000000 --- a/languages/apex/highlights.scm +++ /dev/null @@ -1,299 +0,0 @@ -;; attempting to match concepts represented here: -;; https://code.visualstudio.com/api/language-extensions/semantic-highlight-guide - -[ - "[" - "]" - "{" - "}" - "?" - ";" -] @punctuation - -;; Methods - -(method_declaration - name: (identifier) @method) -(method_declaration - type: (type_identifier) @type) - -(method_invocation - name: (identifier) @method) -(argument_list - (identifier) @variable) -(super) @function.defaultLibrary - -(explicit_constructor_invocation - arguments: (argument_list - (identifier) @variable )) - -;; Annotations - -(annotation - name: (identifier) @decorator) - -"@" @operator - -(annotation_key_value - (identifier) @variable) - - -;; Types - -;; because itendifying it when declared doesn't carry to use -;; leans on the convention that "screaming snake case" is a const -((identifier) @variable.readonly - (#match? @variable.readonly "^_*[A-Z][A-Z\\d_]+$")) - -(interface_declaration - name: (identifier) @interface) -(class_declaration - name: (identifier) @class) -(class_declaration - (superclass) @class) -(enum_declaration - name: (identifier) @enum) -(enum_constant - name: (identifier) @enumMember) - -(interfaces - (type_list - (type_identifier) @interface )) - -(local_variable_declaration - (type_identifier) @type ) - -( expression_statement (_ (identifier)) @variable) - -(type_arguments "<" @punctuation) -(type_arguments ">" @punctuation) - -; (identifier) @variable - -((field_access - object: (identifier) @type)) ;; don't know what type of thing it is - -(generic_type - (type_identifier) @type) -(type_arguments (type_identifier) @type) - -(field_access - field: (identifier) @property) - -((scoped_identifier - scope: (identifier) @type) - (#match? @type "^[A-Z]")) -((method_invocation - object: (identifier) @type) - (#match? @type "^[A-Z]")) - - -(field_declaration - type: (type_identifier) @type) - -(formal_parameter - type: (type_identifier) @type - (identifier) @variable) - -(method_declaration - (formal_parameters - (formal_parameter - name: (identifier) @parameter))) - -(enhanced_for_statement - type: (type_identifier) @type - name: (identifier) @variable ) - -(enhanced_for_statement - value: (identifier) @variable) - -(enhanced_for_statement - name: (identifier) @variable) - -(object_creation_expression - type: (type_identifier) @type) - -(array_creation_expression - type: (type_identifier) @type) - -(array_type - element: (type_identifier) @type) - -(return_statement - (identifier) @variable) - -(local_variable_declaration - (variable_declarator - name: (identifier) @variable )) - -(for_statement - condition: (binary_expression - (identifier) @variable)) - -(for_statement - update: (update_expression - (identifier) @variable)) - -(constructor_declaration - name: (identifier) @class) - -(dml_type) @function.defaultLibrary - -(bound_apex_expression - (identifier) @variable) - -(assignment_operator) @operator - -(update_operator) @operator - -(instanceof_expression - left: (identifier) @variable - right: (type_identifier) @type ) - -(cast_expression - type: (type_identifier) @type - value: (identifier) @variable) - -(switch_expression - condition: (identifier) @variable) - -(switch_rule - (switch_label - (identifier) @enumMember )) - -(when_sobject_type - (type_identifier) @type - (identifier) @variable ) - -(trigger_declaration - name: (identifier) @type - object: (identifier) @type - (trigger_event) @keyword - ("," (trigger_event) @keyword)*) - -(binary_expression - operator: [ - ">" - "<" - ">=" - "<=" - "==" - "===" - "!=" - "!==" - "&&" - "||" - "+" - "-" - "*" - "/" - "&" - "|" - "^" - "%" - "<<" - ">>" - ">>>"] @operator) - -(binary_expression - (identifier) @variable) - -(unary_expression - operator: [ - "+" - "-" - "!" - "~" - ]) @operator - -("=>" @operator) - -[ - (boolean_type) - (void_type) -] @type.defaultLibrary - -; Variables - -(field_declaration (variable_declarator - (identifier) @property)) - -(field_declaration - (modifiers (modifier [(final) (static)])(modifier [(final) (static)])) - (variable_declarator - name: (identifier) @variable.readonly)) - -(this) @variable.defaultLibrary - -; Literals - -[ - (int) -] @number - -[ - (string_literal) -] @string - -[ - (line_comment) - (block_comment) -] @comment - -;; Keywords - -[ - (abstract) - (all_rows_clause) - "break" - "catch" - "class" - "continue" - "do" - "else" - "enum" - "extends" - (final) - "finally" - "for" - "get" - (global) - "if" - "implements" - "instanceof" - "interface" - "new" - "on" - (override) - (private) - (protected) - (public) - "return" - "set" - (static) - "switch" - (testMethod) - (webservice) - "throw" - (transient) - "try" - "trigger" - (virtual) - "when" - "while" - (with_sharing) - (without_sharing) - (inherited_sharing) -] @keyword - -(assignment_expression - left: (identifier) @variable) - -; (type_identifier) @type ;; not respecting precedence... -;; I don't love this but couldn't break them up right now -;; can't figure out how to let that be special without conflicting -;; in the grammar -"System.runAs" @method.defaultLibrary - -(scoped_type_identifier - (type_identifier) @type) diff --git a/languages/apex/language.json b/languages/apex/language.json deleted file mode 100644 index 26f01c75b..000000000 --- a/languages/apex/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "apex", - "crate": "tree-sitter-sfapex", - "version": "2.4.0", - "source": "apex/src", - "symbol": "tree_sitter_apex", - "revision": "9247e2cd12490f1aac71cb212f726ef1a2b5f7f9", - "repository": "https://github.com/aheber/tree-sitter-sfapex", - "variant": "Apex", - "feature": "lang-apex", - "fixture": { - "prefix": "apex", - "extension": "trigger" - }, - "smoke": "public class Example { public Integer f(Integer x) { return x + 1; } }\n" -} diff --git a/languages/build.py b/languages/build.py index e5f092d7a..5ce56e5fa 100644 --- a/languages/build.py +++ b/languages/build.py @@ -74,6 +74,19 @@ def main(): subprocess.run(['strip', '-x' if system == 'darwin' else '--strip-unneeded', str(package / filename)], check=True) abi = int(re.search(r'#define LANGUAGE_VERSION (\d+)', (source / 'parser.c').read_text())[1]) libraries.append(dict(id=definition['id'], file=filename, symbol=definition['symbol'], abi=abi)) + highlights = [] + for query in definition['highlights']: + if 'local' in query: + path = ROOT / query['local'] + else: + dependency = next(p for p in metadata['packages'] if p['name'] == query['crate']) + dependency_root = Path(dependency['manifest_path']).parent + path = dependency_root / query['path'] + if query['crate'] != definition['crate']: + license = ROOT / query['license'] if 'license' in query else dependency_root / 'LICENSE' + shutil.copyfile(license, package / (query['crate'] + '.LICENSE')) + highlights.append(path.read_text()) + (package / (definition['id'] + '.highlights.scm')).write_text(''.join(highlights)) shutil.copyfile(ROOT / 'languages' / definition['id'] / 'LICENSE', package / (definition['id'] + '.LICENSE')) write(package / 'manifest.json', dict(schema=1, pack='extra', version=template['version'], target=args.target, libraries=libraries)) write(package / 'package.json', dict(template, name=name, os=[system], cpu=[cpu], **({'libc': ['glibc']} if system == 'linux' else {}))) @@ -84,7 +97,8 @@ def main(): assert {(d['id'], d['symbol']) for d in manifest['libraries']} == expected, 'stale package: rebuild the language libraries' expected_files = {'manifest.json', 'package.json'} for library in manifest['libraries']: - expected_files.update([library['file'], library['id'] + '.LICENSE']) + expected_files.update([library['file'], library['id'] + '.LICENSE', library['id'] + '.highlights.scm']) + expected_files.update(q['crate'] + '.LICENSE' for d in definitions for q in d['highlights'] if 'crate' in q and q['crate'] != d['crate']) assert {p.name for p in package.iterdir()} == expected_files, 'unexpected or missing package files' files = {p.name: dict(size=p.stat().st_size, sha256=digest(p.read_bytes())) for p in sorted(package.iterdir())} archive = output / (name.split('/')[1] + '-' + template['version'] + '.tgz') diff --git a/languages/definitions.py b/languages/definitions.py index efc510a01..630e83daa 100644 --- a/languages/definitions.py +++ b/languages/definitions.py @@ -8,22 +8,22 @@ def definitions(root=ROOT): - return [json.loads(path.read_text()) for path in sorted((root / 'languages').glob('*/language.json'))] + return json.loads((root / 'languages/extra.json').read_text()) def validate(items, metadata, root=ROOT): package = next(p for p in metadata['packages'] if p['name'] == 'diffr-cli') dependencies = {d['name']: d for d in package['dependencies']} - registry = set(re.findall(r'(\w+)\s*=>\s*\("([^"]+)",\s*"([^"]+)"\)', (root / 'src/parse/optional.rs').read_text())) - expected = {(d['variant'], d['id'], d['feature']) for d in items} - assert registry == expected, 'optional Rust registry and language definitions differ' + builtins = set(re.findall(r'^ (\w+) => \{', (root / 'src/parse/tree_sitter_parser.rs').read_text(), re.MULTILINE)) assert len({d['id'] for d in items}) == len(items), 'duplicate language ID' + assert len({d['variant'] for d in items}) == len(items), 'duplicate language variant' assert {d['crate'] for d in items} == {name for name, dep in dependencies.items() if dep['optional'] and name.startswith(('tree-sitter-', 'ts-parser-'))}, 'optional Cargo dependencies and definitions differ' assert {d['feature'] for d in items} == {name for name in package['features'] if name.startswith('lang-')}, 'language Cargo features and definitions differ' for definition in items: id, feature, crate = (definition[k] for k in ('id', 'feature', 'crate')) assert re.fullmatch(r'[a-z][a-z0-9-]*', id), f'{id}: invalid ID' assert definition['variant'].isidentifier(), f'{id}: invalid enum variant' + assert definition['variant'] not in builtins, f'{id}: remove duplicate built-in config' assert package['features'][feature] == ['dep:' + crate], f'{id}: incorrect Cargo feature' assert feature in package['features']['all-languages'], f'{id}: missing from all-languages' assert dependencies[crate]['optional'], f'{id}: dependency must be optional' @@ -31,8 +31,21 @@ def validate(items, metadata, root=ROOT): assert not source.is_absolute() and '..' not in source.parts, f'{id}: invalid source path' assert re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9]*', definition['symbol']), f'{id}: invalid symbol' assert definition['smoke'].strip(), f'{id}: missing parse smoke input' - for filename in ['LICENSE', 'highlights.scm']: - assert (root / 'languages' / id / filename).is_file(), f'{id}: missing {filename}' + assert re.fullmatch(r'\w+(::\w+)+', definition['parser']), f'{id}: invalid parser constant' + assert (root / 'languages' / id / 'LICENSE').is_file(), f'{id}: missing LICENSE' + for query in definition['highlights']: + if 'license' in query: + assert (root / query['license']).is_file(), f'{id}: missing query license' + if 'local' in query: + relative = Path(query['local']) + assert not relative.is_absolute() and '..' not in relative.parts, f'{id}: invalid local query path' + path = root / relative + assert path.is_file(), f'{id}: missing local query' + else: + assert query['crate'] in dependencies, f'{id}: unknown query crate' + assert re.fullmatch(r'\w+(::\w+)*', query['constant']), f'{id}: invalid query constant' + path = Path(query['path']) + assert not path.is_absolute() and '..' not in path.parts, f'{id}: invalid query path' fixture = definition['fixture'] for side in (1, 2): path = root / 'sample_files' / f"{fixture['prefix']}_{side}.{fixture['extension']}" diff --git a/languages/extra.json b/languages/extra.json new file mode 100644 index 000000000..714840d17 --- /dev/null +++ b/languages/extra.json @@ -0,0 +1,436 @@ +[ + { + "id": "apex", + "crate": "tree-sitter-sfapex", + "version": "2.4.0", + "source": "apex/src", + "symbol": "tree_sitter_apex", + "revision": "9247e2cd12490f1aac71cb212f726ef1a2b5f7f9", + "repository": "https://github.com/aheber/tree-sitter-sfapex", + "variant": "Apex", + "feature": "lang-apex", + "fixture": { + "prefix": "apex", + "extension": "trigger" + }, + "smoke": "public class Example { public Integer f(Integer x) { return x + 1; } }\n", + "parser": "tree_sitter_sfapex::apex::LANGUAGE", + "atoms": [ + "string_literal", + "null_literal", + "boolean", + "int", + "decimal_floating_point_literal", + "date_literal", + "currency_literal" + ], + "delimiters": [ + [ + "[", + "]" + ], + [ + "(", + ")" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-sfapex", + "constant": "apex::HIGHLIGHTS_QUERY", + "path": "apex/queries/highlights.scm" + } + ] + }, + { + "id": "fortran", + "crate": "tree-sitter-fortran", + "version": "0.6.0", + "source": "src", + "symbol": "tree_sitter_fortran", + "revision": "48593829df8a929bfe58528eedcf405171921005", + "repository": "https://github.com/stadelmanma/tree-sitter-fortran", + "variant": "Fortran", + "feature": "lang-fortran", + "fixture": { + "prefix": "fortran", + "extension": "f90" + }, + "smoke": "program hello\n implicit none\n print *, \"hello\"\nend program hello\n", + "parser": "tree_sitter_fortran::LANGUAGE", + "atoms": [ + "string_literal", + "number_literal" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "(/", + "/)" + ], + [ + "[", + "]" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-fortran", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ] + }, + { + "id": "fsharp", + "crate": "tree-sitter-fsharp", + "version": "0.3.0", + "source": "fsharp/src", + "symbol": "tree_sitter_fsharp", + "revision": "5247c1197cb290fcaea0e0a793d32829c1396831", + "repository": "https://github.com/ionide/tree-sitter-fsharp", + "variant": "FSharp", + "feature": "lang-fsharp", + "fixture": { + "prefix": "f_sharp", + "extension": "fs" + }, + "smoke": "module Example\nlet f x =\n let y = x + 1\n y * 2\n", + "parser": "tree_sitter_fsharp::LANGUAGE_FSHARP", + "atoms": [ + "string", + "triple_quoted_string" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-fsharp", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ] + }, + { + "id": "haskell", + "crate": "tree-sitter-haskell", + "version": "0.23.1", + "source": "src", + "symbol": "tree_sitter_haskell", + "revision": "c30d812bc90827f1a54106a25bc9a6307f5cdcec", + "repository": "https://github.com/tree-sitter/tree-sitter-haskell", + "variant": "Haskell", + "feature": "lang-haskell", + "fixture": { + "prefix": "haskell", + "extension": "hs" + }, + "smoke": "module Example where\nf x =\n let y = x + 1\n in y * 2\n", + "parser": "tree_sitter_haskell::LANGUAGE", + "atoms": [ + "qualified_variable" + ], + "delimiters": [ + [ + "[", + "]" + ], + [ + "(", + ")" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-haskell", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ] + }, + { + "id": "julia", + "crate": "tree-sitter-julia", + "version": "0.23.1", + "source": "src", + "symbol": "tree_sitter_julia", + "revision": "a8e1262997d5a45520a06cbe1b86c0737d507054", + "repository": "https://github.com/tree-sitter/tree-sitter-julia", + "variant": "Julia", + "feature": "lang-julia", + "fixture": { + "prefix": "julia", + "extension": "jl" + }, + "smoke": "function f(x)\n x + 1\nend\n", + "parser": "tree_sitter_julia::LANGUAGE", + "atoms": [ + "string_literal", + "prefixed_string_literal", + "command_literal", + "character_literal" + ], + "delimiters": [ + [ + "{", + "}" + ], + [ + "[", + "]" + ], + [ + "(", + ")" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "local": "vendored_parsers/highlights/julia.scm" + } + ] + }, + { + "id": "ocaml", + "crate": "tree-sitter-ocaml", + "version": "0.25.0", + "source": "grammars/ocaml/src", + "symbol": "tree_sitter_ocaml", + "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", + "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", + "variant": "OCaml", + "feature": "lang-ocaml", + "fixture": { + "prefix": "ocaml", + "extension": "ml" + }, + "smoke": "module M = struct\n let f x = x + 1\nend\n", + "parser": "tree_sitter_ocaml::LANGUAGE_OCAML", + "atoms": [ + "character", + "string", + "quoted_string", + "tag", + "type_variable", + "attribute_id" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-ocaml", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ] + }, + { + "id": "ocaml-interface", + "crate": "tree-sitter-ocaml", + "version": "0.25.0", + "source": "grammars/interface/src", + "symbol": "tree_sitter_ocaml_interface", + "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", + "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", + "variant": "OCamlInterface", + "feature": "lang-ocaml", + "fixture": { + "prefix": "ocaml_interface", + "extension": "mli" + }, + "smoke": "type t = { value : int }\nval f : int -> t\n", + "parser": "tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE", + "atoms": [ + "character", + "string", + "quoted_string", + "tag", + "type_variable", + "attribute_id" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "ignore_trailing": [], + "highlights": [] + }, + { + "id": "qml", + "crate": "tree-sitter-qmljs", + "version": "0.3.0", + "source": "src", + "symbol": "tree_sitter_qmljs", + "revision": "0bec4359a7eb2f6c9220cd57372d87d236f66d59", + "repository": "https://github.com/yuja/tree-sitter-qmljs", + "variant": "Qml", + "feature": "lang-qml", + "fixture": { + "prefix": "qml", + "extension": "qml" + }, + "smoke": "import QtQuick\nItem { property int count: 1; function f(x) { return x + count; } }\n", + "parser": "tree_sitter_qmljs::LANGUAGE", + "atoms": [ + "string", + "template_string", + "regex" + ], + "delimiters": [ + [ + "{", + "}" + ], + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "<", + ">" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-javascript", + "constant": "HIGHLIGHT_QUERY", + "path": "queries/highlights.scm" + }, + { + "crate": "tree-sitter-typescript", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm", + "license": "languages/typescript/LICENSE" + }, + { + "crate": "tree-sitter-qmljs", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/highlights.scm" + } + ] + }, + { + "id": "verilog", + "crate": "tree-sitter-verilog", + "version": "1.0.3", + "source": "src", + "symbol": "tree_sitter_verilog", + "revision": "521b535e41a5acd2c6539a922d4649bbe8275110", + "repository": "https://github.com/tree-sitter/tree-sitter-verilog", + "variant": "Verilog", + "feature": "lang-verilog", + "fixture": { + "prefix": "verilog", + "extension": "sv" + }, + "smoke": "module example(input a, output b);\nassign b = a;\nendmodule\n", + "parser": "tree_sitter_verilog::LANGUAGE", + "atoms": [ + "integral_number" + ], + "delimiters": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "begin", + "end" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "local": "vendored_parsers/highlights/verilog.scm" + } + ] + }, + { + "id": "vhdl", + "crate": "tree-sitter-vhdl", + "version": "1.4.0", + "source": "src", + "symbol": "tree_sitter_vhdl", + "revision": "889c521757471093ce5ac83502ca8a89e516404e", + "repository": "https://github.com/jpt13653903/tree-sitter-vhdl", + "variant": "Vhdl", + "feature": "lang-vhdl", + "fixture": { + "prefix": "vhdl", + "extension": "vhd" + }, + "smoke": "entity example is\nend entity example;\narchitecture rtl of example is\nbegin\nend architecture rtl;\n", + "parser": "tree_sitter_vhdl::LANGUAGE", + "atoms": [], + "delimiters": [ + [ + "(", + ")" + ] + ], + "ignore_trailing": [], + "highlights": [ + { + "crate": "tree-sitter-vhdl", + "constant": "HIGHLIGHTS_QUERY", + "path": "queries/Neovim/highlights.scm" + } + ] + } +] diff --git a/languages/fortran/highlights.scm b/languages/fortran/highlights.scm deleted file mode 100644 index a56775ca8..000000000 --- a/languages/fortran/highlights.scm +++ /dev/null @@ -1,204 +0,0 @@ -[ - (identifier) - (module_name) -] @variable - -(string_literal) @string -(number_literal) @number -(statement_label) @number -(statement_label_reference) @number -(boolean_literal) @boolean -(comment) @comment -(custom_directive) @custom_directive - -[ - (derived_type) - (derived_type_statement) - (import_statement) - (intrinsic_type) - (type_name) -] @type - -(intrinsic_type) @type.builtin - -(base_type_specifier - (identifier) @type) - -[ - (module_statement) - (submodule_statement) -] @module - -[ - (abstract_specifier) - (access_specifier) - (block_label) - (block_label_start_expression) - (none) - (procedure_attributes) - (procedure_qualifier) - (type_qualifier) -] @attribute - -[ - "#define" - "#elif" - "#endif" - "#if" - "#ifdef" - (base_type_specifier) - (block_construct) - (contains_statement) - (default) - (end_associate_statement) - (end_block_construct_statement) - (end_block_data_statement) - (end_coarray_critical_statement) - (end_coarray_team_statement) - (end_do_loop_statement) - (end_enum_statement) - (end_enumeration_type_statement) - (end_forall_statement) - (end_function_statement) - (end_if_statement) - (end_interface_statement) - (end_module_procedure_statement) - (end_module_statement) - (end_program_statement) - (end_select_statement) - (end_submodule_statement) - (end_subroutine_statement) - (end_type_statement) - (end_where_statement) - (enum_statement) - (enumeration_type_statement) - (enumerator_statement) - (equivalence_statement) - (function_statement) - (implicit_statement) - (interface_statement) - (keyword_statement) - (language_binding) - (namelist_statement) - (print_statement) - (procedure_statement) - (program_statement) - (subroutine_statement) -] @keyword - -(use_statement "use" @keyword) -(use_statement "intrinsic" @keyword) -(included_items "only" @keyword) -(allocate_statement "allocate" @keyword) -(deallocate_statement "deallocate" @keyword) -(subroutine_call "call" @keyword) -(do_statement "do" @keyword) -(while_statement "while" @keyword) -(if_statement ["if" "then"] @keyword) -(elseif_clause ["else" "if" "elseif"] @keyword) -(else_clause "else" @keyword) -(open_statement "open" @keyword) -(write_statement "write" @keyword) -(private_statement "private" @keyword) -(public_statement "public" @keyword) - - -(select_case_statement "select" @keyword "case" @keyword) -(select_type_statement "select" @keyword "type" @keyword) -(select_rank_statement "select" @keyword "rank" @keyword) -(case_statement "case" @keyword) -(type_statement "type" @keyword) -(rank_statement "rank" @keyword) - - -[ - "*" - "+" - "-" - "/" - "=" - "<" - ">" - "<=" - ">=" - "==" - "/=" - ".and." - ".or." - ".lt." - ".gt." - ".ge." - ".le." - ".eq." - ".eqv." - ".neqv." - ".ne." -] @operator - -;; Brackets -[ - "(" - ")" - "[" - "]" - "<<<" - ">>>" -] @punctuation.bracket - -;; Delimiter -[ - "::" - "," - "%" - ":" -] @punctuation.delimiter - -"&" @punctuation.special - -(parameters - (identifier) @variable.parameter) - -(program_statement - (name) @variable) - -(module_statement - (name) @variable) - -(submodule_statement - (module_name) (name) @variable) - -(function_statement - (name) @function) - -(subroutine_statement - (name) @function) - -(module_procedure_statement - (name) @function) - -(end_program_statement - (name) @variable) - -(end_module_statement - (name) @variable) - -(end_submodule_statement - (name) @variable) - -(end_function_statement - (name) @function) - -(end_subroutine_statement - (name) @function) - -(end_module_procedure_statement - (name) @function) - -(subroutine_call - (identifier) @function) - -(keyword_argument - name: (identifier) @keyword) - -(derived_type_member_expression - (type_member) @property) diff --git a/languages/fortran/language.json b/languages/fortran/language.json deleted file mode 100644 index 317c6ade3..000000000 --- a/languages/fortran/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "fortran", - "crate": "tree-sitter-fortran", - "version": "0.6.0", - "source": "src", - "symbol": "tree_sitter_fortran", - "revision": "48593829df8a929bfe58528eedcf405171921005", - "repository": "https://github.com/stadelmanma/tree-sitter-fortran", - "variant": "Fortran", - "feature": "lang-fortran", - "fixture": { - "prefix": "fortran", - "extension": "f90" - }, - "smoke": "program hello\n implicit none\n print *, \"hello\"\nend program hello\n" -} diff --git a/languages/fsharp/highlights.scm b/languages/fsharp/highlights.scm deleted file mode 100644 index be6f2294c..000000000 --- a/languages/fsharp/highlights.scm +++ /dev/null @@ -1,421 +0,0 @@ -;; ---------------------------------------------------------------------------- -;; Literals and comments - -[ - (line_comment) - (xml_doc) - (block_comment) -] @comment @spell - -(xml_doc) @comment.documentation @spell - -(const - [ - (_) @constant - (unit) @constant.builtin - ]) - -(primary_constr_args (_) @variable.parameter) - -(class_as_reference - (_) @variable.parameter.builtin) - - -((argument_patterns (long_identifier (identifier) @character.special)) - (#match? @character.special "^\_.*")) - -;; ---------------------------------------------------------------------------- -;; Punctuation - -(type_name type_name: (_) @type.definition) -(exception_definition exception_name: (_) @type.definition) - -[ - (_type) - (atomic_type) -] @type - -(member_signature - . - (identifier) @function.member - (curried_spec - (arguments_spec - "*"* @operator - (argument_spec - (argument_name_spec - "?"? @character.special - name: (_) @variable.parameter))))) - -(union_type_case (identifier) @constant) - -(rules - (rule - pattern: (_) @constant - block: (_))) - -(wildcard_pattern) @character.special - -(identifier_pattern - . - (_) @constant - . - (_) @variable) - -(optional_pattern - "?" @character.special) - -(fsi_directive_decl . (string) @module) - -(import_decl . (_) @module) -(named_module - name: (_) @module) -(namespace - name: (_) @module) -(module_defn - . - (_) @module) - -(ce_expression - . - (_) @constant.macro) - -(field_initializer - field: (_) @property) - -(record_fields - (record_field - . - (identifier) @property)) - -(value_declaration_left . (_) @variable) - -(function_declaration_left - . (_) @function) - -(argument_patterns) @variable.parameter -(typed_pattern - (_pattern) @variable.parameter - (_type) @type) - -(member_defn - (method_or_prop_defn - [ - (property_or_ident) @function - (property_or_ident - instance: (identifier) @variable.parameter.builtin - method: (identifier) @function.method) - ] - args: (_)* @variable.parameter)) - - -(dot_expression - . - (_) @variable.member - . - (_)) - -(application_expression - . - (long_identifier_or_op - (identifier) @function.call) - . - (_) @variable) - -(application_expression - . - (dot_expression - field: (long_identifier_or_op - (identifier) @function.call)) - . - (_) @variable) - -(application_expression - . - (typed_expression - (long_identifier_or_op - (identifier) @function.call) - (_)) - . - (_) @variable) - -(application_expression - . - (typed_expression - (dot_expression - field: (long_identifier_or_op - (identifier) @function.call)) - (_)) - . - (_) @variable) - -((infix_expression - . - (_) - . - (infix_op) @operator - . - (_) @function.call - ) - (#eq? @operator "|>") - ) - -((infix_expression - . - (_) @function.call - . - (infix_op) @operator - . - (_) - ) - (#eq? @operator "<|") - ) - -[ - (xint) - (int) - (int16) - (uint16) - (int32) - (uint32) - (int64) - (uint64) - (nativeint) - (unativeint) -] @number - -[ - (ieee32) - (ieee64) - (float) - (decimal) -] @number.float - -(bool) @boolean - -([ - (string) - (triple_quoted_string) - (verbatim_string) - (char) - (format_string) - (format_triple_quoted_string) -] @spell @string) - -(compiler_directive_decl) @keyword.directive - -(preproc_line - "#line" @keyword.directive) - -(attribute - target: (identifier)? @keyword - (_type) @attribute) - -[ - "(" - ")" - "{" - "}" - "[" - "]" - "[|" - "|]" - "{|" - "|}" -] @punctuation.bracket - -[ - "[<" - ">]" -] @punctuation.special - -(format_string_eval - [ - "{" - "}" - ] @punctuation.special) - -[ - "," - ";" - ":" - "." -] @punctuation.delimiter - -[ - "|" - "=" - ">" - "<" - "-" - "~" - "->" - "<-" - "&" - "&&" - "|" - "||" - ":>" - ":?>" - ".." - (infix_op) - (prefix_op) - (op_identifier) -] @operator - -(generic_type - [ - "<" - ">" - ] @punctuation.bracket) - -(typed_expression - ">" @punctuation.bracket) - -[ - "if" - "then" - "else" - "elif" - "when" - "match" - "match!" -] @keyword.conditional - -[ - "and" - "or" - "not" - "upcast" - "downcast" -] @keyword.operator - -[ - "return" - "return!" - "yield" - "yield!" -] @keyword.return - -[ - "for" - "while" - "downto" - "to" -] @keyword.repeat - - -[ - "open" - "#r" - "#load" -] @keyword.import - -[ - "abstract" - "delegate" - "extern" - "static" - "inline" - "mutable" - "override" - "rec" - "global" - (access_modifier) -] @keyword.modifier - -[ - "let" - "let!" - "use" - "use!" - "and!" - "member" -] @keyword.function - -[ - "enum" - "type" - "exception" - "inherit" - "interface" - "and" - "class" - "struct" -] @keyword.type - -((identifier) @keyword.exception - (#any-of? @keyword.exception "failwith" "failwithf" "raise" "reraise")) - -[ - "as" - "assert" - "begin" - "end" - "done" - "default" - "in" - "do" - "do!" - "fun" - "function" - "get" - "set" - "lazy" - "new" - "of" - "struct" - "val" - "module" - "namespace" - "with" -] @keyword - -[ - "null" -] @constant.builtin - -(match_expression "with" @keyword.conditional) - -(try_expression - [ - "try" - "with" - "finally" - ] @keyword.exception) - -((_type - (simple_type - (long_identifier - (identifier) @type.builtin))) - (#any-of? @type.builtin "bool" "byte" "sbyte" "int16" "uint16" "int" "uint" "int64" "uint64" "nativeint" "unativeint" "decimal" "float" "double" "float32" "single" "char" "string" "unit")) - -(preproc_if - [ - "#if" @keyword.directive - "#endif" @keyword.directive - ] - condition: (_)? @keyword.directive) - -(preproc_else - "#else" @keyword.directive) - -((long_identifier - (identifier)+ @variable.member - . - (identifier))) - -((identifier) @module.builtin - (#any-of? @module.builtin "Array" "Async" "Directory" "File" "List" "Option" "Path" "Map" "Set" "Lazy" "Seq" "Task" "String" "Result" )) - -((value_declaration - (attributes - (attribute - (_type - (simple_type - (long_identifier - (identifier) @attribute))))) - (function_or_value_defn - (value_declaration_left - . - (_) @constant))) - (#eq? @attribute "Literal")) diff --git a/languages/fsharp/language.json b/languages/fsharp/language.json deleted file mode 100644 index bfcab294a..000000000 --- a/languages/fsharp/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "fsharp", - "crate": "tree-sitter-fsharp", - "version": "0.3.0", - "source": "fsharp/src", - "symbol": "tree_sitter_fsharp", - "revision": "5247c1197cb290fcaea0e0a793d32829c1396831", - "repository": "https://github.com/ionide/tree-sitter-fsharp", - "variant": "FSharp", - "feature": "lang-fsharp", - "fixture": { - "prefix": "f_sharp", - "extension": "fs" - }, - "smoke": "module Example\nlet f x =\n let y = x + 1\n y * 2\n" -} diff --git a/languages/haskell/highlights.scm b/languages/haskell/highlights.scm deleted file mode 100644 index a81cced88..000000000 --- a/languages/haskell/highlights.scm +++ /dev/null @@ -1,443 +0,0 @@ -; ---------------------------------------------------------------------------- -; Parameters and variables -; NOTE: These are at the top, so that they have low priority, -; and don't override destructured parameters -(variable) @variable - -(pattern/wildcard) @variable - -(decl/function - patterns: (patterns - (_) @variable.parameter)) - -(expression/lambda - (_)+ @variable.parameter - "->") - -(decl/function - (infix - (pattern) @variable.parameter)) - -; ---------------------------------------------------------------------------- -; Literals and comments -(integer) @number - -(negation) @number - -(expression/literal - (float)) @number.float - -(char) @character - -(string) @string - -(unit) @string.special.symbol ; unit, as in () - -(comment) @comment - -((haddock) @comment.documentation) - -; ---------------------------------------------------------------------------- -; Punctuation -[ - "(" - ")" - "{" - "}" - "[" - "]" -] @punctuation.bracket - -[ - "," - ";" -] @punctuation.delimiter - -; ---------------------------------------------------------------------------- -; Keywords, operators, includes -[ - "forall" - ; "∀" ; utf-8 is not cross-platform safe -] @keyword.repeat - -(pragma) @keyword.directive - -[ - "if" - "then" - "else" - "case" - "of" -] @keyword.conditional - -[ - "import" - "qualified" - "module" -] @keyword.import - -[ - (operator) - (constructor_operator) - (all_names) - (wildcard) - "." - ".." - "=" - "|" - "::" - "=>" - "->" - "<-" - "\\" - "`" - "@" -] @operator - -; TODO broken, also huh? -; ((qualified_module -; (module) @constructor) -; . -; (module)) - -(module - (module_id) @module) - -[ - "where" - "let" - "in" - "class" - "instance" - "pattern" - "data" - "newtype" - "family" - "type" - "as" - "hiding" - "deriving" - "via" - "stock" - "anyclass" - "do" - "mdo" - "rec" - "infix" - "infixl" - "infixr" -] @keyword - -; ---------------------------------------------------------------------------- -; Functions and variables -(decl - [ - name: (variable) @function - names: (binding_list (variable) @function) - ]) - -(decl/bind - name: (variable) @variable) - -; Consider signatures (and accompanying functions) -; with only one value on the rhs as variables -(decl/signature - name: (variable) @variable - type: (type)) - -((decl/signature - name: (variable) @_name - type: (type)) - . - (decl - name: (variable) @variable) - match: (_) - (#eq? @_name @variable)) - -; but consider a type that involves 'IO' a decl/function -(decl/signature - name: (variable) @function - type: (type/apply - constructor: (name) @_type) - (#eq? @_type "IO")) - -((decl/signature - name: (variable) @_name - type: (type/apply - constructor: (name) @_type) - (#eq? @_type "IO")) - . - (decl - name: (variable) @function) - match: (_) - (#eq? @_name @function)) - -((decl/signature) @function - . - (decl/function - name: (variable) @function)) - -(decl/bind - name: (variable) @function - (match - expression: (expression/lambda))) - -; view patterns -(view_pattern - [ - (expression/variable) @function.call - (expression/qualified - (variable) @function.call) - ]) - -; consider infix functions as operators -(infix_id - [ - (variable) @operator - (qualified - (variable) @operator) - ]) - -; decl/function calls with an infix operator -; e.g. func <$> a <*> b -(infix - [ - (variable) @function.call - (qualified - ((module) @module - (variable) @function.call)) - ] - . - (operator)) - -; infix operators applied to variables -((expression/variable) @variable - . - (operator)) - -((operator) - . - [ - (expression/variable) @variable - (expression/qualified - (variable) @variable) - ]) - -; decl/function calls with infix operators -([ - (expression/variable) @function.call - (expression/qualified - (variable) @function.call) - ] - . - (operator) @_op - (#any-of? @_op "$" "<$>" ">>=" "=<<")) - -; right hand side of infix operator -((infix - [ - (operator) - (infix_id (variable)) - ] ; infix or `func` - . - [ - (variable) @function.call - (qualified - (variable) @function.call) - ]) - . - (operator) @_op - (#any-of? @_op "$" "<$>" "=<<")) - -; decl/function composition, arrows, monadic composition (lhs) -( - [ - (expression/variable) @function - (expression/qualified - (variable) @function) - ] - . - (operator) @_op - (#any-of? @_op "." ">>>" "***" ">=>" "<=<")) - -; right hand side of infix operator -((infix - [ - (operator) - (infix_id (variable)) - ] ; infix or `func` - . - [ - (variable) @function - (qualified - (variable) @function) - ]) - . - (operator) @_op - (#any-of? @_op "." ">>>" "***" ">=>" "<=<")) - -; function composition, arrows, monadic composition (rhs) -((operator) @_op - . - [ - (expression/variable) @function - (expression/qualified - (variable) @function) - ] - (#any-of? @_op "." ">>>" "***" ">=>" "<=<")) - -; function defined in terms of a function composition -(decl/function - name: (variable) @function - (match - expression: (infix - operator: (operator) @_op - (#any-of? @_op "." ">>>" "***" ">=>" "<=<")))) - -(apply - [ - (expression/variable) @function.call - (expression/qualified - (variable) @function.call) - ]) - -; function compositions, in parentheses, applied -; lhs -(apply - . - (expression/parens - (infix - [ - (variable) @function.call - (qualified - (variable) @function.call) - ] - . - (operator)))) - -; rhs -(apply - . - (expression/parens - (infix - (operator) - . - [ - (variable) @function.call - (qualified - (variable) @function.call) - ]))) - -; variables being passed to a function call -(apply - (_) - . - [ - (expression/variable) @variable - (expression/qualified - (variable) @variable) - ]) - -; main is always a function -; (this prevents `main = undefined` from being highlighted as a variable) -(decl/bind - name: (variable) @function - (#eq? @function "main")) - -; scoped function types (func :: a -> b) -(signature - pattern: (pattern/variable) @function - type: (quantified_type)) - -; signatures that have a function type -; + binds that follow them -(decl/signature - name: (variable) @function - type: (quantified_type)) - -((decl/signature - name: (variable) @_name - type: (quantified_type)) - . - (decl/bind - (variable) @function) - (#eq? @function @_name)) - -; ---------------------------------------------------------------------------- -; Types -(name) @type - -(type/star) @type - -(variable) @type - -(constructor) @constructor - -; True or False -((constructor) @boolean - (#any-of? @boolean "True" "False")) - -; otherwise (= True) -((variable) @boolean - (#eq? @boolean "otherwise")) - -; ---------------------------------------------------------------------------- -; Quasi-quotes -(quoter) @function.call - -(quasiquote - [ - (quoter) @_name - (_ - (variable) @_name) - ] - (#eq? @_name "qq") - (quasiquote_body) @string) - -(quasiquote - (_ - (variable) @_name) - (#eq? @_name "qq") - (quasiquote_body) @string) - -; namespaced quasi-quoter -(quasiquote - (_ - (module) @module - . - (variable) @function.call)) - -; Highlighting of quasiquote_body for other languages is handled by injections.scm -; ---------------------------------------------------------------------------- -; Exceptions/error handling -((variable) @keyword.exception - (#any-of? @keyword.exception - "error" "undefined" "try" "tryJust" "tryAny" "catch" "catches" "catchJust" "handle" "handleJust" - "throw" "throwIO" "throwTo" "throwError" "ioError" "mask" "mask_" "uninterruptibleMask" - "uninterruptibleMask_" "bracket" "bracket_" "bracketOnErrorSource" "finally" "fail" - "onException" "expectationFailure")) - -; ---------------------------------------------------------------------------- -; Debugging -((variable) @keyword.debug - (#any-of? @keyword.debug - "trace" "traceId" "traceShow" "traceShowId" "traceWith" "traceShowWith" "traceStack" "traceIO" - "traceM" "traceShowM" "traceEvent" "traceEventWith" "traceEventIO" "flushEventLog" "traceMarker" - "traceMarkerIO")) - -; ---------------------------------------------------------------------------- -; Fields - -(field_name - (variable) @variable.member) - -(import_name - (name) - . - (children - (variable) @variable.member)) - - -; ---------------------------------------------------------------------------- -; Spell checking -(comment) @spell diff --git a/languages/haskell/language.json b/languages/haskell/language.json deleted file mode 100644 index 700b40745..000000000 --- a/languages/haskell/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "haskell", - "crate": "tree-sitter-haskell", - "version": "0.23.1", - "source": "src", - "symbol": "tree_sitter_haskell", - "revision": "c30d812bc90827f1a54106a25bc9a6307f5cdcec", - "repository": "https://github.com/tree-sitter/tree-sitter-haskell", - "variant": "Haskell", - "feature": "lang-haskell", - "fixture": { - "prefix": "haskell", - "extension": "hs" - }, - "smoke": "module Example where\nf x =\n let y = x + 1\n in y * 2\n" -} diff --git a/languages/julia/highlights.scm b/languages/julia/highlights.scm deleted file mode 100644 index 79f1c1819..000000000 --- a/languages/julia/highlights.scm +++ /dev/null @@ -1,330 +0,0 @@ -; https://github.com/tree-sitter/tree-sitter-julia/blob/e01c928d11375513138a175a68485c4d53e55ea9/queries/highlights.scm -; MIT license - -; Identifiers -(identifier) @variable - -(field_expression - (identifier) @variable.member .) - -; Symbols -(quote_expression - ":" @string.special.symbol - [ - (identifier) - (operator) - ] @string.special.symbol) - -; Function calls -(call_expression - (identifier) @function.call) - -(call_expression - (field_expression - (identifier) @function.call .)) - -(broadcast_call_expression - (identifier) @function.call) - -(broadcast_call_expression - (field_expression - (identifier) @function.call .)) - -; Macros -(macro_identifier - "@" @function.macro - (_) @function.macro) - -(macro_definition - (signature - (call_expression - . - (identifier) @function.macro))) - -; Built-in functions -; filter(name -> Base.eval(Core, name) isa Core.Builtin, names(Core)) -((identifier) @function.builtin - (#any-of? @function.builtin - "applicable" "fieldtype" "getfield" "getglobal" "invoke" "isa" "isdefined" "modifyfield!" - "modifyglobal!" "nfields" "replacefield!" "replaceglobal!" "setfield!" "setfieldonce!" - "setglobal!" "setglobalonce!" "swapfield!" "swapglobal!" "throw" "tuple" "typeassert" "typeof")) - -; Type definitions -(type_head (_) @type.definition) - -; Type annotations -(parametrized_type_expression - [ - (identifier) @type - (field_expression - (identifier) @type .) - ] - (curly_expression - (_) @type)) - -(typed_expression - (identifier) @type .) - -(unary_typed_expression - (identifier) @type .) - -(where_expression - (_) @type .) - -(binary_expression - (_) @type - (operator) @operator - (_) @type - (#any-of? @operator "<:" ">:")) - -; Built-in types -; filter(name -> typeof(Base.eval(Core, name)) in [DataType, UnionAll], names(Core)) -((identifier) @type.builtin - (#any-of? @type.builtin - "AbstractArray" "AbstractChar" "AbstractFloat" "AbstractString" "Any" "ArgumentError" "Array" - "AssertionError" "Bool" "BoundsError" "Char" "ConcurrencyViolationError" "Cvoid" "DataType" - "DenseArray" "DivideError" "DomainError" "ErrorException" "Exception" "Expr" "Float16" "Float32" - "Float64" "Function" "GlobalRef" "IO" "InexactError" "InitError" "Int" "Int128" "Int16" "Int32" - "Int64" "Int8" "Integer" "InterruptException" "LineNumberNode" "LoadError" "Method" - "MethodError" "Module" "NTuple" "NamedTuple" "Nothing" "Number" "OutOfMemoryError" - "OverflowError" "Pair" "Ptr" "QuoteNode" "ReadOnlyMemoryError" "Real" "Ref" "SegmentationFault" - "Signed" "StackOverflowError" "String" "Symbol" "Task" "Tuple" "Type" "TypeError" "TypeVar" - "UInt" "UInt128" "UInt16" "UInt32" "UInt64" "UInt8" "UndefInitializer" "UndefKeywordError" - "UndefRefError" "UndefVarError" "Union" "UnionAll" "Unsigned" "VecElement" "WeakRef")) - -; Keywords -[ - "const" - "global" - "local" -] @keyword - -(compound_statement - [ - "begin" - "end" - ] @keyword) - -(quote_statement - [ - "quote" - "end" - ] @keyword) - -(let_statement - [ - "let" - "end" - ] @keyword) - -(if_statement - [ - "if" - "end" - ] @keyword.conditional) - -(elseif_clause - "elseif" @keyword.conditional) - -(else_clause - "else" @keyword.conditional) - -(ternary_expression - [ - "?" - ":" - ] @keyword.conditional.ternary) - -(try_statement - [ - "try" - "end" - ] @keyword.exception) - -(catch_clause - "catch" @keyword.exception) - -(finally_clause - "finally" @keyword.exception) - -(for_statement - [ - "for" - "end" - ] @keyword.repeat) - -(for_binding - "outer" @keyword.repeat) - -; comprehensions -(for_clause - "for" @keyword.repeat) - -(if_clause - "if" @keyword.conditional) - -(while_statement - [ - "while" - "end" - ] @keyword.repeat) - -[ - (break_statement) - (continue_statement) -] @keyword.repeat - -(function_definition - [ - "function" - "end" - ] @keyword.function) - -(do_clause - [ - "do" - "end" - ] @keyword.function) - -(macro_definition - [ - "macro" - "end" - ] @keyword) - -(return_statement - "return" @keyword.return) - -(module_definition - [ - "module" - "baremodule" - "end" - ] @keyword.import) - -(export_statement - "export" @keyword.import) - -(public_statement - "public" @keyword.import) - -(import_statement - "import" @keyword.import) - -(using_statement - "using" @keyword.import) - -(import_alias - "as" @keyword.import) - -(selected_import - ":" @punctuation.delimiter) - -(struct_definition - [ - "mutable" - "struct" - "end" - ] @keyword.type) - -(abstract_definition - [ - "abstract" - "type" - "end" - ] @keyword.type) - -(primitive_definition - [ - "primitive" - "type" - "end" - ] @keyword.type) - -; Operators & Punctuation -(operator) @operator - -(adjoint_expression - "'" @operator) - -(range_expression - ":" @operator) - -(arrow_function_expression - "->" @operator) - -[ - "." - "..." - "::" -] @punctuation - -[ - "," - ";" -] @punctuation.delimiter - -[ - "(" - ")" - "[" - "]" - "{" - "}" -] @punctuation.bracket - -; Keyword operators -((operator) @keyword.operator - (#any-of? @keyword.operator "in" "isa")) - -(where_expression - "where" @keyword.operator) - -; Built-in constants -((identifier) @constant.builtin - (#any-of? @constant.builtin "nothing" "missing")) - -((identifier) @variable.builtin - (#any-of? @variable.builtin "begin" "end") - (#has-ancestor? @variable.builtin index_expression)) - -; Literals -(boolean_literal) @boolean - -(integer_literal) @number - -(float_literal) @number.float - -((identifier) @number.float - (#any-of? @number.float "NaN" "NaN16" "NaN32" "Inf" "Inf16" "Inf32")) - -(character_literal) @character - -(escape_sequence) @string.escape - -(string_literal) @string - -(prefixed_string_literal - prefix: (identifier) @function.macro) @string - -(command_literal) @string.special - -(prefixed_command_literal - prefix: (identifier) @function.macro) @string.special - -((string_literal) @string.documentation - . - [ - (abstract_definition) - (assignment) - (const_statement) - (function_definition) - (macro_definition) - (module_definition) - (struct_definition) - ]) - -[ - (line_comment) - (block_comment) -] @comment diff --git a/languages/julia/language.json b/languages/julia/language.json deleted file mode 100644 index d1e7f1101..000000000 --- a/languages/julia/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "julia", - "crate": "tree-sitter-julia", - "version": "0.23.1", - "source": "src", - "symbol": "tree_sitter_julia", - "revision": "a8e1262997d5a45520a06cbe1b86c0737d507054", - "repository": "https://github.com/tree-sitter/tree-sitter-julia", - "variant": "Julia", - "feature": "lang-julia", - "fixture": { - "prefix": "julia", - "extension": "jl" - }, - "smoke": "function f(x)\n x + 1\nend\n" -} diff --git a/languages/ocaml-interface/highlights.scm b/languages/ocaml-interface/highlights.scm deleted file mode 100644 index e69de29bb..000000000 diff --git a/languages/ocaml-interface/language.json b/languages/ocaml-interface/language.json deleted file mode 100644 index 78114b6ee..000000000 --- a/languages/ocaml-interface/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ocaml-interface", - "crate": "tree-sitter-ocaml", - "version": "0.25.0", - "source": "grammars/interface/src", - "symbol": "tree_sitter_ocaml_interface", - "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", - "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", - "variant": "OCamlInterface", - "feature": "lang-ocaml", - "fixture": { - "prefix": "ocaml_interface", - "extension": "mli" - }, - "smoke": "type t = { value : int }\nval f : int -> t\n" -} diff --git a/languages/ocaml/highlights.scm b/languages/ocaml/highlights.scm deleted file mode 100644 index 8f73a9fb4..000000000 --- a/languages/ocaml/highlights.scm +++ /dev/null @@ -1,148 +0,0 @@ -; Punctuation -;------------ - -[ - "," "." ";" ":" "=" "|" "~" "?" "+" "-" "!" ">" "&" - "->" ";;" ":>" "+=" ":=" ".." -] @punctuation.delimiter - -["(" ")" "[" "]" "{" "}" "[|" "|]" "[<" "[>"] @punctuation.bracket - -(object_type ["<" ">"] @punctuation.bracket) - -"%" @punctuation.special - -(attribute ["[@" "]"] @punctuation.special) -(item_attribute ["[@@" "]"] @punctuation.special) -(floating_attribute ["[@@@" "]"] @punctuation.special) -(extension ["[%" "]"] @punctuation.special) -(item_extension ["[%%" "]"] @punctuation.special) -(quoted_extension ["{%" "}"] @punctuation.special) -(quoted_item_extension ["{%%" "}"] @punctuation.special) - -; Keywords -;--------- - -[ - "and" "as" "assert" "begin" "class" "constraint" "do" "done" "downto" "effect" - "else" "end" "exception" "external" "for" "fun" "function" "functor" "if" "in" - "include" "inherit" "initializer" "lazy" "let" "match" "method" "module" - "mutable" "new" "nonrec" "object" "of" "open" "private" "rec" "sig" "struct" - "then" "to" "try" "type" "val" "virtual" "when" "while" "with" -] @keyword - -; Operators -;---------- - -[ - (prefix_operator) - (sign_operator) - (pow_operator) - (mult_operator) - (add_operator) - (concat_operator) - (rel_operator) - (and_operator) - (or_operator) - (assign_operator) - (hash_operator) - (indexing_operator) - (let_operator) - (let_and_operator) - (match_operator) -] @operator - -(match_expression (match_operator) @keyword) - -(value_definition [(let_operator) (let_and_operator)] @keyword) - -["*" "#" "::" "<-"] @operator - -; Constants -;---------- - -(boolean) @constant - -[(number) (signed_number)] @number - -[(string) (character)] @string - -(quoted_string "{" @string "}" @string) @string - -(escape_sequence) @escape - -(conversion_specification) @string.special - -; Variables -;---------- - -[(value_name) (type_variable)] @variable - -(value_pattern) @variable.parameter - -; Properties -;----------- - -[(label_name) (field_name) (instance_variable_name)] @property - -; Functions -;---------- - -(let_binding - pattern: (value_name) @function - (parameter)) - -(let_binding - pattern: (value_name) @function - body: [(fun_expression) (function_expression)]) - -(value_specification (value_name) @function) - -(external (value_name) @function) - -(method_name) @function.method - -(application_expression - function: (value_path (value_name) @function)) - -(infix_expression - left: (value_path (value_name) @function) - operator: (concat_operator) @operator - (#eq? @operator "@@")) - -(infix_expression - operator: (rel_operator) @operator - right: (value_path (value_name) @function) - (#eq? @operator "|>")) - -( - (value_name) @function.builtin - (#match? @function.builtin "^(raise(_notrace)?|failwith|invalid_arg)$") -) - -; Types -;------ - -[(class_name) (class_type_name) (type_constructor)] @type - -( - (type_constructor) @type.builtin - (#match? @type.builtin "^(int|char|bytes|string|float|bool|unit|exn|array|list|option|int32|int64|nativeint|format6|lazy_t)$") -) - -[(constructor_name) (tag)] @constructor - -; Modules -;-------- - -[(module_name) (module_type_name)] @module - -; Attributes -;----------- - -(attribute_id) @tag - -; Comments -;--------- - -[(comment) (line_number_directive) (directive) (shebang)] @comment diff --git a/languages/ocaml/language.json b/languages/ocaml/language.json deleted file mode 100644 index e35339e28..000000000 --- a/languages/ocaml/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ocaml", - "crate": "tree-sitter-ocaml", - "version": "0.25.0", - "source": "grammars/ocaml/src", - "symbol": "tree_sitter_ocaml", - "revision": "6902a86ab5b3b80c622030210aae2d8cb95eb775", - "repository": "https://github.com/tree-sitter/tree-sitter-ocaml", - "variant": "OCaml", - "feature": "lang-ocaml", - "fixture": { - "prefix": "ocaml", - "extension": "ml" - }, - "smoke": "module M = struct\n let f x = x + 1\nend\n" -} diff --git a/languages/package.template.json b/languages/package.template.json index b22de7282..b3aa45794 100644 --- a/languages/package.template.json +++ b/languages/package.template.json @@ -1,5 +1,5 @@ { - "version": "0.2.0", + "version": "0.3.0", "description": "Optional native language parsers for diffr", "license": "MIT", "repository": "https://github.com/devdotfast/diffr" diff --git a/languages/qml/highlights.scm b/languages/qml/highlights.scm deleted file mode 100644 index 04a3e52bd..000000000 --- a/languages/qml/highlights.scm +++ /dev/null @@ -1,61 +0,0 @@ -;;; Annotations - -(ui_annotation - "@" @operator - type_name: [ - (identifier) @attribute - (nested_identifier (identifier) @attribute) - ]) - -;;; Properties - -(ui_object_definition_binding - name: [ - (identifier) @property - (nested_identifier (identifier) @property) - ]) - -(ui_binding - name: [ - (identifier) @property - (nested_identifier (identifier) @property) - ]) - -;; locals query appears not working unless id: isn't a parameter. -(ui_binding - name: (identifier) @property - (#eq? @property "id") - value: (expression_statement (identifier) @variable.parameter)) - -(ui_property - name: (identifier) @property) - -(ui_required - name: (identifier) @property) - -(ui_list_property_type - ["<" ">"] @punctuation.bracket) - -;;; Signals - -(ui_signal - name: (identifier) @function.signal) - -(ui_signal_parameter - (identifier) @variable.parameter) - -;;; Keywords - -[ - "as" - "component" - "default" - "final" - "import" - "on" - "pragma" - "property" - "readonly" - "required" - "signal" -] @keyword diff --git a/languages/qml/language.json b/languages/qml/language.json deleted file mode 100644 index 8e0840a19..000000000 --- a/languages/qml/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "qml", - "crate": "tree-sitter-qmljs", - "version": "0.3.0", - "source": "src", - "symbol": "tree_sitter_qmljs", - "revision": "0bec4359a7eb2f6c9220cd57372d87d236f66d59", - "repository": "https://github.com/yuja/tree-sitter-qmljs", - "variant": "Qml", - "feature": "lang-qml", - "fixture": { - "prefix": "qml", - "extension": "qml" - }, - "smoke": "import QtQuick\nItem { property int count: 1; function f(x) { return x + count; } }\n" -} diff --git a/languages/tests/integration.py b/languages/tests/integration.py index 5b41475b2..092597ed3 100644 --- a/languages/tests/integration.py +++ b/languages/tests/integration.py @@ -149,6 +149,12 @@ def git(*args): assert all(l['availability'] == 'built_in' for l in run(static, 'languages', 'list', '--json')[0]['languages']) installed() assert library.stat().st_size == entry['files'][library.name]['size'] + query = revision / (first_id + '.highlights.scm') + query.write_text('(invalid query') + broken_query = run(native, *compare_args(*fixtures[0])) + assert next(e for e in broken_query if e['type'] == 'file')['diff']['stats']['fallback']['code'] == 'parser_load_failed' + installed() + assert relevant(run(native, *compare_args(*fixtures[0]))) == relevant(run(static, *compare_args(*fixtures[0]))) installed() # idempotent, offline timings = [] for _ in range(10): diff --git a/languages/tests/test_definitions.py b/languages/tests/test_definitions.py index ca02002d3..e850a6ad6 100644 --- a/languages/tests/test_definitions.py +++ b/languages/tests/test_definitions.py @@ -20,8 +20,8 @@ def test_current_definitions_and_shared_ocaml_feature(self): ocaml = [d for d in self.items if d['feature'] == 'lang-ocaml'] self.assertEqual({d['id'] for d in ocaml}, {'ocaml', 'ocaml-interface'}) - def test_missing_registry_and_feature_entries_are_rejected(self): - with self.assertRaisesRegex(AssertionError, 'registry'): + def test_missing_definitions_and_feature_entries_are_rejected(self): + with self.assertRaisesRegex(AssertionError, 'dependencies'): validate(self.items[1:], self.metadata) metadata = copy.deepcopy(self.metadata) package = next(p for p in metadata['packages'] if p['name'] == 'diffr-cli') @@ -35,6 +35,18 @@ def test_missing_fixtures_are_rejected(self): with self.assertRaisesRegex(AssertionError, 'missing fixture'): validate(items, self.metadata) + def test_duplicate_builtin_config_is_rejected(self): + items = copy.deepcopy(self.items) + items[0]['variant'] = 'Ada' + with self.assertRaisesRegex(AssertionError, 'duplicate built-in'): + validate(items, self.metadata) + + def test_query_path_traversal_is_rejected(self): + items = copy.deepcopy(self.items) + items[0]['highlights'][0]['path'] = '../highlights.scm' + with self.assertRaisesRegex(AssertionError, 'invalid query path'): + validate(items, self.metadata) + if __name__ == '__main__': unittest.main() diff --git a/languages/typescript/LICENSE b/languages/typescript/LICENSE new file mode 100644 index 000000000..aa9f858db --- /dev/null +++ b/languages/typescript/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/languages/verilog/highlights.scm b/languages/verilog/highlights.scm deleted file mode 100644 index e5511687a..000000000 --- a/languages/verilog/highlights.scm +++ /dev/null @@ -1,322 +0,0 @@ -;; https://github.com/helix-editor/helix/blob/f6878f62f74430cff188e7978d06c5ed143179e9/runtime/queries/verilog/highlights.scm - -; Keywords - -[ - ; block delimiters - (module_keyword) - "endmodule" - "program" - "endprogram" - "class" - "endclass" - "interface" - "endinterface" - "package" - "endpackage" - "checker" - "endchecker" - - ; these work in helix, but panic on 1.0.3. Unsure why. - ; "config" - ; "endconfig" - - "pure" - "virtual" - "extends" - "implements" - "super" - (class_item_qualifier) - - "parameter" - "localparam" - "defparam" - "assign" - "typedef" - "modport" - "fork" - "join" - "join_none" - "join_any" - "default" - "break" - "assert" - (unique_priority) - "tagged" - "extern" -] @keyword - -[ - "function" - "endfunction" - - "task" - "endtask" -] @keyword.function - -"return" @keyword.control.return - -[ - "begin" - "end" -] @label - -[ - (always_keyword) - "generate" - "for" - "foreach" - "repeat" - "forever" - "initial" - "while" -] @keyword.control - -[ - "if" - "else" - (case_keyword) - "endcase" -] @keyword.control.conditional - -(comment) @comment - -(include_compiler_directive) @keyword.directive -(package_import_declaration - "import" @keyword.control.import) - -(package_import_declaration - (package_import_item - (package_identifier - (simple_identifier) @constant))) - -(text_macro_identifier - (simple_identifier) @keyword.directive) - -(package_scope - (package_identifier - (simple_identifier) @constant)) - -(package_declaration - (package_identifier - (simple_identifier) @constant)) - -(parameter_port_list - "#" @constructor) - -[ - "=" - "-" - "+" - "/" - "*" - "**" - "^" - "&" - "|" - "&&" - "||" - ":" - (unary_operator) - "{" - "}" - "'{" - "<=" - "@" - "==" - "!=" - "===" - "!==" - "-:" - "<" - ">" - ">=" - "%" - ">>" - "<<" - ">>>" - "<<<" - "|=" - (inc_or_dec_operator) - "?" -] @operator -[ - "or" - "and" -] @keyword.operator - -(cast - ["'" "(" ")"] @keyword.operator) - -(edge_identifier) @constant - -(port_direction) @label -(port_identifier - (simple_identifier) @variable) - -(variable_port_header ("var") @type.builtin) -(data_declaration ("var") @type.builtin) -(tf_port_item1 ("var") @type.builtin) -[ - (net_type) - (integer_vector_type) - (integer_atom_type) -] @type.builtin - -[ - "signed" - "unsigned" -] @label - -(data_type - (simple_identifier) @type) - -(method_call_body - (method_identifier) @variable.other.member) - -(interface_identifier - (simple_identifier) @type) - -(modport_identifier - (modport_identifier - (simple_identifier) @variable.other.member)) - -(net_port_type1 - (simple_identifier) @type) - -[ - (double_quoted_string) - (string_literal) -] @string - -[ - (include_compiler_directive) - (default_nettype_compiler_directive) - (timescale_compiler_directive) -] @keyword.directive - -; begin/end label -(seq_block - (simple_identifier) @comment) - -[ - ";" - "::" - "," - "." -] @punctuation.delimiter - - -(default_nettype_compiler_directive - (default_nettype_value) @string) - -(text_macro_identifier - (simple_identifier) @function.macro) - -(module_declaration - (module_header - (simple_identifier) @constructor)) - -(class_constructor_declaration - "new" @constructor) - -(parameter_identifier - (simple_identifier) @variable.parameter) - -[ - (integral_number) - (unsigned_number) - (unbased_unsized_literal) -] @constant.numeric - -(time_unit) @constant - -(checker_instantiation - (checker_identifier - (simple_identifier) @constructor)) - -(module_instantiation - (simple_identifier) @constructor) - -(name_of_instance - (instance_identifier - (simple_identifier) @variable)) - -(interface_port_declaration - (interface_identifier - (simple_identifier) @type)) - -(net_declaration - (simple_identifier) @type) - -(lifetime) @label - -(function_identifier - (function_identifier - (simple_identifier) @function)) - -(function_subroutine_call - (subroutine_call - (tf_call - (simple_identifier) @function))) - -(function_subroutine_call - (subroutine_call - (system_tf_call - (system_tf_identifier) @function.builtin))) - -(task_identifier - (task_identifier - (simple_identifier) @function.method)) - -;;TODO: fixme -;(assignment_pattern_expression - ;(assignment_pattern - ;(parameter_identifier) @variable.other.member)) - -(type_declaration - (data_type ["packed"] @label)) - -(struct_union) @type - -[ - "enum" -] @type - -(enum_name_declaration - (enum_identifier - (simple_identifier) @constant)) - -(type_declaration - (simple_identifier) @type) - -[ - (integer_atom_type) - (non_integer_type) - "genvar" -] @type.builtin - -(struct_union_member - (list_of_variable_decl_assignments - (variable_decl_assignment - (simple_identifier) @variable.other.member))) - -(member_identifier - (simple_identifier) @variable.other.member) - -(struct_union_member - (data_type_or_void - (data_type - (simple_identifier) @type))) - -(type_declaration - (simple_identifier) @type) - -(generate_block_identifier) @comment - -[ - "[" - "]" - "(" - ")" -] @punctuation.bracket diff --git a/languages/verilog/language.json b/languages/verilog/language.json deleted file mode 100644 index 5991efa20..000000000 --- a/languages/verilog/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "verilog", - "crate": "tree-sitter-verilog", - "version": "1.0.3", - "source": "src", - "symbol": "tree_sitter_verilog", - "revision": "521b535e41a5acd2c6539a922d4649bbe8275110", - "repository": "https://github.com/tree-sitter/tree-sitter-verilog", - "variant": "Verilog", - "feature": "lang-verilog", - "fixture": { - "prefix": "verilog", - "extension": "sv" - }, - "smoke": "module example(input a, output b);\nassign b = a;\nendmodule\n" -} diff --git a/languages/vhdl/highlights.scm b/languages/vhdl/highlights.scm deleted file mode 100644 index dc005e8e9..000000000 --- a/languages/vhdl/highlights.scm +++ /dev/null @@ -1,435 +0,0 @@ -;------------------------------------------------------------------------------- -; -; Maintainer: jpt13653903 -; Capture Reference: https://neovim.io/doc/user/treesitter.html#treesitter-highlight-groups -;------------------------------------------------------------------------------- -; -(line_comment - (comment_content) @spell) @comment - -(block_comment - (comment_content) @spell) @comment.documentation - -(identifier) @variable - -[ - "access" - "after" - "alias" - "architecture" - "array" - ; "assume" - "attribute" - "block" - "body" - "component" - "configuration" - "context" - ; "cover" - "disconnect" - "entity" - ; "fairness" - "file" - "force" - "generate" - "generic" - "group" - "label" - "literal" - "map" - "new" - "package" - "parameter" - "port" - "property" - "range" - "reject" - "release" - ; "restrict" - "sequence" - "transport" - "unaffected" - "view" - ; "vmode" - ; "vpkg" - ; "vprop" - "vunit" -] @keyword - -[ - (ALL) - (OTHERS) - "<>" - (DEFAULT) - (OPEN) -] @constant.builtin - -[ - "is" - "begin" - "end" -] @keyword - -(parameter_specification - "in" @keyword) - -[ - "process" - "wait" - "on" - "until" -] @keyword.coroutine - -(end_process - "end" @keyword.coroutine) - -(timeout_clause - "for" @keyword.coroutine) - -[ - "function" - "procedure" -] @keyword.function - -(subprogram_end - "end" @keyword.function) - -[ - "to" - "downto" - "of" -] @keyword.operator - -[ - "library" - "use" -] @keyword.import - -[ - "subtype" - "type" - "record" - "units" - "constant" - "signal" - "variable" -] @keyword.type - -[ - "protected" - "private" - "pure" - "impure" - "inertial" - "postponed" - ; "strong" - "guarded" - "out" - "inout" - "linkage" - "buffer" - "register" - "bus" - "shared" -] @keyword.modifier - -(mode - "in" @keyword.modifier) - -(force_mode - "in" @keyword.modifier) - -[ - "while" - "loop" - "next" - "exit" -] @keyword.repeat - -(end_loop - "end" @keyword.repeat) - -(for_loop - "for" @keyword.repeat) - -(block_configuration - "for" @keyword) - -(configuration_specification - "for" @keyword) - -(component_configuration - "for" @keyword) - -(end_for - "for" @keyword) - -"return" @keyword.return - -[ - "assert" - "report" - "severity" -] @keyword.debug - -[ - "if" - "then" - "elsif" - "case" -] @keyword.conditional - -(end_if - "end" @keyword.conditional) - -(end_case - "end" @keyword.conditional) - -(when_element - "when" @keyword.conditional) - -(case_generate_alternative - "when" @keyword.conditional) - -(else_statement - "else" @keyword.conditional) - -(else_generate - "else" @keyword.conditional) - -[ - "with" - "select" -] @keyword.conditional.ternary - -(when_expression - "when" @keyword.conditional.ternary) - -(else_expression - "else" @keyword.conditional.ternary) - -(else_waveform - "else" @keyword.conditional.ternary) - -(else_expression_or_unaffected - "else" @keyword.conditional.ternary) - -"null" @constant.builtin - -(user_directive) @keyword.directive - -(protect_directive) @keyword.directive - -(warning_directive) @keyword.directive - -(error_directive) @keyword.directive - -(if_conditional_analysis - "if" @keyword.directive) - -(if_conditional_analysis - "then" @keyword.directive) - -(elsif_conditional_analysis - "elsif" @keyword.directive) - -(else_conditional_analysis - "else" @keyword.directive) - -(end_conditional_analysis - "end" @keyword.directive) - -(end_conditional_analysis - "if" @keyword.directive) - -(directive_body) @keyword.directive - -(directive_constant_builtin) @constant.macro - -(directive_error) @comment.error - -(directive_protect) @keyword.directive - -(directive_warning) @comment.warning - -[ - (condition_conversion) - (relational_operator) - (sign) - (adding_operator) - (exponentiate) - (variable_assignment) - (signal_assignment) - "*" - "/" - ":" - "=>" -] @operator - -[ - (unary_operator) - (logical_operator) - (shift_operator) - "mod" - "not" - "rem" -] @keyword.operator - -[ - "'" - "," - "." - ";" -] @punctuation.delimiter - -[ - "(" - ")" - "[" - "]" - "<<" - ">>" -] @punctuation.bracket - -"@" @punctuation.special - -[ - (decimal_integer) - (string_literal_std_logic) -] @number - -(decimal_float) @number.float - -(bit_string_length) @property - -(bit_string_base) @type.builtin - -(bit_string_value) @number - -(based_literal - (based_base) @type.builtin - (based_integer) @number) - -(based_literal - (based_base) @type.builtin - (based_float) @number.float) - -(string_literal) @string @spell - -(character_literal) @character - -(library_constant_std_logic) @constant.builtin - -(library_constant) @constant.builtin - -(library_function) @function.builtin - -(library_constant_boolean) @boolean - -(library_constant_character) @character - -(library_constant_debug) @keyword.debug - -(unit) @keyword.modifier - -(library_constant_unit) @keyword.modifier - -(label) @label - -(generic_map_aspect - "generic" @constructor - "map" @constructor) - -(port_map_aspect - "port" @constructor - "map" @constructor) - -(selection - (identifier) @variable.member) - -(_ - view: (_) @type) - -(_ - type: (_) @type) - -(_ - library: (_) @module) - -(_ - package: (_) @module) - -(_ - entity: (_) @module) - -(_ - component: (_) @module) - -(_ - configuration: (_) @property) - -(_ - architecture: (_) @property) - -(_ - function: (_) @function) - -(_ - procedure: (_) @function.method) - -(_ - attribute: (_) @attribute) - -(_ - constant: (_) @constant) - -(_ - generic: (_) @variable.parameter) - -(_ - view: (name - (_)) @type) - -(_ - type: (name - (_)) @type) - -(_ - entity: (name - (_)) @module) - -(_ - component: (name - (_)) @module) - -(_ - configuration: (name - (_)) @module) - -(library_type) @type.builtin - -[ - (attribute_function) - (attribute_impure_function) - (attribute_mode_view) - (attribute_pure_function) - (attribute_range) - (attribute_signal) - (attribute_subtype) - (attribute_type) - (attribute_value) - (library_attribute) -] @attribute.builtin - -(library_namespace) @module.builtin - -(subtype_declaration - (identifier) @type.definition) - -(type_declaration - (identifier) @type.definition) - -(mode_view_declaration - (identifier) @type.definition) - -; (ERROR) @error diff --git a/languages/vhdl/language.json b/languages/vhdl/language.json deleted file mode 100644 index 22333754c..000000000 --- a/languages/vhdl/language.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "vhdl", - "crate": "tree-sitter-vhdl", - "version": "1.4.0", - "source": "src", - "symbol": "tree_sitter_vhdl", - "revision": "889c521757471093ce5ac83502ca8a89e516404e", - "repository": "https://github.com/jpt13653903/tree-sitter-vhdl", - "variant": "Vhdl", - "feature": "lang-vhdl", - "fixture": { - "prefix": "vhdl", - "extension": "vhd" - }, - "smoke": "entity example is\nend entity example;\narchitecture rtl of example is\nbegin\nend architecture rtl;\n" -} diff --git a/src/languages.rs b/src/languages.rs index 2d54347e3..7c2d378aa 100644 --- a/src/languages.rs +++ b/src/languages.rs @@ -126,6 +126,18 @@ pub(crate) fn verify_library( ); Ok(canonical) } + +pub(crate) fn highlights(package: &Package, id: &str, directory: &Path) -> Result { + let name = format!("{id}.highlights.scm"); + let bytes = read_verified( + &directory.join(&name), + package + .files + .get(&name) + .context("missing catalog highlights")?, + )?; + Ok(String::from_utf8(bytes)?) +} fn verify_install(package: &Package, directory: &Path) -> Result<()> { ensure!( !fs::symlink_metadata(directory)?.file_type().is_symlink(), diff --git a/src/parse/native.rs b/src/parse/native.rs index bfc17826a..9d2d17c0e 100644 --- a/src/parse/native.rs +++ b/src/parse/native.rs @@ -9,15 +9,16 @@ use tree_sitter_language::LanguageFn; struct Loaded { language: tree_sitter::Language, + highlights: String, _library: Library, } -pub(crate) fn load(id: &'static str) -> Result { +pub(crate) fn load(id: &'static str) -> Result<(tree_sitter::Language, String), ParserError> { static LOADED: LazyLock>> = LazyLock::new(|| Mutex::new(DftHashMap::default())); let mut loaded = LOADED.lock().unwrap(); if let Some(grammar) = loaded.get(id) { - return Ok(grammar.language.clone()); + return Ok((grammar.language.clone(), grammar.highlights.clone())); } let package = languages::package().ok_or_else(|| { ParserError::new( @@ -49,6 +50,7 @@ pub(crate) fn load(id: &'static str) -> Result Result ($id:literal, $feature:literal), )*) => { +macro_rules! extra_languages { + ($( $variant:ident => { + id: $id:literal, feature: $feature:literal, parser: $parser:path, + highlights: [$($query:expr),*], atoms: [$($atom:literal),*], + delimiters: [$($delimiter:expr),*], ignore_trailing: [$($trailing:expr),*] + }, )*) => { pub(crate) fn id(language: Language) -> Option<&'static str> { match language { $(Language::$variant => Some($id),)* _ => None } } @@ -10,18 +17,35 @@ macro_rules! optional_languages { pub(crate) fn builtin(language: Language) -> bool { match language { $(Language::$variant => cfg!(feature = $feature),)* _ => true } } + pub(crate) fn config(language: Language) -> Result, ParserError> { + match language { + $(Language::$variant => { + #[cfg(feature = $feature)] + let compiled = { + let queries: &[&str] = &[$($query),*]; + Some((tree_sitter::Language::new($parser), queries.concat())) + }; + #[cfg(not(feature = $feature))] + let compiled = None; + let (language, highlights) = match compiled { + Some(parser) => parser, + None => super::native::load($id)?, + }; + let highlight_query = tree_sitter::Query::new(&language, &highlights) + .map_err(|error| ParserError::new(FallbackCause::ParserLoadFailed, error))?; + Ok(Some(TreeSitterConfig { + language, + highlight_query, + atom_nodes: [$($atom),*].into_iter().collect(), + delimiter_tokens: vec![$($delimiter),*], + ignore_trailing_tokens: vec![$($trailing),*], + sub_languages: vec![], + })) + },)* + _ => Ok(None), + } + } }; } -optional_languages! { - Apex => ("apex", "lang-apex"), - Fortran => ("fortran", "lang-fortran"), - FSharp => ("fsharp", "lang-fsharp"), - Haskell => ("haskell", "lang-haskell"), - Julia => ("julia", "lang-julia"), - OCaml => ("ocaml", "lang-ocaml"), - OCamlInterface => ("ocaml-interface", "lang-ocaml"), - Qml => ("qml", "lang-qml"), - Verilog => ("verilog", "lang-verilog"), - Vhdl => ("vhdl", "lang-vhdl"), -} +include!(concat!(env!("OUT_DIR"), "/extra_languages.rs")); diff --git a/src/parse/tree_sitter_parser.rs b/src/parse/tree_sitter_parser.rs index f85fec002..9e9aef31d 100644 --- a/src/parse/tree_sitter_parser.rs +++ b/src/parse/tree_sitter_parser.rs @@ -57,12 +57,12 @@ pub(crate) struct TreeSitterConfig { /// By forcing the tree-sitter subtree to be a difftastic atom, we /// guarantee a correct diff, at the cost of losing some structure /// in the tree-sitter AST. - atom_nodes: DftHashSet<&'static str>, + pub(super) atom_nodes: DftHashSet<&'static str>, /// We want to consider delimiter tokens as part of lists, not /// standalone atoms. Tree-sitter includes delimiter tokens, so /// mark which token pairs we consider to be delimiters. - delimiter_tokens: Vec<(&'static str, &'static str)>, + pub(super) delimiter_tokens: Vec<(&'static str, &'static str)>, /// Tokens that we may ignore if they occur in a trailing /// position. This ensures that `[1, 2]` and `[1, 2,]` don't show @@ -74,11 +74,11 @@ pub(crate) struct TreeSitterConfig { /// /// Note that core diffing still sees these tokens, and we only /// handle them specially in post-processing. - ignore_trailing_tokens: Vec<(&'static str, &'static str)>, + pub(super) ignore_trailing_tokens: Vec<(&'static str, &'static str)>, /// The tree-sitter query used for syntax highlighting this /// language. - highlight_query: ts::Query, + pub(super) highlight_query: ts::Query, /// Sub-languages in use, if any. pub(crate) sub_languages: Vec, @@ -91,16 +91,6 @@ extern "C" { fn tree_sitter_smali() -> ts::Language; } -// TODO: begin/end and object/end. -const OCAML_ATOM_NODES: [&str; 6] = [ - "character", - "string", - "quoted_string", - "tag", - "type_variable", - "attribute_id", -]; - pub(crate) fn from_language( language: guess::Language, ) -> Result<&'static TreeSitterConfig, crate::languages::ParserError> { @@ -117,19 +107,16 @@ pub(crate) fn from_language( if let Some(config) = cache.get(&language) { return Ok(config); } - let external = if crate::languages::builtin(language) { - None - } else { - Some(super::native::load( - crate::languages::optional_id(language).unwrap(), - )?) + let config = match super::optional::config(language)? { + Some(config) => config, + None => build_config(language), }; - let config = Box::leak(Box::new(build_config(language, external))); + let config = Box::leak(Box::new(config)); cache.insert(language, config); Ok(config) } -fn build_config(language: guess::Language, _external: Option) -> TreeSitterConfig { +fn build_config(language: guess::Language) -> TreeSitterConfig { use guess::Language::*; match language { Ada => { @@ -150,35 +137,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - Apex => { - #[cfg(feature = "lang-apex")] - let language = tree_sitter::Language::new(tree_sitter_sfapex::apex::LANGUAGE); - #[cfg(not(feature = "lang-apex"))] - let language = _external.expect("resolved external grammar"); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: [ - "string_literal", - "null_literal", - "boolean", - "int", - "decimal_floating_point_literal", - "date_literal", - "currency_literal", - ] - .into_iter() - .collect(), - delimiter_tokens: vec![("[", "]"), ("(", ")"), ("{", "}")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/apex/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Asm => { let language_fn = tree_sitter_asm::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -470,44 +428,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - FSharp => { - #[cfg(feature = "lang-fsharp")] - let language = tree_sitter::Language::new(tree_sitter_fsharp::LANGUAGE_FSHARP); - #[cfg(not(feature = "lang-fsharp"))] - let language = _external.expect("resolved external grammar"); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["string", "triple_quoted_string"].into_iter().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/fsharp/highlights.scm"), - ) - .unwrap(), - - sub_languages: vec![], - } - } - Fortran => { - #[cfg(feature = "lang-fortran")] - let language = tree_sitter::Language::new(tree_sitter_fortran::LANGUAGE); - #[cfg(not(feature = "lang-fortran"))] - let language = _external.expect("resolved external grammar"); - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["string_literal", "number_literal"].into_iter().collect(), - delimiter_tokens: vec![("(", ")"), ("(/", "/)"), ("[", "]")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/fortran/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Gleam => { let language_fn = tree_sitter_gleam::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -539,24 +459,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - Haskell => { - #[cfg(feature = "lang-haskell")] - let language = tree_sitter::Language::new(tree_sitter_haskell::LANGUAGE); - #[cfg(not(feature = "lang-haskell"))] - let language = _external.expect("resolved external grammar"); - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["qualified_variable"].into_iter().collect(), - delimiter_tokens: vec![("[", "]"), ("(", ")")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/haskell/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Hcl => { let language_fn = tree_sitter_hcl::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -716,32 +618,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - Julia => { - #[cfg(feature = "lang-julia")] - let language = tree_sitter::Language::new(tree_sitter_julia::LANGUAGE); - #[cfg(not(feature = "lang-julia"))] - let language = _external.expect("resolved external grammar"); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: [ - "string_literal", - "prefixed_string_literal", - "command_literal", - "character_literal", - ] - .into_iter() - .collect(), - delimiter_tokens: vec![("{", "}"), ("[", "]"), ("(", ")")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/julia/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Kotlin => { let language = unsafe { tree_sitter_kotlin() }; TreeSitterConfig { @@ -872,43 +748,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - OCaml => { - #[cfg(feature = "lang-ocaml")] - let language = tree_sitter::Language::new(tree_sitter_ocaml::LANGUAGE_OCAML); - #[cfg(not(feature = "lang-ocaml"))] - let language = _external.expect("resolved external grammar"); - TreeSitterConfig { - language: language.clone(), - atom_nodes: OCAML_ATOM_NODES.iter().copied().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/ocaml/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } - OCamlInterface => { - #[cfg(feature = "lang-ocaml")] - let language = tree_sitter::Language::new(tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE); - #[cfg(not(feature = "lang-ocaml"))] - let language = _external.expect("resolved external grammar"); - TreeSitterConfig { - language: language.clone(), - atom_nodes: OCAML_ATOM_NODES.iter().copied().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("{", "}")], - ignore_trailing_tokens: vec![], - // Preserve the existing empty highlight query for interfaces. - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/ocaml-interface/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Pascal => { let language_fn = tree_sitter_pascal::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -1001,25 +840,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - Qml => { - #[cfg(feature = "lang-qml")] - let language = tree_sitter::Language::new(tree_sitter_qmljs::LANGUAGE); - #[cfg(not(feature = "lang-qml"))] - let language = _external.expect("resolved external grammar"); - - let mut highlight_query = tree_sitter_javascript::HIGHLIGHT_QUERY.to_owned(); - highlight_query.push_str(tree_sitter_typescript::HIGHLIGHTS_QUERY); - highlight_query.push_str(include_str!("../../languages/qml/highlights.scm")); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: ["string", "template_string", "regex"].into_iter().collect(), - delimiter_tokens: vec![("{", "}"), ("(", ")"), ("[", "]"), ("<", ">")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new(&language, &highlight_query).unwrap(), - sub_languages: vec![], - } - } R => { let language_fn = tree_sitter_r::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -1290,43 +1110,6 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } - Verilog => { - #[cfg(feature = "lang-verilog")] - let language = tree_sitter::Language::new(tree_sitter_verilog::LANGUAGE); - #[cfg(not(feature = "lang-verilog"))] - let language = _external.expect("resolved external grammar"); - TreeSitterConfig { - ignore_trailing_tokens: vec![], - language: language.clone(), - atom_nodes: ["integral_number"].into_iter().collect(), - delimiter_tokens: vec![("(", ")"), ("[", "]"), ("begin", "end")], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/verilog/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } - Vhdl => { - #[cfg(feature = "lang-vhdl")] - let language = tree_sitter::Language::new(tree_sitter_vhdl::LANGUAGE); - #[cfg(not(feature = "lang-vhdl"))] - let language = _external.expect("resolved external grammar"); - - TreeSitterConfig { - language: language.clone(), - atom_nodes: [].into_iter().collect(), - delimiter_tokens: vec![("(", ")")], - ignore_trailing_tokens: vec![], - highlight_query: ts::Query::new( - &language, - include_str!("../../languages/vhdl/highlights.scm"), - ) - .unwrap(), - sub_languages: vec![], - } - } Zig => { let language_fn = tree_sitter_zig::LANGUAGE; let language = tree_sitter::Language::new(language_fn); @@ -1343,6 +1126,7 @@ fn build_config(language: guess::Language, _external: Option) -> T sub_languages: vec![], } } + _ => unreachable!("language must be registered in built-in or extra config"), } } diff --git a/tests/cli.rs b/tests/cli.rs index 0c9bc9287..8d0d893b4 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -50,13 +50,9 @@ fn optional_languages_are_recognized_without_compiled_parsers() { } else { "parser_not_installed" }; - for entry in std::fs::read_dir(concat!(env!("CARGO_MANIFEST_DIR"), "/languages")).unwrap() { - let path = entry.unwrap().path().join("language.json"); - if !path.is_file() { - continue; - } - let definition: serde_json::Value = - serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + let definitions: Vec = + serde_json::from_str(include_str!("../languages/extra.json")).unwrap(); + for definition in definitions { let name = definition["id"].as_str().unwrap(); let fixture = definition["fixture"]["prefix"].as_str().unwrap(); let extension = definition["fixture"]["extension"].as_str().unwrap(); From b3b45fe41659fa5c11d9bfaf4fd0269c8a49742f Mon Sep 17 00:00:00 2001 From: thesiti92 Date: Tue, 29 Sep 2026 15:19:43 -0400 Subject: [PATCH 4/4] Collect pinned upstream licenses during language packaging AI-assisted implementation and validation with Codex. --- .github/workflows/languages.yml | 1 + Cargo.toml | 1 - languages/ADDING.md | 13 +++--- languages/apex/LICENSE | 21 --------- languages/build.py | 8 ++-- languages/definitions.py | 8 ++-- languages/extra.json | 71 +++++++++++++++++++++++++------ languages/fortran/LICENSE | 19 --------- languages/fsharp/LICENSE | 21 --------- languages/haskell/LICENSE | 21 --------- languages/julia/LICENSE | 21 --------- languages/licenses.py | 43 +++++++++++++++++++ languages/ocaml-interface/LICENSE | 21 --------- languages/ocaml/LICENSE | 21 --------- languages/qml/LICENSE | 20 --------- languages/tests/test_licenses.py | 49 +++++++++++++++++++++ languages/typescript/LICENSE | 21 --------- languages/verilog/LICENSE | 21 --------- languages/vhdl/LICENSE | 21 --------- 19 files changed, 168 insertions(+), 254 deletions(-) delete mode 100644 languages/apex/LICENSE delete mode 100644 languages/fortran/LICENSE delete mode 100644 languages/fsharp/LICENSE delete mode 100644 languages/haskell/LICENSE delete mode 100644 languages/julia/LICENSE create mode 100644 languages/licenses.py delete mode 100644 languages/ocaml-interface/LICENSE delete mode 100644 languages/ocaml/LICENSE delete mode 100644 languages/qml/LICENSE create mode 100644 languages/tests/test_licenses.py delete mode 100644 languages/typescript/LICENSE delete mode 100644 languages/verilog/LICENSE delete mode 100644 languages/vhdl/LICENSE diff --git a/.github/workflows/languages.yml b/.github/workflows/languages.yml index bceacdaea..b7baa7bac 100644 --- a/.github/workflows/languages.yml +++ b/.github/workflows/languages.yml @@ -65,6 +65,7 @@ jobs: run: | cargo xtask check-languages python3 languages/tests/test_definitions.py + python3 languages/tests/test_licenses.py python3 languages/tests/test_publish.py - name: Build locked grammars run: cargo xtask build-languages --target '${{ matrix.target }}' diff --git a/Cargo.toml b/Cargo.toml index bc71adafa..e29af57a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,6 @@ include = [ "/src/", "/languages/catalog.json", "/languages/extra.json", - "/languages/*/LICENSE", "/wit/", "/plugins/", "/vendored_parsers/highlights/*.scm", diff --git a/languages/ADDING.md b/languages/ADDING.md index 35b59ea8d..5831f5152 100644 --- a/languages/ADDING.md +++ b/languages/ADDING.md @@ -3,7 +3,7 @@ `languages/extra.json` is the single extra-language registry. It drives package building, generated Rust parser configuration, availability reporting, and tests. Generated files stay under `target/`; no generated parser or upstream highlight -query needs to be copied into this repository. +query or license needs to be copied into this repository. 1. Move the language's configuration from `tree_sitter_parser.rs` into an entry in `extra.json`. Preserve its atom nodes, delimiters, and trailing-token rules. @@ -11,20 +11,23 @@ query needs to be copied into this repository. 2. Declare the grammar dependency optional in Cargo, retain a `lang-*` feature, and include it in `all-languages`. Both OCaml grammars share one feature. 3. Supply the pinned crate version/revision, source directory, exported C symbol, - Rust parser constant, license, and a pair of representative fixtures. + Rust parser constant, license path/hash, and a pair of representative fixtures. 4. Describe highlight inputs by upstream crate, Rust constant, and relative file path. The package builder reads the files from Cargo's locked sources; built-in builds use the corresponding constants. QML demonstrates combined queries. Use `local` only for an existing diffr customization, such as Julia or Verilog. An empty list preserves OCaml interfaces' existing behavior. Shared query - dependencies include their licenses; use `license` for a repository copy if - the upstream crate omits that file. + dependencies include their licenses. Set each `license` to its upstream path + and SHA-256. Packaging reads Cargo sources first; when a crate omits its + license, it fetches from the GitHub repository at Cargo's exact VCS revision. + Verified licenses are cached under `target/languages/licenses`; subsequent + packaging can reuse them offline. Published archives always include licenses. 5. Run the checks below and advance the independent pack version when its contents change. Keep language names, globs, and detection unconditional. Do not add another Rust or Python language list. Cargo generates the Rust registry from `extra.json`. `check-languages` rejects duplicate built-in configurations, -missing features, licenses, query inputs, and fixtures. Recognition, annotation +missing features, license metadata, query inputs, and fixtures. Recognition, annotation rules, and structural diff rules remain in diffr; the pack carries compiled parser/scanner libraries and integrity-checked highlight query files. diff --git a/languages/apex/LICENSE b/languages/apex/LICENSE deleted file mode 100644 index 4d78f763e..000000000 --- a/languages/apex/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2012 Anthony Heber - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/languages/build.py b/languages/build.py index 5ce56e5fa..dc059b43e 100644 --- a/languages/build.py +++ b/languages/build.py @@ -11,6 +11,7 @@ import subprocess import tarfile from definitions import check +from licenses import package_licenses ROOT = Path(__file__).resolve().parent.parent TARGETS = { @@ -53,8 +54,8 @@ def main(): package = output / 'package' template = read(ROOT / 'languages/package.template.json') name = '@dev.fast/diffr-languages-extra-' + suffix + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--all-features', '--format-version', '1'], cwd=ROOT)) if args.action == 'build': - metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--all-features', '--format-version', '1'], cwd=ROOT)) if package.exists(): shutil.rmtree(package) package.mkdir(parents=True) @@ -82,12 +83,8 @@ def main(): dependency = next(p for p in metadata['packages'] if p['name'] == query['crate']) dependency_root = Path(dependency['manifest_path']).parent path = dependency_root / query['path'] - if query['crate'] != definition['crate']: - license = ROOT / query['license'] if 'license' in query else dependency_root / 'LICENSE' - shutil.copyfile(license, package / (query['crate'] + '.LICENSE')) highlights.append(path.read_text()) (package / (definition['id'] + '.highlights.scm')).write_text(''.join(highlights)) - shutil.copyfile(ROOT / 'languages' / definition['id'] / 'LICENSE', package / (definition['id'] + '.LICENSE')) write(package / 'manifest.json', dict(schema=1, pack='extra', version=template['version'], target=args.target, libraries=libraries)) write(package / 'package.json', dict(template, name=name, os=[system], cpu=[cpu], **({'libc': ['glibc']} if system == 'linux' else {}))) else: @@ -95,6 +92,7 @@ def main(): assert manifest['target'] == args.target and manifest['version'] == template['version'] expected = {(d['id'], d['symbol']) for d in definitions} assert {(d['id'], d['symbol']) for d in manifest['libraries']} == expected, 'stale package: rebuild the language libraries' + package_licenses(definitions, metadata, package, ROOT / 'target/languages/licenses') expected_files = {'manifest.json', 'package.json'} for library in manifest['libraries']: expected_files.update([library['file'], library['id'] + '.LICENSE', library['id'] + '.highlights.scm']) diff --git a/languages/definitions.py b/languages/definitions.py index 630e83daa..b9fe855f9 100644 --- a/languages/definitions.py +++ b/languages/definitions.py @@ -32,10 +32,12 @@ def validate(items, metadata, root=ROOT): assert re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9]*', definition['symbol']), f'{id}: invalid symbol' assert definition['smoke'].strip(), f'{id}: missing parse smoke input' assert re.fullmatch(r'\w+(::\w+)+', definition['parser']), f'{id}: invalid parser constant' - assert (root / 'languages' / id / 'LICENSE').is_file(), f'{id}: missing LICENSE' + licenses = [definition['license']] + [q['license'] for q in definition['highlights'] if 'crate' in q and q['crate'] != crate] + for license in licenses: + path = Path(license['path']) + assert not path.is_absolute() and '..' not in path.parts, f'{id}: invalid license path' + assert re.fullmatch(r'[0-9a-f]{64}', license['sha256']), f'{id}: invalid license hash' for query in definition['highlights']: - if 'license' in query: - assert (root / query['license']).is_file(), f'{id}: missing query license' if 'local' in query: relative = Path(query['local']) assert not relative.is_absolute() and '..' not in relative.parts, f'{id}: invalid local query path' diff --git a/languages/extra.json b/languages/extra.json index 714840d17..a7abe7eac 100644 --- a/languages/extra.json +++ b/languages/extra.json @@ -45,7 +45,11 @@ "constant": "apex::HIGHLIGHTS_QUERY", "path": "apex/queries/highlights.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "c3d045abbc0a8dec1b00737c0668f6dbb028d248521b74e79778c726671925ed" + } }, { "id": "fortran", @@ -88,7 +92,11 @@ "constant": "HIGHLIGHTS_QUERY", "path": "queries/highlights.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "c0fc02a86ae2b179c4729694c523bbf42da674e5136ed8e12e3cb2f26556ea9f" + } }, { "id": "fsharp", @@ -131,7 +139,11 @@ "constant": "HIGHLIGHTS_QUERY", "path": "queries/highlights.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "f9352b63e87252b282b252bf27530027c8ca69829e0023f8b8b682a736f271fd" + } }, { "id": "haskell", @@ -169,7 +181,11 @@ "constant": "HIGHLIGHTS_QUERY", "path": "queries/highlights.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "2e0110e07abef7c2548b26ec9d6969775617ca539a0dc8dbeeb14d6452c711d1" + } }, { "id": "julia", @@ -212,7 +228,11 @@ { "local": "vendored_parsers/highlights/julia.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "74f800107f200f4e28da33e9c3f86230d37d8f4c0a4916f7111e800c395ee3a2" + } }, { "id": "ocaml", @@ -259,7 +279,11 @@ "constant": "HIGHLIGHTS_QUERY", "path": "queries/highlights.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "f977a20805dd6e275f14bc090a5f8e638401dad4df366841c1060f12c75b74ae" + } }, { "id": "ocaml-interface", @@ -300,7 +324,11 @@ ] ], "ignore_trailing": [], - "highlights": [] + "highlights": [], + "license": { + "path": "LICENSE", + "sha256": "f977a20805dd6e275f14bc090a5f8e638401dad4df366841c1060f12c75b74ae" + } }, { "id": "qml", @@ -346,20 +374,31 @@ { "crate": "tree-sitter-javascript", "constant": "HIGHLIGHT_QUERY", - "path": "queries/highlights.scm" + "path": "queries/highlights.scm", + "license": { + "path": "LICENSE", + "sha256": "2e0110e07abef7c2548b26ec9d6969775617ca539a0dc8dbeeb14d6452c711d1" + } }, { "crate": "tree-sitter-typescript", "constant": "HIGHLIGHTS_QUERY", "path": "queries/highlights.scm", - "license": "languages/typescript/LICENSE" + "license": { + "path": "LICENSE", + "sha256": "49bf33cf78ef5897e4e161ce1517df7de1ae5042a65b6bcfd44401e0fc606559" + } }, { "crate": "tree-sitter-qmljs", "constant": "HIGHLIGHTS_QUERY", "path": "queries/highlights.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "a8fb3a7df5ba9ef3b6511a1d65ad69869de66828fe2d690b7b65913678739738" + } }, { "id": "verilog", @@ -399,7 +438,11 @@ { "local": "vendored_parsers/highlights/verilog.scm" } - ] + ], + "license": { + "path": "LICENSE", + "sha256": "8d2ff8d1e308828099436c43dc8bfbb8b6f368ad2a012ab9802a80ca22b03b53" + } }, { "id": "vhdl", @@ -431,6 +474,10 @@ "constant": "HIGHLIGHTS_QUERY", "path": "queries/Neovim/highlights.scm" } - ] + ], + "license": { + "path": "License.md", + "sha256": "a9323218abe9aabd3429b76634470b40c3be012600e1f96bd16ca254724fa954" + } } ] diff --git a/languages/fortran/LICENSE b/languages/fortran/LICENSE deleted file mode 100644 index d13cc4b26..000000000 --- a/languages/fortran/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/fsharp/LICENSE b/languages/fsharp/LICENSE deleted file mode 100644 index e80345de4..000000000 --- a/languages/fsharp/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Nikolaj Sidorenco - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/haskell/LICENSE b/languages/haskell/LICENSE deleted file mode 100644 index 4b52d191c..000000000 --- a/languages/haskell/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/julia/LICENSE b/languages/julia/LICENSE deleted file mode 100644 index 607d94b7f..000000000 --- a/languages/julia/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld, GitHub - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/licenses.py b/languages/licenses.py new file mode 100644 index 000000000..8bf64ce85 --- /dev/null +++ b/languages/licenses.py @@ -0,0 +1,43 @@ +"""Collect licenses from locked Cargo sources or their exact upstream revision.""" +import hashlib +import json +from pathlib import Path +import re +from urllib.request import urlopen + + +def collect(crate, spec, cache): + root = Path(crate['manifest_path']).parent + path = Path(spec['path']) + assert not path.is_absolute() and '..' not in path.parts, 'invalid license path' + fingerprint = spec['sha256'] + assert re.fullmatch(r'[0-9a-f]{64}', fingerprint), 'invalid license hash' + cached = cache / fingerprint + if (root / path).is_file(): + data = (root / path).read_bytes() + elif cached.is_file(): + data = cached.read_bytes() + else: + revision = json.loads((root / '.cargo_vcs_info.json').read_text())['git']['sha1'] + assert re.fullmatch(r'[0-9a-f]{40}', revision), 'invalid upstream revision' + repository = crate['repository'].removesuffix('.git').removesuffix('/') + assert re.fullmatch(r'https://github.com/[\w.-]+/[\w.-]+', repository), 'unsupported license repository' + url = repository.replace('https://github.com/', 'https://raw.githubusercontent.com/') + '/' + revision + '/' + path.as_posix() + with urlopen(url, timeout=30) as response: + data = response.read(1024 * 1024 + 1) + assert len(data) <= 1024 * 1024, 'license exceeds size limit' + assert hashlib.sha256(data).hexdigest() == fingerprint, f"{crate['name']}: license hash mismatch" + cache.mkdir(parents=True, exist_ok=True) + cached.write_bytes(data) + return data + + +def package_licenses(definitions, metadata, package, cache): + crates = {p['name']: p for p in metadata['packages']} + for definition in definitions: + (package / (definition['id'] + '.LICENSE')).write_bytes( + collect(crates[definition['crate']], definition['license'], cache)) + for query in definition['highlights']: + if 'crate' in query and query['crate'] != definition['crate']: + (package / (query['crate'] + '.LICENSE')).write_bytes( + collect(crates[query['crate']], query['license'], cache)) diff --git a/languages/ocaml-interface/LICENSE b/languages/ocaml-interface/LICENSE deleted file mode 100644 index 82557a14d..000000000 --- a/languages/ocaml-interface/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Max Brunsfeld and Pieter Goetschalckx - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/ocaml/LICENSE b/languages/ocaml/LICENSE deleted file mode 100644 index 82557a14d..000000000 --- a/languages/ocaml/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Max Brunsfeld and Pieter Goetschalckx - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/qml/LICENSE b/languages/qml/LICENSE deleted file mode 100644 index 0b46abcac..000000000 --- a/languages/qml/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2021 Yuya Nishihara - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/languages/tests/test_licenses.py b/languages/tests/test_licenses.py new file mode 100644 index 000000000..d0b5b0437 --- /dev/null +++ b/languages/tests/test_licenses.py @@ -0,0 +1,49 @@ +import hashlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from licenses import collect + + +class LicenseChecks(unittest.TestCase): + def test_locked_download_cache_and_integrity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + revision = 'a' * 40 + (root / '.cargo_vcs_info.json').write_text(json.dumps({'git': {'sha1': revision}})) + crate = dict(name='test', manifest_path=str(root / 'Cargo.toml'), repository='https://github.com/example/grammar') + data = b'Upstream license\n' + spec = dict(path='LICENSE', sha256=hashlib.sha256(data).hexdigest()) + cache = root / 'cache' + with patch('licenses.urlopen', return_value=io.BytesIO(data)) as fetch: + self.assertEqual(collect(crate, spec, cache), data) + fetch.assert_called_once_with(f'https://raw.githubusercontent.com/example/grammar/{revision}/LICENSE', timeout=30) + with patch('licenses.urlopen', side_effect=AssertionError('unexpected network')): + self.assertEqual(collect(crate, spec, cache), data) + (cache / spec['sha256']).write_bytes(b'corrupt') + with self.assertRaisesRegex(AssertionError, 'hash mismatch'): + collect(crate, spec, cache) + (root / 'LICENSE').write_bytes(data) + self.assertEqual(collect(crate, spec, cache), data) + with self.assertRaisesRegex(AssertionError, 'invalid license path'): + collect(crate, dict(spec, path='../LICENSE'), cache) + + def test_failed_download_does_not_cache(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / '.cargo_vcs_info.json').write_text(json.dumps({'git': {'sha1': 'b' * 40}})) + crate = dict(name='test', manifest_path=str(root / 'Cargo.toml'), repository='https://github.com/example/grammar') + with patch('licenses.urlopen', return_value=io.BytesIO(b'wrong license')): + with self.assertRaisesRegex(AssertionError, 'hash mismatch'): + collect(crate, dict(path='LICENSE', sha256='0' * 64), root / 'cache') + self.assertFalse((root / 'cache').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/languages/typescript/LICENSE b/languages/typescript/LICENSE deleted file mode 100644 index aa9f858db..000000000 --- a/languages/typescript/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/verilog/LICENSE b/languages/verilog/LICENSE deleted file mode 100644 index d7a5632c8..000000000 --- a/languages/verilog/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018-2023 Aliaksei Chapyzhenka - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/languages/vhdl/LICENSE b/languages/vhdl/LICENSE deleted file mode 100644 index 14dda3100..000000000 --- a/languages/vhdl/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -Copyright (c) 2024 John-Philip Taylor - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE.