diff --git a/.claude/settings.json b/.claude/settings.json index 61132827..4fd0f615 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,22 +1,9 @@ { - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "bash .claude/hooks/enforce-dco-signoff.sh" - } - ] - } - ] - }, "enabledPlugins": { "rust-analyzer-lsp@claude-plugins-official": true, "security-guidance@claude-plugins-official": true, "rust-cli-developer@geoffjay-claude-plugins": true, "systems-programming@claude-code-workflows": true, - "everything-claude-code@everything-claude-code": true + "ecc@ecc": true } -} \ No newline at end of file +} diff --git a/.claude/skills/.gitignore b/.claude/skills/.gitignore new file mode 100644 index 00000000..cc4c9c93 --- /dev/null +++ b/.claude/skills/.gitignore @@ -0,0 +1,8 @@ +**/* +!.gitignore +!api-design/ +!tdd-workflow/ +!security-review/ +!strategic-compact/ +!verification-loop/ +!SKILL.md diff --git a/.codex/config.toml b/.codex/config.toml deleted file mode 100644 index 43343249..00000000 --- a/.codex/config.toml +++ /dev/null @@ -1,4 +0,0 @@ -[mcp_servers.tessl] -type = "stdio" -command = "tessl" -args = [ "mcp", "start" ] diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000..e109892b --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,18 @@ +# cargo-nextest configuration. +# +# Serialize the integration test binaries that spawn child processes (the CLI +# under test, the system `file` binary) or manipulate the process working +# directory. Running these concurrently causes subprocess-spawn contention and +# cwd races -- most visibly the planted-magic CLI security regression tests, +# which resolve behavior from a temp cwd. Under heavy full-suite parallelism +# these produced rare, load-dependent flakes once additional process-spawning +# binaries (the `system_magic_dir` differential-parity suite that shells out to +# GNU `file`) were added. Pinning them to a single-threaded group makes the +# suite deterministic while leaving the ~1500 in-process unit tests fully +# parallel. +[test-groups] +serial-integration = { max-threads = 1 } + +[[profile.default.overrides]] +filter = 'binary(compatibility_tests) or binary(cli_integration) or binary(json_integration_test) or binary(security_regression) or binary(system_magic_dir)' +test-group = 'serial-integration' diff --git a/.gemini/settings.json b/.gemini/settings.json deleted file mode 100644 index c4f22417..00000000 --- a/.gemini/settings.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "mcpServers": { - "tessl": { - "type": "stdio", - "command": "tessl", - "args": ["mcp", "start"] - } - } -} diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index cca75ea1..f924bb63 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -16,6 +16,13 @@ on: # Allow manual triggering workflow_dispatch: +# Cancel superseded runs on the same ref so rapid pushes don't pile up +# concurrent cold-cache `mise-action` installs racing on the shared cache +# (matches ci.yml / security.yml). +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: CARGO_TERM_COLOR: always @@ -28,9 +35,12 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + # Only rust is needed (cargo bench); avoid the full mise tool + # set whose no-prebuilt cargo tools fail to install on runners. + install_args: rust cache: true github_token: ${{ secrets.GITHUB_TOKEN }} @@ -47,7 +57,7 @@ jobs: ${{ runner.os }}-cargo- - name: Run benchmarks - run: cargo bench --bench parser_bench --bench evaluation_bench --bench io_bench + run: mise x -- cargo bench --bench parser_bench --bench evaluation_bench --bench io_bench - name: Upload benchmark results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -68,9 +78,12 @@ jobs: with: fetch-depth: 0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + # Only rust is needed (cargo bench); avoid the full mise tool + # set whose no-prebuilt cargo tools fail to install on runners. + install_args: rust cache: true github_token: ${{ secrets.GITHUB_TOKEN }} @@ -93,7 +106,7 @@ jobs: - name: Run benchmarks on PR branch run: | - cargo bench --bench parser_bench --bench evaluation_bench --bench io_bench -- --save-baseline pr + mise x -- cargo bench --bench parser_bench --bench evaluation_bench --bench io_bench -- --save-baseline pr - name: Checkout main branch run: git checkout origin/main @@ -102,7 +115,7 @@ jobs: continue-on-error: true id: main-bench run: | - cargo bench --bench parser_bench --bench evaluation_bench --bench io_bench -- --save-baseline main + mise x -- cargo bench --bench parser_bench --bench evaluation_bench --bench io_bench -- --save-baseline main - name: Compare benchmarks if: steps.main-bench.outcome == 'success' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 467e7aa7..c3256548 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,34 +29,50 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + # Install only the tools this job uses. Installing the full + # mise tool set pulls in no-prebuilt cargo tools (cargo-audit, + # mdbook-*) whose cargo-install fallback fails on hosted + # runners; the audit/docs workflows install those where needed. + install_args: rust just cache: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Ensure rustfmt and clippy are installed - run: rustup component add rustfmt clippy - + # rustfmt + clippy are installed by mise's rust component set + # (`components = "...,rustfmt,clippy"`). A second `rustup component + # add` here re-added them and collided with mise's install on the + # hosted runner (`detected conflict` / `Directory not empty`), so it + # is intentionally removed. - name: Check formatting run: just lint-rust test: runs-on: ubuntu-latest + # Gate on `quality` so it populates the shared mise/toolchain cache + # FIRST. Without this, `quality` and `test` start together and both run + # a cold-cache `mise-action` install concurrently, racing on the same + # cache key (`detected conflict` / `Directory not empty`). Serializing + # them means `quality` writes the cache and every downstream job + # (`test`, and transitively `coverage` / `test-cross-platform`) restores + # the warm cache instead of racing to populate it. + needs: quality timeout-minutes: 20 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + install_args: rust cargo:cargo-nextest cache: true github_token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests (all features) - run: cargo nextest run --all-features + run: mise x -- cargo nextest run --all-features - name: Build release - run: cargo build --release --all-features + run: mise x -- cargo build --release --all-features test-cross-platform: strategy: @@ -74,16 +90,29 @@ jobs: runs-on: ${{ matrix.os }} needs: test timeout-minutes: 20 + # `mise.toml` sets `exec_auto_install = true`, so `mise x` would + # otherwise auto-install the ENTIRE tool set before running. Several + # `cargo:` dev tools (cargo-audit, mdbook-*, git-cliff, ...) have no + # prebuilt binary on macOS/Windows and mise's cargo-install fallback + # is disabled, so that auto-install fatally fails on those runners + # (it only ever "passed" on ubuntu because every tool has a prebuilt, + # and this job never ran before -- it is gated on `test`, which was + # red until now). Disable exec-time auto-install so `mise x` uses only + # the scoped `install_args` tools (rust + cargo-nextest), which is all + # these steps need. + env: + MISE_EXEC_AUTO_INSTALL: "false" steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + install_args: rust cargo:cargo-nextest cache: true github_token: ${{ secrets.GITHUB_TOKEN }} - - run: cargo nextest run --all-features - - run: cargo build --release --all-features + - run: mise x -- cargo nextest run --all-features + - run: mise x -- cargo build --release --all-features coverage: runs-on: ubuntu-latest @@ -91,22 +120,23 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + install_args: rust cargo:cargo-nextest cargo:cargo-llvm-cov cache: true github_token: ${{ secrets.GITHUB_TOKEN }} - name: Generate coverage # Use --test-threads=1 to prevent race conditions in stdin-mocking tests # that manipulate file descriptors (dup/dup2) which aren't thread-safe - run: cargo llvm-cov --all-features --no-report -- --test-threads=1 + run: mise x -- cargo llvm-cov --all-features --no-report -- --test-threads=1 - name: Combine coverage reports - run: cargo llvm-cov report --lcov --output-path lcov.info + run: mise x -- cargo llvm-cov report --lcov --output-path lcov.info - name: Enforce 85% coverage threshold - run: cargo llvm-cov report --fail-under-lines 85 + run: mise x -- cargo llvm-cov report --fail-under-lines 85 - name: Upload to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.yml index 7e7fc8b4..f23a8b66 100644 --- a/.github/workflows/compatibility.yml +++ b/.github/workflows/compatibility.yml @@ -10,6 +10,13 @@ on: - cron: "0 2 * * *" workflow_dispatch: +# Cancel superseded runs on the same ref so rapid pushes don't pile up +# concurrent cold-cache `mise-action` installs racing on the shared cache +# (matches ci.yml / security.yml). +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: CARGO_TERM_COLOR: always @@ -17,7 +24,12 @@ jobs: compatibility-tests: name: Compatibility Tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} - timeout-minutes: 20 + # 30m (matching benchmarks.yml) because the `cargo build --release` + # step on the windows-latest runner alone routinely approaches 20m on a + # cold dependency cache; the compatibility test itself runs in <1s. The + # fast Linux/macOS legs finish in ~10m, so the higher ceiling only ever + # affects the slow Windows leg. See PR #376. + timeout-minutes: 30 strategy: fail-fast: false @@ -27,9 +39,12 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true + # Only rust + just are needed here; avoid the full mise tool + # set whose no-prebuilt cargo tools fail to install on runners. + install_args: rust just cache: true github_token: ${{ secrets.GITHUB_TOKEN }} @@ -37,10 +52,10 @@ jobs: run: just verify-compatibility-tests - name: Build rmagic - run: cargo build --release + run: mise x -- cargo build --release - name: Run compatibility tests - run: cargo test test_compatibility_with_original_libmagic -- --ignored --nocapture + run: mise x -- cargo test test_compatibility_with_original_libmagic -- --ignored --nocapture - name: Upload test results on failure if: failure() diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index cbedcf5f..85c0b255 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -9,6 +9,13 @@ on: paths: - .github/workflows/copilot-setup-steps.yml +# Cancel superseded runs on the same ref so rapid pushes don't pile up +# concurrent cold-cache `mise-action` installs racing on the shared cache +# (matches ci.yml / security.yml). +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: copilot-setup-steps: runs-on: ubuntu-latest @@ -19,7 +26,7 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true cache: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0f8b062b..5f11d256 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true cache: true @@ -41,16 +41,16 @@ jobs: use-toc: true use-admonish: true - name: Install mdbook plugins - run: cargo binstall mdbook-tabs mdbook-i18n-helpers mdbook-alerts mdbook-yml-header mdbook-image-size --no-confirm + run: mise x -- cargo binstall mdbook-tabs mdbook-i18n-helpers mdbook-alerts mdbook-yml-header mdbook-image-size --no-confirm - name: Build mdBook run: | cd docs - mdbook build + mise x -- mdbook build - name: Build rustdoc and copy into book run: | - cargo doc --no-deps --document-private-items --target-dir target + mise x -- cargo doc --no-deps --document-private-items --target-dir target mkdir -p docs/book/api cp -r target/doc/* docs/book/api/ diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index 0fc9f413..974d518c 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -19,11 +19,11 @@ jobs: fetch-depth: 0 # persist-credentials required for pushing signed tags via git CLI persist-credentials: true - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true - name: Run release-plz - uses: release-plz/action@e8792575c7f2366cf6ff3ccc33ead9ace5b691c7 # v0.5.130 + uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5.131 with: command: release env: @@ -45,11 +45,11 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true - name: Run release-plz - uses: release-plz/action@e8792575c7f2366cf6ff3ccc33ead9ace5b691c7 # v0.5.130 + uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5.131 with: command: release-pr env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1561ce76..e4a1fb1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,7 +56,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: persist-credentials: false submodules: recursive @@ -64,7 +64,7 @@ jobs: # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh" + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.32.0/cargo-dist-installer.sh | sh" - name: Cache dist uses: actions/upload-artifact@v7.0.1 with: @@ -120,7 +120,7 @@ jobs: - name: enable windows longpaths run: | git config --global core.longpaths true - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: persist-credentials: false submodules: recursive @@ -151,7 +151,7 @@ jobs: dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json echo "dist ran successfully" - name: Attest - uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 with: subject-path: "target/distrib/*${{ join(matrix.targets, ', ') }}*" - id: cargo-dist @@ -185,7 +185,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: persist-credentials: false submodules: recursive @@ -254,7 +254,7 @@ jobs: outputs: val: ${{ steps.host.outputs.manifest }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: persist-credentials: false submodules: recursive @@ -319,7 +319,7 @@ jobs: GITHUB_EMAIL: "admin+bot@axo.dev" if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: persist-credentials: true repository: "EvilBit-Labs/homebrew-tap" @@ -366,7 +366,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 4d4a1b40..183894a2 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -74,6 +74,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0 + uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 with: sarif_file: results.sarif diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index d333a952..2cf51534 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -28,14 +28,14 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1 + - uses: jdx/mise-action@dad1bfd3df957f44999b559dd69dc1671cb4e9ea # v4.2.1 with: install: true cache: true github_token: ${{ secrets.GITHUB_TOKEN }} - name: Run cargo deny check - run: cargo deny check + run: mise x -- cargo deny check - name: Run cargo outdated (informational only) # Dropping --exit-code=1: a plain upstream dep release is @@ -43,8 +43,8 @@ jobs: # eroded trust in the security dashboard. cargo audit and # cargo deny above are the authoritative gates. (Review # finding D-M2.) - run: cargo outdated --depth=1 + run: mise x -- cargo outdated --depth=1 continue-on-error: true - name: Run dist plan - run: dist plan + run: mise x -- dist plan diff --git a/.gitignore b/.gitignore index 944468cc..7759ed5f 100644 --- a/.gitignore +++ b/.gitignore @@ -124,8 +124,6 @@ docs/book/ .envrc .direnv/ -megalinter-reports/ - # Override global gitignore !bin/ # Added by goreleaser init: @@ -141,13 +139,9 @@ docs/plans/ .agents/ .augment/ .context/ -.cursor/ -.roo/ .full-review/ SECURITY_AUDIT.md todos/ -**/tessl__* -.tessl/tiles/ -.tessl/RULES.md .claude/instincts/ .claude/scheduled_tasks.lock +.dosu/audit.json diff --git a/.mcp.json b/.mcp.json deleted file mode 100644 index c4f22417..00000000 --- a/.mcp.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "mcpServers": { - "tessl": { - "type": "stdio", - "command": "tessl", - "args": ["mcp", "start"] - } - } -} diff --git a/.mdformat.toml b/.mdformat.toml index 22c2e4ee..a9758605 100644 --- a/.mdformat.toml +++ b/.mdformat.toml @@ -1,19 +1,3 @@ -# mdformat configuration - consistent with Prettier and markdownlint -exclude = [ - "venv/**", - "**/node_modules/**", - "**/*.txt", - "**/*.mdc", - "**/*.tpl.md", - "**/CHANGELOG.md", - "target/**", - "megalinter-reports/**", - "**/*.result", - "**/*.testfile", - "**/SKILL.md", # AI stuff - ".claude/**/*", # AI stuff - ".tessl/**/*", # AI stuff -] validate = true number = true wrap = "no" diff --git a/.serena/.gitignore b/.serena/.gitignore deleted file mode 100644 index 14d86ad6..00000000 --- a/.serena/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/cache diff --git a/.serena/project.yml b/.serena/project.yml deleted file mode 100644 index b34e8afb..00000000 --- a/.serena/project.yml +++ /dev/null @@ -1,138 +0,0 @@ - - -# list of languages for which language servers are started; choose from: -# al ansible bash clojure cpp -# cpp_ccls crystal csharp csharp_omnisharp dart -# elixir elm erlang fortran fsharp -# go groovy haskell haxe hlsl -# java json julia kotlin lean4 -# lua luau markdown matlab msl -# nix ocaml pascal perl php -# php_phpactor powershell python python_jedi python_ty -# r rego ruby ruby_solargraph rust -# scala solidity swift systemverilog terraform -# toml typescript typescript_vts vue yaml -# zig -# (This list may be outdated. For the current list, see values of Language enum here: -# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py -# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) -# Note: -# - For C, use cpp -# - For JavaScript, use typescript -# - For Free Pascal/Lazarus, use pascal -# Special requirements: -# Some languages require additional setup/installations. -# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers -# When using multiple languages, the first language server that supports a given file will be used for that file. -# The first language is the default language and the respective language server will be used as a fallback. -# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. -languages: -- rust - -# the encoding used by text files in the project -# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings -encoding: "utf-8" - -# whether to use project's .gitignore files to ignore files -ignore_all_files_in_gitignore: true - -# list of additional paths to ignore in this project. -# Same syntax as gitignore, so you can use * and **. -# Note: global ignored_paths from serena_config.yml are also applied additively. -ignored_paths: [] - -# whether the project is in read-only mode -# If set to true, all editing tools will be disabled and attempts to use them will result in an error -# Added on 2025-04-18 -read_only: false - -# list of tool names to exclude. -# This extends the existing exclusions (e.g. from the global configuration) -# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -excluded_tools: [] - -# initial prompt for the project. It will always be given to the LLM upon activating the project -# (contrary to the memories, which are loaded on demand). -initial_prompt: "" -# the name by which the project can be referenced within Serena -project_name: "libmagic-rs" - -# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). -# This extends the existing inclusions (e.g. from the global configuration). -# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -included_optional_tools: [] - -# list of mode names to that are always to be included in the set of active modes -# The full set of modes to be activated is base_modes + default_modes. -# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. -# Otherwise, this setting overrides the global configuration. -# Set this to [] to disable base modes for this project. -# Set this to a list of mode names to always include the respective modes for this project. -base_modes: - -# list of mode names that are to be activated by default, overriding the setting in the global configuration. -# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. -# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. -# Otherwise, this overrides the setting from the global configuration (serena_config.yml). -# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply -# for this project. -# This setting can, in turn, be overridden by CLI parameters (--mode). -# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes -default_modes: - -# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. -# This cannot be combined with non-empty excluded_tools or included_optional_tools. -# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -fixed_tools: [] - -# time budget (seconds) per tool call for the retrieval of additional symbol information -# such as docstrings or parameter information. -# This overrides the corresponding setting in the global configuration; see the documentation there. -# If null or missing, use the setting from the global configuration. -symbol_info_budget: - -# The language backend to use for this project. -# If not set, the global setting from serena_config.yml is used. -# Valid values: LSP, JetBrains -# Note: the backend is fixed at startup. If a project with a different backend -# is activated post-init, an error will be returned. -language_backend: - -# line ending convention to use when writing source files. -# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) -# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. -line_ending: - -# list of regex patterns which, when matched, mark a memory entry as read‑only. -# Extends the list from the global configuration, merging the two lists. -read_only_memory_patterns: [] - -# list of regex patterns for memories to completely ignore. -# Matching memories will not appear in list_memories or activate_project output -# and cannot be accessed via read_memory or write_memory. -# To access ignored memory files, use the read_file tool on the raw file path. -# Extends the list from the global configuration, merging the two lists. -# Example: ["_archive/.*", "_episodes/.*"] -ignored_memory_patterns: [] - -# advanced configuration option allowing to configure language server-specific options. -# Maps the language key to the options. -# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. -# No documentation on options means no options are available. -ls_specific_settings: {} - -# list of mode names to be activated additionally for this project, e.g. ["query-projects"] -# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. -# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes -added_modes: - -# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). -# Paths can be absolute or relative to the project root. -# Each folder is registered as an LSP workspace folder, enabling language servers to discover -# symbols and references across package boundaries. -# Currently supported for: TypeScript. -# Example: -# additional_workspace_folders: -# - ../sibling-package -# - ../shared-lib -additional_workspace_folders: [] diff --git a/AGENTS.md b/AGENTS.md index bd4be9ef..b100b8b5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -218,10 +218,11 @@ cargo test --doc # Test documentation examples - **Operators**: `=` (equal), `!=` and bare `!` (not equal), `<` (less than), `>` (greater than), `<=` (less equal), `>=` (greater equal), `&` (bitwise AND with optional mask), `^` (bitwise XOR), `~` (bitwise NOT), `x` (any value). magic(5) bare `!` prefix (e.g., `!0xb8c0078e`) is treated as `NotEqual`. - **Nested Rules**: Hierarchical rule evaluation with proper indentation - **String Matching**: Exact string matching with null-termination and Pascal string (length-prefixed) support -- **Regex type**: Binary-safe regex matching via `regex::bytes::Regex`. Full flag support: `/c` (case-insensitive), `/s` (anchor advances to match-start instead of match-end), `/l` (scan window is measured in lines instead of bytes). Flags combine in any order (`regex/cs`, `regex/csl`, `regex/lc`). Numeric counts are honored: `regex/100` scans at most 100 bytes; `regex/1l` scans at most 1 line. Multi-line regex matching is always on (matching libmagic's unconditional `REG_NEWLINE`), so `^` and `$` match at line boundaries regardless of `/l`. Every scan window is capped at 8192 bytes (`FILE_REGEX_MAX`) regardless of the user's count. -- **Search type**: Bounded literal pattern scan via `memchr::memmem::find`; `search/N` caps the scan window to `N` bytes from the offset. The range is **mandatory** and stored as `NonZeroUsize`, so bare `search` and `search/0` are parse errors (matching GNU `file` magic(5)). Anchor advance follows GNU `file` semantics (match-end, not window-end) so relative-offset children resolve to the byte immediately after the matched pattern. Full flag support (issue #235): `/s` (start-anchor — anchor lands at match-START instead of match-END, required for TGA footer and sfnt name table rules), `/c`/`/C` (asymmetric case-insensitive matching, pattern-controlled fold direction — see GOTCHAS S6.5), `/w` (whitespace-optional parallel-walk), `/W` (whitespace-required compact), `/T` (`STRING_TRIM` — leading/trailing whitespace stripped from pattern at evaluation time), `/f` (`STRING_FULL_WORD` — post-match word boundary check), `/t` and `/b` (parsed and captured as MIME-output hints; comparison-time effect deferred to `!:mime` evaluation in #51). `/B` on search is an alias for `/b` (binary hint) — distinct from `/B` on pstring which is the 1-byte length-width letter. Flag dispatcher splits on category: anchor-only flags (`/s`/`/t`/`/b`) keep the SIMD-accelerated `memchr::memmem::find` fast path; comparison-altering flags (`/c`/`/C`/`/w`/`/W`/`/T`/`/f`) trigger byte-by-byte walk via `compare_string_with_flags` through `SearchFlags::to_string_flags()`. See GOTCHAS S2.6 for the `/s` anchor-advance contract. -- **Meta-type directives**: `default`, `clear`, `name `, `use `, `indirect`, and `offset` are fully implemented. `name` blocks are hoisted into a `NameTable` at load time (`parser::name_table::extract_name_table`). `use` invokes subroutines at the resolved offset via `RuleEnvironment` threaded through `EvaluationContext::rule_env`; subroutine-local absolute offsets resolve relative to the use-site base (tracked via `EvaluationContext::base_offset`). `default` fires only when no sibling at the same level has matched; `clear` resets the per-level sibling-matched flag so a later `default` can fire. `indirect` re-applies the root rule set at the resolved offset, bounded by `EvaluationConfig::max_recursion_depth`. `offset` reports the resolved file offset as `Value::Uint(pos)` for format-string rendering. Continuation siblings (`recursion_depth > 0`) see the parent-level anchor on each iteration rather than chaining -- matching libmagic's `ms->c.li[cont_level]` model. Top-level siblings still chain (documented in GOTCHAS S3.8). -- **Printf-style format substitution**: Rule messages support `%d`, `%i`, `%u`, `%x`, `%X`, `%o`, `%s`, `%c`, and `%%`, along with width/padding modifiers (`%05d`, `%-5d`) and length modifiers (`l`, `ll`, `h`, etc. -- parsed and ignored). Hex specifiers respect the rule's `TypeKind::bit_width()` to mask sign-extended signed reads (so a signed byte carrying `-1` renders as `ff`, not `ffffffffffffffff`). Implemented in `src/output/format.rs::format_magic_message` and wired into `MagicDatabase::build_result`. Unrecognized specifiers pass through literally with a `debug!` log. +- **Regex type**: Binary-safe regex matching via `regex::bytes::Regex`. Full flag support: `/c` (case-insensitive), `/s` (anchor advances to match-start instead of match-end), `/l` (scan window is measured in lines instead of bytes). Flags combine in any order (`regex/cs`, `regex/csl`, `regex/lc`). Numeric counts are honored: `regex/100` scans at most 100 bytes; `regex/1l` scans at most 1 line. Multi-line regex matching is always on (matching libmagic's unconditional `REG_NEWLINE`), so `^` and `$` match at line boundaries regardless of `/l`. Every scan window is capped at 8192 bytes (`FILE_REGEX_MAX`) regardless of the user's count. Unquoted (bareword) `regex` patterns are captured as a GNU `file` `getstr`-resolved `Value::String`: magic(5) escapes (`\t`, octal `\NNN`, hex `\xNN`, named control escapes `\a`/`\b`/`\f`/`\n`/`\r`/`\v`) resolve to their byte; an unrecognized escape drops the backslash (`\^` -> `^`), so `\d`/`\s`/`\w` demote to literal `d`/`s`/`w` (genuine upstream libmagic behavior, not a bug); resolved bytes `>= 0x80` are re-encoded as regex-native `\xHH` in the pattern text. Quoted regex values are unaffected. The evaluator additionally accepts a `Value::Bytes` pattern operand for `Regex` (mirroring `Search`, see GOTCHAS S2.4/S2.12), decoding it via lossy UTF-8 with a `warn!` on real substitution, as a floor-tier safety net for any pattern that still reaches evaluation as `Bytes`. +- **Search type**: Bounded literal pattern scan via `memchr::memmem::find`; `search/N` caps the scan window to `N` bytes from the offset. The range is stored as `Option`: `Some(n)` is the `search/N` form, and `None` is a bare `search` (no `/N` suffix) which scans from the offset to end-of-buffer. magic(5) documents the count as **required**, but the reference `file` binary accepts the bare form and treats it as `str_range == 0` (scan-to-EOF), so rmagic follows the implementation rather than the spec (real system magic relies on this -- e.g. pdf's `>8 search /Count` page-count chain, bioinformatics `0 search ##fileformat=VCFv`, ICC, sfnt font name tables). `search/0` remains a parse error (a zero-width scan is unrepresentable via `NonZeroUsize`). Anchor advance follows GNU `file` semantics (match-end, not window-end) so relative-offset children resolve to the byte immediately after the matched pattern. Full flag support (issue #235): `/s` (start-anchor — anchor lands at match-START instead of match-END, required for TGA footer and sfnt name table rules), `/c`/`/C` (asymmetric case-insensitive matching, pattern-controlled fold direction — see GOTCHAS S6.5), `/w` (whitespace-optional parallel-walk), `/W` (whitespace-required compact), `/T` (`STRING_TRIM` — leading/trailing whitespace stripped from pattern at evaluation time), `/f` (`STRING_FULL_WORD` — post-match word boundary check), `/t` and `/b` (parsed and captured as MIME-output hints; comparison-time effect deferred to `!:mime` evaluation in #51). `/B` on search is an alias for `/b` (binary hint) — distinct from `/B` on pstring which is the 1-byte length-width letter. Flag dispatcher splits on category: anchor-only flags (`/s`/`/t`/`/b`) keep the SIMD-accelerated `memchr::memmem::find` fast path; comparison-altering flags (`/c`/`/C`/`/w`/`/W`/`/T`/`/f`) trigger byte-by-byte walk via `compare_string_with_flags` through `SearchFlags::to_string_flags()`. See GOTCHAS S2.6 for the `/s` anchor-advance contract. +- **Meta-type directives**: `default`, `clear`, `name `, `use `, `indirect`, and `offset` are fully implemented. `name` blocks are hoisted into a `NameTable` at load time (`parser::name_table::extract_name_table`). `use` invokes subroutines at the resolved offset via `RuleEnvironment` threaded through `EvaluationContext::rule_env`; subroutine-local absolute offsets resolve relative to the use-site base (tracked via `EvaluationContext::base_offset`). The magic(5) `\^` prefix (`use \^name`, e.g. big-endian TIFF's `>(4.L) use \^tiff_ifd`) is implemented for value reads: it flips little/big endianness of every `short`/`long`/`quad`/`float`/`double`/date read inside the subroutine (via `evaluator::types::flip_type_endian`, mirroring libmagic `cvt_flip`), toggling per invocation and propagating into nested `use` calls (tracked on `EvaluationContext::flip_endian`, threaded by `SubroutineScope`). This gives byte-for-byte parity on big-endian TIFF detail (direntries + width/height/compression) and single-arch Mach-O dylib classification. Indirect-offset pointer reads inside a flipped subroutine are NOT yet flipped (see GOTCHAS S16.3; the remaining Mach-O universal gap stays under #378). A `name` line's own description (e.g. Mach-O universal `0 name mach-o \b [`, `0 name matlab4 Matlab v4 mat-file`) is emitted at the `use` site ahead of the subroutine body, attached with no separating space -- matching GNU `file`; a `use` site's own trailing message is dropped (see GOTCHAS S14.4). `default` fires only when no sibling at the same level has matched; `clear` resets the per-level sibling-matched flag so a later `default` can fire, and -- matching libmagic's `FILE_CLEAR` -- also emits its own message when it carries one (c-lang's `>>&0 clear x program text` contributes the "program text" fragment of `c program text`; bare `clear x` flag-resets emit nothing -- see GOTCHAS S13.6). `indirect` re-applies the root rule set at the resolved offset, bounded by `EvaluationConfig::max_recursion_depth`. `offset` reports the resolved file position as `Value::Uint(pos)` for format-string rendering and, with a comparison operator (`offset >N`/`>-0 offset >48`); `offset x` always matches. The magic(5) `-0` offset (0 bytes from end) resolves to the EOF position `buffer.len()` via `OffsetSpec::FromEnd(0)`. `stop_at_first_match` short-circuits only the top-level classification loop; child/continuation sibling lists and `use` subroutine bodies always evaluate every matching sibling. Strength sorting is applied to top-level entries only (`sort_rules_by_strength`, non-recursive) -- continuation rules and `name`-block bodies stay in file order, matching libmagic's `apprentice_sort` (see GOTCHAS S13.4/S13.5). Continuation siblings (`recursion_depth > 0`) see the parent-level anchor on each iteration rather than chaining -- matching libmagic's `ms->c.li[cont_level]` model. Top-level siblings still chain (documented in GOTCHAS S3.8). +- **Printf-style format substitution**: Rule messages support `%d`, `%i`, `%u`, `%x`, `%X`, `%o`, `%s`, `%c`, and `%%`, along with width/padding modifiers (`%05d`, `%-5d`), `.precision` (honored for `%s` -- `%.3s` truncates to 3 chars, char-wise; ignored for numeric conversions), and length modifiers (`l`, `ll`, `h`, etc. -- parsed and ignored). Hex specifiers respect the rule's `TypeKind::bit_width()` to mask sign-extended signed reads (so a signed byte carrying `-1` renders as `ff`, not `ffffffffffffffff`). For a `string` rule with an ordering operator (`<`/`>`/`<=`/`>=`), the substituted `%s`/`%.Ns` value is the FULL string field (read until NUL/EOF), matching libmagic's `p->s` render, even though the comparison is prefix-limited to `pattern.len()` (see GOTCHAS S14.3). Implemented in `src/output/format.rs::format_magic_message` and wired into `MagicDatabase::build_result`. Unrecognized specifiers pass through literally with a `debug!` log. +- **Text/data fallback**: When magic-rule evaluation produces no usable description (no rule matched, or every match that did occur carries no message text), `MagicDatabase::build_result` falls back to a minimal GNU `file` `file_ascmagic`-style content classification (`src/output/ascmagic.rs::classify_fallback`): `"empty"` for a zero-byte buffer, `"ASCII text"` for printable-ASCII/common-whitespace content, `"UTF-8 Unicode text"` for valid non-ASCII UTF-8, and `"data"` for anything else. This is what keeps the CLI from ever printing a blank description for a readable file (GOTCHAS S13.3). Also fixed alongside this: a top-level rule that matches but contributes no message text no longer terminates evaluation under `stop_at_first_match: true` -- it only stops once a message-bearing match (direct or via a descendant) is found (GOTCHAS S13.2). See **Development Phases** below for the planned roadmap of features not yet implemented (Aho-Corasick multi-pattern optimization and `!:mime`/`!:ext`/`!:apple` directive evaluation). @@ -230,7 +231,7 @@ See **Development Phases** below for the planned roadmap of features not yet imp ### Type System - 64-bit integer types: `quad`/`uquad`, `bequad`/`ubequad`, `lequad`/`ulequad` are implemented; `qquad` (128-bit) is not yet supported -- `string` evaluation has two read modes selected by whether a comparison pattern is supplied: pattern-comparison uses `read_string_exact` (reads exactly `pattern.len()` bytes byte-for-byte, with NO NUL truncation -- required for patterns containing embedded NULs like `0 string PNCIHISK\0 ...`), while `string x` (any-value) uses `read_string` which reads until first NUL or end-of-buffer for printable-prefix output. The `x` path also fires when `read_string` is called directly without a pattern. Bareword string values interpret magic(5) escape sequences (`\0`, `\n`, `\r`, `\t`, `\\`, `\"`, `\'`, octal `\NNN`, hex `\xNN`); `max_length: Some(_)` is supported programmatically (via the AST) and is now used by the dispatcher for explicit-length reads. See `docs/solutions/logic-errors/magic-string-rule-matching-3-bug-fix-2026-04-25.md` for the load-bearing details. +- `string` evaluation has two read modes selected by whether a comparison pattern is supplied: pattern-comparison uses `read_string_exact` (reads exactly `pattern.len()` bytes byte-for-byte, with NO NUL truncation -- required for patterns containing embedded NULs like `0 string PNCIHISK\0 ...`), while `string x` (any-value) uses `read_string` which reads until first NUL or end-of-buffer for printable-prefix output. The `x` path also fires when `read_string` is called directly without a pattern. Bareword string values interpret magic(5) escape sequences (`\0`, `\n`, `\r`, `\t`, `\\`, `\"`, `\'`, octal `\NNN`, hex `\xNN`); `max_length: Some(_)` is supported programmatically (via the AST) and is now used by the dispatcher for explicit-length reads. A bareword value on a string-family type is **never** parsed as a number: `0 string >0` compares against the literal ASCII byte `'0'` and `>0.6.1` against the literal characters `0.6.1` (via `parse_string_family_value` -- quoted -> `String`, hex/escape -> `Bytes`, everything else -> `String`, with the numeric branches omitted). A prior revision captured these as `Value::Uint`/`Value::Float`, so the pervasive `>0 ... %s` idiom (`name %s`/`face %s`/`palette %s`) and version compares silently never matched (see GOTCHAS S3.6). See `docs/solutions/logic-errors/magic-string-rule-matching-3-bug-fix-2026-04-25.md` for the load-bearing details. - `string` type modifier flags `/c /C /w /W /b /t /T /f` are fully implemented (issue #234, PR #288). `/c` and `/C` perform ASCII case-insensitive matching with the libmagic asymmetric contract (the pattern character controls fold direction — see GOTCHAS S6.5); `/w` and `/W` give whitespace-optional and whitespace-required-compact parallel-walk matching; `/T` trims leading/trailing ASCII whitespace from the pattern at evaluation time (`STRING_TRIM`); `/f` enforces a post-match word boundary (`STRING_FULL_WORD`). `/t` (`STRING_TEXTTEST`) and `/b` (`STRING_BINTEST`) are MIME-output hints — parsed and stored, but with no comparison effect today (output-side wiring deferred to the `!:mime` evaluation work in #51). Default-flag `string` rules continue to take the byte-exact fast path. `/B` is **not** a `string` flag in libmagic — it is exclusively the `pstring` 1-byte length-width letter — so `string/B` is rejected at parse time. See GOTCHAS S6.5 (asymmetric `/c`/`/C`) and S6.6 (`/B` exclusion). - `pstring` supports 1-byte (`/B`), 2-byte big-endian (`/H`), 2-byte little-endian (`/h`), 4-byte big-endian (`/L`), and 4-byte little-endian (`/l`) length prefixes, plus the `/J` flag (stored length includes prefix width). All flags are combinable (e.g., `pstring/HJ`) and fully implemented. @@ -378,6 +379,7 @@ This pattern ensures build-time failures (e.g., invalid magic files) are properl - Graceful degradation - Skip problematic rules and continue with others - Maintain evaluation context for nested rules +- A pattern-bearing rule (`regex`, `search`, or a flagged `string`) evaluated without a usable `String`/`Bytes` pattern operand, or a regex rule that fails to compile (including the `REGEX_COMPILE_SIZE_LIMIT` DoS guard), is skipped as a non-match rather than aborting the rest of the file's evaluation -- logged at `debug!` for the ordinary case and `warn!` for a compile failure. This is a narrow, allowlisted exception; other evaluator capability gaps (an unwired `TypeKind`, an unsupported operator on a pattern-bearing type) still propagate as errors. See GOTCHAS S2.1 for the exact predicate. ### IO Errors @@ -583,3 +585,16 @@ This project has the OSSF Best Practices passing badge. Maintain these standards ## Agent Rules @.tessl/RULES.md follow the [instructions](.tessl/RULES.md) + + + +## Dosu + +Shared team knowledge lives in [Dosu](https://dosu.dev), via the Dosu MCP server. + +- Before a task, and for any codebase or docs questions: pull context with `read_knowledge` before digging through source. +- After a task: save durable learnings with `write_knowledge`. + +Missing these tools? Run `dosu setup --help` — it covers agent-assisted setup. + + diff --git a/CHANGELOG.md b/CHANGELOG.md index 76022bb7..6d89d59c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,21 @@ All notable changes to this project will be documented in this file. - ## [Unreleased] +## [0.11.0] - 2026-07-20 + +### Bug Fixes + +- **deps**: Drop env_logger humantime feature to remove bitflags dup ([#373](https://github.com/EvilBit-Labs/libmagic-rs/pull/373)) +- **evaluator,parser**: Gracefully and correctly handle regex rules from system magic DB ([#376](https://github.com/EvilBit-Labs/libmagic-rs/pull/376)) + +### Miscellaneous Tasks + +- Regenerate mise.lock so rust@latest resolves under --locked ([#342](https://github.com/EvilBit-Labs/libmagic-rs/pull/342)) +- **settings**: Add cargo settings for binstall_only option + + ## [0.10.0] - 2026-07-17 ### Bug Fixes @@ -14,6 +26,7 @@ All notable changes to this project will be documented in this file. ### Security - **security**: Actually enforce workspace lints via `[lints] workspace = true` ([#353](https://github.com/EvilBit-Labs/libmagic-rs/pull/353)) + ## [0.9.0] - 2026-06-08 @@ -25,6 +38,7 @@ All notable changes to this project will be documented in this file. ### Documentation - Correct stale &+N relative-offset parsing TODO claims ([#287](https://github.com/EvilBit-Labs/libmagic-rs/pull/287)) ([#306](https://github.com/EvilBit-Labs/libmagic-rs/pull/306)) + ## [0.8.0] - 2026-05-28 @@ -36,6 +50,7 @@ All notable changes to this project will be documented in this file. ### Documentation - **third_party**: Update NOTICE.md for clarity and accuracy + ## [0.7.0] - 2026-05-27 @@ -51,6 +66,7 @@ All notable changes to this project will be documented in this file. ### Refactor - **tests**: Split flagged-string engine tests into submodule ([#295](https://github.com/EvilBit-Labs/libmagic-rs/pull/295)) + ## [0.6.1] - 2026-05-26 @@ -63,6 +79,7 @@ All notable changes to this project will be documented in this file. - **mergify**: Upgrade configuration to current format ([#252](https://github.com/EvilBit-Labs/libmagic-rs/pull/252)) - **markdownlint**: Disable MD025 (single-h1) ([#283](https://github.com/EvilBit-Labs/libmagic-rs/pull/283)) + ## [0.6.0] - 2026-04-25 @@ -96,6 +113,7 @@ All notable changes to this project will be documented in this file. - **mergify**: Upgrade configuration to current format ([#205](https://github.com/EvilBit-Labs/libmagic-rs/pull/205)) - Resolve all pending TODO items ([#212](https://github.com/EvilBit-Labs/libmagic-rs/pull/212)) - **mergify**: Upgrade configuration to current format ([#231](https://github.com/EvilBit-Labs/libmagic-rs/pull/231)) + ### Security @@ -144,14 +162,15 @@ All notable changes to this project will be documented in this file. ### Features - **parser**: Implement float and double types with endian variants ([#162](https://github.com/EvilBit-Labs/libmagic-rs/pull/162)) - + ## [0.4.3] - 2026-03-07 ### Refactor - **tests**: Extract CLI tests from main.rs into integration tests ([#159](https://github.com/EvilBit-Labs/libmagic-rs/pull/159)) + ## [0.4.2] - 2026-03-06 @@ -159,6 +178,7 @@ All notable changes to this project will be documented in this file. ### Refactor - **evaluator**: Reorganize types module into submodules ([#156](https://github.com/EvilBit-Labs/libmagic-rs/pull/156)) + ## [0.4.1] - 2026-03-06 @@ -166,6 +186,7 @@ All notable changes to this project will be documented in this file. ### Refactor - **evaluator**: Split mod.rs into focused submodules ([#153](https://github.com/EvilBit-Labs/libmagic-rs/pull/153)) + ## [0.4.0] - 2026-03-06 @@ -177,6 +198,7 @@ All notable changes to this project will be documented in this file. ### Documentation - Add comprehensive contributing guide ([#149](https://github.com/EvilBit-Labs/libmagic-rs/pull/149)) + ## [0.3.2] - 2026-03-05 @@ -184,6 +206,7 @@ All notable changes to this project will be documented in this file. ### Refactor - Pre-create evaluator submodules for v0.2.0 features ([#142](https://github.com/EvilBit-Labs/libmagic-rs/pull/142)) + ## [0.3.1] - 2026-03-05 @@ -191,6 +214,7 @@ All notable changes to this project will be documented in this file. ### Refactor - **evaluator**: Rename test functions to match RuleMatch type ([#60](https://github.com/EvilBit-Labs/libmagic-rs/pull/60)) ([#139](https://github.com/EvilBit-Labs/libmagic-rs/pull/139)) + ## [0.3.0] - 2026-03-02 @@ -198,6 +222,7 @@ All notable changes to this project will be documented in this file. ### Features - **parser**: Implement quad 64-bit integer type with endian variants ([#133](https://github.com/EvilBit-Labs/libmagic-rs/pull/133)) + ## [0.2.1] - 2026-03-02 @@ -215,6 +240,7 @@ All notable changes to this project will be documented in this file. - **Mergify**: Configuration update ([#124](https://github.com/EvilBit-Labs/libmagic-rs/pull/124)) - **Mergify**: Configuration update ([#126](https://github.com/EvilBit-Labs/libmagic-rs/pull/126)) - **mergify**: Streamline auto-approval and merging rules for bots + ## [0.2.0] - 2026-03-01 @@ -229,6 +255,7 @@ All notable changes to this project will be documented in this file. - Add Mergify merge queue and simplify CI ([#78](https://github.com/EvilBit-Labs/libmagic-rs/pull/78)) - Mergify merge queue, dependabot integration, and CI simplification ([#79](https://github.com/EvilBit-Labs/libmagic-rs/pull/79)) - **release**: Add regex for version bumping based on commit types + ## [0.1.1] - 2026-02-15 @@ -236,6 +263,7 @@ All notable changes to this project will be documented in this file. ### Miscellaneous Tasks - Regenerate changelog with v0.1.0 tag ([#72](https://github.com/EvilBit-Labs/libmagic-rs/pull/72)) + ## [0.1.0] - 2026-02-15 @@ -263,4 +291,5 @@ All notable changes to this project will be documented in this file. - Add Contributor Covenant Code of Conduct ([#2](https://github.com/EvilBit-Labs/libmagic-rs/pull/2)) - Add release-plz for crates.io publishing ([#69](https://github.com/EvilBit-Labs/libmagic-rs/pull/69)) - Regenerate CHANGELOG.md to fix duplicate entries ([#70](https://github.com/EvilBit-Labs/libmagic-rs/pull/70)) + diff --git a/Cargo.lock b/Cargo.lock index 36fd7bb9..3e3b500e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,12 +121,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.1" @@ -383,37 +377,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror", -] - [[package]] name = "difflib" version = "0.4.0" @@ -426,7 +389,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.13.1", + "bitflags", "block2", "libc", "objc2", @@ -463,7 +426,6 @@ dependencies = [ "anstream", "anstyle", "env_filter", - "jiff", "log", ] @@ -626,31 +588,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jiff" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" -dependencies = [ - "defmt", - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", -] - -[[package]] -name = "jiff-static" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "js-sys" version = "0.3.103" @@ -670,7 +607,7 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libmagic-rs" -version = "0.10.0" +version = "0.11.0" dependencies = [ "assert_cmd", "cfg-if", @@ -693,6 +630,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "testing_logger", "thiserror", ] @@ -729,7 +667,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -836,21 +774,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "portable-atomic-util" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" -dependencies = [ - "portable-atomic", -] - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -907,7 +830,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.13.1", + "bitflags", "num-traits", "rand", "rand_chacha", @@ -1038,7 +961,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -1169,6 +1092,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "testing_logger" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d92b727cb45d33ae956f7f46b966b25f1bc712092aeef9dba5ac798fc89f720" +dependencies = [ + "log", +] + [[package]] name = "thiserror" version = "2.0.18" diff --git a/Cargo.toml b/Cargo.toml index b9ed71f4..c8d3d294 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libmagic-rs" -version = "0.10.0" +version = "0.11.0" edition = "2024" authors = ["UncleSp1d3r "] description = "A pure-Rust implementation of libmagic for file type identification" @@ -32,11 +32,6 @@ exclude = [ "/fuzz", ] -# Opt this package into the workspace lint table below. Without this, -# [workspace.lints] is inert and none of the security lints apply. -[lints] -workspace = true - # Workspace lint configuration [workspace.lints.rust] # Security: Deny unsafe code globally. "deny" rather than "forbid" because the @@ -197,7 +192,15 @@ clap = { version = "4.6.1", features = ["derive"] } clap-stdin = "0.8.1" clap_complete = "4.6.5" ctrlc = { version = "3.5.2", features = ["termination"] } -env_logger = "0.11" +# `humantime` (an env_logger default feature) is the sole path that pulls in +# `jiff` -> `defmt` -> `bitflags 1.3.2`, which collides with the workspace's +# `bitflags 2.x` and trips `clippy::multiple_crate_versions`. We only call +# `env_logger::init()` and never render log timestamps, so we drop it and keep +# color + RUST_LOG regex filtering. +env_logger = { version = "0.11", default-features = false, features = [ + "auto-color", + "regex", +] } log = "0.4" memchr = "2.8.1" memmap2 = "0.9.10" @@ -221,6 +224,12 @@ nix = { version = "0.31.3", features = ["fs"] } predicates = "3.1.4" proptest = "1.11.0" tempfile = "3.27.0" +testing_logger = "0.1.1" + +# Opt this package into the workspace lint table below. Without this, +# [workspace.lints] is inert and none of the security lints apply. +[lints] +workspace = true [[bench]] name = "parser_bench" diff --git a/GOTCHAS.md b/GOTCHAS.md index a6b2d798..7083d8e6 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -28,6 +28,8 @@ Serialization functions live in `src/parser/codegen.rs`, shared by both `build.r Adding a variant to `TypeKind` requires updating exhaustive matches in 10+ files: `ast`, `grammar`, `types`, `codegen` (`serialize_type_kind` -- easy to forget; build.rs is a separate compilation unit so the error surfaces there first), `strength`, `property_tests`, `evaluator/types/mod.rs` (`read_typed_value`, `coerce_value_to_type`, **`bytes_consumed`** -- variable-width variants must be matched explicitly or relative-offset anchors will silently corrupt), `output/mod.rs` (2 length matches), `output/json.rs` (`format_value_as_hex`), and `grammar/tests.rs` (stale assertions). For string-family variants (anything that semantically holds a byte sequence rather than a fixed-width number), also add the variant to `is_string_family_type` in `parser/grammar/mod.rs::parse_magic_rule` so bareword (unquoted) value parsing falls back through the string parser instead of failing strict `parse_value` -- `String16` was caught missing from this helper during the PR #233 review (see `docs/solutions/design-patterns/multi-agent-review-surfaces-cross-cutting-consistency-gaps-2026-04-25.md` Gap 5 bonus fix). Note: `coerce_value_to_type`, output matches, and `bytes_consumed` use catch-all `_ =>` so they compile without changes but may need semantic updates -- `bytes_consumed` will fire a `debug_assert` in test/dev builds for unhandled variable-width variants. +**`TypeReadError::UnsupportedType` propagation has exactly one narrow, allowlisted exception.** The top-level dispatch match in `evaluator/engine/mod.rs::evaluate_rules` carries the comment `TypeReadError::UnsupportedType is intentionally NOT caught here (except the narrow exception in the arm immediately below)`. That exception, and the equivalent arm in the other two engine catch sites (`evaluate_children_or_warn`, and the inline child-recursion match under a successful parent), is gated on `types::is_missing_pattern_operand(type_name)` (the `type_name` string is exactly one of `"regex without string pattern"`, `"search without string/bytes pattern"`, or `"string with flags requires string/bytes pattern"` -- an exhaustive allowlist, not a substring match) or `types::is_regex_compile_failure(type_name)` (`type_name.starts_with("regex compile error:")`, which includes the `REGEX_COMPILE_SIZE_LIMIT` CWE-1333 DoS guard). When either predicate matches, the rule is skipped (non-match) instead of aborting the whole file's evaluation, logged via the shared `log_pattern_operand_skip` helper: `debug!` for the ordinary missing-pattern case, `warn!` for a regex compile failure (so a malicious or pathological magic file's rejection stays visible in logs even though evaluation of the rest of the file continues). Any OTHER `UnsupportedType` cause -- an unwired `TypeKind` variant, a non-Equal/NotEqual operator applied to a pattern-bearing type (S2.4), a `Meta` variant read as a value -- still falls through and propagates fatally at all three sites. Do not widen the predicate to match on the variant alone; that would silently swallow genuine capability gaps this contract exists to surface. + **Meta variants (`TypeKind::Meta(MetaType::...)`)** are structurally different. They do not read bytes, so `read_typed_value`, `coerce_value_to_type`, and `bytes_consumed` treat them as no-ops (catch-all or zero-width arms). The live dispatch for control-flow directives happens in `evaluator/engine/mod.rs::evaluate_rules` -- the *loop body*, not in `evaluate_single_rule_with_anchor`. `MetaType::Use`, `MetaType::Default`, `MetaType::Clear`, and `MetaType::Indirect` are all handled inline there so subroutine / control-flow effects can be spliced into the caller's match vector; `MetaType::Name` is hoisted out of the rule list at parse time by `parser::name_table::extract_name_table` and should normally never reach the evaluator at all. The evaluator's leaked-`Name` arm uses `debug!` (not `debug_assert!`) because `prop_arbitrary_rule_evaluation_never_panics` synthesizes arbitrary `TypeKind` values and a panic there would break the never-panics invariant. **This applies to every inline-dispatched meta variant that has a "not configured" fallback** (currently `Use` and `Indirect` for the no-`RuleEnvironment` case): property tests run with `rule_env = None`, so any `debug_assert!` on that path will fire. When adding a new `MetaType` variant: decide whether it needs inline loop-level dispatch (add to `evaluate_rules`) or is a silent no-op (add to the `_` arm of the `Meta(_)` match in `evaluate_single_rule_with_anchor`). See `docs/solutions/integration-issues/meta-type-subroutine-dispatch-architecture.md` for the full pattern. ### 2.2 `Operator` Exhaustive Matches @@ -46,6 +48,8 @@ Adding a variant to `Value` requires updating: `ast`, `codegen`, `strength`, `pr `TypeKind::Regex` and `TypeKind::Search` are evaluated by **logical match** in `evaluate_single_rule_with_anchor` (`src/evaluator/engine/mod.rs`), not by string equality against `rule.value`. The engine calls `types::read_pattern_match`, which returns `Result, _>`: `Some(v)` means the pattern matched (possibly zero-width) and `None` means it did not. The engine translates that `Option` directly into `Equal`/`NotEqual`. Comparing matched text to the pattern literal via `apply_operator` would fail for any regex with metacharacters (e.g., matched `"123"` vs pattern `"[0-9]+"`). **Non-equality operators on pattern-bearing types are rejected as `TypeReadError::UnsupportedType`** — an earlier revision fell through to `apply_operator` and silently produced lexicographic ordering comparisons against the pattern source text. If you add a new pattern-bearing `TypeKind` variant, add its arm to both `read_pattern_match` and `bytes_consumed_with_pattern`; the engine's special-case match is keyed on the `Regex | Search` pair so you must add new variants there too. +**`Regex` and `Search` both accept `Some(Value::String(_))` and `Some(Value::Bytes(_))` as the pattern operand** — the two types are symmetric, not asymmetric. (An earlier revision had `Regex` accept only `Value::String`, rejecting `Value::Bytes` with `UnsupportedType { "regex without string pattern" }` while `Search` already accepted both; that gap is what let escape-heavy `regex` patterns like `\^[\040\t]{0,50}\\.asciiz` -- parsed as `Value::Bytes` by `parse_value`'s hex/mixed-ascii branch -- fatally abort evaluation of the entire system magic DB. See the fix-system-magic-regex-graceful plan.) Three call sites in `src/evaluator/types/mod.rs` were made symmetric: the `Regex` arms of `read_typed_value_with_pattern` and `read_pattern_match` both now match `Some(Value::Bytes(b)) => Cow::Owned(decode_regex_bytes_pattern(b))` alongside the `Value::String` fast path (`Cow::Borrowed`), and `bytes_consumed_with_pattern`'s `Regex` arm decodes the same way before re-running `regex_bytes_consumed` so the anchor-advance side agrees with the read side. `decode_regex_bytes_pattern` decodes via `String::from_utf8_lossy` and `warn!`s only when `str::from_utf8` on the same bytes actually fails (i.e., a real substitution occurred, not just that the input happened to be `Bytes`) — a lossy substitution silently desyncs the compiled regex (which now contains U+FFFD) from the target buffer (still matched by raw bytes), so the warning is the visibility guard for that divergence. In practice this backstop is now a floor-tier safety net rather than the common path: the parser's getstr resolver (see the new sub-point below) captures unquoted `regex` patterns as `Value::String` directly, so a `Value::Bytes` regex pattern should be rare going forward. + ### 2.5 Zero-Width Regex Matches vs Misses `read_regex` returns `Ok(Some(Value::String("")))` for a legitimate zero-width match (`^`, `a*`, `.{0}`) and `Ok(None)` for a genuine miss. (The Rust `regex` crate does not support look-around assertions such as lookaheads or lookbehinds -- they are excluded to preserve linear-time matching guarantees.) An earlier revision collapsed both cases to `Value::String(String::new())` and distinguished them by `is_empty()`, which broke every pattern that legitimately matches zero bytes. The structured `Option` is the invariant — do not re-flatten it. `read_typed_value_with_pattern` does collapse `None` to `Value::String(String::new())` for back-compat with its single-`Value` return shape, but the engine does not go through that function for pattern types; it calls `read_pattern_match` directly. @@ -89,11 +93,42 @@ Adding a variant to the `MetaType` enum (nested inside `TypeKind::Meta`) require **Evaluator decision rule.** The evaluator has two possible locations for a `MetaType` arm and the choice is load-bearing: -- **Inline loop-level dispatch in `evaluate_rules` (in `src/evaluator/engine/mod.rs`)** — use this when the variant has side-effects that the single-rule path cannot express. `MetaType::Default` (needs the per-level `sibling_matched` flag), `MetaType::Clear` (mutates `sibling_matched`), `MetaType::Use` (splices subroutine matches into the caller's match vector from `RuleEnvironment::name_table`), and `MetaType::Indirect` (recurses into `RuleEnvironment::root_rules` with a reset anchor and a sliced buffer) all live here. These variants are dispatched **before** the loop body calls `evaluate_single_rule_with_anchor`. +- **Inline loop-level dispatch in `evaluate_rules` (in `src/evaluator/engine/mod.rs`)** — use this when the variant has side-effects that the single-rule path cannot express. `MetaType::Default` (needs the per-level `sibling_matched` flag), `MetaType::Clear` (resets `sibling_matched` AND emits its own message when non-empty -- see S13.6), `MetaType::Use` (splices subroutine matches into the caller's match vector from `RuleEnvironment::name_table`), and `MetaType::Indirect` (recurses into `RuleEnvironment::root_rules` with a reset anchor and a sliced buffer) all live here. These variants are dispatched **before** the loop body calls `evaluate_single_rule_with_anchor`. - **Silent no-op via the `Meta(_)` wildcard arm in `evaluate_single_rule_with_anchor`** — use this for variants that produce no match, do not mutate `sibling_matched`, and do not recurse. `MetaType::Name` falls here (it is also defensively scrubbed at load time by `parser::name_table::extract_name_table`, so the evaluator arm is a `debug!`-and-return safety net rather than a normal code path). Similarly, `strength.rs::calculate_default_strength` and `types/mod.rs::bytes_consumed_with_pattern` treat all `Meta(_)` variants uniformly (zero strength contribution, zero bytes consumed) via catch-alls, so they do not need per-variant arms. +### 2.12 Unquoted `regex` Patterns Are getstr-Resolved to `Value::String`, Not Passed Through `parse_hex_bytes` + +`parser/grammar/mod.rs::parse_magic_rule` special-cases `TypeKind::Regex` **ahead of** the generic `is_string_family_type` bareword fallback (S3.6): a quoted regex value (`regex/c "hello" ...`) still goes through `parse_value`'s existing `parse_quoted_string` path unchanged, but an unquoted (bareword) regex pattern is routed to `parser::grammar::getstr::parse_regex_getstr_value` instead of `parse_hex_bytes`/`parse_bare_string_value`. This exists because `parse_value`'s `alt(...)` tries the hex/mixed-ascii branch before any string interpretation, so a pattern beginning with a magic(5) escape (`\^`, `\040`, `\t`, `\x..`) was previously captured as `Value::Bytes` — the root cause of the system-magic-DB fatal abort this fix resolves (see S2.4). If the getstr resolver's own parse fails, `parse_magic_rule` falls back to `parse_value` so previously-working non-escaped and quoted forms are unaffected. + +The resolver (`src/parser/grammar/getstr/mod.rs`) replicates GNU `file`'s `apprentice.c::getstr` escape table, verified against the upstream source (not inferred): + +- `\a`, `\b`, `\f`, `\n`, `\r`, `\t`, `\v` resolve to their C control-character byte (`0x07`, `0x08`, `0x0C`, `0x0A`, `0x0D`, `0x09`, `0x0B`). +- `\0`-`\7` starts a 1-3 digit **octal** escape (`\NNN`), truncated to a byte (matches getstr's `CAST(char, val)`). +- `\x` starts a 0-2 digit **hex** escape (`\xNN`); with no valid hex digit following, getstr's quirky default is the literal byte `'x'` (`0x78`) — replicated here for fidelity, not "fixed." +- Every other escaped character — relation characters (`> < & ^ = !`), a literal backslash, an escaped `.`, an escaped space, an escaped literal tab character, and any otherwise-unrecognized escape such as `\^` — falls through getstr's `default:` case: **the backslash is dropped and the character is kept literally** (`\^` -> `^`, a regex anchor). `parse_bare_string_value` (used for non-regex string-family barewords) now follows the **same** drop-the-backslash rule for an unknown escape (`\<` -> `<`, `\^` -> `^`, `\ ` -> a literal space that continues the token), matching GNU `file`. An earlier revision had it **keep** the backslash, which broke real rules like sgml's `0 string \= 0x80` handling, but neither flattens through `from_utf8_lossy` any longer -- getstr re-encodes a high byte as `\xHH` regex-native text (KTD3 above), while `parse_bare_string_value` returns `Value::Bytes` holding the raw bytes when the resolved token is not valid UTF-8 (mirroring `read_string_exact`; see S6.7). Both preserve the high byte, but in different representations -- getstr as regex-source text (required because the pattern must compile), `parse_bare_string_value` as raw bytes (correct for a plain `string` comparison) -- so regex must still not reuse `parse_bare_string_value` unchanged. The bullet list above is a summary of the notable named escapes, not an exhaustive taxonomy of this fallthrough bucket -- anything not named-control/octal/hex lands here, including plain punctuation that happens to be escaped for readability in a magic file. + +**String-family bareword beginning with a backslash (now fixed -- third drop-backslash site).** A string-family bareword whose value *begins* with a backslash (e.g. sgml's `0 string \ `<`, `\^` -> `^`, `\ ` -> a literal space that CONTINUES the token instead of terminating it). A lone **trailing** backslash (nothing follows) still stays a literal `0x5c` byte -- required by the `0 regex \` recovery contract (`parse_magic_rule_regex_lone_trailing_backslash_pattern_recovers_via_value_fallback`). This resolves XML/plist detection: `\15 string/t >\0 %.3s document text`. This had TWO independent root causes, both now fixed: (1) the ordering-operator string read returned only the compared prefix (`pattern.len()` bytes = 1) as the display value; (2) `format_magic_message` discarded `%s` precision. See S14.3 for the compare/render decoupling and the precision support. + +- getstr does **not** special-case `\d`, `\s`, or `\w` as PCRE-style character classes; they fall into the "everything else" bucket above and demote to the bare literal character (`\d` -> `d`, `\s` -> `s`, `\w` -> `w`). This is genuine upstream libmagic behavior, not a bug in the resolver. +- This resolver deliberately does **not** replicate getstr's `file_magwarn` lint diagnostics (e.g. "unnecessary escape" / "no such escape" warnings emitted by upstream `apprentice.c` while authoring a magic file). Those are magic-file-*authoring* diagnostics, orthogonal to escape *resolution* -- this module's job is producing the correct resolved byte sequence, not linting the source `.magic` file for style. + +**`>= 0x80` re-encoding (KTD3):** a resolved byte `>= 0x80` cannot be pushed onto a Rust `String` as a raw `char` (not a valid standalone Unicode scalar value), so `push_resolved_byte` re-encodes it as a regex-native `\xHH` hex escape *in the output text* instead — the byte lives in the regex the engine compiles, never as a raw byte inside the `String`. This is required because `parse_text_magic_file` is fail-fast (S3.11): a raw `>= 0x80` byte pushed into a `String` would either panic or (via lossy handling) corrupt the pattern, and either way one such rule would abort the entire file load rather than just failing that rule. Named control escapes (`\a`/`\b`/`\f`/`\n`/`\r`/`\t`/`\v`) resolve to a single raw byte, not to escape *syntax*, so they are never reinterpreted by the downstream regex compiler as a shorthand (e.g. a resolved `0x08` byte is an ordinary literal, not the two-character sequence `\` + `b` that the `regex` crate would read as a word-boundary assertion) — no `\xHH` re-encoding is needed for these regardless of value. + +### 2.13 Bare `search` (No `/N`) Is Accepted and Scans to EOF (`range: None`) + +`TypeKind::Search::range` is `Option`, not `NonZeroUsize`. `Some(n)` is the `search/N` form (scan at most `n` bytes); `None` is a **bare `search`** (no `/N` suffix, e.g. pdf's `>8 search /Count`) and scans from the offset to end-of-buffer. magic(5) documents the count as *required*, but the reference GNU `file` binary accepts the bare form and treats it as `str_range == 0` (scan-to-EOF), so rmagic follows the **implementation, not the spec**. `search/0` is still rejected — an explicit zero-width scan is unrepresentable via `NonZeroUsize`. + +- **Parse-side disambiguation** (`parse_type_and_operator` in `grammar/mod.rs`): a ranged suffix attaches `/` directly to the keyword (`search/256`), while a bare search is followed by whitespace then the value (`search\t/Count`). `input.strip_prefix('/')` distinguishes them; when it fails, `search_range` stays `None`. `parse_search_suffix` (`type_suffix.rs`) still handles ONLY the ranged form and is never reached for a bare search. +- **Eval-side** (`evaluator/types/search.rs`): `find_match`/`read_search`/`search_bytes_consumed` take `Option`; `window_len = range.map_or(remaining.len(), |r| r.get().min(remaining.len()))`. `None` -> whole remaining buffer. +- **Why this was load-bearing:** before this change rmagic rejected bare `search` at parse time, so the tolerant loader (S3.11) dropped the rule *and its subtree*. That silently lost the pdf page-count grandchild (`>>&0 regex [0-9]+ \b, %s pages`), VCF/SAM bioinformatics signatures, ICC, sfnt name tables, and more. Only two top-level bare searches exist in the whole system DB (`0 search ##fileformat=VCFv`, `0 search ..OMR`); the rest are children gated behind a matched parent. +- **Accepted edge-case regression (magic.mgc):** because `/usr/share/file/magic.mgc` (the compiled magic DB) literally embeds magic *source text*, the top-level bare `search ##fileformat=VCFv` now matches it, and the strength-formula divergence (issue #379) mis-orders that search above the `lelong` mgc-header rule, so rmagic prints `Variant Call Format (VCF)` where `file` prints `magic binary file ...`. This is the *sole* regression from bare-search acceptance (verified: 39 oracle-confirmed improvements, 1 regression across a 3189-file before/after differential); it affects only `magic.mgc`, is also a compiled-in `.mgc` detector case (#377), and both top-level bare-search patterns have zero collisions with real binaries in `/usr/bin`/`/usr/lib`/`/System/Library`. Regression tests: `tests/search_bare_range_tests.rs` (pdf page count, scan-to-EOF, bounded-window negative control) and `src/evaluator/types/search.rs::test_read_search_bare_none_scans_to_eof`. + ## 3. Parser Architecture ### 3.1 Type Keyword Parsing Split @@ -121,7 +156,15 @@ The nom `tuple` combinator is deprecated. Use bare tuple syntax `(a, b, c)` dire ### 3.6 `parse_value` Requires Quoted Strings (But `parse_magic_rule` Has a Bare-Word Fallback) -`parse_value()` itself does not accept bare unquoted strings -- `parse_value("xyz")` still returns `Err` (see `test_parse_value_invalid_input` in `grammar/tests/mod.rs`, which pins this behavior). However, `parse_magic_rule` adds a bare-word fallback (`parse_bare_string_value`) when the rule's type is string-family (`String`, `PString`, `Regex`, `Search`), so `0 string TEST` and `>0 search/12 ABC` parse successfully without quotes -- matching libmagic magic(5) surface syntax and allowing real-world fixtures like `third_party/tests/searchbug.magic` to load. For non-string-family types (byte/short/long/etc.), bare words still fail; integration tests exercising `parse_value` directly (as opposed to going through `parse_magic_rule`) must still quote string literals. +`parse_value()` itself does not accept bare unquoted strings -- `parse_value("xyz")` still returns `Err` (see `test_parse_value_invalid_input` in `grammar/tests/mod.rs`, which pins this behavior). However, `parse_magic_rule` adds a bare-word fallback when the rule's type is string-family (`String`, `String16`, `PString`, `Regex`, `Search`), so `0 string TEST` and `>0 search/12 ABC` parse successfully without quotes -- matching libmagic magic(5) surface syntax and allowing real-world fixtures like `third_party/tests/searchbug.magic` to load. For non-string-family types (byte/short/long/etc.), bare words still fail; integration tests exercising `parse_value` directly (as opposed to going through `parse_magic_rule`) must still quote string literals. + +**String-family barewords never parse as numbers (`parse_string_family_value`).** For a string-family type (except `Regex`, which has its own getstr branch -- S2.12), `parse_magic_rule` does **not** call `parse_value` at all. It calls `parse_string_family_value`, which runs a leading `multispace0` trim and then a three-way `alt`: quoted string (-> `Value::String`), hex/escape bytes (-> `Value::Bytes`, e.g. gzip's `\037\213`, `\177ELF`), then `parse_bare_string_value` (-> `Value::String`) for everything else. Crucially it **omits** `parse_value`'s float and integer branches. libmagic never treats a `string` comparison value as numeric: `0 string >0` compares against the literal ASCII byte `'0'` (0x30), and `>0.6.1` against the literal characters `0.6.1`. An earlier revision routed these through `parse_value` first, which captured bareword `0` as `Value::Uint(0)` and `0.6.1` as `Value::Float(0.6)` (leaking `.1` into the message); the subsequent `String`-vs-number comparison is incomparable (`compare_values` returns `None`), so the rule **silently never matched** -- breaking the pervasive `>0 ... %s` idiom (`\b, name %s`, `face %s`, `palette %s`) and version compares (`>0.6.1 ... version %s`). This applies to **both equality and ordering** operators -- `parse_string_family_value` never inspects the operator, so `0 string 000` also flips from `Value::Uint(0)` to `Value::String("000")` (correct per libmagic). + +Two invariants must be preserved by anything that touches this function: (1) the leading `multispace0` is load-bearing -- it makes the hex branch see byte-identical input to what `parse_value` fed it, so an escape-heavy value cannot fall through to the lossy-UTF-8 `parse_bare_string_value` path and corrupt a high byte (see the high-byte-UTF-8 corruption class); and (2) hex-*letter* barewords (`>AB`, `cafebabe`) still resolve to `Value::Bytes` via the unchanged hex branch (S3.12) -- only the numeric subset changed. This composes with S14.3: a rule parsed as `0 string >0.6.1` now correctly renders the full field (`0.6.2 release`) while comparing only the `pattern.len()` prefix (`0.6.10` does NOT match, because prefix `0.6.1` equals the pattern). + +**`search` + ordering-op interaction (S2.4).** `Search` is pattern-bearing, so a non-equality operator on it is an intentional `UnsupportedType` fatal gap (S2.4, not allowlisted). Before this change a numeric bareword like `0 search/16 >100` parsed to `Value::Uint(100)`, which -- having no String/Bytes pattern operand -- hit the *allowlisted* `"search without string/bytes pattern"` graceful skip. Now it parses to `Value::String("100")`, so the pattern operand IS present and the `>` operator trips the (deliberate) S2.4 fatal path instead. This is not a real-world regression: no magic rule in the GNU `file` DB uses `search`/`regex` with an ordering operator on a numeric/string bareword (the only real hit, lex's `search/1 >\0`, is a `\0`->`Value::Bytes` value on the unchanged hex branch, and was already on the S2.4 fatal path before and after this change). Do NOT "fix" the skip->fatal flip by widening the S2.1 allowlist to cover non-equality ops on pattern-bearing types -- S2.4 exists precisely to surface that capability gap. Regex is unaffected regardless (it uses its own getstr branch, S2.12, not `parse_string_family_value`). + +Regressions: `test_string_family_bareword_numeric_value_parses_as_string` (covers equality + ordering; the `search` case uses an equality op deliberately), `test_string_family_value_preserves_quoted_and_hex_bytes`, and end-to-end `test_string_numeric_ordering_end_to_end_composes_with_full_field_render`. The system-DB integration tests (`system_magic_dir`, `compatibility_tests`) that load the real magic DB are the actual regression net; the ad-hoc 525-file `file`-differential showed zero *observable changes* (no regressions and no visible top-level improvements -- the affected `>0 %s` rules are deep children whose parents did not match in that sample), so the real-file proof is the synthetic before/after plus those passing DB-loading tests. ### 3.7 Indirect Offset Pointer Specifiers Follow GNU `file` Semantics @@ -148,9 +191,18 @@ The load-bearing invariant is that the anchor is updated *before recursing into Inside a `MetaType::Use` subroutine body, `OffsetSpec::Absolute(n)` with `n >= 0` resolves to `base_offset + n`, where `base_offset` is the use-site offset. `EvaluationContext::base_offset` tracks this; `evaluate_use_rule` saves and restores it around the subroutine call, alongside `last_match_end`. This matches magic(5) / libmagic semantics: a subroutine written as `>0 search/12 ABC` invoked via `>>64 use part2` scans starting at file position 64, not 0. Negative absolute offsets (`FromEnd`-style), `Indirect` pointer reads, and `Relative(&N)` offsets are unaffected -- they already have well-defined reference frames. Callers of `offset::resolve_offset_with_base` pass `context.base_offset()`; the `base_offset` field is gated behind `pub(crate)` accessors so external consumers cannot inject arbitrary offset bias. -### 3.11 `parse_text_magic_file` is Fail-Fast, Not Skip-on-Error +### 3.11 Parsing Is Strict at Build Time, Line-Tolerant at Runtime + +There are **two** hierarchy builders, differing only in how they treat a rule line that fails to parse: + +- **Strict (`build_rule_hierarchy` / `parse_text_magic_file`)** -- propagates any `parse_magic_rule_line` error immediately, so a single unparseable rule causes the **entire** call to fail with `ParseError::InvalidSyntax`. This is used by **build-time codegen** (`src/build_helpers.rs`) so a malformed rule in the crate's OWN builtin magic fails the build loudly. +- **Tolerant (`build_rule_hierarchy_tolerant` / `parse_text_magic_file_tolerant`)** -- skips an unparseable rule **and its deeper-indented subtree** with a `warn!` and keeps the rest of the file, matching GNU `file`. This is used by the **runtime** loaders (`load_magic_file`, `load_magic_directory`). Real system magic databases routinely mix rules this parser fully supports with a handful using constructs it does not yet handle (`guid`, `der`, middle-endian and quad-date variants, some indirect-offset forms). Without tolerance, ONE such rule dropped the whole file -- e.g. losing all of `compress`'s gzip/bzip2 detection over its lone `ustring` XZ rule, or all of `jpeg`/`pdf`/`linux`/`windows`. A prior audit found ~21 real system files being discarded this way. -`build_rule_hierarchy` propagates any `parse_magic_rule_line` error immediately, so a single unparseable rule (e.g., a string value using unquoted `$VAR` substitution on a non-string-family type -- see S3.6) causes the **entire file load** to fail with `ParseError::InvalidSyntax`. There is no skip-and-continue mode. When writing corpus tests against third_party `.magic` files that mix supported and unsupported syntax, bypass the parser and build the equivalent `MagicRule` tree programmatically via the AST; the runtime evaluator can still be exercised end-to-end against the real testfile buffer. See `tests/evaluator_tests.rs::test_regex_eol_corpus` for a worked example. +**Subtree drop, not single-line skip:** when a rule line fails, its descendants (lines with a deeper `>`-level, counted by `leading_indent_level`) are dropped too, so an orphaned child cannot silently re-attach to the wrong parent. A pending `!:strength` directive orphaned by the skip is discarded. + +**Directory-level "all failed" contract preserved:** `load_magic_directory` still errors when a non-empty directory loads nothing usable. A file counts as success only if it contributes at least one rule or name-table entry; a content-bearing file whose rules were ALL skipped is tracked (via `has_rule_lines`) and, if every file in the directory is like that (or fails preprocessing), the load fails with the same truncated "All N ... failed to parse" report. Genuinely empty or comment-only files are valid and contribute nothing. + +When writing corpus tests against third_party `.magic` files that mix supported and unsupported syntax, you can either use the tolerant runtime path (which now loads them) or bypass the parser and build the equivalent `MagicRule` tree programmatically via the AST. See `tests/evaluator_tests.rs::test_regex_eol_corpus` for the AST-build approach and `src/parser/loader.rs::tests::test_load_magic_file_tolerates_unparseable_rule_and_keeps_valid_ones` for the tolerant contract. ### 3.10 `parse_text_magic_file` Returns `ParsedMagic`, Not `Vec` @@ -159,10 +211,14 @@ Inside a `MetaType::Use` subroutine body, `OffsetSpec::Absolute(n)` with `n >= 0 - **Callers must destructure at the boundary.** Codegen consumers (`build.rs`, `src/build_helpers.rs`) use `parsed.rules` and discard the table. Runtime consumers (`MagicDatabase::load_from_file_with_config`) pattern-match `let ParsedMagic { rules, name_table } = ...;` and wrap both in `Arc`s before handing them to the database. - **Directory loads merge name tables with first-wins semantics.** `load_magic_directory` merges per-file name tables alphabetically (same ordering as the rule merge); duplicate-name warnings during merge are distinct from per-file duplicate-name warnings during extraction. - **Evaluator consumption is via `RuleEnvironment`** (`pub(crate)` in `evaluator/mod.rs`), threaded as an optional `Arc` on `EvaluationContext`. Low-level callers (`evaluate_rules`, `evaluate_rules_with_config`) leave this as `None` and `Use` rules no-op silently -- this preserves the low-level API for property tests and fuzz harnesses that construct rule trees by hand. `MagicDatabase::evaluate_buffer_internal` attaches the environment before dispatching. -- **Each subroutine in the name table must be strength-sorted** the same way top-level rules are (via `sort_rules_by_strength_recursive`), otherwise `use`-site evaluation is non-deterministic with respect to source order inside the `name` block. `MagicDatabase::load_from_file_with_config` iterates `name_table.values_mut()` to do this after the top-level sort. +- **Subroutine bodies in the name table are NOT strength-sorted** -- they stay in FILE order, exactly like continuation rules (see S13.4/S13.5 for the full rationale). `MagicDatabase::load_from_file_with_config` sorts only the top-level rule list (`sort_rules_by_strength`, non-recursive) and leaves `name_table` bodies untouched; the old `NameTable::sort_subroutines` / `values_mut()` re-sort machinery was **removed** in the strength-sort fix. This mirrors libmagic's `apprentice_sort`, which orders whole magic entries by their first line and never reorders the lines inside a block, so `use`-site evaluation follows source order inside the `name` block by design. (An earlier revision of this section claimed subroutines *must* be strength-sorted via `sort_rules_by_strength_recursive`; that was made false by the fix documented in S13.4 and is corrected here.) See `docs/solutions/integration-issues/meta-type-subroutine-dispatch-architecture.md` for the full three-layer pattern (parse-time hoist, `ParsedMagic` return type, optional `RuleEnvironment`). +### 3.12 `parse_hex_bytes_no_prefix` Requires a Token Boundary After the Hex Run + +`parse_hex_bytes_no_prefix` (in `src/parser/grammar/value.rs`, one of the three `alt(...)` branches inside `parse_hex_bytes`) is used for a bareword (no `\` prefix) value token that looks like a run of pure hex digit pairs. Before the U7 audit (fix-system-magic-regex-graceful plan), it captured hex digits via `take_while(is_ascii_hexdigit)` and stopped there, returning `Ok` with the truncated byte vector and leaking the remainder as the parser's "remaining input" -- e.g. `parse_hex_bytes("4a[42")` returned `Ok(("[42", vec![0x4a]))`, silently dropping to a 1-byte value and leaking `"[42"` back to the caller, where `parse_magic_rule` would misinterpret it as the start of the rule's message text. The fix adds `is_hex_token_boundary`: after the hex-digit run, the immediately-following character must be end-of-input, whitespace, or a closing quote (`"`), or the whole token is rejected (`Err`) rather than truncated. This only affects the no-prefix branch -- `parse_mixed_hex_ascii` (used when the token starts with `\`) already consumed the whole token via its `none_of(" \t\n\r")` fallback and needed no fix; see `src/parser/grammar/tests/hex_bytes_truncation.rs` for the confirmed-bug regression test and the differential coverage pinning `parse_mixed_hex_ascii`'s pre-existing whole-token behavior. `TypeKind::Regex` no longer routes through this code path at all (S2.12), but `TypeKind::Search` and any other string-family type whose bareword value happens to look like hex bytes still does. + ## 4. Module Visibility & Re-exports ### 4.1 Private Engine Module @@ -242,6 +298,14 @@ Use `u8::to_ascii_lowercase` / `to_ascii_uppercase`, not full Unicode case-foldi magic(5) and libmagic `src/file.h` reserve `/B` (`CHAR_PSTRING_1_BE`) as a `pstring` length-width letter. It is **not** a `string` flag — `string/B` is rejected at parse time. An earlier draft of the grammar (PR #233) included `'B'` in the string-flag set; that was wrong and has been corrected. The pstring suffix parser (`src/parser/grammar/type_suffix.rs::parse_pstring_suffix`) is the only place `/B` should be accepted. +### 6.7 `parse_bare_string_value` Returns `Value::Bytes` on Non-UTF-8 (Parse-Side Mirror of `read_string_exact`) + +`parse_bare_string_value` (`src/parser/grammar/mod.rs`) is the bareword fallback for string-family types (`string`, `pstring`, `search`; **not** `regex`, which has its own getstr branch -- S2.12). It resolves the token's magic(5) escape sequences into a `Vec` and then chooses the `Value` variant by UTF-8 validity: valid UTF-8 -> `Value::String` (so `%s` output renders normally), non-UTF-8 -> `Value::Bytes` holding the raw bytes. This exactly mirrors `read_string_exact` (S6.4), which was already fixed to return `Value::Bytes` on a non-UTF-8 slice. + +**Why the old `from_utf8_lossy` -> `Value::String` was a silent-never-match bug.** A bareword whose resolved bytes contain a byte `>= 0x80` that isn't part of a valid UTF-8 sequence -- e.g. OS/2 INF's top-level signature `0 string HSP\x01\x9b\x00 OS/2 INF` (bytes `48 53 50 01 9b 00`, where `0x9b` is invalid UTF-8) -- was lossy-decoded to `"HSP\x01\u{FFFD}\x00"`. That does two things, both fatal to matching: (1) it **inflates the byte length** from 6 to 8 (U+FFFD is 3 bytes `EF BF BD`), so the dispatcher's `read_string_exact(buffer, offset, pattern.len())` reads 8 bytes instead of 6; and (2) it **changes the byte value** at that position (`9b` -> `EF`), so even the read prefix mismatches. The rule fell through to the ascmagic `data`/`ASCII text` fallback. The signature `HSP` begins with literal ASCII, so it does **not** start with `\` and therefore bypasses `parse_mixed_hex_ascii` (which unconditionally emits `Value::Bytes` -- S2.12) -- `parse_bare_string_value` was the only high-byte-preserving gap left after `read_string_exact` was fixed. + +Cross-type `String`/`Bytes` equality and ordering (S2.3) compare by byte sequence, so the returned variant (`String` for the all-UTF-8 case, `Bytes` for the high-byte case) always compares correctly against the read value regardless of which variant the read side produced. The fix covers **both** escape forms that can carry a high byte in a bareword: `\x9b` (hex) and `\376` (octal). Regression: `parse_bare_string_value_high_byte_returns_bytes_not_lossy_string` (parse) and `os2_inf_high_byte_signature_matches` (end-to-end). See the high-byte-utf8-corruption recurring-bug class. + ## 7. Testing ### 7.1 Doctest Import Paths @@ -337,15 +401,57 @@ All tags and commits MUST be signed -- use `git tag -s` and `git commit -s -S`. - **Validation:** `timeout_ms` is clamped to `MAX_SAFE_TIMEOUT_MS` (5 minutes) by config validation and must be `> 0` if specified -- see the validation logic in `src/config.rs`. - **Note:** `Default` cannot be changed to set a timeout without breaking API expectations of callers who deliberately want no timeout (e.g., CLI one-shot invocations). The gotcha is that the unsafe default is the ergonomic choice; document the tradeoff prominently in any new consumer-facing docs. -### 13.2 `EvaluationConfig::default()` Stops at First Top-Level Match +### 13.2 `EvaluationConfig::default()` Stops at First Top-Level Match -- But Only Once That Match Produces Output + +`EvaluationConfig::default()` sets `stop_at_first_match: true`, so once any top-level rule produces a match **that carries usable description text**, the evaluator halts further sibling iteration. Integration tests that rely on later siblings running (e.g., `default`/`clear`/`default` chains, `use`-followed-by-sibling continuation, anything exercising the per-level `sibling_matched` flag across the full chain) must use `MagicDatabase::load_from_file_with_config` with `EvaluationConfig::default().with_stop_at_first_match(false)`. Unit tests that go through `evaluate_rules` directly hit the same issue -- override the config at construction time. The existing `Use` tests (`test_use_child_rules_evaluated_after_subroutine`, `test_use_stop_at_first_match_short_circuits_siblings`) document both sides of this contract. + +**Refined contract (the "message-less shadowing" fix).** A top-level rule that matches but contributes NO usable description text -- an empty message, a whitespace-only message, or a `\b`-only message (see S14.1), and no message-bearing descendant match either -- does **not** count as the terminating match. Evaluation continues to the next top-level sibling instead. This matters because real-world magic files routinely use message-less top-level rules purely as gating conditions for children (e.g. `/usr/share/file/magic/c-lang`'s `0 search/8192 "#include"` -> `>0 regex \^#include c` chain). Under the OLD (pre-fix) contract, if such a gating rule happened to match a buffer on its own (no matching child) BEFORE a later, more specific rule was tried in strength order, the whole evaluation would halt right there and the CLI would print a **blank description** -- this was a real, maintainer-reported bug (an assembler-source file starting with a leading tab and `.asciiz`, and even plain ASCII text with no special content, both produced blank output against the real system magic DB). The fix is implemented via `has_message_bearing_match` / `is_message_bearing` in `src/evaluator/engine/mod.rs`, checked at every one of the five successful-match dispatch sites in `evaluate_rules` (the plain value-rule path, `Default`, `Indirect`, `Offset`, and `Use`) before the `should_stop_at_first_match()` break fires. `sibling_matched` (the `default`/`clear` dispatch flag) is UNAFFECTED by this refinement -- a message-less match still counts as "a sibling matched" for that purpose; only the `stop_at_first_match` early-exit is gated on message-bearing output. + +See S13.3 for the companion text/data fallback that also had to be added for this fix to fully resolve the reported bug (a message-less rule not stopping evaluation is necessary but not sufficient -- if NO rule ever produces output, something must still replace the blank description). + +### 13.3 Text/Data Fallback When No Rule Produces a Description + +`MagicDatabase::build_result` (`src/lib.rs`) falls back to `crate::output::ascmagic::classify_fallback(buffer)` whenever rule evaluation produces no usable description -- either because `matches` is empty, or because every match's message is empty after concatenation (see S13.2's refined stop-at-first-match contract; the fallback is what makes "continue past message-less matches" actually resolve to a non-blank final answer instead of an empty string once every top-level rule is message-less or none match at all). This mirrors GNU `file`'s `file_ascmagic` basic content classification: `"empty"` for a zero-byte buffer, `"ASCII text"` for buffers containing only printable ASCII plus common text control characters (tab, LF, CR, VT, FF, and the legacy bell/backspace/escape bytes GNU `file` still treats as text), `"UTF-8 Unicode text"` for valid non-ASCII UTF-8, and `"data"` for anything else (binary content). + +This is a deliberately narrow subset of GNU `file`'s real charset detection (no ISO-8859 variants, no UTF-16, no "with CRLF line terminators"/"with escape sequences" qualifiers -- see `src/output/ascmagic.rs`'s module doc for the full list of what is intentionally not replicated). Every classification it produces is a true subset of what GNU `file` would print for the same input (e.g. `file` says `"ASCII text, with CRLF line terminators"` where this crate says plain `"ASCII text"`), so substring-based differential tests against the real `file` binary still hold even though byte-for-byte parity is not attempted. + +`MagicDatabase::concatenate_messages` also skips any match whose rendered (post `format_magic_message`) text is empty, rather than only special-casing the leading-position and `\b`-prefixed cases -- otherwise a message-less match sandwiched between two message-bearing ones would introduce a stray double space in the final description. + +**`apply_bitwise_and` semantics fix (found while validating this fallback, fixed in the same change).** `src/evaluator/operators/bitwise.rs::apply_bitwise_and` (the `Operator::BitwiseAnd`, i.e. bare `&VALUE` relational test, as opposed to the `type&MASK` attached-mask form which uses `BitwiseAndMask`/`apply_bitwise_and_mask`) previously implemented `(value & operand) != 0` ("any masked bit set"). Empirical A/B testing against the real `file` binary on `/usr/share/file/magic/ibm6000`'s `4 belong &0x0feeddb0` entry (a bare mask with no explicit comparison -- magic(5) shorthand for `belong &0x0feeddb0 =0x0feeddb0`) proved the correct libmagic semantics is `(value & operand) == operand` ("all masked bits set", matching GNU `file`'s `magiccheck()` in `src/softmagic.c`): an all-zero masked value and a partial-bit-overlap (nonzero, but not equal to the full mask) value both correctly produce "data" from real `file`, but the old implementation spuriously matched the partial-overlap case -- and because the ibm6000 mask has ~17 bits set, this false-positive fired for the overwhelming majority of random 32-bit values at that offset (nearly any binary blob evaluated against the full system magic DB used to trigger a bogus "AIX core file" classification). `apply_bitwise_and` now implements `(left & right) == right` for all four `Value` integer-type combinations. Consequences worth knowing: + +- **Not commutative in general.** `apply_bitwise_and(a, b)` asks "does `a` have every bit of `b` set" -- swapping operands generally changes the answer (`apply_bitwise_and(0xFF, 0x0F)` is true, `apply_bitwise_and(0x0F, 0xFF)` is false). It IS trivially true for self-AND (`a & a == a` always). The old "any bit set" semantics genuinely was commutative, so `test_apply_bitwise_and_symmetry` was renamed to `test_apply_bitwise_and_is_not_commutative_in_general` and rewritten to pin the asymmetry rather than assert it away. +- **A zero mask is vacuously true.** `(v & 0) == 0` always holds, regardless of `v` -- there are no required bits, so none can fail to be set. This matches libmagic's literal equality test; nobody writes real magic-file rules with a `&0` mask, but the semantics is still worth knowing. +- **Single-bit masks are unaffected.** For a mask with exactly one bit set, "any bit set" and "all bits set" coincide (the value is either 0 or the mask). The overwhelming majority of real magic-file bare-`&` usage is single-bit flag checks (`>6 leshort &0x0001 \b, encrypted`, `>3 byte &0x01 \b, ASCII text`, etc. -- see `docs/MAGIC_FORMAT.md`), so this fix does not change their behavior. Only bare `&` with a genuinely multi-bit mask (rare; the ibm6000 entry above is the corpus example that surfaced this) is affected, and only in the direction of becoming *more* correct (stricter -- it can only remove false-positive matches, never add new ones, since "all bits set" implies "some bit set" whenever the mask is nonzero). + +### 13.4 `stop_at_first_match` Is Top-Level-Only, and Continuation Rules Are NOT Strength-Sorted (they interlock) + +Two libmagic-parity rules that must be maintained together (both landed in the fix-system-magic-regex-graceful work; together they make multi-fragment descriptions like gzip's `gzip compressed data, ..., from Unix, original size modulo 2^32 N` render correctly and in order): + +- **`stop_at_first_match` short-circuits only the TOP-LEVEL classification loop.** `evaluate_rules` computes `stop_at_first_match_applies = !is_child_sibling_list` at entry (`is_child_sibling_list = recursion_depth > 0 && !is_indirect_reentry`) and gates every one of the five `break` sites on it. Inside a child/continuation sibling list or a `use` subroutine body the break NEVER fires -- **all** matching siblings render, matching libmagic where continuation levels always evaluate every entry and only the outer `match()` loop stops at first success. An indirect re-entry is treated as a fresh top-level classification (it consumes `take_indirect_reentry()` so `is_child_sibling_list` is false), so its own sibling list DOES stop at first match, but the children of a rule matched inside it do not. An earlier revision applied the break at every recursion level, silently truncating a description after its first message-bearing child (dropping gzip's `max compression`, `from Unix`, `original size ...`). The `EvaluationConfig::stop_at_first_match` doc already promised top-level-only; the code now matches. Regression: `test_stop_at_first_match_does_not_truncate_child_siblings`. +- **`sort_rules_by_strength` is non-recursive; only top-level entries are sorted.** `MagicDatabase` load (`src/lib.rs`, both the builtin and from-file paths) calls `sort_rules_by_strength` (NOT `sort_rules_by_strength_recursive`) and does **not** sort `name`-block subroutine bodies at all. This mirrors libmagic's `apprentice_sort`, which orders whole magic entries by their first line's strength and never reorders continuation lines. Child rules and subroutine bodies stay in FILE order because their order is load-bearing: (a) `default`/`clear` fire based on whether an **earlier-in-file** sibling matched -- strength-sorting a comparison-bearing sibling ahead of a low-strength `default` wrongly suppresses the `default`; and (b) multi-fragment descriptions must render in source order. The two fixes interlock: once stop-at-first-match no longer truncates children, child evaluation order only affects fragment order + `default`/`clear`, and file order is exactly what makes both correct. The old `NameTable::sort_subroutines` machinery was removed (subroutine bodies must not be sorted). Regressions: `test_sort_rules_by_strength_preserves_child_file_order`, and the end-to-end `test_gzip_multipart_description_end_to_end`. -`EvaluationConfig::default()` sets `stop_at_first_match: true`, so once any top-level rule produces a match the evaluator halts further sibling iteration. Integration tests that rely on later siblings running (e.g., `default`/`clear`/`default` chains, `use`-followed-by-sibling continuation, anything exercising the per-level `sibling_matched` flag across the full chain) must use `MagicDatabase::load_from_file_with_config` with `EvaluationConfig::default().with_stop_at_first_match(false)`. Unit tests that go through `evaluate_rules` directly hit the same issue -- override the config at construction time. The existing `Use` tests (`test_use_child_rules_evaluated_after_subroutine`, `test_use_stop_at_first_match_short_circuits_siblings`) document both sides of this contract. +### 13.5 The `offset` Pseudo-Type Compares the Resolved Position; `-0` Is EOF (`FromEnd(0)`) + +The magic(5) `offset` pseudo-type (`TypeKind::Meta(MetaType::Offset)`) treats the **resolved offset itself** as the read value and applies the rule's comparison operator against it -- it does NOT read bytes at that position. `offset x` is a bare AnyValue placeholder that always matches (used to report the position via `%lld`); `offset >48` / `offset <48` / `offset =N` compare the resolved position against N. gzip's `>>-0 offset >48` gates the trailing "original size modulo 2^32" child on the file being long enough. The engine's `Offset` dispatch (`src/evaluator/engine/mod.rs`) skips the rule as a non-match when the comparison fails (so the false branch and its children do not render). An earlier revision only supported `x` and routed everything else through the meta no-operand parse path, which turned `>48` into literal message text and made the compare a no-op. + +**`-0` parses to `OffsetSpec::FromEnd(0)`, not `Absolute(0)`.** magic(5) `-0` means "0 bytes from end of file" = the EOF **position** (`buffer.len()`), one past the last readable byte -- but `-0 == 0` in a signed integer, so the sign is otherwise lost. `parse_offset` (`src/parser/grammar/mod.rs`) detects the leading `-` explicitly and, for a zero magnitude, emits `FromEnd(0)`; other negatives (`-4`) keep their `Absolute(-4)` encoding (which the evaluator already resolves from the end). The `FromEnd` resolver arm special-cases `FromEnd(0)` to return `buffer.len()` (valid as a position for the read-less `offset` type; `resolve_absolute_offset(0)` would wrongly send it through the positive path and yield start-of-file). Negative `FromEnd` deltas keep the shared from-end resolution. Regressions: `test_resolve_from_end_zero_is_eof_position`, `test_offset_from_end_zero_compares_against_file_size`, and the `parse_offset("-0") == FromEnd(0)` assertion in `test_parse_offset_edge_cases`. + +### 13.6 A Message-Bearing `clear` Emits Its Message (Not Just a Flag Reset) + +The magic(5) `clear` directive (`TypeKind::Meta(MetaType::Clear)`) does two things in libmagic's `FILE_CLEAR`, and rmagic must do both: + +1. **Reset the per-level "sibling matched" flag** (`ms->c.li[cont_level].got_match = 0`) so a later `default` sibling at the same level can fire even if an earlier sibling matched. The flag is unconditionally reset and **never re-set to `true`** by the clear itself (clear does not participate in the "a sibling matched" chain -- unlike a normal matching rule or a fired `default`, both of which set the flag). +2. **Count as a match and print its description when non-empty.** `clear`'s conventional `x` test always succeeds, and libmagic's `mprint` renders the description whenever it is non-empty. So `>>&0 clear x program text` (c-lang) emits the "program text" fragment. An earlier revision only did step 1 (reset-and-`continue`), so rmagic printed `c` where GNU `file` prints `c program text` for a plain C source file. + +The engine's `Clear` dispatch (`src/evaluator/engine/mod.rs::evaluate_rules`) guards emission on a **non-empty message**: the ten bare `clear x` flag-reset directives in the system DB (`apple`, `coff`, `elf`, `pmem`, `pgf`, `yara`, `sniffer`, `pgp-binary-keys`, plus two in `images` -- none of which carry a message or children) stay pure flag-resets with no emitted fragment and no anchor advance, exactly as before. `c-lang`'s `>>&0 clear x program text` is the **only** message-bearing `clear` directive in the entire system DB, so the observable blast radius of this change is C-family source files gaining their "program text" suffix (verified: a monotonic before/after differential -- `c` -> `c program text` for C/`.h`/no-`class` `.cc` files, every clear-bearing binary rule ELF/TIFF/apple unchanged). `clear` is 0-width, so the previous-match anchor is **not** advanced in either case. Children of a message-bearing clear are evaluated for libmagic fidelity (`evaluate_children_or_warn` is a no-op when there are none -- and none exist today). The coexistence (message emitted AND trailing `default` still fires) was verified against real `file` (file-5.41): a `MATCH-A / default / clear x MSG / default` child chain prints `MATCH-A DEF-ONE MSG DEF-TWO`. Regressions: `test_clear_with_message_emits_and_still_resets_flag` and `test_message_less_clear_emits_nothing` in `tests/meta_types_integration.rs`; the pre-existing `test_default_clear_synthetic_scenario` (message-less `0 clear`) is unchanged and still pins the flag-reset path. Note `images:603`/`images:614` (`string ... clear text ...`) and `images:1100` (`\b, clear first`) are `string`/`byte` TYPE rules where "clear" is literal message text, NOT `clear` directives -- do not confuse them when grepping. ## 14. Output Formatting ### 14.1 `\b` (Backspace) Prefix in Rule Messages Suppresses Leading Space -`MagicDatabase::build_result` concatenates rule messages with a space separator, **except** when a message starts with `\u{0008}` (backspace / `\b`), in which case the backspace is stripped and no leading space is inserted. This mirrors GNU `file`'s description formatting (used by rules like `>&1 regex/1l ... \b, version %s` to produce `Ansible Vault text, version 1.1` instead of `Ansible Vault text , version 1.1`). Tests that manually simulate the concatenation path (e.g., corpus tests that bypass `load_from_file` -- see S3.11) must honor this convention or their assertions will diverge from the real evaluator output. +`MagicDatabase::build_result` (via `concatenate_messages`) concatenates rule messages with a space separator, **except** when a message starts with the backspace no-separator marker, in which case the marker is stripped and no leading space is inserted. This mirrors GNU `file`'s description formatting (used by rules like `>&1 regex/1l ... \b, version %s` to produce `Ansible Vault text, version 1.1` instead of `Ansible Vault text , version 1.1`). + +**The marker has two forms, and both are stripped.** The parser (`parse_message`) preserves description text verbatim -- matching GNU `file`, which keeps the desc literal and special-cases a leading `\b` at print time -- so a magic-file message like `\b, for MS Windows` (msdos) or `\b]` (Mach-O universal binary) reaches concatenation as the **literal two-character sequence `\b` (backslash + `'b'`)**, NOT a `\u{0008}` byte. `concatenate_messages` strips a leading `\u{0008}` byte (for programmatically-constructed messages / older tests) *or* a leading literal `"\\b"`. An earlier revision only stripped the `\u{0008}` byte, so real system-DB rules rendered the literal marker into the output (`PE STUB \b, for MS Windows`, `...architectures: \b]`). A leading `\b` in a *description* is always the marker (LaTeX-style `\begin` etc. appear only in string *values*, never descriptions). Tests that manually simulate the concatenation path (e.g., corpus tests that bypass `load_from_file` -- see S3.11) must honor this convention or their assertions will diverge from the real evaluator output. ### 14.2 Printf-Style Format Specifiers Are Substituted by `format_magic_message` @@ -356,3 +462,79 @@ Hex specifiers mask the value to the natural width of the rule's `TypeKind` -- a Substitution runs BEFORE the backspace check in S14.1, so a rule emitting `\b, version %s` correctly composes with the preceding match without an intervening space after the value is substituted. Tests that manually simulate the concatenation path must run their message strings through `format_magic_message` or construct `RuleMatch::message` strings that contain no `%` metacharacters. A literal `%` in user-facing data should be escaped as `%%`. + +### 14.3 String Ordering Ops Render the Full Field; Comparison Stays Prefix-Limited (`%.Ns` Precision) + +For a `string` rule compared with an **ordering** operator (`<`/`>`/`<=`/`>=`), libmagic **compares** only the first `pattern.len()` bytes (`src/softmagic.c::magiccheck` -> `file_strncmp(m->value.s, p->s, m->vallen, ...)`) but **renders** the full string field (`p->s`, read until NUL/EOF). These two are decoupled and BOTH matter -- the sgml XML rule `>15 string/t >\0 %.3s document text` is the canonical case: comparing `>\0` reads 1 byte, but `%.3s` must render the full field `1.0` (from `1.0" encoding=...`) to produce `XML 1.0 document text`. + +- **The decoupling lives in `evaluate_value_rule` (`src/evaluator/engine/mod.rs`), NOT in the read dispatcher.** `matched` is computed first from the untouched exact-length read (`read_typed_value_with_pattern` -> `read_string_exact(pattern.len())`), so the match decision is byte-identical to before. Then `string_ordering_display_value` re-reads the full field via `read_string(buffer, offset, Some(max_string_length))` for the RETURNED display value only, when (and only when) the type is `TypeKind::String { .. }` AND the operator is one of `< > <= >=`. Every other type/operator returns the compared value unchanged. +- **Do NOT "simplify" this by reading the full field for the comparison too.** Verified against the real `file` binary (file-5.41): buffer `0.6.10` does NOT match `>0.6.1` because the compared prefix `0.6.1` equals the pattern (`file_strncmp` limited to `vallen=5`), even though the full string `0.6.10` sorts after `0.6.1`. A full-field comparison would spuriously match. Regression guard: `test_string_ordering_comparison_stays_prefix_limited`. +- **The full-field display read does not move the relative-offset anchor.** `bytes_consumed_with_pattern` re-derives the advance from the PATTERN (`pattern.len()`, +1 for a trailing NUL), never from the display value, so a `>>&N` child of a `string >\0` rule resolves to the same offset. Regression guard: `test_string_ordering_full_field_display_does_not_move_relative_anchor`. +- **On a display-side read error after a successful match, the compared value is returned** rather than propagating -- a matched rule must not abort on a display-only read. +- **Scope:** `TypeKind::String` only. `PString` (`read_pstring`) and `String16` (`read_string16`) already read their full field independent of `pattern.len()`. Equal/NotEqual display is left as the compared prefix (libmagic renders full-field there too, but that is unreported, rarer, and expands regression surface -- deferred). + +**`%.Ns` precision** (`src/output/format.rs`): `Spec` now carries `precision: Option`, parsed from the `.` run (capped at `MAX_FORMAT_WIDTH`; a bare `.` is precision 0). It is honored **only for `%s`** (`Conv::Str`) via `render_str_spec`, which truncates the rendered string to at most `precision` **characters** (char-wise, not byte-wise: C's `%.Ns` is byte-wise, but our value is a Rust `String` and a byte cut could split a multi-byte UTF-8 sequence; identical for the ASCII version/name fields that use precision). Width padding is applied AFTER truncation, so `%4.4s`/`%-.4s` now render correctly (`%s` previously dropped width entirely). Numeric conversions still ignore precision. Regression guard: `test_string_precision_truncation`. + +This is display-detail only -- the match DECISION is unchanged. The remaining `, ASCII text` suffix that real `file` appends (`XML 1.0 document text, ASCII text`) is combined magic + `file_ascmagic` text classification, a separate feature (see S13.3). + +### 14.4 `use ` Emits the Subroutine's `name`-Line Message (No Separator); `use`'s Own Message Is Dropped + +A magic(5) `name` line can carry its own description, and GNU `file` emits that description when the subroutine is invoked via `use` -- ahead of the body's matches and attached with **no separating space**. rmagic previously dropped it at **two** layers, so subroutine name-line fragments silently vanished: Mach-O universal `0 name mach-o \b [` (the leading `[` of `[x86_64:...]`), `0 name matlab4 Matlab v4 mat-file`, `0 name algol_68 Algol 68 source text`, `0 name msdos-driver DOS executable (`, `0 name xbase-prf dBase Printer Form`, etc. (107 of 338 `name` directives in the system DB carry a message.) + +The fix spans three layers; all three are load-bearing: + +1. **Parse (`parser/grammar/mod.rs::parse_name_or_use_meta`).** The trailing text after the identifier is handled **differently for `name` vs `use`**, matching GNU `file`: `name` PRESERVES it (positioned so the caller's `parse_message` captures it as `rule.message`), while `use` still DROPS it up to end-of-line (a use-site has no message slot -- verified: `0 use foo bar` renders no `bar`). The previous code dropped it for both, so `rule.message` was empty for every `name` line and the later layers had nothing to emit. +2. **Hoist (`parser/name_table.rs::extract_name_table`).** The `NameTable` value is a `Subroutine { name_message: String, rules: Arc<[MagicRule]> }` -- the name-line message is captured alongside the body. `NameTable::get(name)` still returns `Arc<[MagicRule]>` (unchanged contract); `NameTable::name_message(name)` returns the description (`None` for a bare `name ` or unknown name). +3. **Emit (`evaluator/engine/mod.rs::evaluate_use_rule`).** After the subroutine body is evaluated, a synthetic `RuleMatch` carrying the name message is **prepended** to the returned matches (document order: name line first). It uses the use-site offset and a dummy value, and does **NOT** advance the previous-match anchor (`terminal_anchor` still comes solely from the body's evaluation) -- the name line reads no bytes. + +**Spacing model (verified against file-5.41).** The name message attaches with no leading separator: `0 name sub SUBMSG` renders `ParentSUBMSG`, not `Parent SUBMSG`. This is reproduced by ensuring the emitted message begins with the `\b` no-separator marker that `concatenate_messages` already strips (S14.1): if the name message does not already start with a marker (`\u{0008}` or literal `\b`), `evaluate_use_rule` prepends a literal `\b`. A message that ALREADY starts with one (Mach-O's `\b [`) is left as-is so it is not double-marked (`\b [` -> strip -> ` [` -> `Parent [`). The `use` rule's own message is never emitted (rmagic already treated a `use` site as producing no surface `RuleMatch`), matching `file` where `0 use foo USEMSG` drops `USEMSG`. + +**Stop-at-first-match interaction.** The synthetic name-message match is message-bearing, so `has_message_bearing_match` (S13.2) now counts it -- a `use` site whose subroutine's only output was the name line can now satisfy `stop_at_first_match`. This matches `file` (which would also emit and stop there). A before/after differential over the system DB confirmed the change is monotonic: the only observable diffs were 278 Mach-O universal binaries improving from malformed `x86_64] arm64e]` to balanced `[ x86_64] [ arm64e]`, with zero non-mach-o divergences. + +**Scope / remaining Mach-O gap.** This fix balances the Mach-O universal brackets but does NOT close the 275-file mach-o diff to full parity. Two mach-o-specific gaps remain, tracked as a separate cluster (issue #378): (a) the extra space in `[ x86_64` (the `>0 use mach-o-cpu \b` line's own `\b` marker is not applied as a no-separator control on the subroutine's first output -- distinct from the name-message `\b`), and (b) the missing inner `:Mach-O 64-bit executable x86_64`, which comes from `>(8.L) indirect x \b:` -- an indirect pointer read whose base offset is not applying the subroutine's `base_offset` (it reads arch[0].cputype at file offset 8 instead of arch[0].offset at 16), plus bare `(N)` indirect-offset parsing, the `indirect/r` flag, and indirect-offset flipping inside a `\^`-flipped subroutine (the value-read half of `\^` endian-flip #236 is now implemented -- see S16 -- but the indirect-offset-pointer-read half that Mach-O universal needs is not, per S16.3). Regression tests: `test_use_emits_name_line_message_with_no_separator` and `test_use_nested_name_message_balances_brackets` (`tests/meta_types_integration.rs`), `test_extract_captures_name_line_message` (`parser/name_table.rs`), and the `name`/`use` message split in `test_parse_magic_rule_meta_name_use_reject_malformed_identifiers` (`parser/grammar/tests/meta_types.rs`). + +## 15. Offset Resolution at EOF + +### 15.1 `offset == buffer.len()` Is a Valid Resolution Target (Width Enforced at Read Time) + +`resolve_absolute_offset` (`src/evaluator/offset/absolute.rs`) permits `offset == buffer_len` -- the EOF position -- and rejects only `offset > buffer_len`. The positive branch uses `>` (not `>=`). This mirrors libmagic's structure: **offset resolution is permissive; each type read enforces its own width.** Do not "tighten" this back to `>=` -- it silently drops legitimate EOF-anchored children. + +**Verified against real `file` (file-5.41)** with a custom magic file (`0 string XY PARENT` + `>2 byte x byte=%d` + `>2 string x [%s,`): + +- On a 2-byte buffer (child offset 2 == EOF): `file` prints `PARENT [,` -- the **numeric** child (`byte x`) is **dropped** (its width-checked read has no bytes at EOF -> non-match), while the **`string x`** child renders an **empty** `%s`. +- On a 3-byte buffer (child offset 2 < EOF): both children render. + +The motivating real-world case is LUKS: `luks:10` is `>8 string x [%s,`, and on a header truncated to exactly 8 bytes GNU `file` prints `LUKS encrypted file, ver 1 [,`. Before this fix, offset resolution rejected `offset == 8` and the trailing detail was silently dropped (rmagic printed only `LUKS encrypted file, ver 1`). The empty-field cascade continues at `>40`/`>72` for headers truncated exactly at those offsets (`[, ,`, `[, , ]`), matching `file` byte-for-byte. + +**Why it is safe (blast radius):** the only two production callers are in `offset/mod.rs` (`Absolute(N)` and negative-`FromEnd`), and only the positive branch changed (`FromEnd` uses the negative branch, already `>`). At `offset == len`: + +- **Fixed-width readers** (`byte`/`short`/`long`/`quad`/`float`/`double`/`date`/`qdate`) use bounds-safe `.get()` / `read_bytes_at` -> `BufferOverrun` -> non-match. Outcome is **unchanged** from before (the child was dropped either way). +- **`read_string`** (`src/evaluator/types/string.rs`) also relaxed its entry guard to `offset > buffer.len()`; at EOF `&buffer[len..]` is a valid empty slice -> `Value::String("")`. This is the ONLY reader whose *outcome* changes (empty render instead of drop). +- **`read_string16`** and **`read_pstring`** keep their `>=`/prefix-length guards, so they return `BufferOverrun` (non-match) at EOF -- they need at least one code unit / their length prefix, matching libmagic. Not relaxed (no observed rule needs it; YAGNI). +- **Anchor-advance** (`string_bytes_consumed`, GOTCHAS S3.8) uses `buffer.get(offset..)` and already returns `0` at `offset == len`, so the anchor lands exactly at `len` -- consistent with the 0-byte read. No divergent guard. + +**Empty-buffer edge:** `resolve_absolute_offset(0, b"")` now returns `Ok(0)` (0 == 0). A top-level `0 string x` would therefore match an empty file with an empty string -- but **no system-DB rule uses a top-level `string`-family `x`** (verified by grep), and GNU `file` still classifies an empty file as `empty`. rmagic's ascmagic "empty" fallback (S13.3) is unaffected. A numeric top-level rule on an empty buffer still fails its width-checked read (non-match), so empty-file classification is unchanged. + +Corpus differential (before/after, 394 small/truncated real files vs GNU `file`): exactly 2 rows changed (both truncated LUKS fixtures), both moving from a dropped detail to exact `file` parity. Zero regressions. The change is monotonic -- it can only *add* an empty string-child render at EOF, never remove or alter existing output. Regressions: `test_resolve_absolute_offset_at_eof_is_permitted` and the updated resolver tests (`absolute.rs`); `test_read_string_offset_at_buffer_end_is_empty_string` (`string.rs`); end-to-end `child_at_eof_numeric_dropped_string_renders_empty` and `luks_truncated_header_renders_empty_cipher_field` (`tests/parser_integration_tests.rs`). + +## 16. `use \^name` Endian-Flip (issue #236) + +### 16.1 `\^` Toggles a Threaded Flip State; It Does Not Live on a Struct Field + +The magic(5) `\^` prefix on a `use` target (`>(4.L) use \^tiff_ifd`, images:314) flips the little/big endianness of every endian-bearing read inside the invoked subroutine. rmagic implements this exactly as libmagic's `softmagic.c` does: + +- **`cvt_flip` equivalent** is `evaluator::types::flip_type_endian` -- it swaps `Endianness::Little` \<-> `Endianness::Big` for `Short`/`Long`/`Quad`/`Float`/`Double`/`Date`/`QDate` and returns everything else unchanged. `Endianness::Native` is left alone (libmagic's `cvt_flip` has no `FILE_SHORT`/`FILE_LONG` case), and `String16` is deliberately NOT flipped (also absent from `cvt_flip`). `signed`/`utc` attributes are preserved. +- **The flip is evaluation state, not AST shape.** libmagic threads an `int flip` parameter through `mget`/`match`; rmagic stores it on `EvaluationContext::flip_endian` and the engine's `SubroutineScope` guard saves/sets/restores it around the subroutine call (alongside `last_match_end` and `base_offset`). A `use \^name` site **toggles** (`saved_flip ^ use_flip`), it does not set -- so a `\^use` nested inside an already-flipped subroutine un-flips. The toggle **propagates into nested plain `use` calls** automatically because the nested `evaluate_use_rule` reads the now-current context flip as its own base. This is load-bearing for TIFF: `\^tiff_ifd` reads `direntries` big-endian AND its nested `>2 use tiff_entry` inherits the flip so the width/height/compression tag walk also reads big-endian. Regression: `nested_flip_use_toggles_back_to_unflipped` and the two TIFF-shape tests in `tests/endian_flip_use_tests.rs`. +- **Applied at read time only.** `evaluate_value_rule` computes a flipped `Cow` for the `read_typed_value_with_pattern` call when `flip_endian` is set; `bit_width()`, `coerce_value_to_type` (the literal's numeric value is endian-invariant), the relative-offset `bytes_consumed` advance, and the string-ordering display read all keep `rule.typ` (endian-invariant for their purposes). The common unflipped path is a cheap `Cow::Borrowed` no-op. + +### 16.2 `MetaType::Use` Is a Struct Variant Carrying `flip_endian` (S2.11 impact) + +`MetaType::Use(String)` became `MetaType::Use { name: String, flip_endian: bool }` (issue #236). This is the standard `MetaType` exhaustive-match change (S2.11): the variant definition + serde/debug/bit_width test fixtures in `ast.rs`, the parse tag in `grammar/mod.rs::parse_name_or_use_meta` (which now records `\^` instead of `warn!`-and-discarding it), `codegen.rs::serialize_type_kind`, `strength.rs`, `property_tests.rs::arb_type_kind`, and every `MetaType::Use(name)` match site (engine dispatch, name_table lookup, test helpers) were updated. Plain `use name` parses to `flip_endian: false`. + +### 16.3 Scope Boundary: Indirect Offsets Inside a Flipped Subroutine Are NOT Flipped + +libmagic also applies `cvt_flip` to indirect-offset pointer-type reads inside a flipped subroutine. rmagic's implementation flips only the **value read**, not the indirect-offset pointer read. This is sufficient for the TIFF oracle (tiff_ifd/tiff_entry use only absolute offsets `>0`/`>2`/`>>8`; the `(4.L)`/`(4.l)` IFD pointer is resolved at the use SITE, before the flip toggles, so it keeps its declared `.L`/`.l` endianness correctly). A subroutine that uses an indirect offset like `(N.s)` *inside* a `\^`-flipped body would not have that pointer read flipped -- no system-DB rule the differential surfaced needs it, and the Mach-O universal per-arch inner classification (which does use indirect offsets inside flipped bodies) stays tracked under #378/#236. Do not assume indirect-offset flip works until it is implemented and tested. + +### 16.4 Blast Radius: TIFF + Mach-O dylib + +A 798-file corpus differential (before commit vs after, vs GNU `file -b` on the system magic DB) showed **40 improvements, 0 regressions** (monotonic): big-endian TIFF (`direntries=%d` + the width/height/compression tag walk went from byte-swapped garbage to exact `file` parity) and single-arch Mach-O dylibs (`filetype=100663296`/`architecture=117440513` -- the byte-swapped MH_DYLIB=6 / x86_64 cputype -- became `dynamically linked shared library x86_64`). The Mach-O dylib fix is a verified bonus, not the target: it uses the same `\^` flip on `use \^macho-*` subroutines. Full Mach-O universal parity is NOT closed by this (see S14.4 / #378). diff --git a/dist-workspace.toml b/dist-workspace.toml index 3fee7174..6e61373c 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -4,7 +4,7 @@ members = [ "cargo:." ] # Config for 'dist' [dist] # The preferred dist version to use in CI (Cargo.toml SemVer syntax) -cargo-dist-version = "0.31.0" +cargo-dist-version = "0.32.0" # CI backends to support ci = "github" # The installers to generate for each app @@ -52,5 +52,5 @@ publish-jobs = [ "homebrew" ] [dist.github-action-commits] "actions/checkout" = "v6.0.2" "actions/download-artifact" = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" -"actions/attest-build-provenance" = "a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32" +"actions/attest-build-provenance" = "0f67c3f4856b2e3261c31976d6725780e5e4c373" "actions/upload-artifact" = "v7.0.1" diff --git a/docs/MAGIC_FORMAT.md b/docs/MAGIC_FORMAT.md index a7aabcd1..7a459a5c 100644 --- a/docs/MAGIC_FORMAT.md +++ b/docs/MAGIC_FORMAT.md @@ -340,16 +340,33 @@ offset regex[/count[unit]][flags] pattern message - `regex/100` - scan up to 100 bytes - `regex/10l` - scan up to 10 lines -- Bare `regex` or `regex/0` are parse errors (range is mandatory per GNU `file` magic(5)) +- Bare `regex` is accepted (unbounded scan, capped at the 8192-byte `FILE_REGEX_MAX`); only `regex/0` is a parse error (the byte count, when given, must be non-zero) Every scan window is capped at 8192 bytes (`FILE_REGEX_MAX`). Multi-line matching is always enabled (matching libmagic's unconditional `REG_NEWLINE`). Anchor advance follows GNU `file` semantics (match-end, not window-end). +**Bareword (unquoted) pattern escape handling:** + +Bareword regex patterns undergo escape resolution using GNU `file`'s getstr escape table before the pattern is compiled. Quoted patterns (`"..."`) preserve escapes verbatim without getstr resolution. + +Supported bareword escape sequences: + +- `\t`, `\n`, `\r`, `\b`, `\f`, `\v` - standard C control sequences (tab, newline, carriage return, backspace, form feed, vertical tab) +- `\NNN` - octal escape sequences (1-3 digits, matching getstr's look-ahead; e.g. `\040` resolves to space, `\0` to NUL) +- `\xNN` - hexadecimal escape sequences (up to 2 hex digits; e.g. `\x20` resolves to space; `\x` with no following hex digit resolves to the literal character `x`, matching getstr) +- `\\` - escaped backslash (resolves to a single backslash) +- Unrecognized escape sequences drop the backslash (e.g. `\^` becomes `^`) + +Bytes >= 0x80 produced by escape resolution are re-encoded as `\xHH` format for the regex engine. Regex shorthand sequences like `\d`, `\s`, `\w` that are not recognized by getstr are demoted to literals (e.g. `\d` becomes `d`). + +This behavior matches GNU `file`'s handling of bareword regex patterns and ensures compatibility with system magic databases like `/usr/share/file/magic/`. + Examples: ```text 0 regex/100 [A-Z]+ Found uppercase letters 0 regex/10l/c error Found "error" (case-insensitive, 10-line cap) 0 regex/500/s ^BEGIN Found BEGIN at start (anchor advances to match-start) +0 regex/50 \^[\040\t]{0,50}\\.asciiz assembler source text (getstr-resolved pattern) ``` ### Search Type @@ -362,7 +379,7 @@ Bounded literal pattern scan. Searches for a literal byte pattern within a speci offset search/range[/flags] pattern message ``` -The range is MANDATORY (`NonZeroUsize`). Bare `search` and `search/0` are parse errors per GNU `file` magic(5). Anchor advance follows GNU `file` semantics (match-end, not window-end) unless `/s` is set. +The range is optional (`Option`): a bare `search` (`None`) scans from the offset to end-of-buffer, matching the reference `file` binary's `str_range == 0` behavior; `search/N` caps the scan to `N` bytes. Only `search/0` is a parse error (the count, when given, must be non-zero). magic(5) documents the count as mandatory, but rmagic follows the reference implementation, which accepts the bare form. Anchor advance follows GNU `file` semantics (match-end, not window-end) unless `/s` is set. **Search Flags:** diff --git a/docs/solutions/integration-issues/loading-real-system-magic-db-getstr-rawbyte-graceful-skip-2026-07-17.md b/docs/solutions/integration-issues/loading-real-system-magic-db-getstr-rawbyte-graceful-skip-2026-07-17.md new file mode 100644 index 00000000..ad87bf9b --- /dev/null +++ b/docs/solutions/integration-issues/loading-real-system-magic-db-getstr-rawbyte-graceful-skip-2026-07-17.md @@ -0,0 +1,87 @@ +--- +title: Loading the real system magic DB without fatal aborts -- getstr fidelity, raw-byte regex, and narrow graceful-skip +date: 2026-07-17 +category: integration-issues +module: parser-grammar/getstr + evaluator-types/regex + evaluator-engine +problem_type: integration_issue +component: parser_evaluator_boundary +severity: high +applies_when: + - loading a real-world third-party magic file / directory (e.g. /usr/share/file/magic) into the strict parser-evaluator + - a bareword (unquoted) pattern value begins with a magic(5) escape sequence + - compiling a user-supplied pattern for binary-safe byte matching via regex::bytes::Regex + - deciding whether an evaluator capability gap should abort a whole file or skip one rule + - a message-less gating rule can win a stop_at_first_match race against a more specific rule + - implementing magic(5) bare `&MASK` relational tests +related_components: + - parser-grammar + - evaluator-types + - evaluator-engine + - evaluator-operators + - output-formatting +tags: + - libmagic-compatibility + - getstr-escape-resolution + - regex-bytes-unicode-false + - graceful-degradation + - stop-at-first-match + - bitwise-and-semantics + - verify-against-canonical-source +--- + +# Loading the real system magic DB without fatal aborts + +## Context + +Loading the macOS/Linux system magic database (`/usr/share/file/magic/`) into libmagic-rs fatally aborted with `TypeReadError::UnsupportedType { "regex without string pattern" }`. A single unparseable/unsupported rule aborts the **entire** file load (`parse_text_magic_file` is fail-fast, GOTCHAS S3.11) or the entire evaluation, so one escape-heavy `regex` rule took the whole DB down. The root cause was not one bug but a chain of four distinct correctness gaps between "a strict, internally-consistent parser+evaluator" and "the messy, real-world magic files GNU `file` actually ships." Each gap is individually reusable. + +The driving example: the assembler magic file's `0 regex \^[\040\t]{0,50}\.asciiz assembler source text` -- an unquoted regex whose pattern begins with the magic(5) escape `\^`. + +## Guidance + +### 1. Bareword pattern values that begin with an escape are captured as `Value::Bytes`, not `Value::String` + +`parse_value`'s `alt(...)` tries the hex/mixed-ASCII branch (`parse_mixed_hex_ascii`) **before** any string interpretation. A bareword regex pattern like `\^[\040\t]{0,50}\.asciiz` starts with `\`, so it was captured as `Value::Bytes` -- and the evaluator's `Regex` arm only accepted `Value::String`, so it fatally aborted. Two independent fixes were needed: + +- **Parser (primary):** special-case `TypeKind::Regex` in `parse_magic_rule` *ahead of* the generic `is_string_family_type` bareword fallback, routing unquoted regex patterns through a dedicated getstr resolver (see #2) that produces a `Value::String`. Quoted regex values are unaffected (they already go through `parse_quoted_string`). If the resolver's own parse fails, fall back to `parse_value` so previously-working forms are untouched. (GOTCHAS S2.12) +- **Evaluator (floor-tier backstop):** make the `Regex` arms symmetric with `Search` -- accept **both** `Value::String` and `Value::Bytes` at all three call sites (`read_typed_value_with_pattern`, `read_pattern_match`, `bytes_consumed_with_pattern`), decoding `Bytes` via `String::from_utf8_lossy` with a `warn!` **only** when a real lossy substitution occurred (`str::from_utf8` actually fails). The warn is the visibility guard for a genuine desync: a lossy substitution puts U+FFFD in the compiled regex while the target buffer still holds raw bytes. (GOTCHAS S2.4) + +The lesson: when a parser has an ordered `alt(...)` where an earlier branch can greedily capture a token the later branch was meant to own, the type-specific dispatch must run **before** the generic `alt`, not rely on it. + +### 2. Replicate GNU `file`'s `getstr` escape table exactly -- verified against upstream, not inferred + +The regex getstr resolver (`src/parser/grammar/getstr/mod.rs`) replicates `apprentice.c::getstr` **verified against the upstream C source**, including its quirks: + +- Named control escapes `\a \b \f \n \r \t \v` -> their C control byte. +- `\0`-`\7` -> 1-3 digit **octal**, truncated to a byte. +- `\x` -> 0-2 digit **hex**; `\x` with *no* following hex digit is the literal byte `'x'` (0x78) -- getstr's quirky `default`, replicated for fidelity, not "fixed." +- **Every other escaped char drops the backslash and keeps the char literally** (`\^` -> `^`, a regex anchor). This is the *opposite* of `parse_bare_string_value` (non-regex string-family barewords), which *keeps* the backslash on an unknown escape. +- getstr does **not** special-case `\d`/`\s`/`\w` as PCRE classes; they fall into the "everything else" bucket and demote to bare `d`/`s`/`w`. **This is genuine upstream libmagic behavior, not a resolver bug** -- a reviewer will flag it; the answer is "yes, real `file` does this too." + +The reusable meta-lesson: for compatibility work, escape-resolution quirks must be copied from the canonical implementation and each surprising one annotated with "verified upstream," or the next reviewer (or you, six months later) will "fix" a deliberate quirk back into a bug. + +### 3. `regex::bytes::Regex` matches `\xHH` against the UTF-8 *encoding* unless you set `unicode(false)` + +A resolved byte `>= 0x80` cannot be pushed onto a Rust `String` as a raw `char` (not a valid scalar value), so the getstr resolver re-encodes it as regex-native `\xHH` text (GOTCHAS S2.12, KTD3). **But** `regex::bytes::RegexBuilder` defaults to `unicode(true)`, under which the pattern `\xff` matches the *two-byte UTF-8 encoding* of U+00FF (`0xC3 0xBF`), **not** the raw byte `0xff`. For binary-safe byte matching against a file buffer you MUST call `.unicode(false)` on the builder (`build_regex` in `src/evaluator/types/regex.rs`). This is a silent correctness bug: the regex compiles fine and matches *some* inputs, so tests that only use ASCII patterns never catch it. Regression coverage must include a `>= 0x80` byte pattern matched against the raw byte. + +### 4. Keep the graceful-skip predicate *narrow* -- an allowlist, never a variant-wildcard + +To stop one bad rule from aborting the whole DB, the engine skips (as a non-match) exactly two classes at three catch sites: `is_missing_pattern_operand(type_name)` (an **exhaustive string allowlist** of `"regex without string pattern"`, `"search without string/bytes pattern"`, `"string with flags requires string/bytes pattern"`) and `is_regex_compile_failure(type_name)` (`starts_with("regex compile error:")`, which includes the CWE-1333 compile-size DoS guard). Everything else -- an unwired `TypeKind`, a non-equality operator on a pattern-bearing type, a `Meta` variant read as a value -- **still propagates fatally**. Do not widen the predicate to match on the error *variant* alone; that silently swallows genuine capability gaps this contract exists to surface. Log `debug!` for the ordinary missing-pattern case, `warn!` for a compile failure (so a pathological/malicious file's rejection stays visible). (GOTCHAS S2.1) + +### 5. A message-less gating rule must not shadow a specific rule under `stop_at_first_match` + +Once #1-#4 let the DB load and evaluate, a *second* class of bug surfaced: real magic files use message-less top-level rules purely as gating conditions for children (e.g. c-lang's `0 search/8192 "#include"`). Under `stop_at_first_match: true` (the CLI default), if such a gating rule matched a buffer on its own -- before a later, more specific rule was tried in strength order -- evaluation halted and the CLI printed a **blank description**. Fix: gate every `should_stop_at_first_match()` break site on `has_message_bearing_match` (the match, or a descendant, must carry real description text). The `default`/`clear` `sibling_matched` flag is **unaffected** -- a message-less match still counts as "a sibling matched"; only the early-exit is gated. Companion: a `classify_fallback` (`empty`/`ASCII text`/`UTF-8 Unicode text`/`data`, a strict subset of GNU `file`'s `file_ascmagic`) so a readable file never yields blank output. (GOTCHAS S13.2, S13.3) + +### 6. magic(5) bare `&MASK` is "all bits set," not "any bit set" -- found by differential testing + +While building the differential-parity test, a random binary blob spuriously matched an `AIX core file` rule (`4 belong &0x0feeddb0`, a ~17-bit bare mask). `apply_bitwise_and` implemented `(v & l) != 0` ("any masked bit set"); real libmagic's `magiccheck()` is `(v & l) == l` ("all masked bits set"). Verified by A/B testing against the real `file` binary. The fix is strictly stricter (removes false positives, never adds matches); single-bit flag rules -- the overwhelming majority of real bare-`&` usage -- are provably unaffected since the two interpretations coincide for a one-bit mask. **This is why differential parity tests must run against un-doctored inputs:** a fixture engineered to dodge a latent bug proves nothing. (GOTCHAS S13.3) + +## Discovery mechanism + +Gaps #1-#4 were the planned fix; #5 surfaced only when the DB actually evaluated end-to-end against real files (macOS host caught it, an integration test pinned it); #6 surfaced only when a differential-parity test was run against the *real* `file` binary with an un-doctored blob. The meta-lesson: **strict-vs-real-world integration bugs come in chains** -- fixing the load-abort merely exposes the next layer (blank output), which exposes the next (false-positive matches). Budget for the chain, and use differential testing against the canonical binary as the floor. + +## Cross-references + +- GOTCHAS S2.1 (narrow graceful-skip allowlist), S2.4 (Regex/Search Bytes symmetry), S2.12 (getstr resolver + `>= 0x80` re-encoding), S3.11 (fail-fast loader), S13.2 (message-bearing stop-at-first-match), S13.3 (text/data fallback + bitwise-AND) +- `src/parser/grammar/getstr/mod.rs`, `src/evaluator/types/regex.rs::build_regex`, `src/evaluator/engine/mod.rs` (graceful-skip + `has_message_bearing_match`), `src/output/ascmagic.rs`, `src/evaluator/operators/bitwise.rs` +- `tests/system_magic_dir.rs` (differential parity against GNU `file`), `tests/regex_getstr_fixtures.rs` (getstr + `unicode(false)` fixtures) diff --git a/docs/src/evaluator.md b/docs/src/evaluator.md index a4cf4ef1..50279bcc 100644 --- a/docs/src/evaluator.md +++ b/docs/src/evaluator.md @@ -125,8 +125,8 @@ The types module is organized into submodules for numeric, floating-point, date/ - **QDate**: 64-bit Unix timestamps (signed seconds since epoch) with configurable endianness and UTC/local time formatting - **String**: Byte sequences with length limits and flag-based comparison semantics (case-insensitive, whitespace handling, etc.) when flags are present - **PString**: Pascal-style length-prefixed strings with 1-byte (`/B`), 2-byte (`/H` or `/h`), or 4-byte (`/L` or `/l`) length prefixes, supporting big-endian and little-endian byte order -- **Regex**: Binary-safe regex matching via `regex::bytes::Regex`. `/c` enables case-insensitive matching, `/s` advances the relative-offset anchor to match-start instead of match-end, and `/l` measures the scan window in lines instead of bytes (with an 8192-byte hard cap). Multi-line regex matching is always on via unconditional `REG_NEWLINE`, so `^`/`$` match at line boundaries regardless of `/l`. -- **Search**: Bounded literal pattern scan with flag support. `search/N` caps the scan window to `N` bytes from the offset; range is mandatory and non-zero (`NonZeroUsize`). Accepts nine flag suffixes (`/s`, `/c`, `/C`, `/w`, `/W`, `/T`, `/f`, `/t`, `/b`) that control scan behavior and anchor advancement. When only anchor-only flags (`/s`, `/t`, `/b`) are set or no flags are present, the SIMD-accelerated `memchr::memmem::find` fast path is used. When comparison-altering flags (`/c`, `/C`, `/w`, `/W`, `/T`, `/f`) are set, a byte-by-byte comparison through `compare_string_with_flags` is used. The `/s` flag sets the previous-match anchor for relative-offset children to match-START instead of match-END. +- **Regex**: Binary-safe regex matching via `regex::bytes::Regex` with `unicode(false)`, so hex escapes like `\xHH` match the literal byte value for all `HH` (including bytes `>= 0x80`). `/c` enables case-insensitive matching, `/s` advances the relative-offset anchor to match-start instead of match-end, and `/l` measures the scan window in lines instead of bytes (with an 8192-byte hard cap). Multi-line regex matching is always on via unconditional `REG_NEWLINE`, so `^`/`$` match at line boundaries regardless of `/l`. Bareword (unquoted) regex patterns are resolved using GNU `file`'s `getstr` escape table before compilation, so octal escapes like `\040` and control characters like `\t` produce the bytes GNU `file` would. The evaluator accepts both `Value::String` and `Value::Bytes` as pattern operands (bytes are lossily decoded to UTF-8 with a `warn!` on real substitution). When a regex rule lacks a valid pattern operand, the evaluator gracefully skips that rule rather than aborting the entire evaluation. +- **Search**: Bounded literal pattern scan with flag support. The range is optional (`Option`): `search/N` caps the scan window to `N` bytes from the offset, while a bare `search` (`None`) scans to end-of-buffer (matching the reference `file` binary's `str_range == 0`); `search/0` is a parse error. Accepts nine flag suffixes (`/s`, `/c`, `/C`, `/w`, `/W`, `/T`, `/f`, `/t`, `/b`) that control scan behavior and anchor advancement. When only anchor-only flags (`/s`, `/t`, `/b`) are set or no flags are present, the SIMD-accelerated `memchr::memmem::find` fast path is used. When comparison-altering flags (`/c`, `/C`, `/w`, `/W`, `/T`, `/f`) are set, a byte-by-byte comparison through `compare_string_with_flags` is used. The `/s` flag sets the previous-match anchor for relative-offset children to match-START instead of match-END. - **Bounds checking**: Prevents buffer overruns The type-reading functions are internal (`pub(crate)`) engine helpers. External library users evaluate rules through `evaluate_rules` or `evaluate_rules_with_config`. @@ -426,6 +426,8 @@ The evaluator uses graceful degradation: - **Invalid offsets**: Skip rule, continue with others - **Type mismatches**: Skip rule, continue with others +- **Missing pattern operand**: When a pattern-bearing type (`Regex`, `Search`, flagged `String`) lacks a valid `String`/`Bytes` pattern operand, skip that rule rather than aborting the entire evaluation +- **Regex compile failure**: When a regex pattern fails to compile (including the 1 MiB `REGEX_COMPILE_SIZE_LIMIT` denial-of-service guard), skip that rule and log a `warn!` rather than aborting the entire evaluation - **Timeout exceeded**: Return error (partial results are not preserved) - **Recursion limit**: Stop descent, continue siblings diff --git a/docs/src/security-assurance.md b/docs/src/security-assurance.md index 4ad0b4f3..722e09b2 100644 --- a/docs/src/security-assurance.md +++ b/docs/src/security-assurance.md @@ -77,16 +77,16 @@ All data crossing the trust boundary (file contents, magic file syntax, CLI argu ## 4. Secure Design Principles (Saltzer and Schroeder) -| Principle | How Applied | -| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Economy of mechanism** | Pure Rust with minimal dependencies. Simple parser-evaluator pipeline. No plugin system, no scripting, no network I/O. | +| Principle | How Applied | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Economy of mechanism** | Pure Rust with minimal dependencies. Simple parser-evaluator pipeline. No plugin system, no scripting, no network I/O. | | **Fail-safe defaults** | Workspace lint `unsafe_code = "deny"` enforced project-wide via `Cargo.toml` (one vetted, SAFETY-commented memmap2 exception in `src/io/mod.rs`). Buffer access defaults to bounds-checked `.get()` returning `None` rather than panicking; `clippy::indexing_slicing` enforces this, with invariant-documented allows at provably-safe sites. Invalid magic rules are skipped, not executed. | -| **Complete mediation** | Every buffer access is bounds-checked. Every magic file is validated during parsing. Every CLI argument is validated by `clap`. | -| **Open design** | Fully open source (Apache-2.0). Security does not depend on obscurity. All security mechanisms are publicly documented. | -| **Separation of privilege** | Parser and evaluator are separate modules with distinct responsibilities. Parse errors cannot bypass evaluation safety checks. | -| **Least privilege** | The tool only reads files; it never writes, executes, or modifies them. No network access. No elevated permissions required. | -| **Least common mechanism** | No shared mutable state between file evaluations. Each evaluation operates on its own data. No global caches that could leak information. | -| **Psychological acceptability** | CLI follows GNU `file` conventions. Error messages are descriptive and actionable. Default behavior is safe (built-in rules, no network). | +| **Complete mediation** | Every buffer access is bounds-checked. Every magic file is validated during parsing. Every CLI argument is validated by `clap`. | +| **Open design** | Fully open source (Apache-2.0). Security does not depend on obscurity. All security mechanisms are publicly documented. | +| **Separation of privilege** | Parser and evaluator are separate modules with distinct responsibilities. Parse errors cannot bypass evaluation safety checks. | +| **Least privilege** | The tool only reads files; it never writes, executes, or modifies them. No network access. No elevated permissions required. | +| **Least common mechanism** | No shared mutable state between file evaluations. Each evaluation operates on its own data. No global caches that could leak information. | +| **Psychological acceptability** | CLI follows GNU `file` conventions. Error messages are descriptive and actionable. Default behavior is safe (built-in rules, no network). | ## 5. Common Weakness Countermeasures @@ -94,7 +94,7 @@ All data crossing the trust boundary (file contents, magic file syntax, CLI argu | CWE | Weakness | Countermeasure | Status | | ------- | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| CWE-787 | Out-of-bounds write | Rust ownership prevents writes to unowned memory. Workspace-level lints in `Cargo.toml` deny unsafe code (single vetted read-only memmap2 exception) and eliminate raw pointer writes. | Mitigated | +| CWE-787 | Out-of-bounds write | Rust ownership prevents writes to unowned memory. Workspace-level lints in `Cargo.toml` deny unsafe code (single vetted read-only memmap2 exception) and eliminate raw pointer writes. | Mitigated | | CWE-79 | XSS | Not applicable (no web output). | N/A | | CWE-89 | SQL injection | Not applicable (no database). | N/A | | CWE-416 | Use after free | Rust ownership/borrowing system prevents use-after-free at compile time. | Mitigated | diff --git a/mise.lock b/mise.lock index cb6ffaf3..df7606f4 100644 --- a/mise.lock +++ b/mise.lock @@ -118,54 +118,6 @@ url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.14/bun-windows- checksum = "sha256:538f9c846355d9e847b2671bc00c47da4229a0befb24df3282b739770f3b475f" url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.14/bun-windows-x64-baseline.zip" -[[tools.bun]] -version = "1.3.12" -backend = "core:bun" - -[tools.bun."platforms.linux-arm64"] -checksum = "sha256:c40bc0ebca11bde7d75af497a654a874d0c7fd8d6a8d6031c173c10c9064297b" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-linux-aarch64.zip" - -[tools.bun."platforms.linux-arm64-musl"] -checksum = "sha256:731baab945bc471c17248ea375e66f71442879d2595c54045b3e861f4e8b9ab1" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-linux-aarch64-musl.zip" - -[tools.bun."platforms.linux-x64"] -checksum = "sha256:11dc3ee11bc1695e149737c6ca3d5619302cf4346e6b8a6ec7988967ef01ddc5" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-linux-x64.zip" - -[tools.bun."platforms.linux-x64-baseline"] -checksum = "sha256:f8bb377a9ae93d44697ff91a2611164d2aedc9263415d623b0c3af24a6f55dab" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-linux-x64-baseline.zip" - -[tools.bun."platforms.linux-x64-musl"] -checksum = "sha256:5a9f9a2102d4bd0d5210b4f6bd345151d2310623947085177c1b306e8587dce6" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-linux-x64-musl.zip" - -[tools.bun."platforms.linux-x64-musl-baseline"] -checksum = "sha256:a95e079aef96f1387b86e27b69f9a6babbd08154d9a59483f29d9de285b8e3ad" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-linux-x64-musl-baseline.zip" - -[tools.bun."platforms.macos-arm64"] -checksum = "sha256:6c4bb87dd013ed1a8d6a16e357a3d094959fd5530b4d7061f7f3680c3c7cea1c" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-darwin-aarch64.zip" - -[tools.bun."platforms.macos-x64"] -checksum = "sha256:0f58c53a3e7947f1e626d2f8d285f97c14b7cadcca9c09ebafc0ae9d35b58c3d" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-darwin-x64.zip" - -[tools.bun."platforms.macos-x64-baseline"] -checksum = "sha256:cc4e22130c2bc2d944d3a286de08f2ed37fa74136e59760f3a4661e610246474" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-darwin-x64-baseline.zip" - -[tools.bun."platforms.windows-x64"] -checksum = "sha256:841ff9c5dffcaa3a2620d1e3f87ee500f32a4ca830b001cade7a3479609d4a89" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-windows-x64.zip" - -[tools.bun."platforms.windows-x64-baseline"] -checksum = "sha256:2d3d5f88a95943563f56f3643c8f4e2422261018f6d915329bb2fb33f7256ba2" -url = "https://github.com/oven-sh/bun/releases/download/bun-v1.3.12/bun-windows-x64-baseline.zip" - [[tools.cargo-binstall]] version = "1.20.1" backend = "aqua:cargo-bins/cargo-binstall" @@ -444,121 +396,62 @@ version = "3.14.6" backend = "core:python" [tools.python."platforms.linux-arm64"] -checksum = "sha256:f177d40ca931df03f660fc006f86ad8cd2ac6e7d6b5d54edbc625103464fc4aa" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.linux-arm64-musl"] -checksum = "sha256:11d463be3e2d34ea67722acfd8f3f2d6d6e5b6b4d4ab27240f220628134a0141" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-aarch64-unknown-linux-musl-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.linux-x64"] -checksum = "sha256:c172314f4a8ec137a8f605289010c3d19c8b56867d968f0095074cc68efa1d29" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.linux-x64-baseline"] -checksum = "sha256:c172314f4a8ec137a8f605289010c3d19c8b56867d968f0095074cc68efa1d29" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.linux-x64-musl"] -checksum = "sha256:f25064ecb3b07cfe2440b178e72001cf1e0d69a5e53625ca3a32b7ae4e2fdcc6" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-unknown-linux-musl-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.linux-x64-musl-baseline"] -checksum = "sha256:f25064ecb3b07cfe2440b178e72001cf1e0d69a5e53625ca3a32b7ae4e2fdcc6" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-unknown-linux-musl-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.macos-arm64"] -checksum = "sha256:35d774f61d63c1fd4f1bc9495a7ada92e500dc4382a0df8a9910eb87ea48e8cf" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-aarch64-apple-darwin-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.macos-x64"] -checksum = "sha256:12fb3ea3d6a855fe6ca1c2241702b06cb7e5939fbef09a5f54bf326e480f66af" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-apple-darwin-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.macos-x64-baseline"] -checksum = "sha256:12fb3ea3d6a855fe6ca1c2241702b06cb7e5939fbef09a5f54bf326e480f66af" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-apple-darwin-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.windows-x64"] -checksum = "sha256:ec05b628ad749682d06d225780fbc02e7bbb5ce2146c9bd8e74a3659b14b693a" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[tools.python."platforms.windows-x64-baseline"] -checksum = "sha256:ec05b628ad749682d06d225780fbc02e7bbb5ce2146c9bd8e74a3659b14b693a" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.14.6+20260623-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" -provenance = "github-attestations" - -[[tools.python]] -version = "3.14.4" -backend = "core:python" - -[tools.python."platforms.linux-arm64"] -checksum = "sha256:b8b597fdb2f8dccdc502c11947b60a4b65eb6bce79cfa60c7ccf9b6e8352c60a" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz" +checksum = "sha256:c2a2caa03aa9beb77a5bbb5a009c8fe77b96a56925da2dba18095237e30faf04" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.linux-arm64-musl"] -checksum = "sha256:a10687b226e0941632569836bc1d8fa6353a8e3e8424316467ca9cdf220b983d" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-aarch64-unknown-linux-musl-install_only_stripped.tar.gz" +checksum = "sha256:ff7da86be2a3d7fdf0124da7b408ad2de3c7ca8d6c19e6c449ec2cddedd01a8b" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-aarch64-unknown-linux-musl-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.linux-x64"] -checksum = "sha256:fe9a9c32d13870af632cbac3dfc7528ae53597e94472aa4c7d6a42e8166136cd" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" +checksum = "sha256:86bf107f65fc30b56f2b263b26797fcbb1661f5315910cdbf27f733eb8738b74" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.linux-x64-baseline"] -checksum = "sha256:fe9a9c32d13870af632cbac3dfc7528ae53597e94472aa4c7d6a42e8166136cd" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" +checksum = "sha256:86bf107f65fc30b56f2b263b26797fcbb1661f5315910cdbf27f733eb8738b74" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.linux-x64-musl"] -checksum = "sha256:d6005226cd24e780630626232c7a63243d4885fdf975dcf930a0758a0759ce14" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-unknown-linux-musl-install_only_stripped.tar.gz" +checksum = "sha256:2c27fc15bde497fbd78a8d5d38559ee1a8c93110952d5c2b390bc6481e2be415" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-unknown-linux-musl-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.linux-x64-musl-baseline"] -checksum = "sha256:d6005226cd24e780630626232c7a63243d4885fdf975dcf930a0758a0759ce14" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-unknown-linux-musl-install_only_stripped.tar.gz" +checksum = "sha256:2c27fc15bde497fbd78a8d5d38559ee1a8c93110952d5c2b390bc6481e2be415" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-unknown-linux-musl-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.macos-arm64"] -checksum = "sha256:6f304f4ec30854611f23316578302235fb517cd970519ecdd11a8c4db87fd843" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-aarch64-apple-darwin-install_only_stripped.tar.gz" +checksum = "sha256:5e8e0746cb0108d138f510cfbb28c4b031b9ed63bbbe3e43a34c77d4663c4fa1" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-aarch64-apple-darwin-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.macos-x64"] -checksum = "sha256:d51250a32fa5d9f0799c7bcb71720c27b10a3afd4a7de288120f96085d508a5a" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-apple-darwin-install_only_stripped.tar.gz" +checksum = "sha256:2d03154bc83c6d2d048bba23bb808b020e352028f5bd541a740e54467fc9cc75" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-apple-darwin-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.macos-x64-baseline"] -checksum = "sha256:d51250a32fa5d9f0799c7bcb71720c27b10a3afd4a7de288120f96085d508a5a" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-apple-darwin-install_only_stripped.tar.gz" +checksum = "sha256:2d03154bc83c6d2d048bba23bb808b020e352028f5bd541a740e54467fc9cc75" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-apple-darwin-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.windows-x64"] -checksum = "sha256:a976991dcd085c1bb5d9a8084823a6bc8b7f9b079d8c432574a6ddd68c3a6fe1" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" +checksum = "sha256:0c5c9f231704fb49149bc8f2b9d9dbe43eeaf6c54f362ef0fc2066ce64b4d10f" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" provenance = "github-attestations" [tools.python."platforms.windows-x64-baseline"] -checksum = "sha256:a976991dcd085c1bb5d9a8084823a6bc8b7f9b079d8c432574a6ddd68c3a6fe1" -url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" +checksum = "sha256:0c5c9f231704fb49149bc8f2b9d9dbe43eeaf6c54f362ef0fc2066ce64b4d10f" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260718/cpython-3.14.6+20260718-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" provenance = "github-attestations" [[tools.rust]] -version = "1.97.0" +version = "stable" backend = "core:rust" [tools.rust.options] @@ -566,14 +459,6 @@ components = "cargo,clippy,llvm-tools,rustfmt" profile = "default" targets = "aarch64-apple-darwin,aarch64-pc-windows-msvc,aarch64-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-msvc,x86_64-unknown-linux-gnu,x86_64-unknown-linux-musl" -[[tools.rust]] -version = "stable" -backend = "core:rust" - -[tools.rust.options] -components = "clippy,llvm-tools-preview,rustfmt" -targets = "aarch64-apple-darwin,x86_64-apple-darwin,x86_64-pc-windows-msvc,x86_64-unknown-linux-gnu" - [[tools.scorecard]] version = "5.5.0" backend = "aqua:ossf/scorecard" diff --git a/mise.toml b/mise.toml index 666307e8..3bdcf21a 100644 --- a/mise.toml +++ b/mise.toml @@ -1,14 +1,21 @@ -# Rust toolchain is declared "latest" here but the concrete version -# lives in `mise.lock` (see `[[tools.rust]] version = ...`). -# `mise.lock` is committed to the repo so every developer and CI run -# resolves to the same Rust version regardless of when they install. -# When bumping rustc: run `mise upgrade rust`, then commit the -# updated `mise.lock`. Review finding D-M1 clarified that mise.lock -# is the reproducibility source of truth, not mise.toml or -# rust-toolchain.toml. +# Rust toolchain tracks `stable`, and this MUST stay consistent with +# `rust-toolchain.toml` (also `stable`). mise honors rust-toolchain.toml as an +# idiomatic version file (`idiomatic_version_file_enable_tools = ["rust"]`), so +# the two files declaring DIFFERENT channels previously made mise resolve two +# separate rust toolchains, and a redundant `rustup component add` step re-added +# rustfmt/clippy a second time -- the duplicate component installs collided on +# the hosted CI runner (`detected conflict` / `Directory not empty`), breaking +# every job at toolchain setup. Keeping a single consistent `stable` +# declaration -- one `[[tools.rust]]` entry in mise.lock, no redundant +# component-add step -- is what avoids that. `mise.lock` is committed so a +# given commit resolves a consistent toolchain across machines; note `stable` +# is a moving channel, so this is stable-tracking rather than a frozen version +# pin -- the resolved rustc advances whenever the lockfile is refreshed. When +# bumping rustc: run `mise upgrade rust` and commit `mise.lock`; update +# Cargo.toml `rust-version` if the MSRV moves. [tools] -rust = { version = "latest", components = "llvm-tools,cargo,rustfmt,clippy", profile = "default", targets = "aarch64-apple-darwin,aarch64-unknown-linux-gnu,aarch64-pc-windows-msvc,x86_64-apple-darwin,x86_64-unknown-linux-gnu,x86_64-unknown-linux-musl,x86_64-pc-windows-msvc" } +rust = { version = "stable", components = "llvm-tools,cargo,rustfmt,clippy", profile = "default", targets = "aarch64-apple-darwin,aarch64-unknown-linux-gnu,aarch64-pc-windows-msvc,x86_64-apple-darwin,x86_64-unknown-linux-gnu,x86_64-unknown-linux-musl,x86_64-pc-windows-msvc" } cargo-binstall = "1.20.1" cargo-insta = "1.48.0" "cargo:cargo-audit" = "0.22.2" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 551cdb98..c0c1a9af 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,16 +1,24 @@ # This file is honored by rustup users who do not go through mise. -# CI and the mise-managed developer path pin the toolchain via -# `mise.lock` (see `[[tools.rust]] version = "1.94.1"`), which -# overrides this file's channel. Review finding D-M1 asked for a -# reproducible pin; the answer is mise.lock -- not this file. Keeping -# `channel = "stable"` here means rustup users always track the -# latest stable compiler, while CI and mise users get the exact -# pinned version from the lockfile. +# CI and the mise-managed developer path resolve the toolchain via +# `mise.lock` (see `[[tools.rust]] version = "stable"`), which overrides +# this file's channel. Both this file and mise track the `stable` channel, +# so rustup, CI, and mise users all build on the latest stable compiler. +# `mise.lock` is committed so a given commit resolves a consistent toolchain +# across machines -- but note `stable` is a moving channel, so this is +# stable-tracking, not a frozen version pin (the resolved version advances +# when the lockfile is refreshed). # -# When bumping the pinned rustc version, update BOTH `mise.lock` -# (via `mise upgrade rust`) and `Cargo.toml` `rust-version` if the -# declared MSRV moves. +# When bumping the toolchain, update `mise.lock` (via `mise upgrade rust`) +# and `Cargo.toml` `rust-version` if the declared MSRV moves. [toolchain] +# Tracks `stable`, and MUST stay consistent with mise.toml's rust channel +# (also `stable`). mise honors this file as an idiomatic version file +# (`idiomatic_version_file_enable_tools = ["rust"]`), so if this channel and +# mise.toml disagree, mise resolves two separate rust toolchains and their +# component installs collide on hosted CI runners (`detected conflict` / +# `Directory not empty`). Keep both on `stable` with a single `[[tools.rust]]` +# entry in mise.lock. When bumping rustc: run `mise upgrade rust`, commit the +# refreshed mise.lock, and update Cargo.toml `rust-version` if the MSRV moves. channel = "stable" components = [ "rustfmt", "clippy", "llvm-tools-preview" ] targets = [ diff --git a/skills-lock.json b/skills-lock.json new file mode 100644 index 00000000..efabdc81 --- /dev/null +++ b/skills-lock.json @@ -0,0 +1,431 @@ +{ + "version": 1, + "skills": { + "address-sanitizer": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/address-sanitizer/SKILL.md", + "computedHash": "1f3f9f4b086781b421040a6a33aa44aff5c543e455d4e2e566e03aa9616dff4d" + }, + "agentic-actions-auditor": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/agentic-actions-auditor/skills/agentic-actions-auditor/SKILL.md", + "computedHash": "b1845ded071fb1248fe997e312c79c9af0a1f3360fb94132f884e467c881fac0" + }, + "ask-questions-if-underspecified": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/ask-questions-if-underspecified/skills/ask-questions-if-underspecified/SKILL.md", + "computedHash": "e78cf8dd5cb5861bd606ce450ed305051bd09998429f81c33eaab9c87db96272" + }, + "audit-context-building": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/audit-context-building/skills/audit-context-building/SKILL.md", + "computedHash": "47ae283f86574c2f21778b9b1b4979dbc81af4f9c018bd9585535b2744266e51" + }, + "cargo-fuzz": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/cargo-fuzz/SKILL.md", + "computedHash": "d3f0be4bd295400e7f8ba8a0fb695cde53bc7958357edf4e5876c798bbc598ba" + }, + "codeql": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/static-analysis/skills/codeql/SKILL.md", + "computedHash": "439ce7e08165b579fd4943aee0b9a7cb04c0538e1b421b76c8b42bd5b6779321" + }, + "coding-guidelines": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/coding-guidelines/SKILL.md", + "computedHash": "0742312a0de5eb6bd84e093c1815b1db98126130b92713ea5fc2cf902338aa3b" + }, + "core-actionbook": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/core-actionbook/SKILL.md", + "computedHash": "52f2dc1a62a5a9fe071265e14cd29dc2e6a01492fe7e434af21bcd833d84d541" + }, + "core-agent-browser": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/core-agent-browser/SKILL.md", + "computedHash": "e373a42fc0a9b586463bc26c0416b64b43a62bd9e064215fa82ee19c116b9d27" + }, + "core-dynamic-skills": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/core-dynamic-skills/SKILL.md", + "computedHash": "7663df4de3177c4ec2395afdb2c5b769cf35b05711040bec1b745c5f94f53fe5" + }, + "core-fix-skill-docs": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/core-fix-skill-docs/SKILL.md", + "computedHash": "3794d89554650ac94bc9eef7330bc0806a17b1c907fac9508a1caafbdb7f0d2c" + }, + "coverage-analysis": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/coverage-analysis/SKILL.md", + "computedHash": "672be3b126ee75a4847c60e97cdc40e9330da7ee1bbb550ad7efaf93129b7503" + }, + "designing-workflow-skills": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/workflow-skill-design/skills/designing-workflow-skills/SKILL.md", + "computedHash": "eaecc1acac4114545700f4f70b0c43a82b99ed7211a3e66cdb4eaf97e6799b84" + }, + "devcontainer-setup": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/devcontainer-setup/skills/devcontainer-setup/SKILL.md", + "computedHash": "e676adf231701ce233dde757273a2a0a726d2570eafbf625495ae3f0da536b2c" + }, + "diagramming-code": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/trailmark/skills/diagramming-code/SKILL.md", + "computedHash": "39c27a7101511f441182cce5f50964fb951360e67400b8820a9d86e37d4f1a00" + }, + "differential-review": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/differential-review/skills/differential-review/SKILL.md", + "computedHash": "d997520fbef18003be46d460cc002a3e41cfdd889e2d7bc4818e4e0e29667a45" + }, + "domain-cli": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-cli/SKILL.md", + "computedHash": "88a6efe282b5105161c8ddb320f09ca0ffed19323fe403dfa7adb4aaa708b8dc" + }, + "domain-cloud-native": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-cloud-native/SKILL.md", + "computedHash": "10a6e60e8d2a1b977fc5d5935976c17b498335ee2f19255438e45fd15a8811bc" + }, + "domain-embedded": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-embedded/SKILL.md", + "computedHash": "3d32e5d2d602f775f4a3d6bc0222d56526ec0dbc882771c80430dffdc2be6dd2" + }, + "domain-fintech": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-fintech/SKILL.md", + "computedHash": "c2fbfc41eafd81d828dead9f6988e63a544697d6961e673e563d08d47b7a17f5" + }, + "domain-iot": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-iot/SKILL.md", + "computedHash": "4a8aaf6e1bebbd1984faf04e6dc0cb3b309409628b090ce72b63c807e9b76e4e" + }, + "domain-ml": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-ml/SKILL.md", + "computedHash": "e53c7de1916555fe903c40270208a88e2350f8e5c2f81c5115059ccf4278cca1" + }, + "domain-web": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/domain-web/SKILL.md", + "computedHash": "dd853ce97875f139da437df6e09eaca8b326931a98f3eb8dc02669670c12161d" + }, + "dwarf-expert": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/dwarf-expert/skills/dwarf-expert/SKILL.md", + "computedHash": "cbbd372f31df208821c2b0240cd924b5a2e9ea43578221b773413c3465abb666" + }, + "fuzzing-dictionary": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/fuzzing-dictionary/SKILL.md", + "computedHash": "bc271ce9708ea69c70fd2372bf8df359b00d41996b289106f058f74e9ab245aa" + }, + "fuzzing-obstacles": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/fuzzing-obstacles/SKILL.md", + "computedHash": "c9d9339e6c7404c62ab4ee0cec3f634efdcaad082ec2d7bddd158d026911e158" + }, + "gh-cli": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/gh-cli/skills/gh-cli/SKILL.md", + "computedHash": "f6bf4536185c6d4db16f240d3f0aa8a4170657c94f5a12000d6cf0c55cf07610" + }, + "git-cleanup": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/git-cleanup/skills/git-cleanup/SKILL.md", + "computedHash": "4c73e965ecffd8b161108591139d90a27fb50222d9f47b6a05675d33bebff929" + }, + "harness-writing": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/harness-writing/SKILL.md", + "computedHash": "19f60e595afc5f211ab74b048e12816e3d0082ce4a85111330dad59c267ac74f" + }, + "insecure-defaults": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/insecure-defaults/skills/insecure-defaults/SKILL.md", + "computedHash": "eb4a6733b54d483ae8e7280fe4831e0788ce0a36f4c6059ba184d88be9653abd" + }, + "m01-ownership": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m01-ownership/SKILL.md", + "computedHash": "69f57ede558e32e703e468ef5328f5bcc6610adff9d99b49984a5b3507d31b5b" + }, + "m02-resource": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m02-resource/SKILL.md", + "computedHash": "b79e4a7702adb7c7622dfab7818dd81710352d448d285885946652e5dc73fd5b" + }, + "m03-mutability": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m03-mutability/SKILL.md", + "computedHash": "78eb8bc5e8b9bbeb5a59375b74b773c8fe824bb1644fa0fb77e99a1255a272ab" + }, + "m04-zero-cost": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m04-zero-cost/SKILL.md", + "computedHash": "8bc6733618b217b608fe5011f4c74b7ceb8c0765ba60702e30fa8fa4824584f2" + }, + "m05-type-driven": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m05-type-driven/SKILL.md", + "computedHash": "06a52b0f35951cd01e21b2906917061f3994ece6b8c6b055dca65bbe14820283" + }, + "m06-error-handling": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m06-error-handling/SKILL.md", + "computedHash": "e6639bd5913e63bd8315f7b36f5dd22a029cae2404135a75ac24c0627a93e4e5" + }, + "m07-concurrency": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m07-concurrency/SKILL.md", + "computedHash": "6ff3b31bae02e84238a75845008150122071ba449b23edb9ea8a50e3635fffb7" + }, + "m09-domain": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m09-domain/SKILL.md", + "computedHash": "ce0ed9b5b5c12775c6c04423c79c8a542638629237ad2447a3c6569accf1a15a" + }, + "m10-performance": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m10-performance/SKILL.md", + "computedHash": "c0e093f630ec71079aa7a138af376821e3cc24dfb129f7162e82d854e26a1087" + }, + "m11-ecosystem": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m11-ecosystem/SKILL.md", + "computedHash": "7ac5dd7206c339379d0aafe304a473a2e54465374be269ffc97d9e615325fc69" + }, + "m12-lifecycle": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m12-lifecycle/SKILL.md", + "computedHash": "e360c0db5b5bf8e47d418e0a4a6ce33de7725b40eb126e6266f9fedfbccff291" + }, + "m13-domain-error": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m13-domain-error/SKILL.md", + "computedHash": "fc90102a877b830f24735d649c5042ad74423e04ed64f1f1e44df776c7880932" + }, + "m14-mental-model": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m14-mental-model/SKILL.md", + "computedHash": "6c53f050376c04a4abb5edce769a7052d93f605cc06299e657d9fce5b01edbab" + }, + "m15-anti-pattern": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/m15-anti-pattern/SKILL.md", + "computedHash": "d9f5d80bd2ab49f063a61d90a96100298788ea0e1e5bd5e917be3c7e124078a2" + }, + "meta-cognition-parallel": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/meta-cognition-parallel/SKILL.md", + "computedHash": "6e3006df76c15c95d64b2b52a39735ec14c6eb398192b3129aebda6fa923fd2f" + }, + "ossfuzz": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/ossfuzz/SKILL.md", + "computedHash": "dbb962113f15480f211de40b45456db323bf069d4adfb9965dfbdd432ed1e132" + }, + "property-based-testing": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/property-based-testing/skills/property-based-testing/SKILL.md", + "computedHash": "30146f00ecab4e49019c40b5e0f18869addfbfa4c448f1382aa288ce8b7c0417" + }, + "rust-call-graph": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-call-graph/SKILL.md", + "computedHash": "0999fbe783c81b87ba1dc1347758bd87b5880dca0d66c69be53bbac133a4c888" + }, + "rust-code-navigator": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-code-navigator/SKILL.md", + "computedHash": "c5c5b9521be30a3b5a3d6fa122bf9df809af3b28760e06fa317d6afab5bb3939" + }, + "rust-daily": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-daily/SKILL.md", + "computedHash": "016d958078944cba2cbd3a7beff9c328abd22aebd9f3206c95fdaedac4d7103a" + }, + "rust-deps-visualizer": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-deps-visualizer/SKILL.md", + "computedHash": "b7cb6cbb693df52c43cae43149469ba44b05ad0747c5826c3d65f5e4b4ad3d74" + }, + "rust-learner": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-learner/SKILL.md", + "computedHash": "12e6434334fc6f131f09f35cbb53871d0d4c753fd404c1337e9c48ad5a9a1dbf" + }, + "rust-refactor-helper": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-refactor-helper/SKILL.md", + "computedHash": "614d54b2b16a57965412a3fff048c6d36e6f1535fcbb9ac2cf71e7a987e5a119" + }, + "rust-review": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/rust-review/skills/rust-review/SKILL.md", + "computedHash": "91afb92b6d8ca0ba99e7fca314fc9ce04a34953a6badae88090fa29aaa5a77a8" + }, + "rust-router": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-router/SKILL.md", + "computedHash": "787e5244b55d9d943abc49a81318bde7d1bbf88dd43bc60dc58ec74023ba3788" + }, + "rust-skill-creator": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-skill-creator/SKILL.md", + "computedHash": "0314bb7b61f6e7be24a498b7fb45969f78924e81c2f74723badb056724697119" + }, + "rust-symbol-analyzer": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-symbol-analyzer/SKILL.md", + "computedHash": "1e6b2fd668a9a9574b89cf90797bcd825636163d0f3b5fc9e678fd71ce4869b9" + }, + "rust-trait-explorer": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/rust-trait-explorer/SKILL.md", + "computedHash": "2feac2b9bb17689a7227fcae62c8a20351bb5afb271a47d49c27ba87ba9e7e26" + }, + "sarif-parsing": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/static-analysis/skills/sarif-parsing/SKILL.md", + "computedHash": "94412d423e7a13940adbd30802632687346e8d4cd49c7e4e38a44ccfe155649c" + }, + "seatbelt-sandboxer": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/seatbelt-sandboxer/skills/seatbelt-sandboxer/SKILL.md", + "computedHash": "e2ef7a63e928bd847906e554fc7e1b3772a049bee253f9e094227de92fa6eeac" + }, + "second-opinion": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/second-opinion/skills/second-opinion/SKILL.md", + "computedHash": "f5dedb068e5c6a71514b65127f3b4c00a579a68ceae00faf49f4748ec7cadf7c" + }, + "secure-workflow-guide": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/building-secure-contracts/skills/secure-workflow-guide/SKILL.md", + "computedHash": "9974c020a5916ec5067c8e5befb0ae867e41f55d0286386289dc0d690a7d2246" + }, + "semgrep": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/static-analysis/skills/semgrep/SKILL.md", + "computedHash": "89123d70706874709ea688ce3565942fdc64f95046fcd94b97bd5d7db5b9e2ef" + }, + "semgrep-rule-creator": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/semgrep-rule-creator/skills/semgrep-rule-creator/SKILL.md", + "computedHash": "8fd82365ec87cd07e82da8cebb3409eae8fc435b56fce2a161d07d158da6ade7" + }, + "semgrep-rule-variant-creator": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/semgrep-rule-variant-creator/skills/semgrep-rule-variant-creator/SKILL.md", + "computedHash": "5b4e342f27670cb703b014de5c7282b4b1528a89c65d22bbd8507fe0f8dea6d7" + }, + "skill-improver": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/skill-improver/skills/skill-improver/SKILL.md", + "computedHash": "3d252246fb92ba463a6c148e53d26540a1526d85314769a56019e3c6ca42f666" + }, + "spec-to-code-compliance": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/spec-to-code-compliance/skills/spec-to-code-compliance/SKILL.md", + "computedHash": "8cc0fc37847fe8398fd433590cfa23989a582b80510ab751cef09b918790d913" + }, + "supply-chain-risk-auditor": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/supply-chain-risk-auditor/skills/supply-chain-risk-auditor/SKILL.md", + "computedHash": "483d9ce83cd95ef1b343c879c608e87ab30ec8ede5873f8ea22707e6ebab033a" + }, + "testing-handbook-generator": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/testing-handbook-skills/skills/testing-handbook-generator/SKILL.md", + "computedHash": "fe36b3af4e3a8758489077072ed1eba92c5fe91d9f690a424ace0e352b658c7f" + }, + "unsafe-checker": { + "source": "actionbook/rust-skills", + "sourceType": "github", + "skillPath": "skills/unsafe-checker/SKILL.md", + "computedHash": "b76d79853846ea40ad42092c61de26da4561299b176768599751d5bcc13cf5db" + }, + "zeroize-audit": { + "source": "trailofbits/skills", + "sourceType": "github", + "skillPath": "plugins/zeroize-audit/skills/zeroize-audit/SKILL.md", + "computedHash": "21b386e86a308bc7573605f4fddb25837f376daa1d3f1913128147b637c18be4" + } + } +} diff --git a/src/build_helpers.rs b/src/build_helpers.rs index 6ee9b900..68725d89 100644 --- a/src/build_helpers.rs +++ b/src/build_helpers.rs @@ -650,18 +650,34 @@ mod tests { fn test_serialize_search_codegen_has_no_expect_panic_marker() { use std::num::NonZeroUsize; + // Bounded `search/512`: `range` is `Option`, so + // `NonZeroUsize::new(512)` already yields `Some(_)` -- no panicking + // conversion and no `.unwrap_or` fallback is needed (the old + // `NonZeroUsize`-typed field required `.unwrap_or(MIN)`; the Option + // field does not). let typ = TypeKind::Search { - range: NonZeroUsize::new(512).unwrap(), + range: NonZeroUsize::new(512), flags: SearchFlags::default(), }; let generated = serialize_type_kind(&typ); assert!( - !generated.contains(".expect("), - "serialize_type_kind must not emit .expect() (AGENTS.md panic-marker ban); got:\n{generated}" + !generated.contains(".expect(") && !generated.contains(".unwrap()"), + "serialize_type_kind must not emit a panic marker (AGENTS.md ban); got:\n{generated}" ); assert!( - generated.contains(".unwrap_or(::std::num::NonZeroUsize::MIN)"), - "serialize_type_kind must emit .unwrap_or(NonZeroUsize::MIN); got:\n{generated}" + generated.contains("range: ::std::num::NonZeroUsize::new(512)"), + "bounded search must serialize range as NonZeroUsize::new(N) (an Option); got:\n{generated}" + ); + + // Bare `search` (`range: None`) must serialize as a literal `None`. + let bare = TypeKind::Search { + range: None, + flags: SearchFlags::default(), + }; + let bare_gen = serialize_type_kind(&bare); + assert!( + bare_gen.contains("range: None"), + "bare search must serialize range as None (scan-to-EOF); got:\n{bare_gen}" ); } @@ -675,7 +691,7 @@ mod tests { use std::num::NonZeroUsize; let typ = TypeKind::Search { - range: NonZeroUsize::new(256).expect("256 is nonzero"), + range: NonZeroUsize::new(256), flags: SearchFlags::default(), }; let generated = serialize_type_kind(&typ); @@ -720,7 +736,7 @@ mod tests { use std::num::NonZeroUsize; let typ = TypeKind::Search { - range: NonZeroUsize::new(1024).expect("1024 is nonzero"), + range: NonZeroUsize::new(1024), flags: SearchFlags::default() .with_start_anchor(true) .with_ignore_lowercase(true), @@ -788,7 +804,7 @@ mod tests { for flags in cases { let typ = TypeKind::Search { - range: NonZeroUsize::new(64).expect("64 is nonzero"), + range: NonZeroUsize::new(64), flags, }; let generated = serialize_type_kind(&typ); @@ -800,11 +816,13 @@ mod tests { ); } - // The `unwrap_or` safe-fallback variant is allowed (it is not - // a panic marker) and is required by the NonZeroUsize idiom. + // `range` is now `Option`, so `NonZeroUsize::new(64)` + // already yields `Some(_)` -- no `.unwrap_or(MIN)` fallback is + // needed (or emitted) any more. The panic-marker checks above are + // the real safety property. assert!( - generated.contains(".unwrap_or(::std::num::NonZeroUsize::MIN)"), - "missing safe-fallback `unwrap_or`; got:\n{generated}" + generated.contains("range: ::std::num::NonZeroUsize::new(64)"), + "bounded search must serialize range as an Option-yielding NonZeroUsize::new(N); got:\n{generated}" ); } } diff --git a/src/evaluator/engine/mod.rs b/src/evaluator/engine/mod.rs index 6a46b4a5..3bc0b951 100644 --- a/src/evaluator/engine/mod.rs +++ b/src/evaluator/engine/mod.rs @@ -85,18 +85,27 @@ struct SubroutineScope<'a> { context: &'a mut EvaluationContext, saved_anchor: usize, saved_base: usize, + saved_flip: bool, } impl<'a> SubroutineScope<'a> { - fn enter(context: &'a mut EvaluationContext, use_site: usize) -> Self { + /// Enter a subroutine body. `flip_use` is the `\^` prefix on the + /// invoking `use` site; the effective flip inside the body is the + /// caller's flip state XOR `flip_use` -- matching libmagic's `flip = + /// !flip` toggle, which nests: a `\^use` inside an already-flipped + /// subroutine un-flips. Restored on drop along with anchor and base. + fn enter(context: &'a mut EvaluationContext, use_site: usize, flip_use: bool) -> Self { let saved_anchor = context.last_match_end(); let saved_base = context.base_offset(); + let saved_flip = context.flip_endian(); context.set_last_match_end(use_site); context.set_base_offset(use_site); + context.set_flip_endian(saved_flip ^ flip_use); Self { context, saved_anchor, saved_base, + saved_flip, } } @@ -109,6 +118,7 @@ impl Drop for SubroutineScope<'_> { fn drop(&mut self) { self.context.set_last_match_end(self.saved_anchor); self.context.set_base_offset(self.saved_base); + self.context.set_flip_endian(self.saved_flip); } } @@ -207,6 +217,7 @@ fn evaluate_single_rule_with_anchor( last_match_end: usize, base_offset: usize, max_string_length: usize, + flip_endian: bool, ) -> Result, LibmagicError> { use crate::parser::ast::TypeKind; @@ -246,7 +257,7 @@ fn evaluate_single_rule_with_anchor( ); return Ok(None); } - TypeKind::Meta(MetaType::Use(_)) => { + TypeKind::Meta(MetaType::Use { .. }) => { // `Use` is dispatched inline by `evaluate_rules` so it can // push the subroutine's matches into the caller's match // vector. Reaching this arm means the rule went through the @@ -260,18 +271,38 @@ fn evaluate_single_rule_with_anchor( } // Flagged `string` rules route through the pattern-bearing path // (see GOTCHAS S2.4 for the contract) so `compare_string_with_flags` - // can do the case-fold / whitespace-flexible match in one pass. - // Default-flag strings (the common case) take the existing - // value-rule fast path with byte-exact `apply_equal`. - TypeKind::String { flags, .. } if !flags.is_empty() => { + // can do the case-fold / whitespace-flexible match in one pass -- + // but ONLY for the equality operators the pattern path supports. + // An ORDERING operator on a flagged string (e.g. the ubiquitous + // `string/t >\0` / `string/b >\0` "non-empty text here, print it with + // %s" idiom in `varied.script`, `sgml`, `linux`, ...) is a + // lexicographic comparison, not a pattern match; routing it to the + // pattern path made it a fatal `UnsupportedType` abort that killed the + // whole file's evaluation. The `/t`/`/b` flags are MIME-output hints + // with no comparison effect, so such a rule behaves like an unflagged + // `string >VALUE` and belongs on the value path. Default-flag strings + // (the common case) also take that value-rule fast path. + TypeKind::String { flags, .. } + if !flags.is_empty() + && matches!( + rule.op, + crate::parser::ast::Operator::Equal | crate::parser::ast::Operator::NotEqual + ) => + { evaluate_pattern_rule(rule, buffer, absolute_offset, max_string_length)? } - _ => evaluate_value_rule(rule, buffer, absolute_offset, max_string_length)?, + _ => evaluate_value_rule( + rule, + buffer, + absolute_offset, + max_string_length, + flip_endian, + )?, }; Ok(matched.then_some((absolute_offset, read_value))) } -/// Evaluate a `TypeKind::Meta(MetaType::Use(name))` rule inline. +/// Evaluate a `TypeKind::Meta(MetaType::Use { name, .. })` rule inline. /// /// Looks up `name` in the context's rule environment, temporarily sets the /// GNU `file` previous-match anchor to the resolved offset, and recursively @@ -294,6 +325,7 @@ fn evaluate_single_rule_with_anchor( fn evaluate_use_rule( rule: &MagicRule, name: &str, + flip_endian: bool, buffer: &[u8], context: &mut EvaluationContext, ) -> Result<(Option, Vec), LibmagicError> { @@ -317,6 +349,13 @@ fn evaluate_use_rule( warn!("use directive references unknown name '{name}'"); return Ok((None, Vec::new())); }; + // The `name` line can carry its own description (e.g. Mach-O universal + // `0 name mach-o \b [`, `0 name matlab4 Matlab v4 mat-file`). GNU `file` + // emits it ahead of the subroutine body, attached with no separating + // space. Capture it here while the env borrow is live; the owned `String` + // lets us drop that borrow before mutating the context below. `None` for + // a bare `name `. + let name_message = env.name_table.name_message(name); // `NameTable::get` returns an `Arc<[MagicRule]>`, so this clone is a // reference-count increment rather than a deep copy of the rule tree. // The Arc is cloned here to release the immutable borrow of `context` @@ -347,14 +386,47 @@ fn evaluate_use_rule( // anchor INSIDE the scope (before Drop restores the caller's value) // preserves it for the caller. let (subroutine_matches, terminal_anchor) = { - let mut scope = SubroutineScope::enter(context, absolute_offset); + let mut scope = SubroutineScope::enter(context, absolute_offset, flip_endian); let mut guard = RecursionGuard::enter(scope.context())?; let matches = evaluate_rules(&subroutine_rules, buffer, guard.context())?; let terminal = guard.context().last_match_end(); (matches, terminal) }; - Ok((Some(terminal_anchor), subroutine_matches)) + // Prepend the `name` line's own description (if any) ahead of the body's + // matches, matching GNU `file`: `use mach-o` emits the mach-o subroutine's + // `\b [` before the per-arch body. The name line reads no bytes, so this + // synthetic match carries a dummy value and does NOT touch the anchor + // (`terminal_anchor` still comes from the body's evaluation). To reproduce + // `file`'s no-separator attachment (`ParentSUBMSG`, not `Parent SUBMSG`), + // ensure the message begins with the `\b` no-separator marker + // (`concatenate_messages` strips a leading literal `\b` / U+0008); a name + // message that already starts with one (mach-o's `\b [`) is left as-is so + // it is not double-marked. + let matches = match name_message { + Some(msg) if !msg.is_empty() => { + let attached = if crate::evaluator::strip_no_separator_marker(&msg).is_some() { + msg + } else { + format!("\\b{msg}") + }; + let name_match = RuleMatch::new( + attached, + absolute_offset, + rule.level, + crate::parser::ast::Value::Uint(0), + rule.typ.clone(), + RuleMatch::calculate_confidence(rule.level), + ); + let mut combined = Vec::with_capacity(subroutine_matches.len() + 1); + combined.push(name_match); + combined.extend(subroutine_matches); + combined + } + _ => subroutine_matches, + }; + + Ok((Some(terminal_anchor), matches)) } /// Evaluate a pattern-bearing rule (`TypeKind::Regex` / `TypeKind::Search`). @@ -419,11 +491,25 @@ fn evaluate_value_rule( buffer: &[u8], absolute_offset: usize, max_string_length: usize, + flip_endian: bool, ) -> Result<(bool, crate::parser::ast::Value), LibmagicError> { + // Apply the `use \^name` endian flip (issue #236) at read time, exactly + // as libmagic's `cvt_flip(m->type, flip)` does in `softmagic.c`. Only the + // typed READ needs the flipped endianness -- `bit_width()`, + // `coerce_value_to_type` (the literal's numeric value is endian-invariant), + // the relative-offset `bytes_consumed` advance, and the string-ordering + // display read are all endian-invariant, so they keep `rule.typ`. + // `flip_type_endian` is a cheap no-op clone for the common `flip == false` + // path. + let read_typ: std::borrow::Cow<'_, crate::parser::ast::TypeKind> = if flip_endian { + std::borrow::Cow::Owned(types::flip_type_endian(&rule.typ)) + } else { + std::borrow::Cow::Borrowed(&rule.typ) + }; let read_value = types::read_typed_value_with_pattern( buffer, absolute_offset, - &rule.typ, + read_typ.as_ref(), Some(&rule.value), max_string_length, ) @@ -450,9 +536,173 @@ fn evaluate_value_rule( expected_ref, rule.typ.bit_width(), ), + // Masked equality (`type&MASK VALUE`) must re-normalize the masked + // result to the type's natural width so a signed read whose high bits + // are cleared by the mask still compares equal to the sign-extended + // rule literal (e.g. the Mach-O `0 lelong&0xfffffffe 0xfeedface` + // rule). See `apply_bitwise_and_mask_with_width`. + crate::parser::ast::Operator::BitwiseAndMask(mask) => { + operators::apply_bitwise_and_mask_with_width( + *mask, + &transformed_value, + expected_ref, + rule.typ.bit_width(), + ) + } op => operators::apply_operator(op, &transformed_value, expected_ref), }; - Ok((matched, transformed_value)) + + // libmagic renders the FULL string field (`p->s`) for a matched string + // comparison, while the comparison itself is prefix-limited to + // `pattern.len()` (`file_strncmp` with `vallen`). The comparison above + // already read exactly `pattern.len()` bytes -- correct and unchanged -- + // but for an ORDERING operator the rendered detail needs the whole field, + // not the compared prefix. sgml's `>15 string/t >\0 %.3s document text` + // is the motivating case: comparing `>\0` reads 1 byte ("1"), but the + // `%.3s` must render the full field ("1.0") to produce `XML 1.0 ...`. + // Re-read the full field for DISPLAY only; the `matched` decision above is + // left byte-identical, so this cannot change any match result. + let display_value = string_ordering_display_value( + rule, + buffer, + absolute_offset, + max_string_length, + transformed_value, + ); + Ok((matched, display_value)) +} + +/// Compute the DISPLAY value for a value-rule match, decoupling it from the +/// value used in the comparison. +/// +/// For a `string` rule compared with an ORDERING operator (`<`/`>`/`<=`/`>=`), +/// libmagic renders the full string field (`p->s`, read until NUL/EOF) even +/// though the comparison is prefix-limited to `pattern.len()`. This re-reads +/// that full field so `%s`/`%.Ns` format specifiers render the whole value +/// rather than only the compared prefix. For every other type or operator the +/// compared value already IS the field libmagic renders, so `compared` is +/// returned unchanged. +/// +/// Only `TypeKind::String` needs this: `PString` (`read_pstring`) and +/// `String16` (`read_string16`) already read their full field independent of +/// `pattern.len()`, and numeric types render the whole value. +/// +/// On a display-side read error after a successful match, the compared value +/// is returned rather than propagating -- a matched rule must not abort on a +/// display-only read. +fn string_ordering_display_value( + rule: &MagicRule, + buffer: &[u8], + absolute_offset: usize, + max_string_length: usize, + compared: crate::parser::ast::Value, +) -> crate::parser::ast::Value { + use crate::parser::ast::Operator::{GreaterEqual, GreaterThan, LessEqual, LessThan}; + + let is_ordering = matches!(rule.op, LessThan | GreaterThan | LessEqual | GreaterEqual); + if is_ordering && matches!(rule.typ, TypeKind::String { .. }) { + match types::read_string(buffer, absolute_offset, Some(max_string_length)) { + Ok(full_field) => full_field, + // A matched rule must not abort on a display-only read (the compared + // prefix was already read successfully at this offset moments ago), + // so fall back to it -- but `debug!` first rather than swallowing the + // error silently, matching this file's graceful-skip logging + // discipline. A latent regression here (e.g. `max_string_length` + // disagreeing with the original read) would otherwise render the + // truncated prefix with no trace of why the full field was dropped. + Err(e) => { + debug!( + "string_ordering_display_value: full-field read failed at offset {absolute_offset} for rule '{}': {e}; rendering compared prefix", + rule.message + ); + compared + } + } + } else { + compared + } +} + +/// Logs the graceful skip of a pattern-bearing-type rule whose +/// `TypeReadError::UnsupportedType` condition falls in the narrow +/// missing-pattern-operand or regex-compile-failure allowlist (see +/// `types::is_missing_pattern_operand` / `types::is_regex_compile_failure`). +/// +/// Shared by all three engine catch sites (`evaluate_children_or_warn`, the +/// top-level dispatch match, and the inline child-recursion match) so a +/// future rewording of the log message only needs one touch point (DRY, +/// AGENTS.md). Split by KTD5 (fix-system-magic-regex-graceful plan): the +/// ordinary missing-pattern case is `debug!`-logged (an expected, +/// low-severity data condition -- e.g. the root-cause parser +/// miscategorization this plan also fixes), while a regex compile failure +/// (which includes the `REGEX_COMPILE_SIZE_LIMIT` CWE-1333 denial-of-service guard) is +/// `warn!`-logged so a malicious or pathological magic file's rejection is +/// not silently invisible, even though the rest of the file's evaluation +/// continues (R1: no fatal abort of the whole evaluation). +fn log_pattern_operand_skip(site_label: &str, rule_message: &str, type_name: &str) { + if types::is_regex_compile_failure(type_name) { + warn!( + "Skipping {site_label} rule '{rule_message}' due to regex compile failure: {type_name} -- this may indicate a malicious or pathological magic file" + ); + } else { + debug!("Skipping {site_label} rule '{rule_message}': {type_name}"); + } +} + +/// Whether `message` carries any usable description text. +/// +/// A message is considered message-less (and thus does not count as +/// "producing output") if, after trimming ASCII/Unicode whitespace and +/// stripping a leading GNU `file` no-separator marker (see GOTCHAS S14.1), +/// nothing remains. This covers three shapes GNU `file` magic files use +/// for structural/gating rules that carry no description of their own: +/// a genuinely empty message (`""`), a whitespace-only message, and a +/// `\b`-only message (used purely to suppress a separator when appended +/// to a sibling's text -- with nothing else to append, it contributes no +/// content either). +/// +/// The marker is recognized in BOTH forms -- the raw byte `U+0008` and the +/// literal `\b` (backslash + `'b'`) -- via the shared +/// [`crate::evaluator::strip_no_separator_marker`], so this predicate agrees +/// with `concatenate_messages`: a message that renders to empty there (e.g. +/// exactly `"\b"`, the literal marker) is classified message-less here and +/// therefore cannot win the `stop_at_first_match` race and shadow a later, +/// more specific rule that would produce real output (the S13.2 bug class). +fn is_message_bearing(message: &str) -> bool { + let trimmed = message.trim_matches(|c: char| c.is_whitespace() || c == '\u{8}'); + let stripped = crate::evaluator::strip_no_separator_marker(trimmed).unwrap_or(trimmed); + !stripped + .trim_matches(|c: char| c.is_whitespace() || c == '\u{8}') + .is_empty() +} + +/// Whether any match in `matches[from..]` carries usable description text +/// (see [`is_message_bearing`]). +/// +/// Used to decide whether a top-level rule's match -- together with any +/// descendant matches produced by its children -- should be treated as +/// the "winning" match for `stop_at_first_match` purposes. GNU `file` +/// magic files commonly use message-less top-level rules purely as +/// gating conditions for child rules (for example the `c-lang` search +/// rules that test for `#include`/`pragma`/etc. before dispatching to a +/// message-bearing regex child); under the old all-or-nothing contract, a +/// message-less rule matching first under `stop_at_first_match: true` +/// would silently shadow a later, more specific rule that actually +/// produces a description (GOTCHAS S13.2, the assembler-source-text / +/// plain-ASCII-text blank-output bug). A rule only "wins" the race if it +/// (or a descendant) contributes real output text; otherwise evaluation +/// continues to the next top-level sibling. +/// +/// Takes `from` (the length of `matches` before this rule's dispatch) and +/// slices via `.get()` (rather than the caller indexing `matches[from..]` +/// directly) so this is panic-free per the project's bounds-checking +/// discipline; `from` is always `<= matches.len()` by construction (it is +/// captured from `matches.len()` earlier in the same call), so `.get()` +/// always returns `Some`, but the panic-free form is required regardless. +fn has_message_bearing_match(matches: &[RuleMatch], from: usize) -> bool { + matches + .get(from..) + .is_some_and(|tail| tail.iter().any(|m| is_message_bearing(&m.message))) } /// Evaluate a rule's children under the standard recursion-guard/graceful-skip discipline. @@ -534,6 +784,22 @@ fn evaluate_children_or_warn( rule_kind, rule.message, e ); } + // Narrow graceful-skip (KTD4): a pattern-bearing type evaluated + // without a usable pattern operand, or a regex compile failure + // (including the REGEX_COMPILE_SIZE_LIMIT DoS guard), must not + // abort the whole evaluation -- see `log_pattern_operand_skip` and + // the top-level dispatch match below for the full contract. This + // arm is defensive: under the current implementation, individual + // child failures are already caught and logged inside the + // recursive `evaluate_rules` call (they never propagate here); it + // guards against a future change to that strategy. + Err(LibmagicError::EvaluationError(crate::error::EvaluationError::TypeReadError( + crate::evaluator::types::TypeReadError::UnsupportedType { ref type_name }, + ))) if types::is_missing_pattern_operand(type_name) + || types::is_regex_compile_failure(type_name) => + { + log_pattern_operand_skip(rule_kind, &rule.message, type_name); + } Err(e) => return Err(e), } Ok(()) @@ -676,6 +942,22 @@ pub fn evaluate_rules( let is_indirect_reentry = context.take_indirect_reentry(); let is_child_sibling_list = context.recursion_depth() > 0 && !is_indirect_reentry; + // `stop_at_first_match` is a TOP-LEVEL classification concept (see the + // `EvaluationConfig::stop_at_first_match` doc): once an outermost rule -- + // or an indirect re-entry, which is itself a fresh top-level + // classification -- produces a message-bearing match, we stop trying + // other top-level candidates. It must NOT short-circuit a child / + // continuation sibling list or a `use` subroutine body: every matching + // sibling there contributes a detail fragment to the description (e.g. + // gzip's "max compression", "from Unix", "original size modulo 2^32 N"), + // and truncating them silently drops multi-part descriptions. This + // mirrors libmagic, where continuation levels always evaluate every + // sibling and only the top-level `match()` loop stops at first success. + // (An earlier revision applied the break at every recursion level, which + // violated the documented top-level-only contract and truncated gzip's + // trailing detail after its first message-bearing child.) + let stop_at_first_match_applies = !is_child_sibling_list; + // Entry-point timeout check: ensures every recursive descent is bounded // and that evaluations of small rule sets (< 16 rules) are still guarded. // Without this, the periodic every-16-rules check below never fires for @@ -708,10 +990,52 @@ pub fn evaluate_rules( // `Clear` resets the per-level "sibling matched" flag so a // subsequent `default` sibling can fire even if an earlier - // sibling matched. It does not produce a match, evaluate - // children, or advance the anchor. + // sibling matched. Matching libmagic's `FILE_CLEAR`, the flag is + // unconditionally reset and NEVER re-set to `true` afterward + // (clear does not participate in the "a sibling matched" chain). + // + // libmagic's `FILE_CLEAR` also COUNTS as a match -- its `x` test + // always succeeds -- and `mprint` renders its description when it + // is non-empty. So a `clear` carrying message text must emit that + // text (c-lang's `>>&0 clear x program text` is the only such rule + // in the system DB, producing the "program text" fragment of + // `c program text`). Verified against real `file` (file-5.41): + // a message-bearing `clear` child prints its message AND still + // resets the flag so a trailing `default` sibling fires. + // + // Emission is guarded on a non-empty message so the many bare + // `clear x` flag-reset directives throughout the system DB (apple, + // coff, elf, pmem, ...) behave exactly as before -- no match, no + // anchor advance. `clear` is 0-width, so the previous-match anchor + // is intentionally not advanced in either case. Children are + // evaluated for a message-bearing clear for libmagic fidelity; + // `evaluate_children_or_warn` is a no-op when there are none. if let TypeKind::Meta(MetaType::Clear) = &rule.typ { sibling_matched = false; + + if !rule.message.is_empty() { + let matches_before = matches.len(); + + let match_result = RuleMatch::new( + rule.message.clone(), + context.last_match_end(), + rule.level, + crate::parser::ast::Value::Uint(0), + rule.typ.clone(), + RuleMatch::calculate_confidence(rule.level), + ); + matches.push(match_result); + + evaluate_children_or_warn(rule, "clear", buffer, context, &mut matches)?; + + if stop_at_first_match_applies + && matches.len() > matches_before + && context.should_stop_at_first_match() + && has_message_bearing_match(&matches, matches_before) + { + break; + } + } continue; } @@ -741,7 +1065,11 @@ pub fn evaluate_rules( sibling_matched = true; - if matches.len() > matches_before && context.should_stop_at_first_match() { + if stop_at_first_match_applies + && matches.len() > matches_before + && context.should_stop_at_first_match() + && has_message_bearing_match(&matches, matches_before) + { break; } } @@ -870,7 +1198,11 @@ pub fn evaluate_rules( // recursion-guard pattern used by every other successful rule. evaluate_children_or_warn(rule, "indirect", buffer, context, &mut matches)?; - if matches.len() > matches_before && context.should_stop_at_first_match() { + if stop_at_first_match_applies + && matches.len() > matches_before + && context.should_stop_at_first_match() + && has_message_bearing_match(&matches, matches_before) + { break; } continue; @@ -888,14 +1220,6 @@ pub fn evaluate_rules( // rather than erroring -- a rogue rule shouldn't poison the rest // of the evaluation. if let TypeKind::Meta(MetaType::Offset) = &rule.typ { - if !matches!(rule.op, crate::parser::ast::Operator::AnyValue) { - debug!( - "offset rule '{}': non-`x` operator {:?} not supported; skipping", - rule.message, rule.op - ); - continue; - } - // Resolve the offset first so a malformed offset surfaces as // a graceful skip rather than a hard error. Mirrors the // `Indirect` dispatch above. @@ -918,6 +1242,26 @@ pub fn evaluate_rules( Err(e) => return Err(e), }; + // The magic(5) `offset` pseudo-type treats the resolved offset + // itself as the read value. `offset x` is a bare AnyValue + // placeholder that always matches (used purely to report the + // position via `%lld`). A comparison operator (`offset >48`, + // `offset <48`, `offset =N`, ...) tests the resolved offset + // against the operand -- e.g. gzip's `>>-0 offset >48` gates + // the trailing "original size modulo 2^32" trailer on the file + // being long enough to carry it, and its `>>-0 offset <48` + // sibling reports "truncated" otherwise. Skip the rule (a + // non-match) when the comparison fails so the false branch and + // its children do not render. + let offset_value = crate::parser::ast::Value::Uint(absolute_offset as u64); + let offset_matched = match &rule.op { + crate::parser::ast::Operator::AnyValue => true, + op => operators::apply_operator(op, &offset_value, &rule.value), + }; + if !offset_matched { + continue; + } + let matches_before = matches.len(); // Advance the anchor BEFORE emitting the match so sibling @@ -930,7 +1274,7 @@ pub fn evaluate_rules( rule.message.clone(), absolute_offset, rule.level, - crate::parser::ast::Value::Uint(absolute_offset as u64), + offset_value, rule.typ.clone(), RuleMatch::calculate_confidence(rule.level), ); @@ -942,7 +1286,11 @@ pub fn evaluate_rules( // by every other successful rule. evaluate_children_or_warn(rule, "offset", buffer, context, &mut matches)?; - if matches.len() > matches_before && context.should_stop_at_first_match() { + if stop_at_first_match_applies + && matches.len() > matches_before + && context.should_stop_at_first_match() + && has_message_bearing_match(&matches, matches_before) + { break; } continue; @@ -960,9 +1308,9 @@ pub fn evaluate_rules( // continuation rules (siblings and descendants of the `use` site) // that depend on the anchor the subroutine left behind; skipping // them produces user-visible false negatives. - if let TypeKind::Meta(MetaType::Use(name)) = &rule.typ { + if let TypeKind::Meta(MetaType::Use { name, flip_endian }) = &rule.typ { let matches_before = matches.len(); - let use_resolved = match evaluate_use_rule(rule, name, buffer, context) { + let use_resolved = match evaluate_use_rule(rule, name, *flip_endian, buffer, context) { Ok((Some(terminal_anchor), subroutine_matches)) => { matches.extend(subroutine_matches); @@ -1015,8 +1363,14 @@ pub fn evaluate_rules( // other successful rule kind: if this `use` site contributed // any matches (either from the subroutine or from its own // children) and the caller configured first-match - // short-circuiting, halt evaluation of further siblings. - if matches.len() > matches_before && context.should_stop_at_first_match() { + // short-circuiting, halt evaluation of further siblings -- + // but only once one of those matches actually carries usable + // description text (see `has_message_bearing_match`). + if stop_at_first_match_applies + && matches.len() > matches_before + && context.should_stop_at_first_match() + && has_message_bearing_match(&matches, matches_before) + { break; } continue; @@ -1031,6 +1385,7 @@ pub fn evaluate_rules( context.last_match_end(), context.base_offset(), context.max_string_length(), + context.flip_endian(), ) { Ok(data) => data, Err( @@ -1046,18 +1401,47 @@ pub fn evaluate_rules( | LibmagicError::IoError(_)), ) => { // Expected data-dependent evaluation errors -- skip gracefully. - // TypeReadError::UnsupportedType is intentionally NOT caught here - // so that evaluator capability gaps propagate as errors. + // TypeReadError::UnsupportedType is intentionally NOT caught + // here (except the narrow exception in the arm immediately + // below) so that evaluator capability gaps propagate as + // errors. debug!("Skipping rule '{}': {}", rule.message, e); continue; } + // Narrow graceful-skip (KTD4, fix-system-magic-regex-graceful + // plan): a pattern-bearing type (`Regex`/`Search`/flagged + // `String`) evaluated without a usable `String`/`Bytes` pattern + // operand, or a regex compile failure (including the + // `REGEX_COMPILE_SIZE_LIMIT` CWE-1333 DoS guard), must not + // abort the whole file's evaluation (R1/R2). This is + // deliberately an exhaustive allowlist keyed on the + // `UnsupportedType` diagnostic string + // (`types::is_missing_pattern_operand` / + // `types::is_regex_compile_failure`), not a broadening of the + // general `UnsupportedType` exclusion above -- any OTHER + // `UnsupportedType` (an unwired `TypeKind` variant, a + // non-Equal/NotEqual operator on a pattern-bearing type, etc.) + // still falls through to the catch-all below and propagates + // (R3). See `log_pattern_operand_skip` for the debug!/warn! + // split. + Err(LibmagicError::EvaluationError(crate::error::EvaluationError::TypeReadError( + crate::evaluator::types::TypeReadError::UnsupportedType { ref type_name }, + ))) if types::is_missing_pattern_operand(type_name) + || types::is_regex_compile_failure(type_name) => + { + log_pattern_operand_skip("top-level", &rule.message, type_name); + continue; + } Err(e) => { - // Unexpected errors (InternalError, UnsupportedType, etc.) should propagate + // Unexpected errors (InternalError, other UnsupportedType + // conditions, etc.) should propagate. return Err(e); } }; if let Some((absolute_offset, read_value)) = match_data { + let matches_before = matches.len(); + // Advance the GNU `file` previous-match anchor BEFORE recursing // into children, so children and their descendants see the new // anchor. The anchor is updated unconditionally to the end of @@ -1131,6 +1515,27 @@ pub fn evaluate_rules( rule.message, e ); } + // Narrow graceful-skip (KTD4): same allowlist as the + // top-level dispatch match above and + // `evaluate_children_or_warn` -- a pattern-bearing type + // evaluated without a usable pattern operand, or a + // regex compile failure, must not abort the parent's + // match. Defensive: individual child failures are + // already caught inside the recursive `evaluate_rules` + // call and never reach here under the current + // implementation; this arm guards against a future + // change to that strategy. + Err(LibmagicError::EvaluationError( + crate::error::EvaluationError::TypeReadError( + crate::evaluator::types::TypeReadError::UnsupportedType { + ref type_name, + }, + ), + )) if types::is_missing_pattern_operand(type_name) + || types::is_regex_compile_failure(type_name) => + { + log_pattern_operand_skip("child", &rule.message, type_name); + } Err(e) => { // Unexpected errors in children (including RecursionLimitExceeded) // should propagate. The guard drops here, decrementing the depth. @@ -1140,8 +1545,16 @@ pub fn evaluate_rules( // `guard` drops here, decrementing the recursion depth. } - // Stop at first match if configured to do so - if context.should_stop_at_first_match() { + // Stop at first match if configured to do so -- but only once + // this rule (or one of its descendants) actually contributed + // usable description text. A message-less match (e.g. a + // gating rule used purely to trigger a child) must not shadow + // a later, more specific top-level rule that would otherwise + // produce real output (GOTCHAS S13.2). + if stop_at_first_match_applies + && context.should_stop_at_first_match() + && has_message_bearing_match(&matches, matches_before) + { break; } } diff --git a/src/evaluator/engine/tests/helpers/meta.rs b/src/evaluator/engine/tests/helpers/meta.rs index 090e6237..d8224c28 100644 --- a/src/evaluator/engine/tests/helpers/meta.rs +++ b/src/evaluator/engine/tests/helpers/meta.rs @@ -21,7 +21,7 @@ pub fn make_context_with_env(name_table: NameTable, root_rules: &[MagicRule]) -> EvaluationContext::new(EvaluationConfig::default()).with_rule_env(env) } -/// Minimal helper: wrap a `TypeKind::Meta(MetaType::Use(name))` rule at +/// Minimal helper: wrap a `TypeKind::Meta(MetaType::Use { name, .. })` rule at /// offset 0 with the given `message` and empty child list. pub fn use_rule(name: &str) -> MagicRule { use_rule_at(name, 0) @@ -33,7 +33,10 @@ pub fn use_rule(name: &str) -> MagicRule { pub fn use_rule_at(name: &str, offset: i64) -> MagicRule { MagicRule { offset: OffsetSpec::Absolute(offset), - typ: TypeKind::Meta(MetaType::Use(name.to_string())), + typ: TypeKind::Meta(MetaType::Use { + name: name.to_string(), + flip_endian: false, + }), op: Operator::Equal, value: Value::Uint(0), message: format!("use {name}"), diff --git a/src/evaluator/engine/tests/meta_default_clear_indirect_tests.rs b/src/evaluator/engine/tests/meta_default_clear_indirect_tests.rs index 62712101..07da815c 100644 --- a/src/evaluator/engine/tests/meta_default_clear_indirect_tests.rs +++ b/src/evaluator/engine/tests/meta_default_clear_indirect_tests.rs @@ -10,6 +10,49 @@ use super::*; +/// GOTCHAS S13.2 (refined): a message-less `default` match must not stop +/// evaluation before a later, message-bearing sibling is reached. +#[test] +fn test_default_message_less_match_does_not_stop_at_first_match() { + let rules = vec![ + default_rule("", vec![]), + byte_eq_rule(0, 0xAA, "Real message"), + ]; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&rules, &[0xAAu8], &mut context).unwrap(); + + assert_eq!( + matches.len(), + 2, + "both the default and the real rule should match" + ); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, "Real message"); +} + +/// Same contract for `indirect`: a message-less indirect directive (here, +/// re-entering an empty root-rule list so the re-entry itself produces +/// nothing) must not shadow a later message-bearing sibling. +#[test] +fn test_indirect_message_less_match_does_not_stop_at_first_match() { + let table = build_name_table(vec![]); + let mut context = make_context_with_env(table, &[]); + let rules = vec![ + indirect_rule(0, "", vec![]), + byte_eq_rule(1, 0xBB, "Real message"), + ]; + let buffer = [0xAAu8, 0xBB]; + let matches = evaluate_rules(&rules, &buffer, &mut context).unwrap(); + + assert_eq!( + matches.len(), + 2, + "both the indirect directive and the real rule should match" + ); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, "Real message"); +} + #[test] fn test_default_fires_when_no_sibling_matched() { let rules = vec![default_rule("DEFAULT-FIRES", vec![])]; diff --git a/src/evaluator/engine/tests/meta_offset_tests.rs b/src/evaluator/engine/tests/meta_offset_tests.rs index f6591844..593e3705 100644 --- a/src/evaluator/engine/tests/meta_offset_tests.rs +++ b/src/evaluator/engine/tests/meta_offset_tests.rs @@ -9,6 +9,27 @@ use super::*; +/// GOTCHAS S13.2 (refined): a message-less `offset` directive must not +/// stop evaluation before a later, message-bearing sibling is reached. +#[test] +fn test_offset_message_less_match_does_not_stop_at_first_match() { + let rules = vec![ + offset_rule(0, "", vec![]), + byte_eq_rule(1, 0xBB, "Real message"), + ]; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let buffer = [0xAAu8, 0xBB]; + let matches = evaluate_rules(&rules, &buffer, &mut context).unwrap(); + + assert_eq!( + matches.len(), + 2, + "both the offset directive and the real rule should match" + ); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, "Real message"); +} + #[test] fn test_offset_emits_match_with_resolved_position() { let rules = vec![offset_rule(5, "pos=%lld", vec![])]; @@ -49,18 +70,71 @@ fn test_offset_out_of_bounds_graceful_skip() { } #[test] -fn test_offset_non_x_operator_is_skipped() { - // magic(5) only allows `x` on an `offset` rule. Anything else is - // semantically undefined -> debug-log + skip. - let mut rule = offset_rule(0, "bogus", vec![]); - rule.op = Operator::Equal; - rule.value = Value::Uint(5); - let rules = vec![rule]; +fn test_offset_comparison_operator_gates_match() { + // The magic(5) `offset` pseudo-type compares the RESOLVED offset value + // against the operand. `offset =N` matches only when the resolved + // position equals N. Here the offset spec resolves to position 0, so + // `offset =5` must NOT match but `offset =0` must. (This replaced the + // earlier "non-x operator is skipped" contract: comparison operators on + // `offset` are now honored -- gzip's `>>-0 offset >48` depends on it.) + let mut miss = offset_rule(0, "no", vec![]); + miss.op = Operator::Equal; + miss.value = Value::Uint(5); let mut context = EvaluationContext::new(EvaluationConfig::default()); - let matches = evaluate_rules(&rules, &[0u8; 4], &mut context).unwrap(); + let matches = evaluate_rules(&[miss], &[0u8; 4], &mut context).unwrap(); + assert!( + matches.is_empty(), + "offset =5 must not match when the resolved offset is 0" + ); + + let mut hit = offset_rule(0, "yes", vec![]); + hit.op = Operator::Equal; + hit.value = Value::Uint(0); + context.reset(); + let matches = evaluate_rules(&[hit], &[0u8; 4], &mut context).unwrap(); + assert_eq!( + matches.len(), + 1, + "offset =0 must match at resolved offset 0" + ); + assert_eq!(matches[0].message, "yes"); +} + +#[test] +fn test_offset_from_end_zero_compares_against_file_size() { + // gzip's trailing-size gate: `>>-0 offset >48` reads the EOF position + // (`buffer.len()`, via FromEnd(0)) and compares it against 48. A file + // longer than 48 bytes matches `>48` (and its "original size" child + // renders); a shorter file matches the sibling `<48` "truncated" branch. + // This pins the -0 -> FromEnd(0) -> buffer.len() resolution together with + // the offset-comparison semantics. + let mut gt = offset_rule(0, "big", vec![byte_eq_rule(0, 0x00, "orig-size")]); + gt.offset = OffsetSpec::FromEnd(0); + gt.op = Operator::GreaterThan; + gt.value = Value::Uint(48); + + let config = EvaluationConfig { + stop_at_first_match: false, + ..EvaluationConfig::default() + }; + + // 64-byte buffer > 48 -> matches, child renders. + let mut context = EvaluationContext::new(config.clone()); + let matches = evaluate_rules(&[gt.clone()], &[0u8; 64], &mut context).unwrap(); + assert_eq!( + matches.len(), + 2, + "offset >48 matches a 64-byte file + child" + ); + assert_eq!(matches[0].value, Value::Uint(64), "resolved offset is EOF"); + assert_eq!(matches[1].message, "orig-size"); + + // 32-byte buffer < 48 -> no match. + let mut context = EvaluationContext::new(config); + let matches = evaluate_rules(&[gt], &[0u8; 32], &mut context).unwrap(); assert!( matches.is_empty(), - "offset rule with non-AnyValue operator must be skipped" + "offset >48 must not match a 32-byte file" ); } diff --git a/src/evaluator/engine/tests/meta_use_tests.rs b/src/evaluator/engine/tests/meta_use_tests.rs index 154f4195..8a1a49a5 100644 --- a/src/evaluator/engine/tests/meta_use_tests.rs +++ b/src/evaluator/engine/tests/meta_use_tests.rs @@ -11,6 +11,40 @@ use super::*; +/// GOTCHAS S13.2 (refined): `use` itself never produces a surface +/// `RuleMatch`, so it only "counts" toward `stop_at_first_match` via the +/// subroutine's own matches. If the subroutine's match carries no +/// message, evaluation must continue to the next top-level sibling +/// rather than treating the `use` site as the terminating match. +#[test] +fn test_use_message_less_subroutine_match_does_not_stop_at_first_match() { + let subroutine = vec![MagicRule { + offset: OffsetSpec::Absolute(3), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(0x42), + message: String::new(), + children: vec![], + level: 1, + strength_modifier: None, + value_transform: None, + }]; + let table = build_name_table(vec![("part2", subroutine)]); + let mut context = make_context_with_env(table, &[]); + + let buffer = [0x00u8, 0x00, 0x00, 0x42, 0xBB]; + let rules = vec![use_rule("part2"), byte_eq_rule(4, 0xBB, "Real message")]; + let matches = evaluate_rules(&rules, &buffer, &mut context).unwrap(); + + assert_eq!( + matches.len(), + 2, + "both the message-less subroutine match and the real rule should match" + ); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, "Real message"); +} + #[test] fn test_use_known_name_evaluates_subroutine() { // The subroutine `part2` reads byte 3 and expects 0x42. diff --git a/src/evaluator/engine/tests/mod.rs b/src/evaluator/engine/tests/mod.rs index c847d345..78565cf5 100644 --- a/src/evaluator/engine/tests/mod.rs +++ b/src/evaluator/engine/tests/mod.rs @@ -34,6 +34,7 @@ fn evaluate_single_rule_legacy( 0, 0, crate::evaluator::types::DEFAULT_MAX_STRING_LENGTH, + false, ) } @@ -716,6 +717,10 @@ fn test_evaluate_single_rule_cross_type_comparison() { #[test] fn test_evaluate_single_rule_bitwise_and_with_shorts() { + // BitwiseAnd requires ALL masked bits to be set (see GOTCHAS S13.3): + // `(value & mask) == mask`, not merely "some bit overlaps". The mask + // here (0xff00) asks "is the entire high byte set" -- so the buffer's + // high byte must genuinely be 0xff for this to match. let rule = MagicRule { offset: OffsetSpec::Absolute(0), typ: TypeKind::Short { @@ -731,13 +736,22 @@ fn test_evaluate_single_rule_bitwise_and_with_shorts() { value_transform: None, }; - let buffer = &[0x34, 0x12]; - let result = evaluate_single_rule_legacy(&rule, buffer).unwrap(); + // Little-endian [0x34, 0xff] -> 0xff34; high byte is 0xff, all mask bits set. + let matching_buffer = &[0x34, 0xff]; + let result = evaluate_single_rule_legacy(&rule, matching_buffer).unwrap(); assert!(result.is_some()); + + // Little-endian [0x34, 0x12] -> 0x1234; high byte is 0x12, not all mask + // bits set, so this must NOT match under the corrected semantics. + let non_matching_buffer = &[0x34, 0x12]; + let result = evaluate_single_rule_legacy(&rule, non_matching_buffer).unwrap(); + assert!(result.is_none()); } #[test] fn test_evaluate_single_rule_bitwise_and_with_longs() { + // Same "all masked bits set" contract as the shorts test above, applied + // to a 32-bit mask spanning the high word. let rule = MagicRule { offset: OffsetSpec::Absolute(0), typ: TypeKind::Long { @@ -753,9 +767,17 @@ fn test_evaluate_single_rule_bitwise_and_with_longs() { value_transform: None, }; - let buffer = &[0x12, 0x34, 0x56, 0x78]; - let result = evaluate_single_rule_legacy(&rule, buffer).unwrap(); + // Big-endian [0xff, 0xff, 0x56, 0x78] -> 0xffff5678; high word is 0xffff, + // all mask bits set. + let matching_buffer = &[0xff, 0xff, 0x56, 0x78]; + let result = evaluate_single_rule_legacy(&rule, matching_buffer).unwrap(); assert!(result.is_some()); + + // Big-endian [0x12, 0x34, 0x56, 0x78] -> 0x12345678; high word is + // 0x1234, not all mask bits set, so this must NOT match. + let non_matching_buffer = &[0x12, 0x34, 0x56, 0x78]; + let result = evaluate_single_rule_legacy(&rule, non_matching_buffer).unwrap(); + assert!(result.is_none()); } #[test] @@ -1227,6 +1249,293 @@ fn test_evaluate_rules_multiple_rules_find_all() { assert_eq!(matches[1].message, "Second match"); } +/// Build a flat, top-level, message-only byte rule matching a distinct +/// value at a distinct offset. Shared by the `stop_at_first_match` +/// message-bearing tests below so each test only needs to state the +/// interesting bit: which offsets carry which messages. +fn message_only_byte_rule(offset: i64, byte: u8, message: &str) -> MagicRule { + MagicRule { + offset: OffsetSpec::Absolute(offset), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(byte)), + message: message.to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + } +} + +/// GOTCHAS S13.2 (refined): a message-less top-level match must not +/// shadow a later, message-bearing sibling under `stop_at_first_match: +/// true`. This is the exact shape of the assembler-source-text / +/// plain-ASCII-text blank-output bug -- a gating rule with no message +/// matches first in strength order and used to terminate evaluation +/// before the real classification rule was ever tried. +#[test] +fn test_evaluate_rules_message_less_match_does_not_stop_at_first_match() { + let buffer = &[0xAA, 0xBB, 0xCC, 0xDD]; + let gating_rule = message_only_byte_rule(0, 0xAA, ""); + let real_rule = message_only_byte_rule(1, 0xBB, "Second match"); + + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules(&[gating_rule, real_rule], buffer, &mut context).unwrap(); + + // Both rules matched: the message-less gating rule did not terminate + // the search, so the message-bearing rule behind it was reached and + // its match is present. + assert_eq!(matches.len(), 2, "both rules should have matched"); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, "Second match"); +} + +/// Reverse of the above: when the message-BEARING rule comes first, the +/// original `stop_at_first_match` short-circuit still applies -- this +/// fix only relaxes the stop condition for message-less matches, it does +/// not disable early-exit for the common (and performance-sensitive) +/// case where the very first top-level rule already produces output. +#[test] +fn test_evaluate_rules_message_bearing_match_still_stops_at_first_match() { + let buffer = &[0xAA, 0xBB, 0xCC, 0xDD]; + let real_rule = message_only_byte_rule(0, 0xAA, "First match"); + let gating_rule = message_only_byte_rule(1, 0xBB, ""); + let never_reached = message_only_byte_rule(2, 0xCC, "Should not be reached"); + + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules( + &[real_rule, gating_rule, never_reached], + buffer, + &mut context, + ) + .unwrap(); + + assert_eq!( + matches.len(), + 1, + "evaluation must still stop right after the first message-bearing match" + ); + assert_eq!(matches[0].message, "First match"); +} + +/// Several message-less matches in a row must all be skipped over (not +/// discarded -- just not treated as terminating) until a message-bearing +/// rule is reached, at which point the usual stop-at-first-match +/// short-circuit applies again. +#[test] +fn test_evaluate_rules_multiple_message_less_matches_before_a_real_one() { + let buffer = &[0xAA, 0xBB, 0xCC, 0xDD]; + let gating_one = message_only_byte_rule(0, 0xAA, ""); + let gating_two = message_only_byte_rule(1, 0xBB, ""); + let real_rule = message_only_byte_rule(2, 0xCC, "Real message"); + let never_reached = message_only_byte_rule(3, 0xDD, "Should not be reached"); + + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules( + &[gating_one, gating_two, real_rule, never_reached], + buffer, + &mut context, + ) + .unwrap(); + + assert_eq!(matches.len(), 3); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, ""); + assert_eq!(matches[2].message, "Real message"); +} + +/// Genuinely-no-usable-output case: every top-level rule matches but +/// none of them carries a message. Under `stop_at_first_match: true` +/// evaluation must run to exhaustion (there is nothing to stop at) -- +/// all matches are collected, and it is the caller's (here: +/// `MagicDatabase::build_result`'s) job to fall back to text/data +/// classification when the resulting description is empty. +#[test] +fn test_evaluate_rules_all_message_less_matches_runs_to_exhaustion() { + let buffer = &[0xAA, 0xBB]; + let gating_one = message_only_byte_rule(0, 0xAA, ""); + let gating_two = message_only_byte_rule(1, 0xBB, ""); + + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules(&[gating_one, gating_two], buffer, &mut context).unwrap(); + + assert_eq!(matches.len(), 2, "both message-less matches are retained"); + assert!(matches.iter().all(|m| m.message.is_empty())); +} + +/// A message consisting solely of whitespace, or solely of the GNU +/// `file` backspace continuation marker (`\b`), is just as +/// "message-less" as an empty string for `stop_at_first_match` purposes +/// -- see `is_message_bearing`'s doc comment for the rationale. +#[test] +fn test_evaluate_rules_whitespace_and_backspace_only_messages_do_not_stop() { + let buffer = &[0xAA, 0xBB, 0xCC]; + let whitespace_only = message_only_byte_rule(0, 0xAA, " "); + let backspace_only = message_only_byte_rule(1, 0xBB, "\u{8}"); + let real_rule = message_only_byte_rule(2, 0xCC, "Real message"); + + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules( + &[whitespace_only, backspace_only, real_rule], + buffer, + &mut context, + ) + .unwrap(); + + assert_eq!(matches.len(), 3); + assert_eq!(matches[2].message, "Real message"); +} + +/// Regression for the `is_message_bearing` literal-`\b`-marker gap (PR #376 +/// review finding). The GNU `file` no-separator marker most often reaches the +/// evaluator as the LITERAL two-character sequence `\b` (backslash + `'b'`), +/// not the raw `U+0008` byte, because the message parser preserves description +/// text verbatim (GOTCHAS S14.1). A rule whose message is exactly the literal +/// marker renders to empty in `concatenate_messages`, so it must be classified +/// message-less here too -- otherwise it could win the `stop_at_first_match` +/// race and shadow a later, more specific rule (the S13.2 blank-output bug +/// class). The prior implementation only trimmed `U+0008` and would have +/// treated `"\\b"` as message-bearing. +#[test] +fn test_literal_backspace_marker_message_is_message_less_and_does_not_stop() { + // Direct predicate: both marker forms (and whitespace-padded variants) are + // message-less; a marker WITH trailing content is message-bearing. + assert!( + !is_message_bearing("\\b"), + "the literal `\\b` marker alone must be message-less" + ); + assert!( + !is_message_bearing("\u{8}"), + "the raw U+0008 marker alone must be message-less" + ); + assert!( + !is_message_bearing(" \\b "), + "a whitespace-padded literal marker must be message-less" + ); + assert!( + is_message_bearing("\\bversion"), + "a literal marker WITH content must be message-bearing" + ); + assert!( + is_message_bearing("plain"), + "plain text must be message-bearing" + ); + + // End-to-end: a literal-`\b`-only gating rule must NOT stop evaluation + // before a later real rule under stop_at_first_match. + let buffer = &[0xAA, 0xBB]; + let literal_marker_only = message_only_byte_rule(0, 0xAA, "\\b"); + let real_rule = message_only_byte_rule(1, 0xBB, "Real message"); + + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + let matches = evaluate_rules(&[literal_marker_only, real_rule], buffer, &mut context).unwrap(); + + assert_eq!( + matches.len(), + 2, + "the literal-marker-only rule must not stop evaluation before the real rule" + ); + assert_eq!(matches[1].message, "Real message"); +} + +/// A message-less top-level rule whose CHILD produces real output text +/// still counts as "producing output" for `stop_at_first_match` +/// purposes -- this is the normal, common shape for gating rules like +/// `c-lang`'s `0 search/8192 "#include"` -> `>0 regex \^#include c` +/// chain, and must keep stopping at the first sibling that (directly or +/// via a descendant) yields a description; only a rule with NO +/// message-bearing output anywhere in its subtree should be skipped +/// past. +#[test] +fn test_evaluate_rules_message_less_parent_with_message_bearing_child_still_stops() { + let child_rule = message_only_byte_rule(1, 0xBB, "child message"); + let mut parent_rule = message_only_byte_rule(0, 0xAA, ""); + parent_rule.children = vec![child_rule]; + + let never_reached = message_only_byte_rule(2, 0xCC, "Should not be reached"); + + let buffer = &[0xAA, 0xBB, 0xCC]; + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules(&[parent_rule, never_reached], buffer, &mut context).unwrap(); + + assert_eq!( + matches.len(), + 2, + "parent (message-less) + child (message-bearing) should be present, \ + and evaluation should stop there" + ); + assert_eq!(matches[0].message, ""); + assert_eq!(matches[1].message, "child message"); +} + +#[test] +fn test_stop_at_first_match_does_not_truncate_child_siblings() { + // Regression guard: `stop_at_first_match` is a TOP-LEVEL classification + // concept. Once a parent matches, ALL of its matching child siblings must + // render even under `stop_at_first_match: true` -- the break must not fire + // inside a child sibling list. (An earlier revision applied the break at + // every recursion level, so the first message-bearing child silently + // truncated the rest -- dropping gzip's "max compression", "from Unix", + // "original size ..." fragments after the first match. See the + // `EvaluationConfig::stop_at_first_match` top-level-only contract.) + let mut parent = message_only_byte_rule(0, 0xAA, "parent"); + parent.children = vec![ + message_only_byte_rule(1, 0xBB, "child-1"), + message_only_byte_rule(2, 0xCC, "child-2"), + message_only_byte_rule(3, 0xDD, "child-3"), + ]; + + let buffer = &[0xAA, 0xBB, 0xCC, 0xDD]; + let config = EvaluationConfig { + stop_at_first_match: true, + ..Default::default() + }; + let mut context = EvaluationContext::new(config); + + let matches = evaluate_rules(&[parent], buffer, &mut context).unwrap(); + let messages: Vec<&str> = matches.iter().map(|m| m.message.as_str()).collect(); + assert_eq!( + messages, + vec!["parent", "child-1", "child-2", "child-3"], + "all matching child siblings must render under stop_at_first_match; \ + the break must not fire inside a child sibling list" + ); +} + #[test] fn test_evaluate_rules_hierarchical_parent_child() { let child_rule = MagicRule { @@ -2569,7 +2878,7 @@ fn test_search_rule_not_equal_succeeds_when_pattern_absent() { let rule = MagicRule { offset: OffsetSpec::Absolute(0), typ: TypeKind::Search { - range: ::std::num::NonZeroUsize::new(64).unwrap(), + range: ::std::num::NonZeroUsize::new(64), flags: SearchFlags::default(), }, op: Operator::NotEqual, @@ -2621,7 +2930,7 @@ fn test_search_rule_with_bitwise_operator_is_rejected() { let rule = MagicRule { offset: OffsetSpec::Absolute(0), typ: TypeKind::Search { - range: ::std::num::NonZeroUsize::new(32).unwrap(), + range: ::std::num::NonZeroUsize::new(32), flags: SearchFlags::default(), }, op: Operator::BitwiseAnd, @@ -2690,6 +2999,620 @@ fn test_regex_parent_advances_anchor_for_relative_child() { assert_eq!(matches[1].message, "first digit"); } +// ============================================================================= +// fix-system-magic-regex-graceful, U2: narrow graceful-skip of the +// missing-pattern-operand `TypeReadError::UnsupportedType` condition. +// +// Before this fix, a regex/search rule whose `value` operand was not a +// `String`/`Bytes` pattern (GOTCHAS S2.4) caused `evaluate_rules` to +// propagate a fatal `Err`, aborting evaluation of the ENTIRE rule set (and, +// via `MagicDatabase`, the entire magic file) rather than skipping just the +// one broken rule. See docs/plans/2026-07-17-001-fix-system-magic-regex- +// graceful-plan.md. +// ============================================================================= + +/// Builds a `TypeKind::Regex` rule whose `value` is `Value::Uint(0)` -- +/// neither `Value::String` nor `Value::Bytes` -- so `read_pattern_match` +/// always returns `Err(UnsupportedType { type_name: "regex without string +/// pattern" })`, regardless of U1's `Value::Bytes` backstop. This isolates +/// U2's engine-level skip from U1's evaluator-level acceptance. +fn broken_pattern_regex_rule(message: &str, level: u32) -> MagicRule { + MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Regex { + flags: crate::parser::ast::RegexFlags::default(), + count: crate::parser::ast::RegexCount::Default, + }, + op: Operator::Equal, + value: Value::Uint(0), + message: message.to_string(), + children: vec![], + level, + strength_modifier: None, + value_transform: None, + } +} + +/// FLOOR ANCHOR (U2 execution note): this is the regression test that must +/// be written and observed to FAIL before the engine fix is wired in. Prior +/// to the fix, `evaluate_rules` returns `Err(LibmagicError::EvaluationError( +/// EvaluationError::TypeReadError(TypeReadError::UnsupportedType { .. })))` +/// for a top-level pattern-less regex rule -- aborting analysis of every +/// target when the system magic DB contains such a rule. The floor +/// requirement (R1/R2) is that `evaluate_rules` must return `Ok` and treat +/// the broken rule as a non-match. +#[test] +fn test_evaluate_rules_skips_pattern_less_regex_rule_gracefully() { + let rule = broken_pattern_regex_rule("broken top-level regex", 0); + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let result = evaluate_rules(&[rule], b"anything to scan", &mut context); + let matches = + result.expect("evaluate_rules must not fatally abort on a pattern-less regex rule"); + assert!( + matches.is_empty(), + "the broken rule must contribute no match, got {matches:?}" + ); +} + +/// All-three-sites parity (a): a broken pattern-less regex at the TOP LEVEL +/// alongside a normal sibling rule -- the sibling must still match and the +/// broken rule must be silently skipped. +#[test] +fn test_pattern_operand_skip_at_top_level_site() { + let broken = broken_pattern_regex_rule("broken top-level regex", 0); + let sibling = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "leading a".to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = + EvaluationContext::new(EvaluationConfig::default().with_stop_at_first_match(false)); + let matches = evaluate_rules(&[broken, sibling], b"abc", &mut context) + .expect("top-level pattern-less regex must be skipped, not fatal"); + assert_eq!( + matches.len(), + 1, + "expected only the sibling match, got {matches:?}" + ); + assert_eq!(matches[0].message, "leading a"); +} + +/// All-three-sites parity (b): a broken pattern-less regex as a CHILD under +/// a matched parent (the inline child-recursion catch arm, ~L1108-1118). +/// The parent match must still be emitted even though its child is broken. +#[test] +fn test_pattern_operand_skip_under_matched_parent_child_recursion_site() { + let broken_child = broken_pattern_regex_rule("broken child regex", 1); + let parent = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "parent byte".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[parent], b"abc", &mut context) + .expect("a broken child regex must not abort evaluation of the parent"); + assert_eq!( + matches.len(), + 1, + "expected only the parent match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "parent byte"); +} + +/// All-three-sites parity (c): a broken pattern-less regex as a child of a +/// `default` rule -- exercises the `evaluate_children_or_warn` path +/// (~L522-530). The `default` match must still be emitted. +#[test] +fn test_pattern_operand_skip_under_default_children_or_warn_site() { + let broken_child = broken_pattern_regex_rule("broken default child regex", 1); + let default_rule = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Meta(MetaType::Default), + op: Operator::AnyValue, + value: Value::Uint(0), + message: "default fallback".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[default_rule], b"anything", &mut context) + .expect("a broken child regex under `default` must not abort evaluation"); + assert_eq!( + matches.len(), + 1, + "expected only the default match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "default fallback"); +} + +/// NEGATIVE (pins R3 narrowness): an `UnsupportedType` condition that is +/// NOT the missing-pattern-operand class -- here, a non-Equal/NotEqual +/// operator on a pattern-bearing type -- must still propagate fatally. +/// This proves U2's skip did not widen into swallowing the whole +/// `UnsupportedType` variant. +#[test] +fn test_evaluate_rules_propagates_non_pattern_missing_unsupported_type() { + let rule = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Regex { + flags: crate::parser::ast::RegexFlags::default(), + count: crate::parser::ast::RegexCount::Default, + }, + op: Operator::GreaterThan, + value: Value::String("[0-9]+".to_string()), + message: "bogus ordering".to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let result = evaluate_rules(&[rule], b"abc123", &mut context); + assert!( + matches!(result, Err(LibmagicError::EvaluationError(_))), + "a non-pattern-missing UnsupportedType must still propagate, got {result:?}" + ); +} + +/// NEGATIVE: a regex whose pattern fails to compile under the +/// `REGEX_COMPILE_SIZE_LIMIT` (1 MiB) CWE-1333 denial-of-service guard is skipped (not +/// fatal) per KTD5, but this is a distinct, louder-logged condition than +/// the ordinary missing-pattern skip. There is no log-capturing test seam +/// in this crate (no `test-log`/`tracing-test` dev-dependency), so this +/// test asserts only the behavioral half of the contract -- the rule is +/// skipped, not fatal -- and the `warn!` vs `debug!` split is verified by +/// code inspection (`log_pattern_operand_skip` in `engine/mod.rs`). +#[test] +fn test_pathological_regex_compile_failure_is_skipped_not_fatal() { + let rule = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Regex { + flags: crate::parser::ast::RegexFlags::default(), + count: crate::parser::ast::RegexCount::Default, + }, + op: Operator::Equal, + value: Value::String("a{1000000}".to_string()), + message: "pathological regex".to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[rule], b"aaaa", &mut context) + .expect("a regex compile-size rejection must be skipped, not fatal"); + assert!( + matches.is_empty(), + "the pathological regex rule must contribute no match, got {matches:?}" + ); +} + +/// IO/offset arms untouched (KTD4 regression guard): a `BufferOverrun` +/// condition (via an anchor pinned to `usize::MAX`) must still be skipped +/// exactly as before, proving U2 added a new arm rather than replacing the +/// pre-existing IO/offset catch set. +#[test] +fn test_buffer_overrun_still_skipped_after_pattern_operand_guard_added() { + let buffer = [0xAA, 0xBB, 0xCC, 0xDD]; + let mut ctx = EvaluationContext::new(EvaluationConfig::default()); + ctx.set_last_match_end(usize::MAX); + + let rule = MagicRule { + offset: OffsetSpec::Relative(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(0xAA), + message: "rel-zero-near-sat-regression".to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let matches = evaluate_rules(&[rule], &buffer, &mut ctx) + .expect("BufferOverrun must still be skipped gracefully, not propagated"); + assert!( + matches.is_empty(), + "Relative(0) at usize::MAX anchor must skip, not match or panic" + ); +} + +// ----------------------------------------------------------------------- +// C2 hardening: the missing-pattern-operand skip is asserted end-to-end +// only for `Regex` above. `Search` and flagged `String` share the SAME +// allowlisted consts (`types::SEARCH_MISSING_PATTERN_MSG` / +// `types::FLAGGED_STRING_MISSING_PATTERN_MSG`) and the same three engine +// catch sites, so this closes R2 for every pattern-bearing type and +// guards the C1 const extraction against silent drift. +// ----------------------------------------------------------------------- + +/// Builds a `TypeKind::Search` rule whose `value` is `Value::Uint(0)` -- +/// neither `Value::String` nor `Value::Bytes` -- so `read_pattern_match` +/// always returns `Err(UnsupportedType { type_name: +/// SEARCH_MISSING_PATTERN_MSG })`. +fn broken_pattern_search_rule(message: &str, level: u32) -> MagicRule { + MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Search { + range: ::std::num::NonZeroUsize::new(16), + flags: SearchFlags::default(), + }, + op: Operator::Equal, + value: Value::Uint(0), + message: message.to_string(), + children: vec![], + level, + strength_modifier: None, + value_transform: None, + } +} + +/// Builds a flagged `TypeKind::String` rule (non-empty `flags`, routing +/// through the pattern-bearing path per GOTCHAS S2.4) whose `value` is +/// `Value::Uint(0)`, so `read_pattern_match` always returns +/// `Err(UnsupportedType { type_name: FLAGGED_STRING_MISSING_PATTERN_MSG })`. +fn broken_pattern_flagged_string_rule(message: &str, level: u32) -> MagicRule { + MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::String { + max_length: None, + flags: StringFlags { + ignore_lowercase: true, + ..StringFlags::default() + }, + }, + op: Operator::Equal, + value: Value::Uint(0), + message: message.to_string(), + children: vec![], + level, + strength_modifier: None, + value_transform: None, + } +} + +/// Top-level site: a pattern-less `search` rule is skipped, not fatal. +#[test] +fn test_pattern_operand_skip_at_top_level_site_search() { + let broken = broken_pattern_search_rule("broken top-level search", 0); + let sibling = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "leading a".to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = + EvaluationContext::new(EvaluationConfig::default().with_stop_at_first_match(false)); + let matches = evaluate_rules(&[broken, sibling], b"abc", &mut context) + .expect("top-level pattern-less search must be skipped, not fatal"); + assert_eq!( + matches.len(), + 1, + "expected only the sibling match, got {matches:?}" + ); + assert_eq!(matches[0].message, "leading a"); +} + +/// Child-recursion site: a pattern-less `search` rule under a matched +/// parent must not abort the parent's match. +#[test] +fn test_pattern_operand_skip_under_matched_parent_child_recursion_site_search() { + let broken_child = broken_pattern_search_rule("broken child search", 1); + let parent = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "parent byte".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[parent], b"abc", &mut context) + .expect("a broken child search must not abort evaluation of the parent"); + assert_eq!( + matches.len(), + 1, + "expected only the parent match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "parent byte"); +} + +/// `evaluate_children_or_warn` site: a pattern-less `search` rule as a +/// child of `default` must not abort the `default` match. +#[test] +fn test_pattern_operand_skip_under_default_children_or_warn_site_search() { + let broken_child = broken_pattern_search_rule("broken default child search", 1); + let default_rule = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Meta(MetaType::Default), + op: Operator::AnyValue, + value: Value::Uint(0), + message: "default fallback".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[default_rule], b"anything", &mut context) + .expect("a broken child search under `default` must not abort evaluation"); + assert_eq!( + matches.len(), + 1, + "expected only the default match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "default fallback"); +} + +/// Top-level site: a pattern-less flagged `string` rule is skipped, not +/// fatal. +#[test] +fn test_pattern_operand_skip_at_top_level_site_flagged_string() { + let broken = broken_pattern_flagged_string_rule("broken top-level flagged string", 0); + let sibling = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "leading a".to_string(), + children: vec![], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = + EvaluationContext::new(EvaluationConfig::default().with_stop_at_first_match(false)); + let matches = evaluate_rules(&[broken, sibling], b"abc", &mut context) + .expect("top-level pattern-less flagged string must be skipped, not fatal"); + assert_eq!( + matches.len(), + 1, + "expected only the sibling match, got {matches:?}" + ); + assert_eq!(matches[0].message, "leading a"); +} + +/// Child-recursion site: a pattern-less flagged `string` rule under a +/// matched parent must not abort the parent's match. +#[test] +fn test_pattern_operand_skip_under_matched_parent_child_recursion_site_flagged_string() { + let broken_child = broken_pattern_flagged_string_rule("broken child flagged string", 1); + let parent = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "parent byte".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[parent], b"abc", &mut context) + .expect("a broken child flagged string must not abort evaluation of the parent"); + assert_eq!( + matches.len(), + 1, + "expected only the parent match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "parent byte"); +} + +/// `evaluate_children_or_warn` site: a pattern-less flagged `string` rule +/// as a child of `default` must not abort the `default` match. +#[test] +fn test_pattern_operand_skip_under_default_children_or_warn_site_flagged_string() { + let broken_child = broken_pattern_flagged_string_rule("broken default child flagged string", 1); + let default_rule = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Meta(MetaType::Default), + op: Operator::AnyValue, + value: Value::Uint(0), + message: "default fallback".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[default_rule], b"anything", &mut context) + .expect("a broken child flagged string under `default` must not abort evaluation"); + assert_eq!( + matches.len(), + 1, + "expected only the default match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "default fallback"); +} + +// ----------------------------------------------------------------------- +// E hardening: the compile-failure (warn!) skip is proven end-to-end only +// at the top-level dispatch site above +// (`test_pathological_regex_compile_failure_is_skipped_not_fatal`). Add +// the two missing sites for 3-site parity with the missing-pattern +// (debug!) coverage. +// ----------------------------------------------------------------------- + +/// Builds a `TypeKind::Regex` rule whose pattern is syntactically valid +/// (`Value::String`, so U1's `Value::Bytes` backstop is irrelevant here) +/// but rejected by the `REGEX_COMPILE_SIZE_LIMIT` (1 MiB) CWE-1333 +/// denial-of-service guard at compile time. +fn pathological_regex_rule(message: &str, level: u32) -> MagicRule { + MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Regex { + flags: crate::parser::ast::RegexFlags::default(), + count: crate::parser::ast::RegexCount::Default, + }, + op: Operator::Equal, + value: Value::String("a{1000000}".to_string()), + message: message.to_string(), + children: vec![], + level, + strength_modifier: None, + value_transform: None, + } +} + +/// Child-recursion site: a regex compile-size rejection under a matched +/// parent must not abort the parent's match. +#[test] +fn test_pathological_regex_compile_failure_skipped_under_matched_parent_child_recursion_site() { + let broken_child = pathological_regex_rule("broken compile child regex", 1); + let parent = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'a')), + message: "parent byte".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[parent], b"aaaa", &mut context) + .expect("a regex compile-size rejection under a matched parent must not abort evaluation"); + assert_eq!( + matches.len(), + 1, + "expected only the parent match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "parent byte"); +} + +/// `evaluate_children_or_warn` site: a regex compile-size rejection as a +/// child of `default` must not abort the `default` match. +#[test] +fn test_pathological_regex_compile_failure_skipped_under_default_children_or_warn_site() { + let broken_child = pathological_regex_rule("broken compile default child regex", 1); + let default_rule = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::Meta(MetaType::Default), + op: Operator::AnyValue, + value: Value::Uint(0), + message: "default fallback".to_string(), + children: vec![broken_child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[default_rule], b"aaaa", &mut context) + .expect("a regex compile-size rejection under default children must not abort evaluation"); + assert_eq!( + matches.len(), + 1, + "expected only the default match (broken child skipped), got {matches:?}" + ); + assert_eq!(matches[0].message, "default fallback"); +} + +// ----------------------------------------------------------------------- +// H hardening: pin the debug!/warn! log-level contract with a real +// log-capture seam (`testing_logger`, which captures the `log` facade +// this crate uses -- not `tracing`). Previously these contracts were +// asserted by code inspection only. +// ----------------------------------------------------------------------- + +/// Test-only helper: `testing_logger::CapturedLog` does not implement +/// `Debug`, so format captured logs manually for failure messages. +fn format_logs(logs: &[testing_logger::CapturedLog]) -> String { + logs.iter() + .map(|l| format!("{:?}: {}", l.level, l.body)) + .collect::>() + .join(", ") +} + +/// The ordinary missing-pattern-operand skip (top-level site) logs at +/// `debug!`, not `warn!` -- it is an expected, low-severity data +/// condition, not a security-relevant signal. +#[test] +fn test_missing_pattern_operand_skip_logs_at_debug_level() { + testing_logger::setup(); + let rule = broken_pattern_regex_rule("broken top-level regex", 0); + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[rule], b"anything to scan", &mut context) + .expect("pattern-less regex must be skipped, not fatal"); + assert!(matches.is_empty()); + testing_logger::validate(|captured_logs| { + let skip_logs: Vec<_> = captured_logs + .iter() + .filter(|l| l.body.contains("Skipping top-level rule")) + .collect(); + assert_eq!( + skip_logs.len(), + 1, + "expected exactly one skip log entry, got {:?}", + format_logs(captured_logs) + ); + assert_eq!( + skip_logs[0].level, + log::Level::Debug, + "missing-pattern-operand skip must log at debug!, not warn! -- \ + got {:?}: {:?}", + skip_logs[0].level, + skip_logs[0].body + ); + }); +} + +/// A regex compile-size rejection (`REGEX_COMPILE_SIZE_LIMIT`, +/// CWE-1333) logs at `warn!`, not `debug!` -- a malicious or pathological +/// magic file's rejection must stay visible in logs even though +/// evaluation of the rest of the file continues. +#[test] +fn test_regex_compile_failure_skip_logs_at_warn_level() { + testing_logger::setup(); + let rule = pathological_regex_rule("pathological regex", 0); + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[rule], b"aaaa", &mut context) + .expect("a regex compile-size rejection must be skipped, not fatal"); + assert!(matches.is_empty()); + testing_logger::validate(|captured_logs| { + let skip_logs: Vec<_> = captured_logs + .iter() + .filter(|l| l.body.contains("regex compile failure")) + .collect(); + assert_eq!( + skip_logs.len(), + 1, + "expected exactly one compile-failure log entry, got {:?}", + format_logs(captured_logs) + ); + assert_eq!( + skip_logs[0].level, + log::Level::Warn, + "regex compile-failure skip must log at warn!, not debug! -- \ + got {:?}: {:?}", + skip_logs[0].level, + skip_logs[0].body + ); + }); +} + /// A child rule with `OffsetSpec::Relative(0)` after a parent search match /// must land at `match_index + pattern.len()` — NOT at `window_end` (the /// pre-fix window-size advance would land on a completely different byte). @@ -2714,7 +3637,7 @@ fn test_search_parent_advances_anchor_to_match_end_not_window_end() { let parent = MagicRule { offset: OffsetSpec::Absolute(0), typ: TypeKind::Search { - range: ::std::num::NonZeroUsize::new(14).unwrap(), + range: ::std::num::NonZeroUsize::new(14), flags: SearchFlags::default(), }, op: Operator::Equal, @@ -2754,7 +3677,7 @@ fn test_search_parent_relative_child_at_positive_offset() { let parent = MagicRule { offset: OffsetSpec::Absolute(0), typ: TypeKind::Search { - range: ::std::num::NonZeroUsize::new(32).unwrap(), + range: ::std::num::NonZeroUsize::new(32), flags: SearchFlags::default(), }, op: Operator::Equal, @@ -2772,6 +3695,79 @@ fn test_search_parent_relative_child_at_positive_offset() { assert_eq!(matches[1].message, "a after"); } +/// Regression (end-to-end): the Mach-O 64-bit signature rule +/// `0 lelong&0xfffffffe 0xfeedface` must match a buffer beginning with the +/// little-endian magic `cf fa ed fe`. `lelong` is signed, so the read is +/// sign-extended to i64; before the width-aware masked-comparison fix the +/// 32-bit mask cleared the high bits (making the result positive) while the +/// rule literal stayed sign-extended (negative), so the two i64 values never +/// compared equal and the rule silently failed -- letting a weak +/// `measure`/Lepton rule win on real Mach-O binaries. See +/// `operators::apply_bitwise_and_mask_with_width`. +#[test] +fn test_signed_masked_long_matches_macho_signature_end_to_end() { + use crate::parser::grammar::parse_magic_rule; + + let (_, rule) = parse_magic_rule("0\tlelong&0xfffffffe\t0xfeedface\tMach-O") + .expect("the Mach-O magic rule must parse"); + // Real Mach-O 64-bit header prefix (0xFEEDFACF little-endian) + padding. + let buffer = [0xcf_u8, 0xfa, 0xed, 0xfe, 0x0c, 0x00, 0x00, 0x01]; + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = + evaluate_rules(&[rule], &buffer, &mut context).expect("evaluation must not fatally error"); + assert!( + matches.iter().any(|m| m.message.contains("Mach-O")), + "signed lelong&0xfffffffe must detect the Mach-O signature, got: {matches:?}" + ); + + // Negative control: a non-Mach-O long must NOT match. + let (_, rule2) = + parse_magic_rule("0\tlelong&0xfffffffe\t0xfeedface\tMach-O").expect("rule must parse"); + let other = [0x12_u8, 0x34, 0x56, 0x78]; + let mut ctx2 = EvaluationContext::new(EvaluationConfig::default()); + let none = evaluate_rules(&[rule2], &other, &mut ctx2).expect("evaluation must not error"); + assert!( + !none.iter().any(|m| m.message.contains("Mach-O")), + "a non-Mach-O buffer must not match the signature" + ); +} + +/// Regression (end-to-end): a `string` rule whose value contains a byte +/// `>= 0x80` (invalid UTF-8) must still match the raw file bytes. The gzip +/// signature `0 string \037\213` (bytes 0x1f 0x8b, `\213` = octal 0x8b) used +/// to silently never match: `read_string_exact` decoded the file bytes via +/// lossy UTF-8, turning 0x8b into U+FFFD, which never equalled the raw-byte +/// pattern -- so gzip (and any high-byte string signature) classified as +/// `data`. `read_string_exact` now returns the raw bytes as `Value::Bytes` +/// for non-UTF-8 slices; `apply_equal` compares Bytes/String by byte sequence. +#[test] +fn test_string_rule_with_high_byte_value_matches_raw_bytes_end_to_end() { + use crate::parser::grammar::parse_magic_rule; + + // gzip magic: 0x1f 0x8b via octal escapes. + let (_, rule) = parse_magic_rule("0\tstring\t\\037\\213\tgzip compressed data") + .expect("gzip magic rule must parse"); + let buffer = [0x1f_u8, 0x8b, 0x08, 0x00]; // real gzip header prefix + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = + evaluate_rules(&[rule], &buffer, &mut context).expect("evaluation must not error"); + assert!( + matches.iter().any(|m| m.message.contains("gzip")), + "a high-byte string value must match the raw file bytes, got: {matches:?}" + ); + + // Negative control: a buffer without the signature must not match. + let (_, rule2) = + parse_magic_rule("0\tstring\t\\037\\213\tgzip compressed data").expect("rule must parse"); + let other = [0x1f_u8, 0x9d, 0x00, 0x00]; // 0x1f 0x9d is compress(1), not gzip + let mut ctx2 = EvaluationContext::new(EvaluationConfig::default()); + let none = evaluate_rules(&[rule2], &other, &mut ctx2).expect("evaluation must not error"); + assert!( + !none.iter().any(|m| m.message.contains("gzip")), + "a non-gzip high byte must not match the gzip signature" + ); +} + // Flagged-string engine dispatch tests are split into // `string_flags_dispatch_tests.rs` (see the submodule declaration at the // bottom of this file). They cover the engine routing layer; lower-level diff --git a/src/evaluator/engine/tests/string_flags_dispatch_tests.rs b/src/evaluator/engine/tests/string_flags_dispatch_tests.rs index 9e0f7151..41d84abe 100644 --- a/src/evaluator/engine/tests/string_flags_dispatch_tests.rs +++ b/src/evaluator/engine/tests/string_flags_dispatch_tests.rs @@ -99,19 +99,46 @@ fn test_flagged_string_not_equal_inverts_match() { } #[test] -fn test_flagged_string_ordering_operator_is_rejected() { - // Same contract as regex/search: only Equal/NotEqual are allowed on - // pattern-bearing rules. GreaterThan must surface as EvaluationError. +fn test_flagged_string_ordering_operator_uses_lexicographic_value_path() { + // An ORDERING operator on a flagged string is a lexicographic comparison, + // not a pattern match, so it routes to the value path (like an unflagged + // `string >VALUE`) instead of the equality-only pattern-bearing path. + // This is the ubiquitous `string/t >\0` / `string/b >\0` "there is + // non-empty text here, print it with %s" idiom (varied.script, sgml, + // linux, ...). Routing it to the pattern path made it a fatal + // `UnsupportedType` that aborted the ENTIRE file's evaluation -- e.g. + // `rmagic` used to error out on a Bourne-Again shell script. The `/t`/`/b` + // flags are MIME-output hints with no ordering effect. (Case-fold flags + // like `/c` are also not applied to ordering, but such rules do not occur + // in real magic files; the value path's byte-lexicographic compare is the + // correct behavior for the flags that actually pair with `<`/`>`.) let rule = make_flagged_string_rule( - "foo", - StringFlags::default().with_ignore_lowercase(true), + "\0", + StringFlags::default().with_text_test(true), Operator::GreaterThan, ); let mut context = EvaluationContext::new(EvaluationConfig::default()); - let result = evaluate_single_rule(&rule, b"FOObar", &mut context); + let matches = evaluate_rules(&[rule], b"FOObar", &mut context) + .expect("ordering operator on a flagged string must not fatally abort"); + assert_eq!( + matches.len(), + 1, + "a non-empty string is lexicographically greater than the null byte" + ); + + // Negative control: an empty buffer has no string > \\0, so no match -- + // and still no abort. + let rule2 = make_flagged_string_rule( + "\0", + StringFlags::default().with_text_test(true), + Operator::GreaterThan, + ); + let mut ctx2 = EvaluationContext::new(EvaluationConfig::default()); + let empty = evaluate_rules(&[rule2], b"", &mut ctx2) + .expect("ordering operator must not abort even on an empty buffer"); assert!( - matches!(result, Err(LibmagicError::EvaluationError(_))), - "expected EvaluationError for ordering operator on flagged string" + empty.is_empty(), + "no string is present in an empty buffer, so `>\\0` must not match" ); } @@ -155,3 +182,115 @@ fn test_flagged_string_w_whitespace_consumes_extra_file_bytes_for_anchor() { assert_eq!(matches.len(), 2, "parent + child must both match"); assert_eq!(matches[1].message, "exclaim"); } + +#[test] +fn test_string_ordering_renders_full_field_not_compared_prefix() { + // libmagic compares a `string >VALUE` rule prefix-limited to + // `pattern.len()` (`file_strncmp` with `vallen`) but RENDERS the full + // string field (`p->s`). The engine must therefore return the full field + // as the match's DISPLAY value, not just the compared prefix -- otherwise + // sgml's `>15 string/t >\0 %.3s ...` renders the XML version `1` instead + // of `1.0`. Here the pattern is 5 bytes ("0.6.1") but the display value + // must be the whole field. + let rule = make_flagged_string_rule("0.6.1", StringFlags::default(), Operator::GreaterThan); + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[rule], b"0.6.2 and more text", &mut context).unwrap(); + assert_eq!(matches.len(), 1, "0.6.2 > 0.6.1 must match"); + assert_eq!( + matches[0].value, + Value::String("0.6.2 and more text".to_string()), + "display value must be the FULL string field, not the compared 5-byte prefix" + ); +} + +#[test] +fn test_string_ordering_comparison_stays_prefix_limited() { + // Guard against a future revert to full-field COMPARISON. libmagic + // compares only the first `pattern.len()` bytes: buffer "0.6.10" has + // prefix "0.6.1" == pattern, so `> 0.6.1` is FALSE even though the full + // string "0.6.10" sorts after "0.6.1". Verified against the real `file` + // binary (file-5.41): `0.6.10` does NOT match `>0.6.1`, but `0.6.2` does. + let rule = make_flagged_string_rule("0.6.1", StringFlags::default(), Operator::GreaterThan); + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let no_match = evaluate_rules(&[rule], b"0.6.10more", &mut context).unwrap(); + assert!( + no_match.is_empty(), + "prefix '0.6.1' equals the pattern, so `> 0.6.1` must be false (prefix-limited compare)" + ); + + // Positive control: a genuinely-greater prefix matches. + let rule2 = make_flagged_string_rule("0.6.1", StringFlags::default(), Operator::GreaterThan); + let mut ctx2 = EvaluationContext::new(EvaluationConfig::default()); + let m = evaluate_rules(&[rule2], b"0.6.2", &mut ctx2).unwrap(); + assert_eq!(m.len(), 1, "0.6.2 > 0.6.1 within the compared prefix"); +} + +#[test] +fn test_string_ordering_full_field_display_does_not_move_relative_anchor() { + // The full-field DISPLAY read must not disturb the relative-offset + // anchor: `bytes_consumed_with_pattern` re-derives the advance from the + // PATTERN (5 bytes for "0.6.1"), never from the display value. A child at + // Relative(0) must resolve to offset 5 (just past the compared prefix), + // NOT to the full-field length. Pins the advisor's anchor-stability + // concern for the decoupled display read. + let child = MagicRule { + offset: OffsetSpec::Relative(0), + typ: TypeKind::Byte { signed: false }, + op: Operator::Equal, + value: Value::Uint(u64::from(b'X')), + message: "at-offset-5".to_string(), + children: vec![], + level: 1, + strength_modifier: None, + value_transform: None, + }; + let parent = MagicRule { + offset: OffsetSpec::Absolute(0), + typ: TypeKind::String { + max_length: None, + flags: StringFlags::default(), + }, + op: Operator::GreaterThan, + value: Value::String("0.6.1".to_string()), + message: "ver".to_string(), + children: vec![child], + level: 0, + strength_modifier: None, + value_transform: None, + }; + let mut context = + EvaluationContext::new(EvaluationConfig::default().with_stop_at_first_match(false)); + // buffer: bytes 0..4 = "0.6.2" (matches >0.6.1), byte 5 = 'X' (child + // target), then a long tail that becomes the full-field display value. + let matches = + evaluate_rules(&[parent], b"0.6.2X and a long trailing field", &mut context).unwrap(); + assert_eq!(matches.len(), 2, "parent + child must both match"); + assert_eq!(matches[1].message, "at-offset-5"); + assert_eq!( + matches[1].offset, 5, + "child must resolve just past the 5-byte pattern, not the full field" + ); +} + +#[test] +fn test_flagged_string_any_value_operator_does_not_panic_end_to_end() { + // Regression: `0 string/b x` -- a flagged string with the AnyValue (`x`) + // operator, a common message-less gating rule (e.g. compress's zlib + // detector `0 string/b x` -> `>0 beshort%31 =0` ...) -- routes to the value + // path (not the equality-only pattern path). Its rule value is the AnyValue + // placeholder `Uint(0)`, not a string. `bytes_consumed_with_pattern` used to + // dispatch ALL flagged strings to the pattern walker regardless of operator + // and hit a `debug_assert!(false)` on the missing string pattern, panicking + // in debug builds (a no-panic-policy violation). It must instead treat the + // read like a normal string and evaluate without panicking. + use crate::parser::grammar::parse_magic_rule; + + let (_, rule) = parse_magic_rule("0\tstring/b\tx\tgating").expect("rule must parse"); + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let matches = evaluate_rules(&[rule], b"\x0b\x30 arbitrary binary bytes", &mut context) + .expect("a flagged string with the AnyValue operator must evaluate without panicking"); + assert!( + matches.iter().any(|m| m.message.contains("gating")), + "AnyValue (`x`) matches any buffer, so the gating rule fires" + ); +} diff --git a/src/evaluator/mod.rs b/src/evaluator/mod.rs index c966dfb8..eef9a286 100644 --- a/src/evaluator/mod.rs +++ b/src/evaluator/mod.rs @@ -18,6 +18,25 @@ pub mod types; pub use engine::{evaluate_rules, evaluate_rules_with_config, evaluate_single_rule}; +/// Strip a single leading GNU `file` no-separator marker from `s`, if present. +/// +/// The marker (GOTCHAS S14.1) suppresses the separating space when a +/// description is appended to a sibling's text. It reaches evaluator code in +/// two forms: the raw byte `U+0008` (backspace), used for programmatically +/// constructed messages, and -- far more commonly -- the literal +/// two-character sequence `\b` (backslash + `'b'`), because the message parser +/// preserves description text verbatim (matching GNU `file`, which keeps the +/// desc literal and special-cases a leading `\b` at print time). +/// +/// Returns `Some(rest)` with the marker removed when one is present, else +/// `None`. This is the single source of truth for the two-form marker so the +/// three call sites (`concatenate_messages` output rendering, +/// `is_message_bearing` gating, and `evaluate_use_rule` name-message +/// attachment) cannot drift on which forms they recognize. +pub(crate) fn strip_no_separator_marker(s: &str) -> Option<&str> { + s.strip_prefix('\u{0008}').or_else(|| s.strip_prefix("\\b")) +} + /// Shared environment attached to an [`EvaluationContext`] so the engine can /// resolve whole-database operations (currently: `Use` subroutine lookups; /// eventually `indirect` whole-tree re-entry). @@ -102,6 +121,18 @@ pub struct EvaluationContext { /// because top-level rules in the re-entered database should /// chain sibling anchors like any other top-level evaluation. indirect_reentry: bool, + /// Endian-flip state for `use \^name` subroutine invocation (magic(5) + /// `\^` prefix; libmagic `softmagic.c` `cvt_flip`). When true, every + /// endian-bearing typed read inside the current subroutine body + /// (`short`/`long`/`quad`/`float`/`double`/date families) has its + /// declared little/big endianness swapped before the read. A `use + /// \^name` site *toggles* this flag (not sets it) and the toggle + /// propagates into nested `use` calls, matching libmagic's `flip = + /// !flip` parameter threading. Saved and restored around the + /// subroutine call by the `SubroutineScope` RAII guard in + /// `engine/mod.rs`. Native-endian types and `String16` are never + /// flipped (they are absent from `cvt_flip`). + flip_endian: bool, } impl EvaluationContext { @@ -154,9 +185,26 @@ impl EvaluationContext { rule_env: None, base_offset: 0, indirect_reentry: false, + flip_endian: false, } } + /// Read-only access to the `use \^name` endian-flip state. True only + /// while evaluating a subroutine body reached through an odd number of + /// `\^`-prefixed `use` invocations. + #[must_use] + pub(crate) const fn flip_endian(&self) -> bool { + self.flip_endian + } + + /// Set the endian-flip state. + /// + /// `pub(crate)` and owned by the engine's `SubroutineScope` RAII guard + /// -- no external caller should set this directly. + pub(crate) fn set_flip_endian(&mut self, flip: bool) { + self.flip_endian = flip; + } + /// Read-only access to the subroutine base offset. Non-zero only /// during a `MetaType::Use` body evaluation. #[must_use] @@ -375,6 +423,7 @@ impl EvaluationContext { self.recursion_depth = 0; self.base_offset = 0; self.indirect_reentry = false; + self.flip_endian = false; } } diff --git a/src/evaluator/offset/absolute.rs b/src/evaluator/offset/absolute.rs index c4c08da2..7d934660 100644 --- a/src/evaluator/offset/absolute.rs +++ b/src/evaluator/offset/absolute.rs @@ -99,9 +99,25 @@ pub fn resolve_absolute_offset(offset: i64, buffer: &[u8]) -> Result= 0 { - // Positive offset from start + // Positive offset from start. + // + // The bound is `>` (not `>=`): `offset == buffer_len` is the EOF + // position and is a VALID resolution target, matching libmagic's + // model where offset resolution is permissive and each type read + // enforces its own width. Verified against real `file` (file-5.41): + // for a rule whose child offset lands exactly at EOF, a numeric + // child (`byte x`, `short x`, ...) is dropped -- its width-checked + // read fails at EOF and produces a non-match -- while a `string x` + // child renders an EMPTY string. LUKS's `>8 string x [%s,` on a + // header truncated to 8 bytes prints `[,` in GNU `file`; without + // permitting `offset == buffer_len` here, that child was silently + // dropped at offset resolution. Width enforcement now lives entirely + // in the readers: fixed-width readers use bounds-safe `.get()` / + // `read_bytes_at` (BufferOverrun -> non-match at EOF), and + // `read_string` returns an empty string at `offset == buffer_len`. + // See GOTCHAS S15.1. let abs_offset = usize::try_from(offset).map_err(|_| OffsetError::ArithmeticOverflow)?; - if abs_offset >= buffer_len { + if abs_offset > buffer_len { return Err(OffsetError::BufferOverrun { offset: abs_offset, buffer_len, @@ -156,17 +172,28 @@ mod tests { assert_eq!(resolve_absolute_offset(-13, buffer).unwrap(), 0); // First character } + #[test] + fn test_resolve_absolute_offset_at_eof_is_permitted() { + let buffer = b"Hello"; // len 5 + + // `offset == buffer_len` is the EOF position and resolves + // successfully -- libmagic permits it, deferring width enforcement + // to the type read (a numeric read fails at EOF -> non-match; a + // `string x` read yields an empty string). See GOTCHAS S15.1. + assert_eq!(resolve_absolute_offset(5, buffer).unwrap(), 5); + } + #[test] fn test_resolve_absolute_offset_out_of_bounds_positive() { - let buffer = b"Hello"; + let buffer = b"Hello"; // len 5 - // Test positive offset beyond buffer - let result = resolve_absolute_offset(5, buffer); + // Strictly beyond EOF (offset > buffer_len) is a genuine overrun. + let result = resolve_absolute_offset(6, buffer); assert!(result.is_err()); match result.unwrap_err() { OffsetError::BufferOverrun { offset, buffer_len } => { - assert_eq!(offset, 5); + assert_eq!(offset, 6); assert_eq!(buffer_len, 5); } _ => panic!("Expected BufferOverrun error"), @@ -201,8 +228,14 @@ mod tests { fn test_resolve_absolute_offset_empty_buffer() { let buffer = b""; - // Any offset in empty buffer should fail - assert!(resolve_absolute_offset(0, buffer).is_err()); + // `offset == buffer_len` holds trivially for an empty buffer + // (0 == 0), so offset 0 resolves to the EOF position 0 -- the type + // read then decides: a numeric read finds no bytes (non-match) and + // a `string x` read yields an empty string. No system-DB rule uses + // a top-level `0 string x`, and GNU `file` still classifies an + // empty file as "empty" (verified). See GOTCHAS S15.1. + assert_eq!(resolve_absolute_offset(0, buffer).unwrap(), 0); + // Strictly past EOF still fails. assert!(resolve_absolute_offset(1, buffer).is_err()); assert!(resolve_absolute_offset(-1, buffer).is_err()); } @@ -214,9 +247,12 @@ mod tests { // Valid cases assert_eq!(resolve_absolute_offset(0, buffer).unwrap(), 0); assert_eq!(resolve_absolute_offset(-1, buffer).unwrap(), 0); + // offset == buffer_len (1) is the EOF position, now permitted + // (width enforced at read time). See GOTCHAS S15.1. + assert_eq!(resolve_absolute_offset(1, buffer).unwrap(), 1); - // Invalid cases - assert!(resolve_absolute_offset(1, buffer).is_err()); + // Invalid cases: strictly past EOF. + assert!(resolve_absolute_offset(2, buffer).is_err()); assert!(resolve_absolute_offset(-2, buffer).is_err()); } @@ -235,8 +271,10 @@ mod tests { assert_eq!(resolve_absolute_offset(-512, &large_buffer).unwrap(), 512); assert_eq!(resolve_absolute_offset(-1024, &large_buffer).unwrap(), 0); - // Test out of bounds - assert!(resolve_absolute_offset(1024, &large_buffer).is_err()); + // offset == buffer_len (1024) is the EOF position, now permitted. + assert_eq!(resolve_absolute_offset(1024, &large_buffer).unwrap(), 1024); + // Strictly past EOF still fails. + assert!(resolve_absolute_offset(1025, &large_buffer).is_err()); assert!(resolve_absolute_offset(-1025, &large_buffer).is_err()); } @@ -252,9 +290,13 @@ mod tests { let result = resolve_absolute_offset(offset, buffer); // Should either succeed with valid offset or fail gracefully if let Ok(resolved) = result { - // If it succeeds, the resolved offset must be within buffer bounds + // If it succeeds, the resolved offset must be at most the + // buffer length. `offset == buffer_len` (the EOF position) + // is a valid resolution target now that width enforcement + // lives in the type read (GOTCHAS S15.1); anything strictly + // greater is a genuine overrun and would have errored. assert!( - resolved < buffer.len(), + resolved <= buffer.len(), "Resolved offset {resolved} exceeds buffer length {}", buffer.len() ); diff --git a/src/evaluator/offset/mod.rs b/src/evaluator/offset/mod.rs index 8685d7e9..dbf41ea4 100644 --- a/src/evaluator/offset/mod.rs +++ b/src/evaluator/offset/mod.rs @@ -171,9 +171,18 @@ pub(crate) fn resolve_offset_with_base( } OffsetSpec::Relative(_) => relative::resolve_relative_offset(spec, buffer, last_match_end), OffsetSpec::FromEnd(offset) => { - // FromEnd is handled the same as negative Absolute offsets. - // Base offset does not apply -- "from end" is always - // relative to the buffer itself. + // `FromEnd(0)` is the magic(5) `-0` form: the end-of-file + // *position* (`buffer.len()`), one past the last readable byte. + // It is valid as a position value for the `offset` pseudo-type + // (which never reads there) even though `resolve_absolute_offset` + // would wrongly send `0` through its positive path and resolve to + // start-of-file. Negative `FromEnd` deltas keep the shared + // from-end resolution (identical to negative `Absolute`); base + // offset never applies -- "from end" is always relative to the + // buffer itself. + if *offset == 0 { + return Ok(buffer.len()); + } resolve_absolute_offset(*offset, buffer).map_err(|e| map_offset_error(&e, *offset)) } } @@ -279,6 +288,31 @@ mod tests { assert_eq!(resolve_offset_with_context(&spec, buffer, 999).unwrap(), 6); } + #[test] + fn test_resolve_from_end_zero_is_eof_position() { + // The magic(5) `-0` form (`FromEnd(0)`) resolves to the end-of-file + // POSITION -- `buffer.len()`, one past the last byte -- NOT offset 0. + // `resolve_absolute_offset(0)` would wrongly send it through the + // positive path and yield 0/start; the FromEnd arm special-cases it. + // Used by gzip's `>>-0 offset >48` trailing-size gate. + let buffer = b"Test data"; // 9 bytes + assert_eq!( + resolve_offset_with_context(&OffsetSpec::FromEnd(0), buffer, 0).unwrap(), + buffer.len(), + "FromEnd(0) must resolve to the EOF position (buffer.len())" + ); + // Distinct from a real absolute 0. + assert_eq!( + resolve_offset_with_context(&OffsetSpec::Absolute(0), buffer, 0).unwrap(), + 0 + ); + // Empty buffer: EOF position is 0, and that must not error. + assert_eq!( + resolve_offset_with_context(&OffsetSpec::FromEnd(0), b"", 0).unwrap(), + 0 + ); + } + #[test] fn test_resolve_offset_with_context_passthrough_indirect() { // Same indirect setup as test_resolve_offset_indirect_success above. diff --git a/src/evaluator/operators/bitwise.rs b/src/evaluator/operators/bitwise.rs index 3db25d52..519df15a 100644 --- a/src/evaluator/operators/bitwise.rs +++ b/src/evaluator/operators/bitwise.rs @@ -39,10 +39,85 @@ use crate::parser::ast::Value; /// ``` #[must_use] pub fn apply_bitwise_and_mask(mask: u64, left: &Value, right: &Value) -> bool { - let masked_left = match left { - Value::Uint(val) => Value::Uint(val & mask), - Value::Int(val) => { - // Convert u64 mask to i64, using bitwise representation for values > i64::MAX + apply_bitwise_and_mask_with_width(mask, left, right, None) +} + +/// Apply bitwise AND with mask, re-normalizing the masked result to the +/// type's natural bit width before comparison. +/// +/// This is the width-aware companion to [`apply_bitwise_and_mask`], required +/// for correct evaluation of masked comparisons on **signed** types. A magic +/// rule like `0 lelong&0xfffffffe 0xfeedface` (the Mach-O 64-bit signature) +/// reads a signed 32-bit long, which the evaluator sign-extends to `i64` +/// (`0xFEEDFACF` -> `0xFFFF_FFFF_FEED_FACF`). Applying the 32-bit mask +/// `0xfffffffe` in `i64` space clears the high 32 bits, yielding a *positive* +/// `0x0000_0000_FEED_FACE`; but the rule literal `0xfeedface`, coerced to the +/// signed type, is stored *sign-extended* as `0xFFFF_FFFF_FEED_FACE` +/// (negative). The two `i64` values differ in their high 32 bits and never +/// compare equal even though the low 32 bits match -- so the Mach-O rule +/// silently fails and lower-strength rules win. +/// +/// When `bit_width` is `Some(w)` with `w < 64` and the value is signed +/// (`Value::Int`), this masks within the `w`-bit window and then re-sign-extends +/// from bit `w-1`, mirroring libmagic's `(int32_t)(v & mask)` in +/// `mconvert`/`magiccheck` (`src/softmagic.c`). Unsigned values are masked +/// within the width (zero-extended). `bit_width == None` (or `>= 64`) preserves +/// the historical width-unaware behavior for callers that lack type context. +/// +/// # Arguments +/// +/// * `mask` - The bitmask to apply to the left value +/// * `left` - The left-hand side value (typically from file data) +/// * `right` - The right-hand side value (typically from the magic rule) +/// * `bit_width` - The natural bit width of the rule's type (`8`/`16`/`32`/`64`) +/// +/// # Examples +/// +/// ``` +/// use libmagic_rs::parser::ast::Value; +/// use libmagic_rs::evaluator::operators::apply_bitwise_and_mask_with_width; +/// +/// // The Mach-O case: signed 32-bit read 0xFEEDFACF (sign-extended to i64), +/// // masked with 0xfffffffe, compared to the sign-extended rule value. +/// let read = Value::Int(0xFFFF_FFFF_FEED_FACF_u64 as i64); // lelong sign-extended +/// let rule = Value::Int(0xFFFF_FFFF_FEED_FACE_u64 as i64); // 0xfeedface coerced to lelong +/// assert!(apply_bitwise_and_mask_with_width(0xffff_fffe, &read, &rule, Some(32))); +/// +/// // Without the width, the historical behavior does NOT match (the bug): +/// assert!(!apply_bitwise_and_mask_with_width(0xffff_fffe, &read, &rule, None)); +/// ``` +#[must_use] +pub fn apply_bitwise_and_mask_with_width( + mask: u64, + left: &Value, + right: &Value, + bit_width: Option, +) -> bool { + let masked_left = match (left, bit_width) { + // Unsigned with a known type width: mask within the width (zero-extended). + (Value::Uint(val), Some(width)) if width < 64 => { + let width_mask = (1u64 << width) - 1; + Value::Uint((val & mask) & width_mask) + } + (Value::Uint(val), _) => Value::Uint(val & mask), + // Signed with a known type width: mask within the width, then + // re-sign-extend from the type's sign bit so the result matches the + // sign-extended rule literal (see the doc comment for the Mach-O case). + (Value::Int(val), Some(width)) if width < 64 => { + let width_mask = (1u64 << width) - 1; + let masked = u64::from_ne_bytes(val.to_ne_bytes()) & mask & width_mask; + let sign_bit = 1u64 << (width - 1); + let extended = if masked & sign_bit != 0 { + masked | !width_mask + } else { + masked + }; + // Bit-reinterpret u64 -> i64 (matches the `from_ne_bytes` idiom + // used for the mask above; avoids `clippy::cast_possible_wrap`). + Value::Int(i64::from_ne_bytes(extended.to_ne_bytes())) + } + // Signed without a known width (or width >= 64): historical behavior. + (Value::Int(val), _) => { let i64_mask = i64::try_from(mask).unwrap_or_else(|_| i64::from_ne_bytes(mask.to_ne_bytes())); Value::Int(val & i64_mask) @@ -55,8 +130,9 @@ pub fn apply_bitwise_and_mask(mask: u64, left: &Value, right: &Value) -> bool { /// Apply bitwise AND operation for pattern matching /// /// Performs bitwise AND operation between two integer values for pattern matching. -/// This is commonly used in magic rules to check if specific bits are set in a value. -/// Only works with integer types (Uint and Int), returns `false` for other types. +/// This implements magic(5)'s bare `&MASK` relational test: the file value must have +/// **every** bit in `right` (the mask) set, not merely *some* bit. Only works with +/// integer types (Uint and Int), returns `false` for other types. /// /// # Arguments /// @@ -65,7 +141,18 @@ pub fn apply_bitwise_and_mask(mask: u64, left: &Value, right: &Value) -> bool { /// /// # Returns /// -/// `true` if the bitwise AND result is non-zero, `false` otherwise or for non-integer types +/// `true` if `(left & right) == right` -- i.e. every bit set in `right` is also set +/// in `left` -- `false` otherwise or for non-integer types. +/// +/// # libmagic Compatibility +/// +/// This mirrors GNU `file`'s `magiccheck()` (`src/softmagic.c`), whose `'&'` relation +/// is `(v & l) == l` (`v` = file value, `l` = the rule's mask), i.e. "all masked bits +/// set" -- not "any masked bit set". A mask of `0` is vacuously satisfied by any file +/// value (every one of the zero required bits is trivially set), matching libmagic's +/// literal equality test. For a single-bit mask the two interpretations coincide, +/// which is why simple flag-check rules (`>6 leshort &0x0001 \b, encrypted`) are +/// unaffected either way -- only multi-bit bare-`&` masks distinguish them. /// /// # Examples /// @@ -77,9 +164,10 @@ pub fn apply_bitwise_and_mask(mask: u64, left: &Value, right: &Value) -> bool { /// assert!(apply_bitwise_and(&Value::Uint(0x01), &Value::Uint(0x01))); /// assert!(!apply_bitwise_and(&Value::Uint(0x02), &Value::Uint(0x01))); /// -/// // Check multiple bits +/// // Multi-bit mask: ALL masked bits must be set, not merely some /// assert!(apply_bitwise_and(&Value::Uint(0xFF), &Value::Uint(0x0F))); /// assert!(!apply_bitwise_and(&Value::Uint(0xF0), &Value::Uint(0x0F))); +/// assert!(!apply_bitwise_and(&Value::Uint(0x8F), &Value::Uint(0xFF))); // partial overlap fails /// /// // Works with signed integers too /// assert!(apply_bitwise_and(&Value::Int(-1), &Value::Int(0x01))); @@ -90,18 +178,24 @@ pub fn apply_bitwise_and_mask(mask: u64, left: &Value, right: &Value) -> bool { #[must_use] pub fn apply_bitwise_and(left: &Value, right: &Value) -> bool { match (left, right) { - // Unsigned integer bitwise AND - (Value::Uint(a), Value::Uint(b)) => (a & b) != 0, + // Unsigned integer bitwise AND: all bits in `right` (the mask) must be set in `left`. + (Value::Uint(a), Value::Uint(b)) => (a & b) == *b, // Signed integer bitwise AND (cast to unsigned for bitwise operations) #[allow(clippy::cast_sign_loss)] - (Value::Int(a), Value::Int(b)) => ((*a as u64) & (*b as u64)) != 0, + (Value::Int(a), Value::Int(b)) => { + let (a, b) = (*a as u64, *b as u64); + (a & b) == b + } // Mixed signed/unsigned integer bitwise AND #[allow(clippy::cast_sign_loss)] - (Value::Uint(a), Value::Int(b)) => (a & (*b as u64)) != 0, + (Value::Uint(a), Value::Int(b)) => { + let b = *b as u64; + (a & b) == b + } #[allow(clippy::cast_sign_loss)] - (Value::Int(a), Value::Uint(b)) => ((*a as u64) & b) != 0, + (Value::Int(a), Value::Uint(b)) => ((*a as u64) & b) == *b, // Non-integer types cannot perform bitwise AND _ => false, @@ -242,10 +336,13 @@ mod tests { #[test] fn test_apply_bitwise_and_uint_edge_cases() { - // Zero cases - assert!(!apply_bitwise_and(&Value::Uint(0), &Value::Uint(0xFF))); // Zero & anything = 0 - assert!(!apply_bitwise_and(&Value::Uint(0xFF), &Value::Uint(0))); // Anything & zero = 0 - assert!(!apply_bitwise_and(&Value::Uint(0), &Value::Uint(0))); // Zero & zero = 0 + // Zero cases. Under the "all masked bits set" semantics (matching + // libmagic's `(v & l) == l`), a zero mask is vacuously satisfied by + // any file value -- there are no required bits to check. Only the + // "value is zero but mask is not" case fails. + assert!(!apply_bitwise_and(&Value::Uint(0), &Value::Uint(0xFF))); // Zero value, nonzero mask: unsatisfied + assert!(apply_bitwise_and(&Value::Uint(0xFF), &Value::Uint(0))); // Zero mask: vacuously true + assert!(apply_bitwise_and(&Value::Uint(0), &Value::Uint(0))); // Zero mask: vacuously true // Maximum values assert!(apply_bitwise_and(&Value::Uint(u64::MAX), &Value::Uint(1))); // Max & 1 @@ -257,15 +354,29 @@ mod tests { #[test] fn test_apply_bitwise_and_uint_specific_patterns() { - // Common magic number patterns - assert!(apply_bitwise_and( + // Common magic number patterns. An 0xFF-per-byte mask asks "is every + // bit in this byte region set" -- it is NOT satisfied just because + // that byte region happens to be the value of interest (0x7F, 0x504B, + // etc. are not all-ones bytes/words). A mask built from the ACTUAL + // bits present in the value (mirroring the value itself in that + // region) is what bare `&` is for; an 0xFF-style "extract this byte" + // mask belongs with `BitwiseAndMask` + an explicit equality compare. + assert!(!apply_bitwise_and( &Value::Uint(0x7F45_4C46), &Value::Uint(0xFF00_0000) - )); // ELF magic high byte + )); // ELF's high byte (0x7F) is not all-ones assert!(apply_bitwise_and( + &Value::Uint(0x7F45_4C46), + &Value::Uint(0x7F00_0000) + )); // Mask matching the ELF high byte exactly does satisfy + assert!(!apply_bitwise_and( &Value::Uint(0x504B_0304), &Value::Uint(0xFFFF_0000) - )); // ZIP magic high word + )); // ZIP's high word (0x504B) is not all-ones + assert!(apply_bitwise_and( + &Value::Uint(0x504B_0304), + &Value::Uint(0x504B_0000) + )); // Mask matching the ZIP high word exactly does satisfy assert!(!apply_bitwise_and( &Value::Uint(0x1234_5678), &Value::Uint(0x0000_0001) @@ -290,10 +401,11 @@ mod tests { #[test] fn test_apply_bitwise_and_int_zero() { - // Zero cases with signed integers - assert!(!apply_bitwise_and(&Value::Int(0), &Value::Int(0xFF))); // Zero & anything = 0 - assert!(!apply_bitwise_and(&Value::Int(0xFF), &Value::Int(0))); // Anything & zero = 0 - assert!(!apply_bitwise_and(&Value::Int(0), &Value::Int(0))); // Zero & zero = 0 + // Zero cases with signed integers. As with the unsigned edge cases, + // a zero mask is vacuously satisfied (see GOTCHAS S13.3). + assert!(!apply_bitwise_and(&Value::Int(0), &Value::Int(0xFF))); // Zero value, nonzero mask: unsatisfied + assert!(apply_bitwise_and(&Value::Int(0xFF), &Value::Int(0))); // Zero mask: vacuously true + assert!(apply_bitwise_and(&Value::Int(0), &Value::Int(0))); // Zero mask: vacuously true } #[test] @@ -303,8 +415,14 @@ mod tests { assert!(apply_bitwise_and( &Value::Int(i64::MIN), &Value::Int(i64::MIN) - )); // Min & Min - assert!(apply_bitwise_and(&Value::Int(i64::MIN), &Value::Int(-1))); // Min & -1 (all bits set) + )); // Min & Min (self-AND is always true) + + // Min (only the sign bit set) does NOT have every bit of an + // all-ones mask set, so `apply_bitwise_and(MIN, -1)` is false -- + // only the reverse direction (does -1 have every bit of MIN set) + // is true, since -1's bit pattern is a superset of MIN's. + assert!(!apply_bitwise_and(&Value::Int(i64::MIN), &Value::Int(-1))); + assert!(apply_bitwise_and(&Value::Int(-1), &Value::Int(i64::MIN))); } #[test] @@ -318,9 +436,12 @@ mod tests { #[test] fn test_apply_bitwise_and_mixed_negative_uint() { - // Negative int with uint (negative numbers have high bits set) - assert!(apply_bitwise_and(&Value::Int(-1), &Value::Uint(1))); // -1 & 1 - assert!(apply_bitwise_and(&Value::Uint(1), &Value::Int(-1))); // 1 & -1 + // Negative int with uint (negative numbers have high bits set). + // -1's bit pattern is all-ones, so it has every bit of 1 set: true. + assert!(apply_bitwise_and(&Value::Int(-1), &Value::Uint(1))); + // But 1 does NOT have every bit of -1 (all-ones) set: false. This is + // the asymmetric case pinned by test_apply_bitwise_and_is_not_commutative_in_general. + assert!(!apply_bitwise_and(&Value::Uint(1), &Value::Int(-1))); assert!(!apply_bitwise_and(&Value::Int(-2), &Value::Uint(1))); // -2 & 1 (bit 0 not set in -2) assert!(!apply_bitwise_and(&Value::Uint(1), &Value::Int(-2))); // 1 & -2 } @@ -397,8 +518,8 @@ mod tests { (0b1111_0000_u64, 0b0000_1111_u64, false), // None of lower 4 bits (0b1010_1010_u64, 0b0101_0101_u64, false), // No overlap (0b1010_1010_u64, 0b1010_1010_u64, true), // Perfect match - (0b1111_1111_u64, 0b0000_0000_u64, false), // Mask is zero - (0b0000_0000_u64, 0b1111_1111_u64, false), // Value is zero + (0b1111_1111_u64, 0b0000_0000_u64, true), // Zero mask: vacuously satisfied (no required bits) + (0b0000_0000_u64, 0b1111_1111_u64, false), // Value is zero, mask is not: unsatisfied ]; for (value, mask, expected) in test_cases { @@ -412,48 +533,84 @@ mod tests { #[test] fn test_apply_bitwise_and_magic_file_patterns() { - // Test patterns commonly found in magic files - - // ELF magic number (0x7F454C46) - check if it's an ELF file + // Test patterns commonly found in magic files. Under the "all masked + // bits set" semantics, a mask only matches when EVERY one of its 1 + // bits is also set in the value -- so a mask must be built from bits + // that are genuinely known-set, not merely "some byte region of + // interest" (that latter pattern belongs to `Operator::Equal` after + // masking with `BitwiseAndMask`, or plain `Equal` on the exact value). + + // ELF magic number (0x7F454C46) is not all-ones, so it can never + // satisfy an all-ones mask via bare `&`; identity is what `Equal` + // is for. This documents why `&0xFFFFFFFF` is NOT how one would + // check "is this exactly the ELF magic" in a real magic file. let elf_magic = Value::Uint(0x7F45_4C46); - let elf_mask = Value::Uint(0xFFFF_FFFF); - assert!(apply_bitwise_and(&elf_magic, &elf_mask)); - - // Check specific bytes in ELF magic - assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x7F00_0000))); // First byte - assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x0045_0000))); // Second byte 'E' - assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x0000_4C00))); // Third byte 'L' - assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x0000_0046))); // Fourth byte 'F' - - // ZIP magic number (0x504B0304) - check if it's a ZIP file + let elf_all_ones_mask = Value::Uint(0xFFFF_FFFF); + assert!(!apply_bitwise_and(&elf_magic, &elf_all_ones_mask)); + + // Masks built from bits that ARE present in elf_magic still match, + // because every 1 bit in the mask is also 1 in the value. + assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x7F00_0000))); // First byte (0x7F, matches exactly) + assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x0045_0000))); // Second byte 'E' (0x45, matches exactly) + assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x0000_4C00))); // Third byte 'L' (0x4C, matches exactly) + assert!(apply_bitwise_and(&elf_magic, &Value::Uint(0x0000_0046))); // Fourth byte 'F' (0x46, matches exactly) + + // ZIP magic number (0x504B0304): mask built from the actual "PK" + // signature bits matches; a bit that's genuinely unset in the value + // (bit 0) does not. let zip_magic = Value::Uint(0x504B_0304); - assert!(apply_bitwise_and(&zip_magic, &Value::Uint(0x504B_0000))); // PK signature + assert!(apply_bitwise_and(&zip_magic, &Value::Uint(0x504B_0000))); // PK signature (matches exactly) assert!(!apply_bitwise_and(&zip_magic, &Value::Uint(0x0000_0001))); // Bit 0 not set - // PDF magic (%PDF) - first few bytes + // PDF magic (%PDF): an 0xFF-style "give me this byte region" mask is + // NOT satisfied unless the value's bits in that region are all 1 -- + // '%' (0x25) and 'P' (0x50) are not all-ones bytes, so a bare 0xFF + // mask over that byte fails. This is the case that distinguishes + // "any bit set" from "all bits set": real magic files would use + // `Operator::Equal` (or `BitwiseAndMask` + explicit compare value) + // to test "this byte equals 0x25", not bare `&0xFF00_0000`. let pdf_magic = Value::Uint(0x2550_4446); // "%PDF" as uint32 - assert!(apply_bitwise_and(&pdf_magic, &Value::Uint(0xFF00_0000))); // '%' character - assert!(apply_bitwise_and(&pdf_magic, &Value::Uint(0x00FF_0000))); // 'P' character + assert!(!apply_bitwise_and(&pdf_magic, &Value::Uint(0xFF00_0000))); // '%' (0x25) is not all-ones + assert!(!apply_bitwise_and(&pdf_magic, &Value::Uint(0x00FF_0000))); // 'P' (0x50) is not all-ones } #[test] - fn test_apply_bitwise_and_symmetry() { - // Test that bitwise AND is commutative for integer types - let test_cases = vec![ - (Value::Uint(0xFF), Value::Uint(0x0F)), - (Value::Int(42), Value::Int(24)), - (Value::Uint(0xAAAA), Value::Int(0x5555)), - (Value::Int(-1), Value::Uint(1)), + fn test_apply_bitwise_and_is_not_commutative_in_general() { + // `apply_bitwise_and(left, right)` tests "does `left` have every bit + // of `right` set" -- this is a genuinely asymmetric relation (`left` + // is the file value, `right` is the rule's mask), matching libmagic's + // `(v & l) == l`. It is NOT commutative in general: swapping which + // operand plays "value" vs "mask" changes the question being asked. + // (It IS trivially symmetric when left == right, or when one side's + // bits are a superset of the other's in both directions -- e.g. two + // equal masks -- but that is not the general case.) An earlier + // revision of this crate implemented "any bit set" (`(a & b) != 0`), + // which genuinely is commutative; that was the wrong semantics (see + // GOTCHAS S13.3) and this test's name/assertions have been corrected + // accordingly rather than deleted, so the asymmetry stays pinned. + let asymmetric_cases = vec![ + (Value::Uint(0xFF), Value::Uint(0x0F)), // 0xFF has all of 0x0F's bits; 0x0F does not have all of 0xFF's + (Value::Uint(1), Value::Int(-1)), // 1 does not have all bits of all-ones; all-ones has bit 0 ]; - for (left, right) in test_cases { + for (left, right) in asymmetric_cases { let left_to_right = apply_bitwise_and(&left, &right); let right_to_left = apply_bitwise_and(&right, &left); - assert_eq!( + assert_ne!( left_to_right, right_to_left, - "Bitwise AND should be commutative: {left:?} & {right:?}" + "expected asymmetric result for {left:?} & {right:?} vs swapped operands" ); } + + // Self-AND is always true regardless of operand order (a value always + // has every one of its own bits set), so swapping identical operands + // trivially agrees. + let self_case = Value::Uint(0x5555); + assert_eq!( + apply_bitwise_and(&self_case, &self_case), + apply_bitwise_and(&self_case, &self_case) + ); + assert!(apply_bitwise_and(&self_case, &self_case)); } #[test] @@ -611,4 +768,100 @@ mod tests { None )); } + + // ------------------------------------------------------------------ + // Regression: signed masked comparison must re-normalize to the type + // width (the Mach-O `0 lelong&0xfffffffe 0xfeedface` miss). + // ------------------------------------------------------------------ + + /// The exact real-world failure: a signed 32-bit `lelong` read of the + /// Mach-O 64-bit magic `0xFEEDFACF` (sign-extended to i64) masked with + /// `0xfffffffe` must equal the sign-extended rule literal `0xfeedface`. + /// Before the width-aware fix this silently returned `false`, so the + /// Mach-O rule never matched and a weak `measure`/Lepton rule won. + #[test] + #[allow(clippy::cast_possible_wrap)] // intentional i64 bit-patterns + fn test_apply_bitwise_and_mask_with_width_matches_signed_macho_signature() { + // lelong read of bytes `cf fa ed fe`, sign-extended to i64. + let read = Value::Int(0xFFFF_FFFF_FEED_FACF_u64 as i64); + // `0xfeedface` coerced to the signed 32-bit type (sign-extended). + let rule = Value::Int(0xFFFF_FFFF_FEED_FACE_u64 as i64); + + // Width-aware (the fix): matches. + assert!( + apply_bitwise_and_mask_with_width(0xffff_fffe, &read, &rule, Some(32)), + "signed lelong&0xfffffffe must equal sign-extended 0xfeedface at 32-bit width" + ); + // Width-unaware (the historical bug): does NOT match -- pins that the + // width is load-bearing, so a future refactor that drops it regresses + // this test rather than silently reintroducing the Mach-O miss. + assert!( + !apply_bitwise_and_mask_with_width(0xffff_fffe, &read, &rule, None), + "without the type width the sign-extension mismatch resurfaces (the bug)" + ); + } + + /// A signed masked read that does NOT match the rule literal must stay a + /// non-match under the width-aware path (guards against the fix being a + /// blanket "always true"). + #[test] + #[allow(clippy::cast_possible_wrap)] // intentional i64 bit-patterns + fn test_apply_bitwise_and_mask_with_width_signed_negative_case() { + let read = Value::Int(0x0000_0000_1234_5678_u64 as i64); // positive lelong + let rule = Value::Int(0xFFFF_FFFF_FEED_FACE_u64 as i64); // 0xfeedface @ lelong + assert!(!apply_bitwise_and_mask_with_width( + 0xffff_fffe, + &read, + &rule, + Some(32) + )); + } + + /// Unsigned masked comparison is unaffected by the fix (it already worked) + /// -- `ulelong&0xfffffffe 0xfeedface` still matches with and without width. + #[test] + fn test_apply_bitwise_and_mask_with_width_unsigned_unaffected() { + let read = Value::Uint(0x0000_0000_FEED_FACF); // ulelong read + let rule = Value::Uint(0x0000_0000_FEED_FACE); // rule literal + assert!(apply_bitwise_and_mask_with_width( + 0xffff_fffe, + &read, + &rule, + Some(32) + )); + assert!(apply_bitwise_and_mask_with_width( + 0xffff_fffe, + &read, + &rule, + None + )); + } + + /// Signed byte width: masking must re-extend from bit 7, not bit 31/63. + #[test] + #[allow(clippy::cast_possible_wrap)] // intentional i64 bit-patterns + fn test_apply_bitwise_and_mask_with_width_signed_byte() { + // byte read 0x81 -> Int(-127) sign-extended; mask 0xfe -> 0x80; at + // 8-bit width bit 7 is set so it re-extends to Int(-128 == 0xFF..80). + let read = Value::Int(0xFFFF_FFFF_FFFF_FF81_u64 as i64); + let rule = Value::Int(0xFFFF_FFFF_FFFF_FF80_u64 as i64); // 0x80 @ signed byte + assert!(apply_bitwise_and_mask_with_width( + 0xfe, + &read, + &rule, + Some(8) + )); + } + + /// The convenience wrapper `apply_bitwise_and_mask` must behave exactly + /// like the width-aware form with `bit_width == None`. + #[test] + fn test_apply_bitwise_and_mask_delegates_to_none_width() { + let read = Value::Uint(0x1234); + let rule = Value::Uint(0x34); + assert_eq!( + apply_bitwise_and_mask(0xFF, &read, &rule), + apply_bitwise_and_mask_with_width(0xFF, &read, &rule, None) + ); + } } diff --git a/src/evaluator/operators/mod.rs b/src/evaluator/operators/mod.rs index cd74054e..29e09c86 100644 --- a/src/evaluator/operators/mod.rs +++ b/src/evaluator/operators/mod.rs @@ -13,8 +13,8 @@ mod comparison; mod equality; pub use bitwise::{ - apply_bitwise_and, apply_bitwise_and_mask, apply_bitwise_not, apply_bitwise_not_with_width, - apply_bitwise_xor, + apply_bitwise_and, apply_bitwise_and_mask, apply_bitwise_and_mask_with_width, + apply_bitwise_not, apply_bitwise_not_with_width, apply_bitwise_xor, }; pub use comparison::{ apply_greater_equal, apply_greater_than, apply_less_equal, apply_less_than, compare_values, @@ -641,11 +641,14 @@ mod tests { "NotEqual should be true for different values: {left:?} != {right:?}" ); - // BitwiseAnd behavior depends on the value types and content + // BitwiseAnd behavior depends on the value types and content. + // Semantics: (left & right) == right -- ALL bits set in `right` + // (the mask) must also be set in `left` (see GOTCHAS S13.3 and + // bitwise.rs::apply_bitwise_and doc comment). let bitwise_result = apply_operator(&Operator::BitwiseAnd, &left, &right); match (&left, &right) { (Value::Uint(a), Value::Uint(b)) => { - let expected = (a & b) != 0; + let expected = (a & b) == *b; assert_eq!( bitwise_result, expected, "BitwiseAnd for Uint({a}) & Uint({b}) should be {expected}" @@ -653,20 +656,33 @@ mod tests { } (Value::Int(a), Value::Int(b)) => { #[allow(clippy::cast_sign_loss)] - let expected = ((*a as u64) & (*b as u64)) != 0; + let expected = { + let (a, b) = (*a as u64, *b as u64); + (a & b) == b + }; assert_eq!( bitwise_result, expected, "BitwiseAnd for Int({a}) & Int({b}) should be {expected}" ); } - (Value::Uint(a), Value::Int(b)) | (Value::Int(b), Value::Uint(a)) => { + (Value::Uint(a), Value::Int(b)) => { #[allow(clippy::cast_sign_loss)] - let expected = (a & (*b as u64)) != 0; + let b = *b as u64; + let expected = (a & b) == b; assert_eq!( bitwise_result, expected, "BitwiseAnd for mixed Uint/Int should be {expected}" ); } + (Value::Int(a), Value::Uint(b)) => { + #[allow(clippy::cast_sign_loss)] + let a = *a as u64; + let expected = (a & b) == *b; + assert_eq!( + bitwise_result, expected, + "BitwiseAnd for mixed Int/Uint should be {expected}" + ); + } _ => { // For non-integer types, BitwiseAnd should always be false assert!( diff --git a/src/evaluator/strength.rs b/src/evaluator/strength.rs index b7985925..482fb493 100644 --- a/src/evaluator/strength.rs +++ b/src/evaluator/strength.rs @@ -140,7 +140,7 @@ pub fn calculate_default_strength(rule: &MagicRule) -> i32 { // `use` dispatches into a subroutine whose specificity is // opaque from the call site -- give it a moderate weight so // it sorts above pure no-ops but below real type-bearing rules. - crate::parser::ast::MetaType::Use(_) => 5, + crate::parser::ast::MetaType::Use { .. } => 5, // `indirect` re-evaluates the root rule set at the resolved // offset; same rationale as `use` for the moderate weight. crate::parser::ast::MetaType::Indirect => 5, @@ -952,6 +952,69 @@ mod tests { assert_eq!(rules[1].message, "byte rule"); } + #[test] + fn test_sort_rules_by_strength_preserves_child_file_order() { + // libmagic's `apprentice_sort` orders whole top-level magic entries by + // their first line's strength but NEVER reorders continuation + // (child) lines. `sort_rules_by_strength` is non-recursive to match: + // it sorts only the top-level slice, leaving each rule's `children` + // in source order. This is load-bearing for `default`/`clear` firing + // and for multi-fragment descriptions (e.g. gzip's detail siblings). + // + // Build one top-level rule whose children are, in file order, a + // low-strength `default` FIRST and a high-strength byte comparison + // SECOND. A recursive sort would swap them (byte outranks default), + // wrongly letting the comparison sibling suppress the `default`. + let low_first = { + let mut r = make_rule( + TypeKind::Meta(crate::parser::ast::MetaType::Default), + Operator::AnyValue, + OffsetSpec::Absolute(0), + Value::Uint(0), + ); + r.message = "default-child".to_string(); + r.level = 1; + r + }; + let high_second = { + let mut r = make_rule( + TypeKind::Long { + endian: crate::parser::ast::Endianness::Big, + signed: false, + }, + Operator::Equal, + OffsetSpec::Absolute(0), + Value::Uint(0xDEAD_BEEF), + ); + r.message = "strong-child".to_string(); + r.level = 1; + r + }; + let mut parent = make_rule( + TypeKind::Byte { signed: true }, + Operator::Equal, + OffsetSpec::Absolute(0), + Value::Uint(0), + ); + parent.message = "parent".to_string(); + parent.children = vec![low_first, high_second]; + + let mut rules = vec![parent]; + sort_rules_by_strength(&mut rules); + + let child_order: Vec<&str> = rules[0] + .children + .iter() + .map(|c| c.message.as_str()) + .collect(); + assert_eq!( + child_order, + vec!["default-child", "strong-child"], + "child rules must stay in file order; the non-recursive sort must \ + not reorder continuation rules by strength" + ); + } + #[test] fn test_sort_rules_by_strength_with_modifier() { let mut rules = vec![ @@ -1146,7 +1209,13 @@ mod tests { #[test] fn test_meta_use_and_indirect_sort_above_default() { use crate::parser::ast::MetaType; - let use_rule = meta_rule(MetaType::Use("sub".to_string()), "use"); + let use_rule = meta_rule( + MetaType::Use { + name: "sub".to_string(), + flip_endian: false, + }, + "use", + ); let indirect_rule = meta_rule(MetaType::Indirect, "indirect"); let default_rule = meta_rule(MetaType::Default, "default"); let clear_rule = meta_rule(MetaType::Clear, "clear"); diff --git a/src/evaluator/types/mod.rs b/src/evaluator/types/mod.rs index 4a38fb79..b2534861 100644 --- a/src/evaluator/types/mod.rs +++ b/src/evaluator/types/mod.rs @@ -26,6 +26,66 @@ pub(crate) use search::read_search; use string::string16_bytes_consumed; pub(crate) use string::{read_pstring, read_string, read_string_exact, read_string16}; +/// Swap the declared endianness of an endian-bearing [`TypeKind`] for the +/// magic(5) `use \^name` endian flip (issue #236). +/// +/// This mirrors libmagic's `cvt_flip` in `src/softmagic.c` exactly: only the +/// explicit little/big-endian numeric, float, and date families are swapped +/// (`short`/`long`/`quad`/`float`/`double` and the `date`/`ldate`/`qdate`/ +/// `qldate` families). `Endianness::Native` is left untouched (libmagic has +/// no `FILE_SHORT`/`FILE_LONG` case in `cvt_flip`), `String16` is deliberately +/// NOT flipped (also absent from `cvt_flip`), and every non-endian type is +/// returned unchanged. The `signed`/`utc` attributes are preserved. +/// +/// The evaluator calls this at read time only when the `\^` flip is active for +/// the current subroutine body, so the common (unflipped) path never allocates +/// a flipped clone. +pub(crate) fn flip_type_endian(typ: &TypeKind) -> TypeKind { + use crate::parser::ast::Endianness; + + /// Swap Little<->Big; leave Native alone (matches `cvt_flip`, which only + /// has explicit BE/LE cases). + const fn swap(e: Endianness) -> Endianness { + match e { + Endianness::Little => Endianness::Big, + Endianness::Big => Endianness::Little, + Endianness::Native => Endianness::Native, + } + } + + match *typ { + TypeKind::Short { endian, signed } => TypeKind::Short { + endian: swap(endian), + signed, + }, + TypeKind::Long { endian, signed } => TypeKind::Long { + endian: swap(endian), + signed, + }, + TypeKind::Quad { endian, signed } => TypeKind::Quad { + endian: swap(endian), + signed, + }, + TypeKind::Float { endian } => TypeKind::Float { + endian: swap(endian), + }, + TypeKind::Double { endian } => TypeKind::Double { + endian: swap(endian), + }, + TypeKind::Date { endian, utc } => TypeKind::Date { + endian: swap(endian), + utc, + }, + TypeKind::QDate { endian, utc } => TypeKind::QDate { + endian: swap(endian), + utc, + }, + // Byte, String, String16, PString, Regex, Search, Meta: unchanged. + // (`String16` is intentionally absent from libmagic's `cvt_flip`.) + ref other => other.clone(), + } +} + /// Reads a fixed-size byte array from the buffer at the given offset. /// /// This is a shared helper for numeric, date, and float type readers that @@ -101,6 +161,63 @@ pub enum TypeReadError { }, } +/// Diagnostic string for `TypeKind::Regex` evaluated without a usable +/// `String`/`Bytes` pattern operand. Shared by every construction site +/// (`read_typed_value_with_pattern` and `read_pattern_match`'s `Regex` +/// arms) AND [`is_missing_pattern_operand`], so the two can never drift +/// out of sync (a hardening item from the multi-agent review of the +/// fix-system-magic-regex-graceful PR: the message previously existed as +/// five independent string-literal copies across the two construction +/// sites and the predicate). +pub(crate) const REGEX_MISSING_PATTERN_MSG: &str = "regex without string pattern"; + +/// Diagnostic string for `TypeKind::Search` evaluated without a usable +/// `String`/`Bytes` pattern operand. See [`REGEX_MISSING_PATTERN_MSG`] for +/// the single-source-of-truth rationale. +pub(crate) const SEARCH_MISSING_PATTERN_MSG: &str = "search without string/bytes pattern"; + +/// Diagnostic string for a flagged `TypeKind::String` evaluated without a +/// usable `String`/`Bytes` pattern operand. See +/// [`REGEX_MISSING_PATTERN_MSG`] for the single-source-of-truth rationale. +pub(crate) const FLAGGED_STRING_MISSING_PATTERN_MSG: &str = + "string with flags requires string/bytes pattern"; + +/// Returns `true` if `type_name` (the free-form diagnostic string carried by +/// [`TypeReadError::UnsupportedType`]) describes a pattern-bearing type +/// (`Regex`, `Search`, or a flagged `String`) that was evaluated without a +/// usable `String`/`Bytes` pattern operand. +/// +/// This is the narrow "missing pattern" condition the engine's graceful skip +/// (GOTCHAS S2.1, S2.4) targets -- deliberately an exhaustive allowlist of +/// the exact messages this module emits for that condition, not a substring +/// heuristic, so a genuine evaluator capability gap (an unwired `TypeKind` +/// variant, or a non-equality operator on a pattern-bearing type) is never +/// accidentally swallowed. See `is_regex_compile_failure` for the sibling +/// "compile failure" condition, and the fix-system-magic-regex-graceful +/// plan (KTD4/KTD5, requirement R3) for the full rationale. +#[must_use] +pub(crate) fn is_missing_pattern_operand(type_name: &str) -> bool { + matches!( + type_name, + REGEX_MISSING_PATTERN_MSG | SEARCH_MISSING_PATTERN_MSG | FLAGGED_STRING_MISSING_PATTERN_MSG + ) +} + +/// Returns `true` if `type_name` describes a regex compile failure, as +/// opposed to a missing pattern operand. +/// +/// Compile failures include the `REGEX_COMPILE_SIZE_LIMIT` (1 MiB) denial-of-service guard +/// (CWE-1333) from `evaluator::types::regex`. Both conditions flow +/// through the same [`TypeReadError::UnsupportedType`] variant and are +/// gracefully skipped by the engine, but a compile failure is logged at +/// `warn!` rather than `debug!` so a malicious or pathological magic file's +/// rejection is not silently invisible even though evaluation of the rest +/// of the file continues (KTD5). +#[must_use] +pub(crate) fn is_regex_compile_failure(type_name: &str) -> bool { + type_name.starts_with("regex compile error:") +} + /// Default `max_string_length` used by [`read_typed_value`] when callers /// do not supply an explicit cap. Matches /// `EvaluationConfig::default().max_string_length` so call sites that @@ -150,6 +267,43 @@ pub(crate) fn read_typed_value( read_typed_value_with_pattern(buffer, offset, type_kind, None, DEFAULT_MAX_STRING_LENGTH) } +/// Decodes a `Value::Bytes` regex pattern operand into a `String` for +/// compilation (KTD1/KTD6 of the fix-system-magic-regex-graceful plan). +/// +/// Regex patterns are normally captured by the parser as `Value::String`, +/// but escape-heavy patterns (e.g. `\^[\040\t]{0,50}\\.asciiz`) can +/// currently be miscategorized as `Value::Bytes` by `parse_value`'s +/// hex/mixed-ascii branch. This backstop lets such patterns compile +/// instead of fatally erroring (GOTCHAS S2.4), mirroring the existing +/// `TypeKind::Search` arms' dual `String`/`Bytes` acceptance. +/// +/// `String::from_utf8_lossy` never panics, but on a real substitution -- +/// any byte `>= 0x80` that is not valid UTF-8 -- it silently replaces the +/// byte with U+FFFD while the target buffer is still matched against its +/// *raw* bytes. The two sides then diverge silently: the compiled regex +/// no longer represents the same bytes the file comparison expects. Detect +/// a real substitution with `str::from_utf8` first and `warn!` so the +/// divergence is visible in logs rather than a silent wrong answer. +fn decode_regex_bytes_pattern(bytes: &[u8]) -> String { + if std::str::from_utf8(bytes).is_err() { + // The pattern originates from an untrusted magic DB and can be + // arbitrarily long, so log only its length and a bounded preview + // rather than the whole buffer -- otherwise a pathological pattern + // could flood logs / churn memory (CWE-117-adjacent). 32 bytes is + // enough to identify the offending rule without dumping it. + const PREVIEW_LEN: usize = 32; + let preview = bytes.get(..PREVIEW_LEN.min(bytes.len())).unwrap_or(bytes); + let truncated = if bytes.len() > PREVIEW_LEN { "..." } else { "" }; + log::warn!( + "regex pattern given as {} raw bytes (preview: {preview:?}{truncated}) is not \ + valid UTF-8; lossily reinterpreting as text before compiling (bytes >= 0x80 \ + become U+FFFD and will not byte-match the target buffer)", + bytes.len() + ); + } + String::from_utf8_lossy(bytes).into_owned() +} + /// Reads bytes according to the specified `TypeKind`, threading a /// `pattern` operand through for non-pattern-bearing types whose /// dispatch arm consults the rule's value operand (e.g. `TypeKind::String` @@ -246,11 +400,16 @@ pub(crate) fn read_typed_value_with_pattern( *length_includes_itself, ), TypeKind::Regex { flags, count } => { - let pattern_str = match pattern { - Some(Value::String(s)) => s.as_str(), + // Dual `String`/`Bytes` acceptance mirrors the `Search` arm + // directly below (GOTCHAS S2.4). The `String` fast path stays + // allocation-free via `Cow::Borrowed`; `Bytes` goes through + // `decode_regex_bytes_pattern` (KTD6 lossy-substitution guard). + let pattern_str: Cow<'_, str> = match pattern { + Some(Value::String(s)) => Cow::Borrowed(s.as_str()), + Some(Value::Bytes(b)) => Cow::Owned(decode_regex_bytes_pattern(b)), _ => { return Err(TypeReadError::UnsupportedType { - type_name: "regex without string pattern".to_string(), + type_name: REGEX_MISSING_PATTERN_MSG.to_string(), }); } }; @@ -259,7 +418,7 @@ pub(crate) fn read_typed_value_with_pattern( // shape. The engine path goes through `read_pattern_match` // directly and preserves the `Option` so it can distinguish a // zero-width match from a miss. - Ok(read_regex(buffer, offset, pattern_str, *flags, *count)? + Ok(read_regex(buffer, offset, &pattern_str, *flags, *count)? .unwrap_or_else(|| Value::String(String::new()))) } TypeKind::Search { range, flags } => { @@ -268,7 +427,7 @@ pub(crate) fn read_typed_value_with_pattern( Some(Value::Bytes(b)) => b.as_slice(), _ => { return Err(TypeReadError::UnsupportedType { - type_name: "search without string/bytes pattern".to_string(), + type_name: SEARCH_MISSING_PATTERN_MSG.to_string(), }); } }; @@ -324,15 +483,19 @@ pub(crate) fn read_pattern_match( } match type_kind { TypeKind::Regex { flags, count } => { - let pattern_str = match pattern { - Some(Value::String(s)) => s.as_str(), + // Dual `String`/`Bytes` acceptance mirrors the `Search` arm + // directly below (GOTCHAS S2.4); see `decode_regex_bytes_pattern` + // for the KTD6 lossy-substitution `warn!` guard. + let pattern_str: Cow<'_, str> = match pattern { + Some(Value::String(s)) => Cow::Borrowed(s.as_str()), + Some(Value::Bytes(b)) => Cow::Owned(decode_regex_bytes_pattern(b)), _ => { return Err(TypeReadError::UnsupportedType { - type_name: "regex without string pattern".to_string(), + type_name: REGEX_MISSING_PATTERN_MSG.to_string(), }); } }; - read_regex(buffer, offset, pattern_str, *flags, *count) + read_regex(buffer, offset, &pattern_str, *flags, *count) } TypeKind::Search { range, flags } => { let pattern_bytes: &[u8] = match pattern { @@ -340,7 +503,7 @@ pub(crate) fn read_pattern_match( Some(Value::Bytes(b)) => b.as_slice(), _ => { return Err(TypeReadError::UnsupportedType { - type_name: "search without string/bytes pattern".to_string(), + type_name: SEARCH_MISSING_PATTERN_MSG.to_string(), }); } }; @@ -368,7 +531,7 @@ pub(crate) fn read_pattern_match( Some(Value::Bytes(b)) => b.as_slice(), _ => { return Err(TypeReadError::UnsupportedType { - type_name: "string with flags requires string/bytes pattern".to_string(), + type_name: FLAGGED_STRING_MISSING_PATTERN_MSG.to_string(), }); } }; @@ -476,18 +639,16 @@ fn flagged_string_bytes_consumed( Some(Value::String(s)) => s.as_bytes(), Some(Value::Bytes(b)) => b.as_slice(), _ => { - // Invariant: the engine only calls `bytes_consumed_with_pattern` - // after a successful `read_pattern_match`, which validated the - // pattern shape. If we reach this arm, a future caller violated - // the contract. Fire the debug-only assert for test/dev builds - // AND a release-time `warn!` so production runs surface the - // anchor-corruption signature (relative-offset child rules - // would read from the un-advanced anchor) rather than going - // silent. Returns 0 to preserve buffer safety. - debug_assert!( - false, - "flagged_string_bytes_consumed: missing string/bytes pattern ({pattern:?}) -- read_pattern_match should have rejected this" - ); + // The dispatcher (`bytes_consumed_with_pattern`) only routes here + // when a string/bytes pattern is present (the Equal/NotEqual + // pattern-match path), so this arm is not normally reachable. It is + // handled defensively rather than with a `debug_assert!(false)`: a + // panic in library code is forbidden by the no-panic policy, and a + // flagged string reaching the consume side without a string pattern + // (e.g. an AnyValue/ordering operator misrouted by a future change) + // must degrade -- warn and return 0 (buffer-safe; the relative- + // offset anchor simply does not advance for that rule) rather than + // crash. log::warn!( "flagged_string_bytes_consumed: missing string/bytes pattern ({pattern:?}); \ relative-offset anchor will not advance for this rule" @@ -673,7 +834,16 @@ pub(crate) fn bytes_consumed_with_pattern( match type_kind { TypeKind::String { max_length, flags } => { - if !flags.is_empty() { + // Route to the flag-aware consumer ONLY when there is a string/bytes + // pattern to walk -- i.e. the Equal/NotEqual pattern-match path. + // For an AnyValue (`x`) or ordering operator on a flagged string + // (e.g. `0 string/b x`, `>15 string/t >\0`), the engine reads the + // string via the plain value path (the /t, /b, /c... flags do not + // change how many bytes are consumed), so fall through to the normal + // string logic. Without this guard the pattern walker hit its + // missing-pattern branch and panicked in debug builds. Mirrors the + // engine's `TypeKind::String { flags }` operator-based dispatch. + if !flags.is_empty() && matches!(pattern, Some(Value::String(_) | Value::Bytes(_))) { return flagged_string_bytes_consumed(buffer, offset, *max_length, *flags, pattern); } // For the (`max_length: None`, string literal pattern) @@ -735,16 +905,33 @@ pub(crate) fn bytes_consumed_with_pattern( Some(Value::String(s)) => { regex::regex_bytes_consumed(buffer, offset, s.as_str(), *flags, *count) } + // U1 (fix-system-magic-regex-graceful): `Value::Bytes` regex + // patterns are now accepted by `read_typed_value_with_pattern`/ + // `read_pattern_match` (GOTCHAS S2.4), so this consume-side + // helper must mirror that acceptance -- otherwise a + // successful Bytes-pattern regex match would hit the + // `debug_assert` below and silently fail to advance the + // anchor. Decode via the same `decode_regex_bytes_pattern` + // helper used by the read paths so both sides agree on what + // the pattern text is. + Some(Value::Bytes(b)) => { + let decoded = decode_regex_bytes_pattern(b); + regex::regex_bytes_consumed(buffer, offset, &decoded, *flags, *count) + } // Invariant: the engine only calls `bytes_consumed_with_pattern` // after a successful `read_typed_value_with_pattern`/`read_pattern_match`, - // which requires `Some(Value::String(_))` for regex. If we land - // here the invariant is broken by a new caller and the anchor - // would silently stall instead of advancing. Fire a debug_assert - // so the mismatch is caught in dev/test builds. + // which requires `Some(Value::String(_) | Value::Bytes(_))` for + // regex, so this arm is not normally reachable. Handle it + // defensively rather than with a `debug_assert!(false)` (which + // would panic in dev/test, violating the no-panic policy and the + // `prop_arbitrary_rule_evaluation_never_panics` invariant): `warn!` + // so a release build's silent stale-anchor is visible in logs, and + // return 0 (buffer-safe; the relative-offset anchor simply does not + // advance for this rule). Mirrors `flagged_string_bytes_consumed`. other => { - debug_assert!( - false, - "bytes_consumed_with_pattern: TypeKind::Regex without Value::String pattern ({other:?}) -- engine invariant violated" + log::warn!( + "bytes_consumed_with_pattern: TypeKind::Regex without Value::String/Bytes pattern ({other:?}); \ + relative-offset anchor will not advance for this rule" ); 0 } @@ -756,10 +943,13 @@ pub(crate) fn bytes_consumed_with_pattern( Some(Value::Bytes(b)) => { search::search_bytes_consumed(buffer, offset, b.as_slice(), *range, *flags) } + // Same invariant and defensive rationale as the `Regex` arm above: + // `warn!` + return 0 rather than `debug_assert!(false)`, so a + // release build's silent stale-anchor is visible in logs. other => { - debug_assert!( - false, - "bytes_consumed_with_pattern: TypeKind::Search without Value::String/Bytes pattern ({other:?}) -- engine invariant violated" + log::warn!( + "bytes_consumed_with_pattern: TypeKind::Search without Value::String/Bytes pattern ({other:?}); \ + relative-offset anchor will not advance for this rule" ); 0 } diff --git a/src/evaluator/types/regex.rs b/src/evaluator/types/regex.rs index 02f33bd0..cf01434f 100644 --- a/src/evaluator/types/regex.rs +++ b/src/evaluator/types/regex.rs @@ -132,11 +132,34 @@ pub(crate) const REGEX_COMPILE_SIZE_LIMIT: usize = 1 << 20; /// The compile-time `size_limit` and `dfa_size_limit` are both bounded by /// [`REGEX_COMPILE_SIZE_LIMIT`] (1 MiB) to bound the worst-case memory /// footprint of attacker-controlled patterns loaded from magic files. +/// +/// # Unicode mode is disabled (`unicode(false)`) +/// +/// This module's docs advertise binary-safe matching against raw bytes, +/// but `regex::bytes::RegexBuilder` defaults `unicode` to `true` even for +/// the bytes API -- in that mode, a hex escape like `\xff` in the pattern +/// is interpreted as the Unicode scalar U+00FF and matched against its +/// **UTF-8 encoding** (`0xC3 0xBF`), not the single raw byte `0xFF`. This +/// silently broke the getstr resolver's `>= 0x80` byte re-encoding +/// contract (see `src/parser/grammar/getstr/mod.rs`, KTD3 in the +/// `fix/system-magic-regex-graceful` plan): the parser emits `\xHH` for +/// escape-produced bytes `>= 0x80` specifically so the *regex engine* +/// carries the byte, but with unicode mode on, that byte was never +/// actually matched -- a `search`/`regex` rule containing a +/// high-byte escape would silently and permanently fail to match the +/// byte it was written for, with no compile error to surface the defect. +/// `unicode(false)` makes `\xHH` match the literal byte value for all +/// `HH`, restores POSIX/byte-oriented semantics (matching libmagic's C +/// `regcomp`, which has no concept of Unicode scalar values), and does +/// not regress ASCII case-insensitive matching or POSIX bracket classes +/// (`[[:space:]]`, `[[:alpha:]]`, ...), which remain fully supported +/// without Unicode tables. fn build_regex(pattern: &str, case_insensitive: bool) -> Result { RegexBuilder::new(pattern) .case_insensitive(case_insensitive) .multi_line(true) .dot_matches_new_line(false) + .unicode(false) .size_limit(REGEX_COMPILE_SIZE_LIMIT) .dfa_size_limit(REGEX_COMPILE_SIZE_LIMIT) .build() @@ -560,6 +583,42 @@ mod tests { assert_eq!(result, Some(Value::String("123".to_string()))); } + /// Regression guard for the `unicode(false)` fix in `build_regex`. + /// + /// A getstr-resolved regex pattern re-encodes escape-produced bytes + /// `>= 0x80` as a regex-native `\xHH` sequence (KTD3 in the + /// `fix/system-magic-regex-graceful` plan, see + /// `src/parser/grammar/getstr/mod.rs`). With `regex::bytes::Regex`'s + /// default unicode mode, `\xff` compiles successfully but silently + /// matches the *UTF-8 encoding* of U+00FF (`0xC3 0xBF`) instead of the + /// single raw byte `0xFF` -- a real, previously undetected bug (no + /// compile error, just permanently-wrong matching). This test pins + /// both halves of the contract: the raw byte matches, and the + /// two-byte UTF-8 encoding of the same code point does not. + #[test] + fn test_read_regex_high_byte_escape_matches_raw_byte_not_utf8_encoding() { + let pattern = r"a\xffb"; + + let raw_byte_buffer: &[u8] = &[b'a', 0xff, b'b']; + let raw_result = + read_regex(raw_byte_buffer, 0, pattern, no_flags(), default_count()).unwrap(); + assert!( + raw_result.is_some(), + "\\xff must match the raw byte 0xFF, got {raw_result:?}" + ); + + // 0xC3 0xBF is the UTF-8 encoding of U+00FF -- this must NOT match, + // proving unicode mode is genuinely disabled rather than merely + // permissive on both encodings. + let utf8_encoded_buffer: &[u8] = &[b'a', 0xc3, 0xbf, b'b']; + let utf8_result = + read_regex(utf8_encoded_buffer, 0, pattern, no_flags(), default_count()).unwrap(); + assert!( + utf8_result.is_none(), + "\\xff must not match the UTF-8 encoding of U+00FF, got {utf8_result:?}" + ); + } + // ------- V1: line-based window ------- #[test] diff --git a/src/evaluator/types/search.rs b/src/evaluator/types/search.rs index a8a9d4a2..d84e010e 100644 --- a/src/evaluator/types/search.rs +++ b/src/evaluator/types/search.rs @@ -7,7 +7,13 @@ //! pattern within a bounded window. Unlike `TypeKind::String`, which only //! matches at the exact offset, `search` advances through the buffer looking //! for the first occurrence of the pattern anywhere in the window. The -//! search window is `buffer[offset..]` capped by the optional `range`. +//! search window is `buffer[offset..]` capped by `range`: `Some(n)` scans at +//! most `n` bytes, while `None` (bare `search` with no `/N` suffix) scans the +//! whole remaining buffer. magic(5) documents the count as required, but the +//! reference `file` binary accepts the bare form and treats it as +//! `str_range == 0` -- scan from the offset to end-of-buffer -- so we follow +//! the implementation. See `parser::grammar::parse_magic_rule` for the parse +//! side and GOTCHAS S2 for the range representation. use super::TypeReadError; use super::string::compare_string_with_flags; @@ -65,7 +71,7 @@ fn find_match( buffer: &[u8], offset: usize, pattern: &[u8], - range: NonZeroUsize, + range: Option, flags: SearchFlags, ) -> Result, TypeReadError> { if offset >= buffer.len() { @@ -76,7 +82,10 @@ fn find_match( } let remaining = &buffer[offset..]; - let window_len = range.get().min(remaining.len()); + // `Some(n)` -> scan at most `n` bytes; `None` (bare `search`, no `/N`) + // -> scan the whole remaining buffer, matching GNU `file`'s + // `str_range == 0` semantics (see this module's doc header). + let window_len = range.map_or(remaining.len(), |r| r.get().min(remaining.len())); let window = &remaining[..window_len]; if flags.needs_byte_compare() { @@ -157,7 +166,7 @@ pub fn read_search( buffer: &[u8], offset: usize, pattern: &[u8], - range: NonZeroUsize, + range: Option, flags: SearchFlags, ) -> Result, TypeReadError> { match find_match(buffer, offset, pattern, range, flags)? { @@ -207,7 +216,7 @@ pub(super) fn search_bytes_consumed( buffer: &[u8], offset: usize, pattern: &[u8], - range: NonZeroUsize, + range: Option, flags: SearchFlags, ) -> usize { if buffer.get(offset..).is_none() { @@ -238,8 +247,14 @@ pub(super) fn search_bytes_consumed( mod tests { use super::*; - fn nz(n: usize) -> NonZeroUsize { - NonZeroUsize::new(n).expect("non-zero in test") + /// Wrap a bounded range as `Some(NonZeroUsize)` for the search readers, + /// which now take `Option` (`None` = scan-to-EOF). Tests + /// exercising the bare-`search` open window pass `None` directly. + // Always returns `Some`; the `Option` return matches the reader API so + // call sites read as `read_search(.., nz(100), ..)` without wrapping. + #[allow(clippy::unnecessary_wraps)] + fn nz(n: usize) -> Option { + Some(NonZeroUsize::new(n).expect("non-zero in test")) } fn default_flags() -> SearchFlags { @@ -282,6 +297,29 @@ mod tests { assert_eq!(result, Some(Value::String("lo".to_string()))); } + #[test] + fn test_read_search_bare_none_scans_to_eof() { + // Bare `search` (range `None`) scans the whole remaining buffer, + // matching GNU `file`'s `str_range == 0` semantics. The needle sits + // far past any typical bounded window. + let mut buffer = vec![b'.'; 5000]; + buffer.extend_from_slice(b"NEEDLE"); + let result = read_search(&buffer, 0, b"NEEDLE", None, default_flags()).unwrap(); + assert_eq!(result, Some(Value::String("NEEDLE".to_string()))); + } + + #[test] + fn test_search_bytes_consumed_bare_none_reports_match_end() { + // The scan-to-EOF window still advances the anchor to match-end, not + // buffer-end. + let buffer = b"abcWorldxyz___more_data"; + assert_eq!( + search_bytes_consumed(buffer, 0, b"World", None, default_flags()), + 8, + "bare search must advance to match-end (8), not buffer length" + ); + } + #[test] fn test_read_search_at_offset() { let buffer = b"junk_prefix_World!"; diff --git a/src/evaluator/types/string.rs b/src/evaluator/types/string.rs index 1ba902bc..d236b21b 100644 --- a/src/evaluator/types/string.rs +++ b/src/evaluator/types/string.rs @@ -62,17 +62,23 @@ const STRING16_MAX_UNITS: usize = 8192; /// /// # Errors /// -/// Returns `TypeReadError::BufferOverrun` if the offset is greater than or equal to the buffer -/// length. -// Slicing is invariant-safe: `offset < buffer.len()` is checked at entry; -// `search_len` and `read_length` are clamped by `min`/`memchr` results. +/// Returns `TypeReadError::BufferOverrun` only when the offset is strictly +/// greater than the buffer length. `offset == buffer.len()` is the EOF +/// position and yields an empty string (0 readable bytes), matching +/// libmagic's `string` behavior: a `string x` child whose offset lands +/// exactly at EOF renders an empty `%s` rather than being dropped (e.g. +/// LUKS's `>8 string x [%s,` on an 8-byte-truncated header prints `[,`). +/// See GOTCHAS S15.1. +// Slicing is invariant-safe: `offset <= buffer.len()` is checked at entry, so +// `&buffer[offset..]` is at worst an empty slice; `search_len` and +// `read_length` are clamped by `min`/`memchr` results. #[allow(clippy::indexing_slicing)] pub fn read_string( buffer: &[u8], offset: usize, max_length: Option, ) -> Result { - if offset >= buffer.len() { + if offset > buffer.len() { return Err(TypeReadError::BufferOverrun { offset, buffer_len: buffer.len(), @@ -125,7 +131,21 @@ pub fn read_string_exact( offset, buffer_len: buffer.len(), })?; - Ok(Value::String(bytes_to_string_fast(slice))) + // This path exists for BYTE-EXACT `string PATTERN` comparison (GOTCHAS + // S6.4), so it must not corrupt the read with lossy UTF-8 replacement. + // When the slice is valid UTF-8, return `Value::String` (so `%s` output on + // the matched value renders normally). When it is NOT -- e.g. a rule with a + // high-byte value like gzip's `0 string \037\213` (bytes 0x1f 0x8b, where + // 0x8b is invalid UTF-8) -- return the RAW bytes as `Value::Bytes` instead. + // A lossy `String` decode would turn 0x8b into U+FFFD (3 bytes 0xEF BF BD), + // which never equals the raw-byte pattern, so the rule would silently never + // match. `apply_equal` / `compare_values` compare `Bytes` and `String` by + // underlying byte sequence (GOTCHAS S2.3), so either variant compares + // correctly against a `String` or `Bytes` pattern operand. + match std::str::from_utf8(slice) { + Ok(valid) => Ok(Value::String(valid.to_string())), + Err(_) => Ok(Value::Bytes(slice.to_vec())), + } } /// Compare a magic-rule `pattern` against `buffer[offset..]` using libmagic @@ -696,21 +716,19 @@ mod tests { } #[test] - fn test_read_string_empty_buffer() { + fn test_read_string_empty_buffer_at_eof_is_empty_string() { + // offset == buffer.len() (0 == 0) is the EOF position: 0 readable + // bytes -> empty string, NOT an error. This is what lets a + // `string x` child whose offset lands exactly at EOF render an + // empty `%s` (GNU `file` parity; GOTCHAS S15.1). let buffer = b""; - let result = read_string(buffer, 0, None); - assert!(result.is_err()); - assert_eq!( - result.unwrap_err(), - TypeReadError::BufferOverrun { - offset: 0, - buffer_len: 0 - } - ); + let result = read_string(buffer, 0, None).unwrap(); + assert_eq!(result, Value::String(String::new())); } #[test] fn test_read_string_offset_out_of_bounds() { + // Strictly past EOF (offset > buffer.len()) is still an overrun. let buffer = b"Hello"; let result = read_string(buffer, 10, None); assert!(result.is_err()); @@ -724,14 +742,22 @@ mod tests { } #[test] - fn test_read_string_offset_at_buffer_end() { - let buffer = b"Hello"; - let result = read_string(buffer, 5, None); - assert!(result.is_err()); + fn test_read_string_offset_at_buffer_end_is_empty_string() { + // offset == buffer.len() yields an empty string (0 readable bytes), + // matching libmagic's `string` behavior at EOF. See GOTCHAS S15.1. + let buffer = b"Hello"; // len 5 + let result = read_string(buffer, 5, None).unwrap(); + assert_eq!(result, Value::String(String::new())); + // With a max_length cap it is still empty. + let capped = read_string(buffer, 5, Some(8)).unwrap(); + assert_eq!(capped, Value::String(String::new())); + // One byte strictly past EOF is a genuine overrun. + let overrun = read_string(buffer, 6, None); + assert!(overrun.is_err()); assert_eq!( - result.unwrap_err(), + overrun.unwrap_err(), TypeReadError::BufferOverrun { - offset: 5, + offset: 6, buffer_len: 5 } ); @@ -861,7 +887,12 @@ mod tests { #[test] fn test_read_string_edge_case_combinations() { let test_cases = [ - (b"" as &[u8], 0, None, true), + // offset == buffer.len() is the EOF position -> empty string, + // no longer an error (GOTCHAS S15.1). + (b"" as &[u8], 0, None, false), + // offset strictly past EOF is still an error. + (b"", 1, None, true), + (b"A", 2, None, true), (b"\x00", 0, None, false), (b"A", 0, Some(0), false), (b"AB", 1, Some(1), false), diff --git a/src/evaluator/types/tests.rs b/src/evaluator/types/tests.rs index 3da6fe93..f0bcb21f 100644 --- a/src/evaluator/types/tests.rs +++ b/src/evaluator/types/tests.rs @@ -1030,6 +1030,194 @@ fn test_bytes_consumed_string_at_past_end_returns_zero() { assert_eq!(bytes_consumed_with_pattern(buf, 10, &typ, None), 0); } +// ============================================================================= +// fix-system-magic-regex-graceful, U1: `Value::Bytes` backstop for +// `TypeKind::Regex`. +// +// The parser can currently miscategorize an escape-heavy regex pattern +// (e.g. `\^[\040\t]{0,50}\\.asciiz`) as `Value::Bytes` instead of +// `Value::String` (see `parse_value`'s hex/mixed-ascii branch). Before this +// fix, both `read_typed_value_with_pattern` and `read_pattern_match` +// rejected `Value::Bytes` regex patterns with `UnsupportedType`, unlike the +// sibling `TypeKind::Search` arms which already accepted both variants +// (GOTCHAS S2.4). See docs/plans/2026-07-17-001-fix-system-magic-regex- +// graceful-plan.md. +// ============================================================================= + +fn regex_type() -> TypeKind { + TypeKind::Regex { + flags: crate::parser::ast::RegexFlags::default(), + count: crate::parser::ast::RegexCount::Default, + } +} + +/// Happy path (regression guard): a `Value::String` pattern still matches +/// through `read_pattern_match`, unaffected by the new `Bytes` arm. +#[test] +fn test_read_pattern_match_regex_string_pattern_still_matches() { + let typ = regex_type(); + let pattern = Value::String("foobar[0-9]+".to_string()); + let result = read_pattern_match(b"prefix foobar123 suffix", 0, &typ, Some(&pattern), 8192) + .expect("read_pattern_match should not error for a valid String pattern"); + assert!( + matches!(result, Some(Value::String(ref s)) if s == "foobar123"), + "expected a match on the String pattern, got {result:?}" + ); +} + +/// A `Value::Bytes` regex pattern (the miscategorized-escape case) must be +/// accepted by `read_pattern_match`, not rejected as `UnsupportedType`. +#[test] +fn test_read_pattern_match_regex_accepts_bytes_pattern() { + let typ = regex_type(); + let pattern = Value::Bytes(b"^[ \t]*\\.asciiz".to_vec()); + let result = read_pattern_match(b"\t.asciiz \"hi\"", 0, &typ, Some(&pattern), 8192) + .expect("read_pattern_match must accept a Value::Bytes regex pattern, not UnsupportedType"); + assert!( + result.is_some(), + "expected the Bytes pattern to match the leading-whitespace buffer, got {result:?}" + ); +} + +/// The same `Value::Bytes` acceptance must hold for +/// `read_typed_value_with_pattern` (the non-engine dispatch entry point), +/// mirroring the `read_pattern_match` arm exactly. +#[test] +fn test_read_typed_value_with_pattern_regex_accepts_bytes_pattern() { + let typ = regex_type(); + let pattern = Value::Bytes(b"[0-9]+".to_vec()); + let result = read_typed_value_with_pattern(b"abc123def", 0, &typ, Some(&pattern), 8192) + .expect("read_typed_value_with_pattern must accept a Value::Bytes regex pattern"); + assert_eq!( + result, + Value::String("123".to_string()), + "expected the matched digits, got {result:?}" + ); +} + +/// Zero-width match contract (GOTCHAS S2.5) must be preserved for a +/// `Value::Bytes` pattern: `^` matches at position 0 with an empty capture, +/// which is `Ok(Some(Value::String("")))`, distinct from a genuine miss. +#[test] +fn test_read_pattern_match_regex_bytes_zero_width_match() { + let typ = regex_type(); + let pattern = Value::Bytes(b"^".to_vec()); + let result = read_pattern_match(b"hello", 0, &typ, Some(&pattern), 8192) + .expect("zero-width Bytes pattern should not error"); + assert_eq!( + result, + Some(Value::String(String::new())), + "zero-width match must be Some(empty string), not None (GOTCHAS S2.5)" + ); +} + +/// A `Value::Bytes` pattern that does not match the buffer must produce a +/// genuine miss (`Ok(None)`), not an error. +#[test] +fn test_read_pattern_match_regex_bytes_pattern_miss() { + let typ = regex_type(); + let pattern = Value::Bytes(b"xyz".to_vec()); + let result = + read_pattern_match(b"abcdef", 0, &typ, Some(&pattern), 8192).expect("miss is not an error"); + assert_eq!(result, None, "non-matching Bytes pattern must be Ok(None)"); +} + +/// KTD6: a `Value::Bytes` regex pattern containing a byte that is not +/// valid UTF-8 must not panic. `String::from_utf8_lossy` substitutes +/// U+FFFD for the invalid byte before compiling -- this test pins the +/// no-panic / graceful-result contract; the `warn!` emission itself is +/// asserted separately via the `testing_logger`-based tests further down +/// this module (see `format_logs` and its callers). +#[test] +fn test_read_pattern_match_regex_bytes_invalid_utf8_does_not_panic() { + let typ = regex_type(); + // 0xFF is never valid UTF-8 in any position. + let pattern = Value::Bytes(vec![0xFF, b'a']); + let result = read_pattern_match(b"\xEF\xBF\xBDa tail", 0, &typ, Some(&pattern), 8192); + // Whether this happens to match the lossily-substituted U+FFFD encoding + // or not is incidental; the load-bearing assertion is that decoding an + // invalid-UTF-8 Bytes pattern never panics and always yields a valid + // Result. + assert!( + result.is_ok(), + "invalid-UTF-8 Bytes pattern must not error, got {result:?}" + ); +} + +/// Missing pattern (`None`) must still be a hard `UnsupportedType` error in +/// both dispatch functions -- U2's engine-level graceful skip depends on +/// this remaining an `Err`, not silently becoming a non-match here. +#[test] +fn test_regex_missing_pattern_still_errors_in_both_dispatch_fns() { + let typ = regex_type(); + + let pattern_match_result = read_pattern_match(b"abc", 0, &typ, None, 8192); + assert!( + matches!( + pattern_match_result, + Err(TypeReadError::UnsupportedType { ref type_name }) if type_name == "regex without string pattern" + ), + "read_pattern_match with no pattern must still error, got {pattern_match_result:?}" + ); + + let typed_value_result = read_typed_value_with_pattern(b"abc", 0, &typ, None, 8192); + assert!( + matches!( + typed_value_result, + Err(TypeReadError::UnsupportedType { ref type_name }) if type_name == "regex without string pattern" + ), + "read_typed_value_with_pattern with no pattern must still error, got {typed_value_result:?}" + ); +} + +// ============================================================================= +// U2 predicate helpers: `is_missing_pattern_operand` / `is_regex_compile_failure` +// ============================================================================= + +#[test] +fn test_is_missing_pattern_operand_recognizes_known_messages() { + let recognized: &[&str] = &[ + "regex without string pattern", + "search without string/bytes pattern", + "string with flags requires string/bytes pattern", + ]; + for msg in recognized { + assert!( + is_missing_pattern_operand(msg), + "expected {msg:?} to be recognized as a missing-pattern-operand condition" + ); + } +} + +#[test] +fn test_is_missing_pattern_operand_rejects_other_unsupported_type_messages() { + // R3 narrowness guard: these must NOT be recognized, or a genuine + // capability gap / operator-misuse error would be silently swallowed. + let not_recognized: &[String] = &[ + "regex compile error: some failure".to_string(), + "meta-type Offset cannot be read as a value".to_string(), + "operator GreaterThan is not supported for pattern-bearing type".to_string(), + "read_pattern_match called on non-pattern type".to_string(), + ]; + for msg in not_recognized { + assert!( + !is_missing_pattern_operand(msg), + "expected {msg:?} to NOT be recognized as a missing-pattern-operand condition" + ); + } +} + +#[test] +fn test_is_regex_compile_failure_matches_only_compile_error_prefix() { + assert!(is_regex_compile_failure( + "regex compile error: Compiled regex exceeds size limit of 1048576 bytes." + )); + assert!(!is_regex_compile_failure("regex without string pattern")); + assert!(!is_regex_compile_failure( + "search without string/bytes pattern" + )); +} + #[test] fn test_bytes_consumed_regex_with_string_pattern() { // Regression guard for GOTCHAS 2.1: variable-width variants must be @@ -1050,6 +1238,33 @@ fn test_bytes_consumed_regex_with_string_pattern() { ); } +/// Regression guard: `bytes_consumed_with_pattern`'s `Regex` arm must +/// accept a `Value::Bytes` pattern, mirroring U1's read-side acceptance +/// (`read_pattern_match` / `read_typed_value_with_pattern`). This was +/// caught by `prop_arbitrary_rule_evaluation_never_panics` firing the +/// `debug_assert` in the pre-fix `other => { debug_assert!(false, ...) }` +/// arm for a `NotEqual` regex rule with a `Value::Bytes` pattern -- a +/// successful Bytes-pattern regex match would advance the anchor by 0 +/// (silently stalling) instead of the correct match-end distance, and the +/// property test additionally caught the `debug_assert!(false, ...)` +/// firing as a panic in debug builds. +#[test] +fn test_bytes_consumed_regex_with_bytes_pattern() { + let buf = b"prefix_World_suffix"; + let typ = TypeKind::Regex { + flags: crate::parser::ast::RegexFlags::default(), + count: crate::parser::ast::RegexCount::Default, + }; + let pattern = Value::Bytes(b"World".to_vec()); + // "World" starts at index 7, length 5, so a scan from offset 0 + // consumes 7+5=12 bytes -- matching the Value::String equivalent in + // `test_bytes_consumed_regex_with_string_pattern`. + assert_eq!( + bytes_consumed_with_pattern(buf, 0, &typ, Some(&pattern)), + 12 + ); +} + #[test] fn test_bytes_consumed_regex_no_match_returns_zero() { let buf = b"abcdef"; @@ -1098,7 +1313,7 @@ fn test_bytes_consumed_search_with_pattern_is_match_end() { // FILE_SEARCH, the anchor advances to `base + match_idx + pattern.len()`. let buf = b"abcWorld_xyz"; let typ = TypeKind::Search { - range: ::std::num::NonZeroUsize::new(10).unwrap(), + range: ::std::num::NonZeroUsize::new(10), flags: SearchFlags::default(), }; let pattern = Value::String("World".to_string()); @@ -1114,7 +1329,7 @@ fn test_bytes_consumed_search_with_pattern_is_match_end() { fn test_bytes_consumed_search_no_match_returns_zero() { let buf = b"abcdefghij"; let typ = TypeKind::Search { - range: ::std::num::NonZeroUsize::new(10).unwrap(), + range: ::std::num::NonZeroUsize::new(10), flags: SearchFlags::default(), }; let pattern = Value::String("XYZ".to_string()); @@ -1128,7 +1343,7 @@ fn test_bytes_consumed_search_bytes_pattern_works() { // Value::String pattern would. let buf = &[0x00, 0xff, 0xde, 0xad, 0xbe, 0xef, 0x11]; let typ = TypeKind::Search { - range: ::std::num::NonZeroUsize::new(7).unwrap(), + range: ::std::num::NonZeroUsize::new(7), flags: SearchFlags::default(), }; let pattern = Value::Bytes(vec![0xde, 0xad, 0xbe, 0xef]); @@ -1219,3 +1434,181 @@ fn test_bytes_consumed_string_with_bytes_pattern_is_exact_length() { "usize::MAX offset must return 0 via checked_add" ); } + +// ----------------------------------------------------------------------- +// H hardening: pin the `decode_regex_bytes_pattern` warn!-on-real- +// substitution contract (KTD6) with a real log-capture seam +// (`testing_logger`), rather than code inspection only. +// ----------------------------------------------------------------------- + +/// Test-only helper: `testing_logger::CapturedLog` does not implement +/// `Debug`, so format captured logs manually for failure messages. +fn format_logs(logs: &[testing_logger::CapturedLog]) -> String { + logs.iter() + .map(|l| format!("{:?}: {}", l.level, l.body)) + .collect::>() + .join(", ") +} + +/// A `Value::Bytes` regex pattern containing a byte `>= 0x80` that is not +/// valid UTF-8 triggers a real lossy substitution (`from_utf8_lossy` +/// replaces it with U+FFFD); `decode_regex_bytes_pattern` must `warn!` +/// because the compiled regex now silently diverges from the raw bytes +/// the target buffer is matched against. +#[test] +fn decode_regex_bytes_pattern_warns_on_real_utf8_substitution() { + testing_logger::setup(); + let decoded = decode_regex_bytes_pattern(&[0xFF, b'a']); + // Sanity: the function is still infallible and produces SOME string + // (the lossy replacement), never panics. + assert!(decoded.contains('a')); + testing_logger::validate(|captured_logs| { + let warn_logs: Vec<_> = captured_logs + .iter() + .filter(|l| l.body.contains("not valid UTF-8")) + .collect(); + assert_eq!( + warn_logs.len(), + 1, + "expected exactly one lossy-substitution warning, got {:?}", + format_logs(captured_logs) + ); + assert_eq!( + warn_logs[0].level, + log::Level::Warn, + "lossy UTF-8 substitution must log at warn!, not another level -- got {:?}", + warn_logs[0].level + ); + }); +} + +/// The converse: valid-UTF-8 bytes must NOT trigger the substitution +/// warning at all -- the guard is keyed on `str::from_utf8` actually +/// failing, not merely on the input being `Value::Bytes`. +#[test] +fn decode_regex_bytes_pattern_does_not_warn_on_valid_utf8() { + testing_logger::setup(); + let decoded = decode_regex_bytes_pattern(b"hello[0-9]+"); + assert_eq!(decoded, "hello[0-9]+"); + testing_logger::validate(|captured_logs| { + assert!( + captured_logs.is_empty(), + "valid UTF-8 bytes must not trigger any log entry, got {:?}", + format_logs(captured_logs) + ); + }); +} + +/// `flip_type_endian` mirrors libmagic `softmagic.c` `cvt_flip`: it swaps +/// only the explicit little/big-endian numeric, float, and date families, +/// leaves `Endianness::Native` untouched, and returns every non-endian type +/// (including `String16`, which is deliberately absent from `cvt_flip`) +/// unchanged. The `signed`/`utc` attributes are preserved. +#[test] +fn test_flip_type_endian_matches_cvt_flip() { + use crate::parser::ast::TypeKind; + + // (input, expected-after-flip). Endian-bearing types swap LE<->BE. + let flipped: &[(TypeKind, TypeKind)] = &[ + ( + TypeKind::Short { + endian: Endianness::Little, + signed: true, + }, + TypeKind::Short { + endian: Endianness::Big, + signed: true, + }, + ), + ( + TypeKind::Long { + endian: Endianness::Big, + signed: false, + }, + TypeKind::Long { + endian: Endianness::Little, + signed: false, + }, + ), + ( + TypeKind::Quad { + endian: Endianness::Little, + signed: true, + }, + TypeKind::Quad { + endian: Endianness::Big, + signed: true, + }, + ), + ( + TypeKind::Float { + endian: Endianness::Big, + }, + TypeKind::Float { + endian: Endianness::Little, + }, + ), + ( + TypeKind::Double { + endian: Endianness::Little, + }, + TypeKind::Double { + endian: Endianness::Big, + }, + ), + ( + TypeKind::Date { + endian: Endianness::Big, + utc: true, + }, + TypeKind::Date { + endian: Endianness::Little, + utc: true, + }, + ), + ( + TypeKind::QDate { + endian: Endianness::Little, + utc: false, + }, + TypeKind::QDate { + endian: Endianness::Big, + utc: false, + }, + ), + ]; + for (input, expected) in flipped { + assert_eq!( + &flip_type_endian(input), + expected, + "endian-bearing type must swap LE<->BE: {input:?}" + ); + } + + // Native-endian and non-endian types must be returned unchanged. + let unchanged: &[TypeKind] = &[ + TypeKind::Short { + endian: Endianness::Native, + signed: true, + }, + TypeKind::Long { + endian: Endianness::Native, + signed: false, + }, + TypeKind::Byte { signed: true }, + // String16 is intentionally NOT in libmagic's cvt_flip. + TypeKind::String16 { + endian: Endianness::Big, + }, + TypeKind::String16 { + endian: Endianness::Little, + }, + ]; + for typ in unchanged { + assert_eq!( + &flip_type_endian(typ), + typ, + "native/non-endian type must be unchanged: {typ:?}" + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index c3c77535..ee086ffb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,7 +153,7 @@ impl From for LibmagicError { pub struct MagicDatabase { /// Named subroutine definitions extracted from magic file `name` rules, /// keyed by identifier. The evaluator consults this table when a rule of - /// type `TypeKind::Meta(MetaType::Use(name))` is reached. + /// type `TypeKind::Meta(MetaType::Use { name, .. })` is reached. name_table: std::sync::Arc, /// Top-level rules as a shared immutable slice. This is the primary rule /// storage for the database. Passed through the evaluation context as part @@ -249,7 +249,15 @@ impl MagicDatabase { pub fn with_builtin_rules_and_config(config: EvaluationConfig) -> Result { config.validate()?; let mut rules = crate::builtin_rules::get_builtin_rules(); - crate::evaluator::strength::sort_rules_by_strength_recursive(&mut rules); + // Sort only the TOP-LEVEL rules by strength (libmagic's + // `apprentice_sort` orders whole magic entries by their first line's + // strength). Continuation/child rules are NOT reordered -- they run + // in file order, which is load-bearing for order-sensitive directives + // like `default`/`clear` and for multi-fragment descriptions whose + // pieces must render in source order (e.g. gzip's "last modified, + // max compression, from Unix"). See the non-recursive contract note + // on `sort_rules_by_strength`. + crate::evaluator::strength::sort_rules_by_strength(&mut rules); let root_rules: std::sync::Arc<[MagicRule]> = std::sync::Arc::from(rules.into_boxed_slice()); Ok(Self { @@ -314,15 +322,20 @@ impl MagicDatabase { })?; let parser::ParsedMagic { mut rules, - mut name_table, + name_table, } = parsed; - crate::evaluator::strength::sort_rules_by_strength_recursive(&mut rules); - // Each named subroutine body must be sorted by the same strength - // ordering so evaluation of a `use` site is deterministic and - // matches the ordering applied to top-level rules. - name_table.sort_subroutines(|rules| { - crate::evaluator::strength::sort_rules_by_strength_recursive(rules); - }); + // Sort only the TOP-LEVEL rules by strength, mirroring libmagic's + // `apprentice_sort` (which orders whole magic entries by their first + // line's strength and never reorders continuation lines). Child rules + // and `name`-block subroutine bodies stay in file order: they are + // continuation-level rules, and their order is load-bearing for + // `default`/`clear` firing and for multi-fragment descriptions that + // must render in source order (e.g. the `gzip-info` subroutine's + // "last modified, max compression, from Unix"). Strength-sorting them + // reorders a comparison-bearing sibling ahead of a low-strength + // `default`, wrongly suppressing the `default` message. See the + // non-recursive contract note on `sort_rules_by_strength`. + crate::evaluator::strength::sort_rules_by_strength(&mut rules); let root_rules: std::sync::Arc<[MagicRule]> = std::sync::Arc::from(rules.into_boxed_slice()); @@ -477,24 +490,46 @@ impl MagicDatabase { // so no `is_empty()` guard is needed here. let matches = evaluate_rules(&self.root_rules, buffer, &mut context)?; - Ok(self.build_result(matches, file_size, start_time)) + Ok(self.build_result(matches, buffer, file_size, start_time)) } /// Build an `EvaluationResult` from match results, file size, and start time. /// /// This is shared between `evaluate_file` and `evaluate_buffer_internal` to /// avoid duplicating the result-construction logic. + /// + /// # Text/data fallback + /// + /// When rule evaluation produces no usable description -- either + /// because no rule matched at all, or because every match that did + /// occur carries no description text (a message-less gating rule + /// that was allowed to proceed past `stop_at_first_match` without a + /// message-bearing rule ever firing behind it, GOTCHAS S13.2) -- the + /// description falls back to [`crate::output::ascmagic::classify_fallback`] + /// against the original buffer, mirroring GNU `file`'s `file_ascmagic` + /// basic text/data classification. This is what keeps the CLI from + /// ever printing a blank description for a readable file. fn build_result( &self, matches: Vec, + buffer: &[u8], file_size: u64, start_time: std::time::Instant, ) -> EvaluationResult { - let (description, confidence) = if matches.is_empty() { - ("data".to_string(), 0.0) + let rule_description = if matches.is_empty() { + String::new() } else { + Self::concatenate_messages(&matches) + }; + + let (description, confidence) = if rule_description.trim().is_empty() { ( - Self::concatenate_messages(&matches), + crate::output::ascmagic::classify_fallback(buffer).to_string(), + 0.0, + ) + } else { + ( + rule_description, matches.first().map_or(0.0, |m| m.confidence), ) }; @@ -527,7 +562,12 @@ impl MagicDatabase { /// Each match's `message` is first run through /// [`crate::output::format::format_magic_message`], which substitutes /// printf-style specifiers (`%lld`, `%02x`, `%s`, etc.) with the - /// rule's read value. The resulting rendered strings are then joined + /// rule's read value. Matches that render to an empty string (a + /// message-less gating rule -- see GOTCHAS S13.2 -- or a + /// `default`/`indirect`/`offset`/`use` directive with no message) + /// contribute nothing and are skipped entirely, so they cannot + /// introduce a stray separating space between the descriptions on + /// either side of them. The remaining rendered strings are joined /// with spaces, except when a rendered string starts with the /// backspace character (`\b`, U+0008) which suppresses both the /// separating space and the backspace itself (GOTCHAS.md S14.1). @@ -542,8 +582,22 @@ impl MagicDatabase { let mut result = String::with_capacity(capacity); for m in matches { let rendered = format_magic_message(&m.message, &m.value, &m.type_kind); - if let Some(rest) = rendered.strip_prefix('\u{0008}') { - // Backspace suppresses the space and the character itself + if rendered.is_empty() { + // No text to contribute -- skip so this match cannot + // introduce a stray separating space. + continue; + } + // GNU `file`'s no-separator convention: a description beginning + // with a backspace suppresses both the separating space and the + // marker itself (GOTCHAS.md S14.1). The marker most often reaches + // us as the literal two-character sequence `\b` (backslash + 'b'), + // because the message parser preserves description text verbatim -- + // matching GNU `file`, which keeps the desc literal and + // special-cases a leading `\b` at print time (e.g. the msdos + // `\b, for MS Windows` and Mach-O universal `\b]` rules). We also + // accept a bare U+0008 for programmatically-constructed messages. + let without_marker = crate::evaluator::strip_no_separator_marker(&rendered); + if let Some(rest) = without_marker { result.push_str(rest); } else if !result.is_empty() { result.push(' '); diff --git a/src/output/ascmagic.rs b/src/output/ascmagic.rs new file mode 100644 index 00000000..4606d9c7 --- /dev/null +++ b/src/output/ascmagic.rs @@ -0,0 +1,151 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal text/data fallback classification, modeled on GNU `file`'s +//! `file_ascmagic` (`src/ascmagic.c`). +//! +//! When no magic rule produces a usable description -- either because no +//! rule matched at all, or because every rule that matched carries no +//! description text (GOTCHAS S13.2) -- GNU `file` never prints a blank +//! line. It falls back to a basic content classification: `"empty"` for a +//! zero-byte file, `"ASCII text"` for plain textual content, a Unicode +//! variant for valid non-ASCII UTF-8, and `"data"` for anything else +//! (binary content). +//! +//! # Scope +//! +//! This is a deliberately narrow subset of GNU `file`'s real charset +//! detection, which additionally distinguishes ISO-8859 variants, UTF-16, +//! line-ending styles, and several "text with X" qualifiers (escape +//! sequences, overstriking, CRLF terminators, byte-order marks, etc. -- +//! see `src/ascmagic.c` and `src/encoding.c` upstream). Replicating that +//! fully is out of scope for this fallback: the goal here is solely to +//! ensure the CLI never emits a blank description for a readable file +//! (the assembler-source-text and plain-ASCII-text bugs this module +//! fixes), not full charset fidelity. Every classification below is a +//! true subset of what GNU `file` would print for the same input -- e.g. +//! `file` prints `"ASCII text, with CRLF line terminators"` for a +//! CRLF-terminated buffer where we print plain `"ASCII text"` -- so +//! differential tests that check for a specific classification (rather +//! than exact byte-for-byte output) still hold. + +/// Bytes GNU `file`'s `ascmagic`/`encoding` text test treats as part of +/// ordinary "text" content: printable ASCII (0x20..=0x7E) plus the common +/// control characters that appear in real-world text files (tab, LF, CR, +/// vertical tab, form feed) and a few legacy terminal-control bytes GNU +/// `file` still classifies as text -- bell, backspace, and escape -- +/// which it reports as `"ASCII text, with ..."` qualifiers rather than +/// reclassifying as binary `"data"`. This fallback does not reproduce +/// those qualifiers (see the module doc), but a buffer containing only +/// these bytes is still `"ASCII text"`, not `"data"`. +fn is_text_safe_byte(b: u8) -> bool { + matches!( + b, + 0x20..=0x7E | b'\t' | b'\n' | b'\r' | 0x0B | 0x0C | 0x07 | 0x08 | 0x1B + ) +} + +/// Classify a buffer using the minimal text/data fallback described in +/// the module doc. +/// +/// Returns one of `"empty"`, `"ASCII text"`, `"UTF-8 Unicode text"`, or +/// `"data"`. This is intentionally infallible -- there is no input for +/// which classification can fail, so the caller never needs to handle +/// an error path here (matching the evaluator's graceful-degradation +/// discipline: a fallback that can itself fail would defeat its purpose). +/// +/// # Examples +/// +/// ``` +/// use libmagic_rs::output::ascmagic::classify_fallback; +/// +/// assert_eq!(classify_fallback(b""), "empty"); +/// assert_eq!(classify_fallback(b"hello world\n"), "ASCII text"); +/// assert_eq!(classify_fallback(&[0x00, 0x01, 0x02, 0xff]), "data"); +/// ``` +#[must_use] +pub fn classify_fallback(buffer: &[u8]) -> &'static str { + if buffer.is_empty() { + return "empty"; + } + + if buffer.iter().all(|&b| is_text_safe_byte(b)) { + return "ASCII text"; + } + + // Valid non-ASCII UTF-8 only counts as text when every low byte (< 0x80) is + // itself text-safe. A NUL or other binary control byte is a strong binary + // signal even inside an otherwise-valid UTF-8 buffer -- GNU `file` + // classifies such input as data, and the ASCII branch above already treats + // these bytes as non-text. High bytes (>= 0x80) are exempt from the check + // because they are the UTF-8 continuation/lead bytes the `from_utf8` gate + // validates. Without this, a valid-UTF-8 buffer carrying an embedded NUL + // would be mislabelled "UTF-8 Unicode text". + let has_non_ascii_byte = buffer.iter().any(|&b| b >= 0x80); + let no_binary_control_byte = buffer.iter().all(|&b| b >= 0x80 || is_text_safe_byte(b)); + if has_non_ascii_byte && no_binary_control_byte && std::str::from_utf8(buffer).is_ok() { + return "UTF-8 Unicode text"; + } + + "data" +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_empty_buffer_as_empty() { + assert_eq!(classify_fallback(b""), "empty"); + } + + #[test] + fn classifies_plain_ascii_as_ascii_text() { + let cases: &[(&str, &[u8])] = &[ + ("simple sentence", b"hello world this is plain text\n"), + ("tabs and newlines", b"a\tb\nc\r\nd\n"), + ("vertical tab and form feed", b"a\x0bb\x0cc\n"), + ("bell, backspace, escape", b"a\x07b\x08c\x1bd\n"), + ]; + for (label, input) in cases { + assert_eq!( + classify_fallback(input), + "ASCII text", + "case {label:?} should classify as ASCII text" + ); + } + } + + #[test] + fn classifies_valid_non_ascii_utf8_as_utf8_unicode_text() { + let cases: &[(&str, &[u8])] = &[ + ("accented latin", "caf\u{e9} r\u{e9}sum\u{e9}\n".as_bytes()), + ("bom prefix", &[0xEF, 0xBB, 0xBF, b'h', b'i']), + ("multi-byte cjk", "\u{4f60}\u{597d}\n".as_bytes()), + ]; + for (label, input) in cases { + assert_eq!( + classify_fallback(input), + "UTF-8 Unicode text", + "case {label:?} should classify as UTF-8 Unicode text" + ); + } + } + + #[test] + fn classifies_binary_content_as_data() { + let cases: &[(&str, &[u8])] = &[ + ("null byte in otherwise-ascii text", b"hello\x00world\n"), + ("random high bytes", &[0x80, 0x81, 0x82, 0x83]), + ("invalid utf8 continuation-only", &[0xC0, 0x80]), + ("elf magic", &[0x7f, b'E', b'L', b'F']), + ]; + for (label, input) in cases { + assert_eq!( + classify_fallback(input), + "data", + "case {label:?} should classify as data" + ); + } + } +} diff --git a/src/output/format.rs b/src/output/format.rs index d0ec88ae..c8e55ae4 100644 --- a/src/output/format.rs +++ b/src/output/format.rs @@ -162,6 +162,11 @@ struct Spec { left_align: bool, alt_form: bool, width: usize, + /// `.` precision. Currently honored only for `%s` (`Conv::Str`), + /// where it truncates the rendered string to at most `precision` + /// characters -- e.g. sgml's `%.3s` renders the XML version `1.0` from a + /// longer field. `None` means no precision was given (no truncation). + precision: Option, conv: Conv, /// Byte index of the character *after* this specifier in the template. end: usize, @@ -222,13 +227,23 @@ fn parse_spec(bytes: &[u8], start: usize) -> Option { i += 1; } - // Precision (`.`): parsed and skipped -- no current consumer - // requires precision handling, and numeric rendering is whole-value. + // Precision (`.`). Captured for `%s` truncation (see `render`); + // numeric rendering is still whole-value and ignores it. A bare `.` with + // no digits means precision 0 (C semantics). The digit run is capped at + // MAX_FORMAT_WIDTH for the same crafted-input protection as `width`. + let mut precision: Option = None; if i < bytes.len() && bytes[i] == b'.' { i += 1; + let mut prec: usize = 0; while i < bytes.len() && bytes[i].is_ascii_digit() { + let digit = (bytes[i] - b'0') as usize; + prec = prec + .saturating_mul(10) + .saturating_add(digit) + .min(MAX_FORMAT_WIDTH); i += 1; } + precision = Some(prec); } // Length modifier (`h`, `hh`, `l`, `ll`, `j`, `z`, `t`). We consume @@ -263,6 +278,7 @@ fn parse_spec(bytes: &[u8], start: usize) -> Option { left_align, alt_form, width, + precision, conv, end: i, }) @@ -273,7 +289,7 @@ fn parse_spec(bytes: &[u8], start: usize) -> Option { fn render(spec: &Spec, value: &Value, type_kind: &TypeKind) -> Option { match spec.conv { Conv::Percent => Some("%".to_string()), - Conv::Str => Some(render_string(value)), + Conv::Str => Some(render_str_spec(spec, value)), Conv::Signed => { let n = coerce_to_i64(value)?; Some(pad_numeric(&n.to_string(), spec)) @@ -324,6 +340,33 @@ fn render(spec: &Spec, value: &Value, type_kind: &TypeKind) -> Option { } } +/// Render a `%s` specifier: base string, then optional `.` +/// truncation, then width padding. +/// +/// Precision truncates to at most `precision` characters. Truncation is +/// **char-wise**, not byte-wise: C's `%.Ns` truncates by bytes, but our value +/// is a Rust `String` and a byte-wise cut could split a multi-byte UTF-8 +/// sequence. For the ASCII version/name fields that use precision in the magic +/// corpus (`%.3s`, `%-.4s`, `%.10s`) the two are identical; char-wise is the +/// safe choice for the rare non-ASCII case. +/// +/// Width padding (via [`pad_non_numeric`]) is applied after truncation so +/// `%4.4s` and `%-.4s` render correctly -- previously `%s` dropped width +/// entirely. +fn render_str_spec(spec: &Spec, value: &Value) -> String { + let base = render_string(value); + // Truncate to `p` chars in a single pass bounded by `p` (no preceding full + // `chars().count()` pass, and no byte slicing -- the repo forbids `&s[n..]` + // for UTF-8 safety). `take(p)` stops after at most `p` chars regardless of + // string length; when `p >= len` this collects an identical copy, which is + // a cheap, correct no-op for the rare precision case. + let truncated = match spec.precision { + Some(p) => base.chars().take(p).collect::(), + None => base, + }; + pad_non_numeric(&truncated, spec) +} + /// Render a [`Value`] for `%s`. Strings pass through; byte sequences are /// converted via lossy UTF-8; numbers render as decimal (GNU `file` does /// the same for mixed-type `%s` substitutions). @@ -590,6 +633,50 @@ mod tests { assert_eq!(out, "data=abc"); } + #[test] + fn test_string_precision_truncation() { + // `%.Ns` truncates the rendered string to at most N characters. + // sgml's `>15 string/t >\0 %.3s document text` is the motivating + // rule: the full XML-version field is `1.0" encoding=...` but `%.3s` + // must render only `1.0` so the description reads `XML 1.0 ...`. + let str_t = TypeKind::String { + max_length: None, + flags: StringFlags::default(), + }; + // (template, value, expected) + let cases: &[(&str, &str, &str)] = &[ + // The XML case: full field truncated to 3 chars. + ( + "%.3s document text", + "1.0\" encoding=\"UTF-8\"?>", + "1.0 document text", + ), + // Precision shorter than the string truncates. + ("%.1s", "1.0", "1"), + // Precision >= length is a no-op (no padding without width). + ("%.10s", "abc", "abc"), + ("%.3s", "abc", "abc"), + // Precision 0 renders the empty string. + ("%.0s", "abc", ""), + // Left-align precision (`-` is a no-op here since no width). + ("%-.4s", "versionX", "vers"), + // Width padding is applied AFTER truncation: `%4.4s` on "ab" + // truncates to "ab" (no-op) then right-aligns to width 4 (pads on + // the LEFT). + ("[%4.4s]", "ab", "[ ab]"), + // `%-4.2s`: truncate "hello" to "he", then `-` left-aligns to + // width 4 (pads on the RIGHT). + ("[%-4.2s]", "hello", "[he ]"), + ]; + for (template, value, expected) in cases { + let out = format_magic_message(template, &Value::String((*value).to_string()), &str_t); + assert_eq!( + out, *expected, + "template {template:?} on value {value:?} should render {expected:?}", + ); + } + } + #[test] fn test_alt_form_prefix_suppressed_on_zero_value() { // C printf special-cases `%#o`, `%#x`, `%#X` with value 0: the diff --git a/src/output/mod.rs b/src/output/mod.rs index 1884fed1..2940ac1b 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -9,6 +9,7 @@ //! The module follows a structured approach where evaluation results contain metadata //! about the evaluation process and a list of matches found during rule processing. +pub mod ascmagic; pub mod format; pub mod json; pub mod text; @@ -1054,7 +1055,10 @@ mod tests { // Verify all matches have proper rule paths for match_result in &result.matches { assert!(!match_result.rule_path.is_empty()); - assert_eq!(match_result.rule_path[0], "elf"); + assert_eq!( + match_result.rule_path.first().map(String::as_str), + Some("elf") + ); } } } diff --git a/src/parser/ast.rs b/src/parser/ast.rs index b566945e..aeef34e2 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -350,14 +350,30 @@ pub enum MetaType { /// previously declared via [`MetaType::Name`]. See magic(5) for the /// "use" type semantics. /// + /// `flip_endian` records the magic(5) `\^` prefix (`use \^name`), + /// which flips the endianness of every endian-bearing read inside the + /// invoked subroutine (libmagic `softmagic.c` `cvt_flip`; the flip + /// toggles and propagates into nested `use` calls). A plain `use name` + /// has `flip_endian: false`. See the evaluator's `flip_type_endian` + /// and the `SubroutineScope` guard for the runtime semantics. + /// /// # Examples /// /// ``` /// use libmagic_rs::parser::ast::MetaType; - /// let meta = MetaType::Use("part2".to_string()); - /// assert_eq!(meta, MetaType::Use("part2".to_string())); + /// let meta = MetaType::Use { name: "part2".to_string(), flip_endian: false }; + /// assert_eq!( + /// meta, + /// MetaType::Use { name: "part2".to_string(), flip_endian: false } + /// ); /// ``` - Use(String), + Use { + /// Identifier of the named subroutine to invoke. + name: String, + /// Whether to flip endianness of reads inside the subroutine + /// (the magic(5) `\^` prefix). + flip_endian: bool, + }, /// `indirect` directive: re-applies the entire magic database at the /// resolved offset. See magic(5) for the "indirect" type semantics. /// @@ -646,14 +662,19 @@ pub enum TypeKind { /// three cases. count: RegexCount, }, - /// Multi-byte pattern search within a bounded range + /// Multi-byte pattern search within a bounded (or open) range + /// + /// Search rules look for a literal byte pattern starting at the offset. + /// Unlike [`TypeKind::String`], which only matches at the exact offset, + /// `search` scans forward for the first occurrence. The window is + /// controlled by `range`: /// - /// Search rules look for a literal byte pattern within `range` bytes of - /// the offset. Unlike [`TypeKind::String`], which only matches at the - /// exact offset, `search` scans forward up to `range` bytes for the - /// first occurrence. The range is **mandatory** per GNU `file`'s - /// magic(5) specification and is stored as a [`NonZeroUsize`] so a - /// zero-range search is unrepresentable. + /// * `Some(n)` -- the `search/N` form -- scans at most `n` bytes. `n` is + /// a [`NonZeroUsize`] so `search/0` is unrepresentable. + /// * `None` -- bare `search` with no `/N` suffix -- scans from the offset + /// to end-of-buffer. magic(5) documents the count as required, but the + /// reference `file` binary accepts the bare form and treats it as + /// `str_range == 0` (scan-to-EOF), so we follow the implementation. /// /// # Examples /// @@ -663,13 +684,20 @@ pub enum TypeKind { /// /// // `search/256` -- scan up to 256 bytes for the literal pattern. /// let bounded = TypeKind::Search { - /// range: NonZeroUsize::new(256).unwrap(), + /// range: NonZeroUsize::new(256), + /// flags: libmagic_rs::parser::ast::SearchFlags::default(), + /// }; + /// + /// // bare `search` -- scan the whole remaining buffer. + /// let open = TypeKind::Search { + /// range: None, /// flags: libmagic_rs::parser::ast::SearchFlags::default(), /// }; /// ``` Search { - /// Scan window width in bytes, starting at the rule's offset. - range: NonZeroUsize, + /// Scan window width in bytes starting at the rule's offset; + /// `None` means scan to end-of-buffer (bare `search`). + range: Option, /// Modifier flags from the `/[sCcWwTtBbf]` suffix on a `search` /// rule. The `/s` flag controls anchor advance (match-START vs /// match-END); the eight `StringFlags`-shared letters alter how @@ -2603,7 +2631,16 @@ mod tests { MetaType::Indirect, MetaType::Offset, MetaType::Name("part2".to_string()), - MetaType::Use("part2".to_string()), + MetaType::Use { + name: "part2".to_string(), + flip_endian: false, + }, + // A `\^`-flipped use must be distinct from the plain form so the + // endian-flip flag participates in equality/serde/round-trip. + MetaType::Use { + name: "part2".to_string(), + flip_endian: true, + }, ]; for (i, variant) in cases.iter().enumerate() { @@ -2643,7 +2680,14 @@ mod tests { MetaType::Indirect, MetaType::Offset, MetaType::Name("foo".to_string()), - MetaType::Use("bar".to_string()), + MetaType::Use { + name: "bar".to_string(), + flip_endian: false, + }, + MetaType::Use { + name: "bar".to_string(), + flip_endian: true, + }, ]; for variant in cases { @@ -2661,7 +2705,14 @@ mod tests { MetaType::Indirect, MetaType::Offset, MetaType::Name("x".to_string()), - MetaType::Use("x".to_string()), + MetaType::Use { + name: "x".to_string(), + flip_endian: false, + }, + MetaType::Use { + name: "x".to_string(), + flip_endian: true, + }, ]; for meta in cases { let kind = TypeKind::Meta(meta); diff --git a/src/parser/codegen.rs b/src/parser/codegen.rs index cef4badf..07d308ef 100644 --- a/src/parser/codegen.rs +++ b/src/parser/codegen.rs @@ -287,8 +287,15 @@ pub fn serialize_type_kind(typ: &TypeKind) -> String { ) } TypeKind::Search { range, flags } => format!( - "TypeKind::Search {{ range: ::std::num::NonZeroUsize::new({}).unwrap_or(::std::num::NonZeroUsize::MIN), flags: crate::parser::ast::SearchFlags {{ compact_whitespace: {}, compact_optional_whitespace: {}, ignore_lowercase: {}, ignore_uppercase: {}, text_test: {}, trim: {}, bin_test: {}, full_word: {}, start_anchor: {} }} }}", - range.get(), + // `range` is `Option`. `NonZeroUsize::new(n)` already + // returns `Some(_)` for n > 0, so a `Some(range)` serializes as a + // `Some(..)` expression; a bare `search` (`None`) serializes as + // `None` (scan-to-EOF). + "TypeKind::Search {{ range: {}, flags: crate::parser::ast::SearchFlags {{ compact_whitespace: {}, compact_optional_whitespace: {}, ignore_lowercase: {}, ignore_uppercase: {}, text_test: {}, trim: {}, bin_test: {}, full_word: {}, start_anchor: {} }} }}", + match range { + Some(r) => format!("::std::num::NonZeroUsize::new({})", r.get()), + None => "None".to_string(), + }, flags.compact_whitespace, flags.compact_optional_whitespace, flags.ignore_lowercase, @@ -308,9 +315,10 @@ pub fn serialize_type_kind(typ: &TypeKind) -> String { "TypeKind::Meta(MetaType::Name(String::from({})))", format_string_literal(id) ), - MetaType::Use(id) => format!( - "TypeKind::Meta(MetaType::Use(String::from({})))", - format_string_literal(id) + MetaType::Use { name, flip_endian } => format!( + "TypeKind::Meta(MetaType::Use {{ name: String::from({}), flip_endian: {} }})", + format_string_literal(name), + flip_endian ), }, } @@ -673,7 +681,10 @@ mod tests { let malicious = r#""; panic!("pwned-from-use"); let _ = ""#; let rule = MagicRule { offset: OffsetSpec::Absolute(0), - typ: TypeKind::Meta(MetaType::Use(malicious.to_string())), + typ: TypeKind::Meta(MetaType::Use { + name: malicious.to_string(), + flip_endian: false, + }), op: Operator::AnyValue, value: Value::Uint(0), message: "meta use rule".to_string(), diff --git a/src/parser/grammar/getstr/mod.rs b/src/parser/grammar/getstr/mod.rs new file mode 100644 index 00000000..e84a5217 --- /dev/null +++ b/src/parser/grammar/getstr/mod.rs @@ -0,0 +1,278 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +//! GNU `file`-compatible `getstr` escape resolution for `regex` pattern values. +//! +//! `TypeKind::Regex` patterns are captured here as a getstr-**resolved** +//! [`Value::String`] instead of falling through the generic +//! `parse_hex_bytes`/`parse_bare_string_value` bareword path used by the +//! other string-family types. Two problems make this its own module: +//! +//! 1. `parse_value`'s `alt(...)` tries `parse_hex_bytes` (see +//! `super::value`) before any string interpretation, so a `regex` +//! pattern beginning with a magic(5) escape (`\^`, `\040`, `\t`, `\x..`) +//! was previously captured as `Value::Bytes` instead of `Value::String` +//! -- the evaluator's `Regex` arms only accepted `Value::String`, +//! producing a fatal `UnsupportedType` error for every such rule. +//! 2. Rust's `regex` crate does not interpret octal escapes (`\040`) the +//! way GNU `file`'s `getstr` does, so handing the RAW escape text to +//! `regex::bytes::Regex::new` is not an option -- the pattern must be +//! getstr-resolved (escapes replaced by their resolved bytes) *before* +//! it reaches the regex compiler, exactly as GNU `file` does before +//! calling `regcomp`. +//! +//! # Escape table (verified against GNU `file` source, not inferred) +//! +//! The table below was read directly from `file/file`'s `src/apprentice.c`, +//! function `getstr` (`master` branch, fetched during this fix; the +//! function body starts with the comment "Convert a string containing C +//! character escapes. Stop at an unescaped space or tab."). The relevant +//! `switch (c = *s++)` after a backslash is seen: +//! +//! - `\a`, `\b`, `\f`, `\n`, `\r`, `\t`, `\v` resolve to the corresponding +//! C control-character byte: `0x07`, `0x08`, `0x0C`, `0x0A`, `0x0D`, +//! `0x09`, `0x0B`. +//! - `\0`-`\7` starts a 1-3 digit **octal** escape (`\NNN`); the resolved +//! value is truncated to a byte exactly like getstr's `CAST(char, val)`. +//! - `\x` starts a 0-2 digit **hex** escape (`\xNN`). If no valid hex +//! digit follows, getstr's `val` variable keeps its initialized default +//! of the literal character `'x'` (`0x78`) -- an intentional getstr +//! quirk, replicated here for fidelity. +//! - **Every other escaped character** -- the "relation" characters +//! `> < & ^ = !`, backslash itself, space, an escaped `.`, an escaped +//! literal tab character, and any genuinely unrecognized escape such as +//! `\^` -- falls through getstr's `default:` case into the shared +//! `case ' ':` body, whose only effect is `*p++ = c;`. In other words: +//! **the backslash is dropped and the following character is kept +//! literally.** This is not special-cased per character in the C +//! source; it is the general fallback for anything not in the named +//! list above -- the bullets above name the notable members, not an +//! exhaustive enumeration of this bucket. +//! +//! Note also: this resolver deliberately does **not** reproduce getstr's +//! `file_magwarn` lint diagnostics (the "unnecessary escape" / "no such +//! escape" warnings `apprentice.c` emits while *compiling* a magic file). +//! Those are magic-file-authoring diagnostics, orthogonal to escape +//! *resolution* -- this module's contract is producing the correct +//! resolved byte sequence for a pattern that already parsed, not linting +//! the source `.magic` file's style. +//! +//! ## Resolved open question: control-char / regex-shorthand collision +//! +//! GNU `file`'s getstr resolves `\b` to the raw control byte `0x08` +//! (backspace) -- Rust's `regex` crate assigns `\b` (the two-character +//! ASCII sequence backslash + `b`) the *different* meaning "word boundary +//! assertion". Confirmed from the `apprentice.c` source above that this is +//! genuinely how getstr behaves (case `'b'`: `*p++ = '\b';`), so the +//! collision the plan flagged is real *at the character level* -- but it +//! does **not** manifest as an actual bug here, because this resolver +//! never re-emits the two-character sequence `\` + `b`. It appends the +//! single resolved byte `0x08` directly to the pattern text. A lone raw +//! `0x08` byte, unescaped, has no special meaning to the `regex` crate -- +//! it is matched as an ordinary literal byte, not reinterpreted as the +//! `\b` escape (that requires the literal two-character source sequence +//! `\b`, which this resolver by construction never produces for a +//! getstr-resolved byte `< 0x80`). The same reasoning applies to any +//! other named control escape (`\a`, `\f`, `\n`, `\r`, `\t`, `\v`): each +//! resolves to a single raw byte, not to escape *syntax*, so it cannot be +//! reinterpreted by the downstream regex compiler. No `\xHH` re-encoding +//! is needed for these -- only bytes `>= 0x80` require it (see +//! [`push_resolved_byte`]), because those cannot be represented as a +//! single-byte Rust `char`/UTF-8 sequence at all. +//! +//! Unrelated but worth noting for anyone reading a `.magic` file with +//! `\d`, `\s`, or `\w` in a `regex` pattern expecting PCRE-style character +//! classes: getstr does **not** special-case those letters, so they fall +//! into the "everything else" bucket above and are demoted to the bare +//! literal character (`\d` -> `d`, `\s` -> `s`, `\w` -> `w`). That is +//! genuine upstream libmagic behavior, not a bug in this resolver. + +use nom::IResult; +use nom::character::complete::multispace0; +use nom::error::Error as NomError; + +use crate::parser::ast::Value; + +/// Parse a single (unquoted) `regex` pattern token, resolving magic(5) +/// `getstr` escapes into the byte sequence GNU `file` would produce, and +/// return it as a [`Value::String`] ready for `regex::bytes::Regex::new`. +/// +/// Token boundary: consumes non-whitespace characters, stopping at the +/// first *unescaped* whitespace character -- mirroring getstr's +/// `isspace(c)` early-exit (see module docs). Quoted (`"..."`) regex +/// values are **not** handled here; callers should keep using +/// [`super::value::parse_value`] for those (see `GOTCHAS.md` S3.6 and the +/// existing quoted-regex round-trip test for why quoting is preserved +/// as-is rather than getstr-resolved). +/// +/// # Errors +/// Returns a nom parsing error if the input is empty, starts with +/// whitespace, or (after escape resolution) yields no characters at all. +pub(super) fn parse_regex_getstr_value(input: &str) -> IResult<&str, Value> { + let (input, _) = multispace0(input)?; + if input.is_empty() { + return Err(nom::Err::Error(NomError::new( + input, + nom::error::ErrorKind::TakeWhile1, + ))); + } + + let mut resolved = String::new(); + let mut remaining = input; + + while let Some(ch) = remaining.chars().next() { + if ch.is_whitespace() { + break; + } + if ch == '\\' { + let after_backslash = remaining.strip_prefix('\\').unwrap_or(remaining); + if let Some(rest) = resolve_getstr_escape(&mut resolved, after_backslash) { + remaining = rest; + } else { + // Incomplete escape at end of token (a lone trailing + // backslash): getstr's `case '\0': ... goto out;` stops + // scanning here without emitting the backslash. + remaining = after_backslash; + break; + } + continue; + } + resolved.push(ch); + remaining = remaining.strip_prefix(ch).unwrap_or(remaining); + } + + if resolved.is_empty() { + return Err(nom::Err::Error(NomError::new( + input, + nom::error::ErrorKind::TakeWhile1, + ))); + } + + Ok((remaining, Value::String(resolved))) +} + +/// Resolve a single getstr escape sequence, appending its resolved +/// representation to `resolved`. `rest` is the input immediately +/// following the triggering backslash. Returns the remaining input after +/// the escape, or `None` if `rest` is empty (an incomplete trailing +/// escape -- see [`parse_regex_getstr_value`]). +fn resolve_getstr_escape<'a>(resolved: &mut String, rest: &'a str) -> Option<&'a str> { + let c = rest.chars().next()?; + let after = rest.strip_prefix(c).unwrap_or(rest); + + // Named single-character control escapes (getstr `case 'a'`..`'v'`). + let named_byte = match c { + 'a' => Some(0x07u8), + 'b' => Some(0x08), + 'f' => Some(0x0C), + 'n' => Some(0x0A), + 'r' => Some(0x0D), + 't' => Some(0x09), + 'v' => Some(0x0B), + _ => None, + }; + if let Some(byte) = named_byte { + push_resolved_byte(resolved, byte); + return Some(after); + } + + if c.is_ascii_digit() && ('0'..='7').contains(&c) { + let (byte, rest_after_octal) = resolve_octal_escape(c, after); + push_resolved_byte(resolved, byte); + return Some(rest_after_octal); + } + + if c == 'x' { + let (byte, rest_after_hex) = resolve_hex_escape(after); + push_resolved_byte(resolved, byte); + return Some(rest_after_hex); + } + + // getstr `default:` / shared `case ' ':` body: drop the backslash, + // keep the literal character. Covers the "relation" characters + // (`> < & ^ = !`), backslash itself, space, an escaped `.`, an + // escaped literal tab, and any unrecognized escape (`\^`, `\d`, ...). + resolved.push(c); + Some(after) +} + +/// Resolve a 1-3 digit octal escape (`\NNN`), matching getstr's +/// look-ahead-up-to-2-more-digits logic. `first` is the digit already +/// consumed by the caller's dispatch (guaranteed `'0'..='7'`). +fn resolve_octal_escape(first: char, rest: &str) -> (u8, &str) { + let mut val: u32 = first.to_digit(8).unwrap_or(0); + let mut remaining = rest; + + for _ in 0..2 { + let Some(next) = remaining.chars().next() else { + break; + }; + if !('0'..='7').contains(&next) { + break; + } + val = (val << 3) | next.to_digit(8).unwrap_or(0); + remaining = remaining.strip_prefix(next).unwrap_or(remaining); + } + + // A 3-digit octal escape can exceed 255 (e.g. `\777` = 511); getstr + // truncates via `CAST(char, val)`. `as u8` matches that truncation. + #[allow(clippy::cast_possible_truncation)] + let byte = val as u8; + (byte, remaining) +} + +/// Resolve a 0-2 digit hex escape (`\xNN`), matching getstr's quirky +/// "no digits following `\x`" default of the literal character `'x'`. +fn resolve_hex_escape(rest: &str) -> (u8, &str) { + let mut chars = rest.chars(); + let Some(first) = chars.next() else { + return (b'x', rest); + }; + let Some(high) = first.to_digit(16) else { + return (b'x', rest); + }; + let after_first = rest.strip_prefix(first).unwrap_or(rest); + + let Some(second) = after_first.chars().next() else { + #[allow(clippy::cast_possible_truncation)] + return (high as u8, after_first); + }; + let Some(low) = second.to_digit(16) else { + #[allow(clippy::cast_possible_truncation)] + return (high as u8, after_first); + }; + + let after_second = after_first.strip_prefix(second).unwrap_or(after_first); + #[allow(clippy::cast_possible_truncation)] + let byte = ((high << 4) | low) as u8; + (byte, after_second) +} + +/// Append a resolved getstr byte to the in-progress pattern text. +/// +/// Bytes `< 0x80` are valid ASCII / single-byte UTF-8 and are pushed +/// directly as a `char`. Bytes `>= 0x80` cannot be pushed as a raw `char` +/// (`0x80`-`0xFF` are not valid standalone Unicode scalar values, and +/// `String` must stay valid UTF-8 -- pushing one would panic or require +/// lossy reinterpretation that corrupts the intended byte value). Per the +/// plan's KTD3, such bytes are instead re-encoded as a regex-native +/// `\xHH` hex escape *in the output text*, so the regex engine -- not the +/// `String` -- carries the byte. This is a Rust-`String`-vs-C-`char*` +/// type-system accommodation, not a getstr behavior: GNU `file`'s C +/// implementation just writes the raw byte into an untyped buffer handed +/// to `regcomp`. +fn push_resolved_byte(resolved: &mut String, byte: u8) { + use std::fmt::Write as _; + + if byte < 0x80 { + resolved.push(byte as char); + } else { + // `write!` to a `String` is infallible; the `Result` only + // exists to satisfy the generic `fmt::Write` trait. + #[allow(clippy::let_underscore_must_use)] + let _ = write!(resolved, "\\x{byte:02x}"); + } +} + +#[cfg(test)] +mod tests; diff --git a/src/parser/grammar/getstr/tests.rs b/src/parser/grammar/getstr/tests.rs new file mode 100644 index 00000000..edafafe8 --- /dev/null +++ b/src/parser/grammar/getstr/tests.rs @@ -0,0 +1,346 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the `getstr` escape resolver ([`super::parse_regex_getstr_value`]) +//! and integration-level guard tests through [`parse_magic_rule`] pinning +//! that `TypeKind::Regex` (and only `Regex`) is routed through it. +//! +//! U4 content-fidelity table: every row asserts the exact resolved +//! **text**, not merely the `Value` variant, so a future refactor of the +//! escape table cannot silently regress correctness (see the plan's +//! Verification Contract: "Accuracy is the core value"). + +use super::parse_regex_getstr_value; +use crate::parser::ast::{TypeKind, Value}; +use crate::parser::grammar::parse_magic_rule; + +/// U4 content-fidelity table: `(input, expected_resolved_regex_text)`. +/// +/// Inputs are raw strings so the magic(5) source escapes (backslashes) +/// are represented literally, one-for-one, with no Rust-level +/// re-interpretation. Expected values are normal strings so Rust's own +/// `\t`/`\\` escapes conveniently produce the exact resolved bytes we +/// want to assert (a raw tab byte, a raw backslash byte) -- this is a +/// test-authoring convenience, not a claim that the resolver itself does +/// any Rust-string-escape interpretation. +#[test] +fn getstr_resolver_content_fidelity_table() { + let cases: &[(&str, &str)] = &[ + // Worked example from the plan (KTD2): `\^` drops the backslash + // (unknown escape) -> anchor `^`; `\040` is octal space; `\t` is + // the named tab escape; `\\.` is backslash-then-backslash -> + // resolves to ONE literal backslash, followed by the unescaped + // `.` -> together `\.` (escaped dot for the regex engine). + (r"\^[\040\t]{0,50}\\.asciiz", "^[ \t]{0,50}\\.asciiz"), + // Octal space in isolation. + (r"[\040]", "[ ]"), + // Tab: named getstr escape -> raw tab byte. + (r"\t", "\t"), + // Unknown escape: drop the backslash, keep the literal char. + (r"\^", "^"), + // Escaped dot: `\\.``(two backslash chars + dot) -> `\.` (one + // backslash byte + dot) -- required so the *regex* engine sees + // an escaped, literal dot rather than the "any character" + // metacharacter. + (r"\\.", "\\."), + // Backslash: four literal backslash chars in magic(5) source + // (two escape pairs) resolve to two literal backslash bytes. + (r"\\\\", "\\\\"), + // Hex passthrough of a low (< 0x80) byte. + (r"\x41", "A"), + // >= 0x80 re-encoding (KTD3): octal `\377` = 0xFF, hex `\xFF` = + // 0xFF -- both must produce the SAME regex-native `\xff` text + // (lowercase, pinned), never a raw 0xFF byte in the `String`. + (r"\377", "\\xff"), + (r"\xFF", "\\xff"), + // Resolved open question (control-char collision): getstr names + // `\b` as backspace (0x08). Confirmed via `apprentice.c` (see + // module docs) that this is a real named escape, not dropped. + // Because the resolver appends the raw byte (not the two-char + // `\b` escape *sequence*), the regex crate's own `\b` + // word-boundary meaning is never triggered -- no `\xHH` + // re-encoding is needed since 0x08 < 0x80. + (r"\b", "\x08"), + // Remaining named control escapes (getstr `case 'a'`/`'f'`/`'n'`/ + // `'r'`/`'v'`) completing the coverage `\t`/`\b` started above -- + // each is a distinct match arm in `resolve_getstr_escape`'s + // `named_byte` table (getstr/mod.rs ~L155-164); a transposition + // between any two of these bytes (e.g. `\n`<->`\r`) must fail its + // own row rather than being masked by only testing two of seven. + (r"\a", "\x07"), + (r"\f", "\x0C"), + (r"\n", "\n"), + (r"\r", "\r"), + (r"\v", "\x0B"), + ]; + + for &(input, expected) in cases { + let (remaining, value) = + parse_regex_getstr_value(input).unwrap_or_else(|e| panic!("{input:?}: {e:?}")); + assert_eq!(remaining, "", "remaining input for {input:?}"); + assert_eq!( + value, + Value::String(expected.to_string()), + "resolved text for input {input:?}" + ); + } +} + +#[test] +fn getstr_resolver_ge_0x80_bytes_are_valid_utf8_never_raw() { + // Belt-and-suspenders on top of the table above: explicitly check + // the resolved String's byte content never contains a raw 0xFF (or + // any byte >= 0x80) and remains valid UTF-8 (guaranteed by the type, + // but asserted here to make the invariant visible in the test). + for input in [r"\377", r"\xFF", r"\x80", r"\200"] { + let (_, value) = parse_regex_getstr_value(input).expect(input); + let Value::String(s) = value else { + panic!("expected Value::String for {input:?}"); + }; + assert!( + s.bytes().all(|b| b < 0x80), + "resolved text for {input:?} must be pure ASCII (the >= 0x80 byte is \ + re-encoded as \\xHH text, never appended raw): got {s:?}" + ); + assert!( + std::str::from_utf8(s.as_bytes()).is_ok(), + "resolved text for {input:?} must be valid UTF-8" + ); + } +} + +#[test] +fn getstr_resolver_hex_escape_with_no_digits_falls_back_to_literal_x() { + // getstr quirk: `\x` immediately followed by a non-hex character + // leaves `val` at its initialized default, the literal character + // `'x'` (0x78). Replicated here for fidelity even though it is a + // corner case unlikely to appear in real magic files. + let (remaining, value) = parse_regex_getstr_value(r"\xZZ").expect(r"\xZZ"); + assert_eq!(value, Value::String("xZZ".to_string())); + assert_eq!(remaining, ""); +} + +#[test] +fn getstr_resolver_stops_at_unescaped_whitespace() { + let (remaining, value) = parse_regex_getstr_value("foobar[0-9]+ trailing message").unwrap(); + assert_eq!(value, Value::String("foobar[0-9]+".to_string())); + assert_eq!(remaining, " trailing message"); +} + +#[test] +fn getstr_resolver_rejects_empty_or_whitespace_only_input() { + assert!(parse_regex_getstr_value("").is_err()); + assert!(parse_regex_getstr_value(" ").is_err()); +} + +/// R2/highest-value coverage gap: a pattern that resolves to NOTHING at +/// all -- a lone trailing backslash with no character following it (an +/// incomplete escape, getstr's `case '\0': ... goto out;`, getstr/mod.rs +/// ~L123-127) -- must hit the `resolved.is_empty()` -> `Err` path +/// (~L135-139), not silently succeed with an empty pattern. An empty +/// `Value::String("")` regex pattern would compile (the empty regex +/// matches everything, zero-width) and every rule using it would +/// "succeed" nonsensically instead of being rejected at parse time. This +/// is exactly the class of fatal-abort-turned-silent-wrong-answer bug +/// this PR exists to close: a graceful `Err` here is the correct, +/// desired outcome, not a bug. +#[test] +fn getstr_resolver_rejects_lone_trailing_backslash_incomplete_escape() { + let result = parse_regex_getstr_value("\\"); + assert!( + result.is_err(), + "a lone trailing backslash with nothing following it must be rejected \ + (incomplete escape resolves to zero characters), got {result:?}" + ); +} + +/// End-to-end guard for the same condition through the full +/// `parse_magic_rule` pipeline (`grammar/mod.rs` ~L1198-1201): when the +/// getstr resolver rejects a bareword regex pattern, `parse_magic_rule` +/// does NOT propagate that error unconditionally -- it retries via +/// `parse_value(input).map_err(|_| orig_err)?`, using `orig_err` (the +/// getstr failure) only if the fallback ALSO fails. +/// +/// For a lone trailing backslash specifically, this test confirms +/// (empirically, not by assumption) that the fallback SUCCEEDS: a bare +/// `\` with nothing following is captured by `parse_value`'s +/// hex/mixed-ascii branch as a single raw byte (`Value::Bytes([0x5c])`), +/// exactly like the analogous `search` cross-wiring guard above +/// (`parse_magic_rule_search_with_same_escaped_pattern_is_unaffected`). +/// The overall pipeline is therefore graceful in the strongest sense -- +/// it recovers to a well-formed `Ok` rule rather than needing to fall +/// back to an `Err` -- and, critically, it never panics. +#[test] +fn parse_magic_rule_regex_lone_trailing_backslash_pattern_recovers_via_value_fallback() { + let input = "0 regex \\"; + let (remaining, rule) = parse_magic_rule(input) + .expect("the parse_value fallback must recover a bare trailing backslash gracefully"); + assert_eq!(remaining, ""); + assert!(matches!(rule.typ, TypeKind::Regex { .. })); + assert_eq!( + rule.value, + Value::Bytes(vec![0x5c]), + "parse_value's hex/mixed-ascii fallback captures the lone backslash as a raw byte" + ); +} + +/// A genuinely-Err round trip: when the value token is truly EMPTY (no +/// pattern at all follows the `regex` keyword -- not even a lone +/// backslash), BOTH the getstr resolver (`input.is_empty()` guard, +/// getstr/mod.rs ~L104) AND the `parse_value` fallback (`parse_value`'s +/// own explicit empty-input check, `grammar/value.rs` ~L366-371) reject +/// it, so `parse_magic_rule` has no recovery path and returns `Err` +/// gracefully -- never a panic. +/// +/// Per GOTCHAS S3.11, `parse_text_magic_file` is fail-fast: in a real +/// magic file, a single rule like this would abort the WHOLE file load +/// rather than being skipped. This test only pins that the failure mode +/// at the single-rule parser level is a clean `Err`; it does not claim +/// any file-level skip-and-continue behavior exists (it does not). +#[test] +fn parse_magic_rule_regex_empty_pattern_errs_gracefully() { + let input = "0 regex "; + let result = parse_magic_rule(input); + assert!( + result.is_err(), + "a regex rule with a completely empty value token must be rejected \ + gracefully (Err), never panic: got {result:?}" + ); +} + +// --------------------------------------------------------------------- +// U3 variant-guard tests (via the full `parse_magic_rule` pipeline) +// --------------------------------------------------------------------- + +#[test] +fn parse_magic_rule_regex_escaped_pattern_yields_string_not_bytes() { + let input = r"0 regex \^[\040\t]{0,50}\\.asciiz assembler source text"; + let (remaining, rule) = parse_magic_rule(input).expect(input); + assert_eq!(remaining, ""); + assert!(matches!(rule.typ, TypeKind::Regex { .. })); + assert_eq!( + rule.value, + Value::String("^[ \t]{0,50}\\.asciiz".to_string()), + "regex pattern must resolve to Value::String, not Value::Bytes" + ); + assert_eq!(rule.message, "assembler source text"); +} + +#[test] +fn parse_magic_rule_regex_non_escaped_pattern_still_value_string() { + // No regression: a plain, non-escaped regex pattern must keep + // parsing as `Value::String` exactly as before this fix. + let input = "0 regex foobar[0-9]+ numeric suffix"; + let (remaining, rule) = parse_magic_rule(input).expect(input); + assert_eq!(remaining, ""); + assert_eq!(rule.value, Value::String("foobar[0-9]+".to_string())); + assert_eq!(rule.message, "numeric suffix"); +} + +#[test] +fn parse_magic_rule_regex_quoted_pattern_unaffected_by_getstr_routing() { + // Quoted regex values are NOT routed through the getstr resolver -- + // they keep using `parse_quoted_string`'s own (different) escape + // table, exactly as before. This guards against regressing the + // pre-existing `test_parse_magic_rule_regex_and_search` coverage in + // `grammar/tests/mod.rs`. + let input = r#"0 regex/c "hello" case-insensitive match"#; + let (remaining, rule) = parse_magic_rule(input).expect(input); + assert_eq!(remaining, ""); + assert_eq!(rule.value, Value::String("hello".to_string())); +} + +#[test] +fn parse_magic_rule_search_with_same_escaped_pattern_drops_unknown_escape_backslash() { + // The SAME escaped source text, used on a `search` rule instead of + // `regex`, still goes through `parse_hex_bytes`/`parse_mixed_hex_ascii` + // (not the getstr resolver, which is regex-only) and still yields + // `Value::Bytes`. But `parse_mixed_hex_ascii`'s unrecognized-escape + // fallback now matches GNU `file` getstr: the standalone backslash on + // `\^` is DROPPED and `^` is kept literally, exactly as the `regex` + // path resolves it (see + // `parse_magic_rule_regex_escaped_pattern_yields_string_not_bytes` + // above -- same bytes, different Value variant). `\040`->space and + // `\t`->tab are recognized escapes (unchanged); `\\.`->`\.` is an + // escaped backslash then a literal dot (unchanged). This is the + // getstr-parity fix that also lets `0 string \ { + assert_eq!( + bytes, b"^[ \t]{0,50}\\.asciiz", + "unrecognized `\\^` escape drops the backslash (getstr parity), \ + matching the regex path's resolved bytes" + ); + } + other => panic!("expected search to still produce Value::Bytes, got {other:?}"), + } + assert_eq!(rule.message, "test"); +} + +#[test] +fn assembler_asciiz_rule_matches_gnu_file_semantics_end_to_end() { + // End-to-end sanity check against the real trigger rule from the + // macOS-shipped system magic DB (`/usr/share/file/magic/assembler:5`): + // + // 0 regex \^[\040\t]{0,50}\\.asciiz assembler source text + // + // The source line is reproduced verbatim here (not read from the + // filesystem) so this test is portable and does not depend on the + // system magic DB being installed. It exercises the full pipeline: + // parse -> getstr-resolve -> compile as `regex::bytes::Regex` -> + // match/no-match against representative inputs (GNU `file` + // semantics: an optional run of up to 50 leading spaces/tabs, then a + // LITERAL `.asciiz`, anchored at line start). + let line = "0\tregex\t\\^[\\040\\t]{0,50}\\\\.asciiz\t\tassembler source text"; + let (_, rule) = parse_magic_rule(line).expect(line); + let Value::String(pattern) = &rule.value else { + panic!("expected Value::String, got {:?}", rule.value); + }; + + let re = regex::bytes::Regex::new(pattern).expect("resolved pattern must compile"); + + assert!( + re.is_match(b"\t.asciiz \"hi\""), + "leading tab then .asciiz must match" + ); + assert!( + re.is_match(b".asciiz \"hi\""), + "zero leading whitespace (lower bound of {{0,50}}) must match" + ); + assert!( + !re.is_match(b"xyz .asciiz"), + "non-whitespace before .asciiz must not match" + ); + assert!( + !re.is_match(b"Xasciiz"), + "escaped dot must be a literal '.', not the any-character wildcard" + ); + + let too_much_padding = format!("{}{}", " ".repeat(51), ".asciiz"); + assert!( + !re.is_match(too_much_padding.as_bytes()), + "more than 50 leading whitespace chars must exceed the {{0,50}} upper bound" + ); +} + +#[test] +fn parse_magic_rule_regex_with_ge_0x80_escape_does_not_abort_parse() { + // R5/KTD3: an escape-produced byte >= 0x80 must not panic, corrupt + // UTF-8, or fail the parse (which would abort the WHOLE fail-fast + // file load per GOTCHAS S3.11 -- relocating the fatal-abort class + // this fix kills from evaluator to parser). + let input = r"0 regex \377 high byte octal escape"; + let (remaining, rule) = parse_magic_rule(input).expect(input); + assert_eq!(remaining, ""); + assert_eq!(rule.value, Value::String("\\xff".to_string())); + + let input_hex = r"0 regex \xFF high byte hex escape"; + let (remaining, rule) = parse_magic_rule(input_hex).expect(input_hex); + assert_eq!(remaining, ""); + assert_eq!(rule.value, Value::String("\\xff".to_string())); +} diff --git a/src/parser/grammar/mod.rs b/src/parser/grammar/mod.rs index 902791c7..6fa11a86 100644 --- a/src/parser/grammar/mod.rs +++ b/src/parser/grammar/mod.rs @@ -24,6 +24,7 @@ use crate::parser::ast::{ TypeKind, Value, }; +mod getstr; mod numbers; mod type_suffix; mod value; @@ -351,8 +352,20 @@ pub fn parse_offset(input: &str) -> IResult<&str, OffsetSpec> { let (rest, _) = multispace0(rest)?; Ok((rest, OffsetSpec::Relative(value))) } else { + // Capture the leading `-` before `parse_number` consumes it: magic(5) + // `-0` means "0 bytes from the end of the file" -- the EOF position + // (`buffer.len()`), NOT absolute offset 0. Because `-0 == 0` in a + // signed integer the sign is otherwise lost, so detect it explicitly + // and encode `FromEnd(0)`. Used by e.g. gzip's `>>-0 offset >48` to + // gate the trailing-size trailer on the file being long enough. + // Other negative offsets (`-4`) keep their `Absolute(-4)` encoding, + // which the evaluator already resolves from the buffer end. + let starts_with_minus = input.starts_with('-'); let (input, offset_value) = parse_number(input)?; let (input, _) = multispace0(input)?; + if starts_with_minus && offset_value == 0 { + return Ok((input, OffsetSpec::FromEnd(0))); + } Ok((input, OffsetSpec::Absolute(offset_value))) } } @@ -513,26 +526,16 @@ fn parse_name_or_use_meta<'a>( // magic(5) allows a `\^` prefix on a `use` identifier to mean "invoke // the named subroutine but flip the endianness of every read inside - // it". We do not yet implement the endian flip semantically (tracked - // as issue #236), but the file must still load: consume the prefix - // and treat the rest of the identifier as a normal `use` reference. - // Emit a warn! so users see why their LE/BE detection paired with - // `use \^name` produces wrong metadata at default log levels. - let input = if type_name == "use" { - if let Some(rest) = input.strip_prefix("\\^") { - warn!( - "use directive with `\\^` prefix: endian-flip semantics \ - are not yet implemented (issue #236). Subroutine reads \ - will use their declared endianness; metadata fields may \ - be incorrect. Identifier: {:?}", - rest.split_whitespace().next().unwrap_or("") - ); - rest - } else { - input - } - } else { + // it" (libmagic `softmagic.c` `cvt_flip`). Consume the prefix and + // record it on `MetaType::Use::flip_endian` so the evaluator can apply + // the flip (issue #236). The `\^` prefix is meaningless on `name` + // declarations, so only `use` is inspected. + let (input, use_flip_endian) = if type_name == "use" { input + .strip_prefix("\\^") + .map_or((input, false), |rest| (rest, true)) + } else { + (input, false) }; let (after_id, id) = @@ -558,37 +561,47 @@ fn parse_name_or_use_meta<'a>( ))); } - // Consume horizontal whitespace after the identifier. Real-world - // magic files sometimes append a descriptive message after a - // `name`/`use` directive (e.g. `0 name xbase-prf dBase Printer - // Form`). magic(5) does not officially document this, but GNU - // `file` tolerates it -- the trailing text is silently ignored - // because `name`/`use` rules don't have a message slot in the - // softmagic struct. We do the same: consume horizontal whitespace - // and a single optional trailing token, stopping at end-of-line. + // Handle the trailing text after the identifier. The two directives + // diverge here, matching GNU `file`: + // + // - `use`: magic(5) has no message slot for a `use` site, and GNU + // `file` never emits a use-site's own description (verified: a + // `use foo BAR` renders no `BAR`). Drop the trailing text up to + // end-of-line so the caller parses an empty message. + // - `name`: the `name` line's OWN description IS emitted when the + // subroutine is invoked via `use` -- Mach-O universal `0 name + // mach-o \b [`, `0 name matlab4 Matlab v4 mat-file`, `0 name + // algol_68 Algol 68 source text`, etc. PRESERVE the trailing text + // so the caller's `parse_message` captures it as the rule message + // (later stored in the name table by `extract_name_table` and + // emitted at the `use` site). Dropping it here is what previously + // made rmagic omit those fragments (e.g. the leading `[` of the + // Mach-O universal bracket detail). + // // We deliberately do NOT reject embedded whitespace inside the // identifier itself (which would be a real malformed rule like - // `part 2`); that's enforced earlier when `take_while` truncates - // the identifier on the first non-id character. + // `part 2`); that's enforced earlier when `take_while` truncates the + // identifier on the first non-id character. let mut tail = after_id; - while let Some(rest) = tail.strip_prefix(' ').or_else(|| tail.strip_prefix('\t')) { - tail = rest; - } - // Skip any trailing text (descriptive label) up to end-of-line. - if let Some(next_char) = tail.chars().next() - && !matches!(next_char, '\n' | '\r') - { - // Drop the rest of the line silently. magic(5)'s `name`/`use` - // directives have no message slot, so anything after the - // identifier is informational only. - let line_end = tail.find(['\n', '\r']).unwrap_or(tail.len()); - tail = &tail[line_end..]; + if type_name == "use" { + while let Some(rest) = tail.strip_prefix(' ').or_else(|| tail.strip_prefix('\t')) { + tail = rest; + } + if let Some(next_char) = tail.chars().next() + && !matches!(next_char, '\n' | '\r') + { + let line_end = tail.find(['\n', '\r']).unwrap_or(tail.len()); + tail = &tail[line_end..]; + } } let meta = if type_name == "name" { MetaType::Name(id.to_string()) } else { - MetaType::Use(id.to_string()) + MetaType::Use { + name: id.to_string(), + flip_endian: use_flip_endian, + } }; let (rest, _) = multispace0(tail)?; Ok((rest, (TypeKind::Meta(meta), None, None))) @@ -677,17 +690,24 @@ pub fn parse_type_and_operator( input = rest; } - // Handle search suffix: required decimal range plus optional flags - // (e.g., `search/256`, `search/256/s`, `search/256/cs`). Per GNU - // `file` magic(5), the range is mandatory. `search/0` and bare - // `search` are rejected at parse time via `NonZeroUsize`. - let mut search_suffix: Option<(::std::num::NonZeroUsize, crate::parser::ast::SearchFlags)> = - None; + // Handle search suffix. `search/N[/flags]` supplies an explicit scan + // window (e.g., `search/256`, `search/256/s`, `search/256/cs`); the + // count `N` is a `NonZeroUsize`, so `search/0` is rejected. A bare + // `search` (no `/` suffix, e.g. `>8 search /Count`) is ALSO accepted: + // its `range` stays `None`, meaning scan-to-end-of-buffer. magic(5) + // documents the count as required, but the reference `file` binary + // accepts the bare form (`str_range == 0`), so we follow the + // implementation rather than the spec. Disambiguation is unambiguous: + // a ranged suffix attaches `/` directly to the keyword, while a bare + // search is followed by whitespace then the value. + let mut search_range: Option<::std::num::NonZeroUsize> = None; + let mut search_flags = crate::parser::ast::SearchFlags::default(); if type_name == "search" && let Some(suffix_rest) = input.strip_prefix('/') { - let (rest, parsed) = parse_search_suffix(input, suffix_rest)?; - search_suffix = Some(parsed); + let (rest, (range, flags)) = parse_search_suffix(input, suffix_rest)?; + search_range = Some(range); + search_flags = flags; input = rest; } @@ -762,13 +782,12 @@ pub fn parse_type_and_operator( flags: regex_flags, count: regex_count, }, - "search" => { - // Mandatory range: reject bare `search` at parse time. - let (range, flags) = search_suffix.ok_or_else(|| { - nom::Err::Error(nom::error::Error::new(input, nom::error::ErrorKind::Tag)) - })?; - TypeKind::Search { range, flags } - } + "search" => TypeKind::Search { + // `None` range = bare `search` = scan-to-EOF (see the suffix + // handling above); `Some(n)` = `search/N`. + range: search_range, + flags: search_flags, + }, _ => { // `type_keyword_to_kind` returns: // * `Ok(Some(kind))` for every fully-specified keyword @@ -1080,18 +1099,27 @@ pub fn parse_magic_rule(input: &str) -> IResult<&str, MagicRule> { // pre-comparison value transform (`+N`/`-N`/`*N`/`/N`/`%N`/`|N`/`^N`). let (input, (typ, attached_op, value_transform)) = parse_type_and_operator(input)?; - // Meta-type directives (default, clear, name, use, indirect, offset) - // conceptually have no operator/value operand, but magic(5) source - // files (including GNU `file`'s own `searchbug.magic`) often write - // them with an `x` (AnyValue) placeholder between the type and the - // message, e.g. `>>&0 offset x at_offset %lld`. Consume an optional - // leading `x` token here so it does not leak into the rendered - // message. + // Meta-type directives (default, clear, name, use, indirect) conceptually + // have no operator/value operand, but magic(5) source files (including + // GNU `file`'s own `searchbug.magic`) often write them with an `x` + // (AnyValue) placeholder between the type and the message, e.g. + // `>>&0 offset x at_offset %lld`. Consume an optional leading `x` token + // here so it does not leak into the rendered message. + // + // `offset` is deliberately EXCLUDED from this no-operand path: unlike the + // other meta-types, the `offset` pseudo-type carries a real comparison + // operand -- `offset >48` / `offset <48` (gzip's trailing-size gate) read + // the resolved offset and compare it against N, and only the `offset x` + // form is a bare AnyValue placeholder. Routing `offset` through the normal + // operator+value+message path below handles both forms; the engine then + // applies the comparison (see the `MetaType::Offset` dispatch in + // `evaluator::engine`). Folding `offset >48` into this block instead + // silently turned `>48` into message text and made the compare a no-op. // // `name`/`use` are handled earlier in parse_type_and_operator and // already consumed their identifier operand, so the `x` stripping // is a no-op for them. - if matches!(typ, TypeKind::Meta(_)) { + if matches!(typ, TypeKind::Meta(_)) && !matches!(typ, TypeKind::Meta(MetaType::Offset)) { // Meta-type directives have no operand, so an attached operator // like `default&0xf` is malformed — reject it here rather than // silently dropping it on the floor. `name`/`use` short-circuit in @@ -1175,11 +1203,32 @@ pub fn parse_magic_rule(input: &str) -> IResult<&str, MagicRule> { ); let (input, value) = if op == Operator::AnyValue { (input, Value::Uint(0)) - } else if is_string_family_type { - match parse_value(input) { - Ok(ok) => ok, - Err(orig_err) => parse_bare_string_value(input).map_err(|_| orig_err)?, + } else if matches!(typ, TypeKind::Regex { .. }) { + // `regex` patterns get special-cased ahead of the generic + // string-family fallback below (issue: getstr fidelity fix). + // Quoted values (`regex/c "hello" ...`) keep using + // `parse_value`'s existing `parse_quoted_string` path unchanged + // -- quoting is a project convenience layered on top of + // magic(5), not part of GNU `file`'s own syntax, and existing + // quoted-regex rules must keep their current (non-getstr) + // escape handling. Bareword (unquoted) patterns are routed + // through the dedicated getstr resolver instead of + // `parse_hex_bytes`/`parse_bare_string_value`: a pattern + // beginning with a magic(5) escape (`\^`, `\040`, `\t`, `\x..`) + // would otherwise be captured by `parse_hex_bytes` as + // `Value::Bytes` before any string interpretation ran, and + // Rust's `regex` crate does not interpret octal escapes the way + // GNU `file`'s `getstr` does -- see `getstr.rs` module docs. + if input.trim_start().starts_with('"') { + parse_value(input)? + } else { + match getstr::parse_regex_getstr_value(input) { + Ok(ok) => ok, + Err(orig_err) => parse_value(input).map_err(|_| orig_err)?, + } } + } else if is_string_family_type { + parse_string_family_value(input)? } else { parse_value(input)? }; @@ -1206,6 +1255,54 @@ pub fn parse_magic_rule(input: &str) -> IResult<&str, MagicRule> { Ok((input, rule)) } +/// Parse the comparison value for a string-family type. +/// +/// libmagic never interprets a `string`/`pstring`/`string16`/`search` +/// comparison value as a number: `0 string >0` compares against the +/// literal ASCII byte `'0'` (0x30), and `>0.6.1` against the literal +/// characters `0.6.1` -- not the integer 0 or the float 0.6. The generic +/// [`parse_value`] tries its float and integer branches (see +/// `value::parse_value`) before falling through, so a bareword like `0` or +/// `0.6.1` was captured as `Value::Uint`/`Value::Float`. A subsequent +/// comparison against the string field read from the file then yields no +/// ordering (`String` vs `Uint`/`Float` is incomparable), so the rule +/// silently never matched -- breaking real `>0` idioms such as +/// `\b, name %s` / `face %s` / `palette %s` and version compares like +/// `>0.6.1 ... version %s`. +/// +/// This parser mirrors [`parse_value`]'s ordering for the two branches +/// that are correct for string-family values -- a leading whitespace trim, +/// then a quoted string (-> `Value::String`), then a hex/escape byte +/// sequence (-> `Value::Bytes`, e.g. gzip's `\037\213` or `\177ELF`) -- but +/// replaces the numeric (float/integer) branches with +/// [`parse_bare_string_value`], so every remaining bareword resolves to a +/// `Value::String`. The leading `multispace0` is load-bearing: it ensures +/// the hex branch sees byte-identical input to what `parse_value` fed it, +/// so an escape-heavy value cannot fall through to the lossy-UTF-8 +/// `parse_bare_string_value` path and corrupt a high byte (see the +/// `high-byte-utf8-corruption-class` note). +/// +/// Hex-*letter* barewords (`>AB`, `cafebabe`) still resolve to +/// `Value::Bytes` via the unchanged hex branch, matching GOTCHAS S3.12 -- +/// only the numeric subset changes here. +/// +/// # Errors +/// Returns a nom parsing error only when the value is empty/whitespace-only +/// (via [`parse_bare_string_value`]); quoted and hex forms are attempted +/// first and never error out of this function on a non-empty token. +fn parse_string_family_value(input: &str) -> IResult<&str, Value> { + // Trim leading whitespace up front so the hex branch below receives the + // same (trimmed) input `parse_value` would have handed it. + let (input, _) = multispace0(input)?; + if let Ok((rest, s)) = value::parse_quoted_string(input) { + return Ok((rest, Value::String(s))); + } + if let Ok((rest, bytes)) = value::parse_hex_bytes(input) { + return Ok((rest, Value::Bytes(bytes))); + } + parse_bare_string_value(input) +} + /// Parse a bare (unquoted) single-token string literal as a `Value::String`. /// /// Used only as a fallback for string-family types (`string`, `pstring`, @@ -1270,11 +1367,27 @@ fn parse_bare_string_value(input: &str) -> IResult<&str, Value> { remaining = rest; continue; } - // Lone `\` not followed by a recognised escape -- treat as - // a literal backslash and continue. This matches GNU `file` - // tolerance for malformed escapes. - bytes.push(b'\\'); - remaining = &remaining[1..]; + // Lone `\` followed by an unrecognised escape char: magic(5) + // getstr DROPS the backslash and keeps the character literally + // (`\<` -> `<`, `\^` -> `^`, `\ ` -> a literal space that + // continues the token because it lands in `bytes` rather than + // being re-examined by the whitespace-terminator check). This + // matches GNU `file`; the earlier "keep the backslash" behavior + // broke real rules like sgml's `0 string \ IResult<&str, Value> { ))); } - // The downstream comparison is `Value::String` against the buffer's - // bytes. Use `from_utf8_lossy` so non-UTF-8 byte sequences (like - // `\xff`) round-trip as best they can; the buffer-side read uses - // the same lossy conversion, so equality still holds. - let value = String::from_utf8_lossy(&bytes).into_owned(); - Ok((remaining, Value::String(value))) + // Mirror `read_string_exact` (evaluator/types/string.rs): when the + // resolved bytes are valid UTF-8, return `Value::String` so `%s` + // output renders normally; when they are NOT -- e.g. a bareword like + // OS/2 INF's `HSP\x01\x9b\x00` (0x9b is invalid UTF-8), or an octal + // form like `AB\376` -- return the RAW bytes as `Value::Bytes`. A + // lossy `String` decode would turn 0x9b into U+FFFD (3 bytes 0xEF BF + // BD), which BOTH inflates the pattern's byte length (6 -> 8, so + // `read_string_exact` reads the wrong number of bytes) AND changes the + // byte value at that position, so the rule would silently never match. + // Cross-type `String`/`Bytes` equality and ordering (GOTCHAS S2.3) + // compare by byte sequence, so either variant compares correctly + // against the read value. This completes the read/parse symmetry: + // `read_string_exact` was fixed to return `Value::Bytes` on non-UTF-8 + // slices, but the parse side kept lossy-decoding until this change. + match String::from_utf8(bytes) { + Ok(s) => Ok((remaining, Value::String(s))), + Err(e) => Ok((remaining, Value::Bytes(e.into_bytes()))), + } } /// Parse a comment line (starts with #) diff --git a/src/parser/grammar/tests/hex_bytes_truncation.rs b/src/parser/grammar/tests/hex_bytes_truncation.rs new file mode 100644 index 00000000..bf39dffb --- /dev/null +++ b/src/parser/grammar/tests/hex_bytes_truncation.rs @@ -0,0 +1,302 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +//! U7 audit: does `parse_hex_bytes` (and its sub-parsers +//! `parse_mixed_hex_ascii` / `parse_hex_bytes_no_prefix`) consume a whole +//! value token, or silently truncate at the first character that doesn't +//! look like a hex digit and leak the remainder back to the caller? +//! +//! `TypeKind::Regex` no longer routes through this code path (see +//! `parser::grammar::getstr`), but `TypeKind::Search` (and any other +//! string-family type whose bareword value happens to look like hex +//! bytes) still does, via `parse_value`'s `alt(...)`. +//! +//! **Finding (confirmed by the tests below, not inferred):** +//! `parse_mixed_hex_ascii` (used whenever the token starts with a +//! backslash) consumes the WHOLE token -- its final fallback branch, +//! `none_of(" \t\n\r")`, accepts virtually any non-whitespace character, +//! so a mixed escape+literal token like `\x41BC[def` is captured in +//! full. `parse_hex_bytes_no_prefix` (used only when the token does +//! **not** start with a backslash and looks like a run of pure hex +//! digits) DOES truncate: it uses `take_while(is_ascii_hexdigit)`, which +//! stops at the first non-hex-digit character and returns `Ok` with the +//! shorter byte vector, leaving the remainder (e.g. `[42`) in the +//! `IResult`'s remaining-input slot. Through `parse_magic_rule`, that +//! leaked remainder is treated as the start of the rule's *message* +//! field -- corrupting the parsed value AND fabricating message text +//! that was never intended, with no parse error to signal the mistake. +//! +//! This is a genuine, if narrow, bug: it requires a bareword (unquoted, +//! no `\` prefix) token that is (a) an even number of hex-digit +//! characters, (b) contains at least one `a`-`f`/`A`-`F` letter (so +//! `parse_hex_bytes_no_prefix`'s "must look like hex, not a decimal +//! number" guard doesn't reject it), and (c) is immediately followed +//! (no whitespace) by more non-whitespace, non-hex-digit characters. +//! Fixed below by requiring the parsed hex run to be immediately +//! followed by whitespace or end-of-input -- i.e. the ENTIRE token must +//! be hex digits, not just a hex-looking prefix of it. + +use super::*; +use crate::EvaluationConfig; +use crate::evaluator::{EvaluationContext, evaluate_rules}; + +#[test] +fn mixed_escape_and_literal_token_is_captured_whole() { + // `\x41BC[def`: hex-prefixed byte, then plain ASCII letters, then a + // literal `[` (not part of any escape), then more plain letters. + // Confirms `parse_mixed_hex_ascii` does NOT truncate at `[`. + let (remaining, bytes) = parse_hex_bytes(r"\x41BC[def").expect(r"\x41BC[def"); + assert_eq!(remaining, "", "whole token must be consumed, none leaked"); + assert_eq!( + bytes, + vec![0x41, b'B', b'C', b'[', b'd', b'e', b'f'], + "all intended bytes must be present, in order" + ); +} + +#[test] +fn mixed_escape_and_literal_token_stops_only_at_whitespace() { + let (remaining, bytes) = + parse_hex_bytes(r"\x7f\x45(LF)[bracket] trailing").expect("mixed token with brackets"); + assert_eq!(remaining, " trailing"); + assert_eq!( + bytes, + { + let mut expected = vec![0x7f, 0x45]; + expected.extend_from_slice(b"(LF)[bracket]"); + expected + }, + "escape + arbitrary literal punctuation must all be captured" + ); +} + +/// Regression test for the confirmed truncation bug in +/// `parse_hex_bytes_no_prefix`: a bareword (no `\` prefix) token that +/// looks like hex digits followed immediately by non-hex-digit +/// characters (no separating whitespace) must NOT silently truncate and +/// leak the remainder. Before the fix, `parse_hex_bytes("4a[42")` +/// returned `Ok(("[42", vec![0x4a]))` -- only the "4a" prefix was +/// captured and "[42" leaked back as if it were separate input. +#[test] +fn no_prefix_hex_token_does_not_truncate_at_trailing_non_hex_chars() { + // A hex-looking token immediately followed by a non-hex-digit + // character with NO separating whitespace is not a valid pure-hex + // token -- it must be rejected outright (Err), not silently + // truncated to a shorter byte vector with the remainder leaked. + assert!( + parse_hex_bytes("4a[42").is_err(), + "a hex-looking prefix immediately followed by non-hex-digit, \ + non-whitespace characters must not silently truncate" + ); + + // A whitespace boundary is still a legitimate token terminator (this + // is how `parse_hex_bytes` is meant to be used inside `parse_value` + // when more input, such as an operator or message, follows). + assert_eq!( + parse_hex_bytes("4a1b more text"), + Ok((" more text", vec![0x4a, 0x1b])) + ); + + // A trailing quote (closing a quoted-value context) is likewise a + // legitimate boundary, matching the existing + // `test_parse_hex_bytes_with_remaining_input` coverage. + assert_eq!(parse_hex_bytes("ab\""), Ok(("\"", vec![0xab]))); +} + +#[test] +fn search_literal_no_prefix_hex_lookalike_round_trips_through_parse_value() { + // Behavioral confirmation at the `parse_value` level (the entry + // point `parse_magic_rule` uses for string-family types): a + // bareword token that looks like hex digits but is immediately + // followed by non-hex-digit characters must not corrupt the parsed + // value or leak text into what would become the rule's message. + // + // `"ab[cd message"` is used (rather than a leading-digit token like + // `"4a[cd message"`) to isolate this assertion to the hex-bytes path + // under audit: a leading decimal digit would also be greedily + // consumed by `parse_numeric_value`'s `digit1` (e.g. `"4a..."` still + // yields `Ok(Value::Uint(4))` with `"a..."` leaking as remaining -- + // a pre-existing, separate greediness in the *numeric* fallback, + // out of scope for this hex-bytes-specific audit). Starting with a + // letter (`'a'`) means neither `parse_float_value` nor + // `parse_numeric_value` can match at all, so the only way + // `parse_value` could have returned `Ok` here was via the hex-bytes + // path -- which, before this fix, DID: `parse_value("ab[cd message")` + // returned `Ok(("[cd message", Value::Bytes(vec![0xab])))`, silently + // truncating the value to one byte and leaking "[cd message" as if + // it were separate input (`parse_magic_rule` would have rendered it + // as the rule's message text). + // + // After the fix: the malformed pure-hex-lookalike token is rejected + // by the hex-bytes path, so `parse_value` itself returns `Err`, + // exactly as it would for any other unrecognized bareword token + // (see `test_parse_value_invalid_input`). The string-family bareword + // fallback (`parse_bare_string_value`) in `parse_magic_rule` then + // takes over and captures the ENTIRE token, brackets included. + assert!(parse_value("ab[cd message").is_err()); + + match parse_value("4a1bcd 0x41 more") { + Ok((remaining, Value::Bytes(bytes))) => { + assert_eq!(remaining, " 0x41 more"); + assert_eq!(bytes, vec![0x4a, 0x1b, 0xcd]); + } + other => panic!("expected whitespace-terminated hex bytes, got {other:?}"), + } +} + +#[test] +fn search_rule_with_malformed_hex_lookalike_literal_captures_whole_value() { + // Full `parse_magic_rule` round-trip (not just `parse_value`): a + // `search` rule whose bareword literal looks like a hex-byte prefix + // but is immediately followed by non-hex characters must parse with + // the ENTIRE literal as the value and the actual trailing words as + // the message -- not a 1-byte value with "[cd real message" mangled + // into the message field. + let input = "0 search/16 ab[cd real message"; + let (remaining, rule) = parse_magic_rule(input).expect(input); + assert_eq!(remaining, ""); + assert_eq!( + rule.value, + Value::String("ab[cd".to_string()), + "the whole malformed-hex-lookalike token must be captured as one value \ + via the bareword string fallback, not truncated to a partial Value::Bytes" + ); + assert_eq!(rule.message, "real message"); +} + +/// U7 (plan scenario "Search literal round-trip"): parse-correctness must +/// translate to MATCH-correctness, not just parse-shape. The corrected +/// whole-token literal from the test above (`ab[cd`, previously truncated +/// to a 1-byte `Value::Bytes` by the confirmed `parse_hex_bytes_no_prefix` +/// bug) is driven through the full evaluator against a buffer that +/// CONTAINS the literal, asserting the rule actually matches. Without this +/// test, a regression that silently reintroduces truncation (e.g. back to +/// a 1-byte value matching `\xab` alone) could still parse successfully +/// while producing wrong evaluation results -- the "non-crashing wrong +/// answer" failure mode the plan's Verification Contract calls out. +#[test] +fn search_rule_with_malformed_hex_lookalike_literal_matches_buffer_containing_it() { + let input = "0 search/16 ab[cd real message"; + let (_, rule) = parse_magic_rule(input).expect(input); + assert_eq!(rule.value, Value::String("ab[cd".to_string())); + + let mut context = EvaluationContext::new(EvaluationConfig::default()); + + // Positive: the buffer contains the whole "ab[cd" literal -- must match. + let matches = evaluate_rules(std::slice::from_ref(&rule), b"xxab[cdyy", &mut context) + .expect("evaluation must not error"); + assert_eq!( + matches.len(), + 1, + "the whole 5-byte literal must match when present in the buffer, got {matches:?}" + ); + assert_eq!(matches[0].message, "real message"); + + // Negative: a buffer containing only the truncated 1-byte value + // (`0xab`) followed by DIFFERENT bytes (not `[cd`) must NOT match -- + // proving the rule requires the full literal, not just its prefix. + context.reset(); + let no_match = evaluate_rules(std::slice::from_ref(&rule), b"xxab99yy", &mut context) + .expect("evaluation must not error"); + assert!( + no_match.is_empty(), + "a buffer containing only the truncated 1-byte prefix must not match, got {no_match:?}" + ); +} + +#[test] +fn bareword_escaped_space_does_not_truncate_the_token() { + // magic(5) `\ ` (backslash-space) is a literal space that must NOT + // terminate a bareword string/search value. The real GNU `file` `tex` + // rule `0 search/4096 %\ -*-latex-*- LaTeX document text` must parse + // with the FULL 13-byte pattern "% -*-latex-*-" as the value and + // "LaTeX document text" as the message. Before the fix the token + // truncated at the escaped space to "%", turning the rule into + // `search "%"` -- which false-matches nearly any binary (a `%` byte + // appears in almost every file), mislabeling gzip/JPEG/etc. as LaTeX. + let input = r"0 search/4096 %\ -*-latex-*- LaTeX document text"; + let (remaining, rule) = parse_magic_rule(input).expect(input); + assert_eq!(remaining, ""); + assert_eq!( + rule.value, + Value::String("% -*-latex-*-".to_string()), + "the escaped space must be a literal space inside the value, not a token terminator" + ); + assert_eq!(rule.message, "LaTeX document text"); + + // Match correctness: the rule must fire ONLY when the full literal is + // present, and must NOT fire on a buffer that merely contains a `%`. + let mut context = EvaluationContext::new(EvaluationConfig::default()); + let hit = evaluate_rules( + std::slice::from_ref(&rule), + b"junk % -*-latex-*- more", + &mut context, + ) + .expect("evaluation must not error"); + assert_eq!(hit.len(), 1, "must match when the full pattern is present"); + assert_eq!(hit[0].message, "LaTeX document text"); + + context.reset(); + let miss = evaluate_rules( + std::slice::from_ref(&rule), + b"100% binary \x1f\x8b data with a percent", + &mut context, + ) + .expect("evaluation must not error"); + assert!( + miss.is_empty(), + "a buffer with a bare `%` (but not the full pattern) must NOT match, got {miss:?}" + ); +} + +#[test] +fn bareword_value_beginning_with_backslash_resolves_full_token_and_matches_xml() { + // The system-DB sgml rule `0 string \ `<` and `\ ` -> a literal space that + // CONTINUES the token, so the full 14-byte pattern `"#, + &mut context, + ) + .expect("evaluation must not error"); + assert_eq!(hit.len(), 1, "a real 0 use part2", 1, - TypeKind::Meta(MetaType::Use("part2".to_string())), + TypeKind::Meta(MetaType::Use { + name: "part2".to_string(), + flip_endian: false, + }), "", ), ]; @@ -103,24 +109,35 @@ fn test_parse_magic_rule_meta_name_use_reject_malformed_identifiers() { } // Identifiers followed by whitespace + descriptive text are accepted: - // real-world magic files use this for human-readable annotations - // (e.g. `/usr/share/file/magic/database` line 588 has - // `0 name xbase-prf dBase Printer Form`, where "dBase Printer Form" - // is a comment, not part of the identifier). The identifier ends at - // the first non-id character (space, tab) and any trailing text on - // the same line is discarded. Verify that this dropping happens - // cleanly: the parsed identifier is just the first id-token. + // real-world magic files use this for human-readable annotations. The + // identifier always ends at the first non-id character (space, tab), so + // `expected_id` is the first token. The trailing text handling then + // DIVERGES by directive, matching GNU `file`: + // - `name`: the trailing text IS the subroutine's own description and + // is preserved as the rule message (emitted at the `use` site, e.g. + // `0 name xbase-prf dBase Printer Form`). + // - `use`: a use-site has no message slot; the trailing text is + // dropped (`0 use foo bar` renders no `bar`). let trailing_text_cases = [ - ("0 name part 2", "part"), - ("0 use part2 extra", "part2"), - ("0 name my id", "my"), - ("0 use foo bar", "foo"), + ("0 name part 2", "part", "2"), + ("0 name my id", "my", "id"), + ( + "0 name xbase-prf dBase Printer Form", + "xbase-prf", + "dBase Printer Form", + ), + // Mach-O universal subroutine: the `\b [` no-separator marker + `[` + // must survive parsing verbatim (the leading `\b` is a literal + // backslash-b, preserved per GOTCHAS S14.1). + ("0 name mach-o \\b [", "mach-o", "\\b ["), + ("0 use part2 extra", "part2", ""), + ("0 use foo bar", "foo", ""), ]; - for (input, expected_id) in trailing_text_cases { + for (input, expected_id, expected_message) in trailing_text_cases { let (_, rule) = parse_magic_rule(input) .unwrap_or_else(|e| panic!("trailing text after id should parse {input:?}: {e:?}")); match &rule.typ { - TypeKind::Meta(MetaType::Name(id) | MetaType::Use(id)) => { + TypeKind::Meta(MetaType::Name(id) | MetaType::Use { name: id, .. }) => { assert_eq!( id, expected_id, "identifier should stop at first whitespace for {input:?}" @@ -128,6 +145,10 @@ fn test_parse_magic_rule_meta_name_use_reject_malformed_identifiers() { } other => panic!("expected Name/Use meta, got {other:?}"), } + assert_eq!( + rule.message, expected_message, + "message mismatch for {input:?} (name preserves trailing text, use drops it)" + ); } // Sanity check: an identifier followed only by trailing whitespace still parses. @@ -137,7 +158,40 @@ fn test_parse_magic_rule_meta_name_use_reject_malformed_identifiers() { TypeKind::Meta(MetaType::Name("part2".to_string())) ); let (_, rule) = parse_magic_rule("0 use part2\t").expect("trailing tab is ok"); - assert_eq!(rule.typ, TypeKind::Meta(MetaType::Use("part2".to_string()))); + assert_eq!( + rule.typ, + TypeKind::Meta(MetaType::Use { + name: "part2".to_string(), + flip_endian: false + }) + ); +} + +#[test] +fn test_parse_use_caret_prefix_sets_flip_endian() { + // magic(5) `use \^name` (the `\^` endian-flip prefix, issue #236) parses + // to `MetaType::Use { flip_endian: true }`; the `\^` is consumed and the + // bare identifier is preserved. A plain `use name` stays `flip_endian: + // false`. This is the real `images` TIFF `>(4.L) use \^tiff_ifd` shape. + let (_, flipped) = parse_magic_rule(">0 use \\^tiff_ifd").expect("flip use parses"); + assert_eq!( + flipped.typ, + TypeKind::Meta(MetaType::Use { + name: "tiff_ifd".to_string(), + flip_endian: true, + }), + "`use \\^name` must set flip_endian and strip the \\^ prefix" + ); + + let (_, plain) = parse_magic_rule(">0 use tiff_ifd").expect("plain use parses"); + assert_eq!( + plain.typ, + TypeKind::Meta(MetaType::Use { + name: "tiff_ifd".to_string(), + flip_endian: false, + }), + "a plain `use name` must leave flip_endian false" + ); } #[test] @@ -203,7 +257,10 @@ fn test_parse_text_magic_file_meta_roundtrip() { assert_eq!( rules[0].typ, - TypeKind::Meta(MetaType::Use("subroutine".to_string())) + TypeKind::Meta(MetaType::Use { + name: "subroutine".to_string(), + flip_endian: false, + }) ); assert_eq!(rules[1].typ, TypeKind::Meta(MetaType::Default)); assert_eq!(rules[2].typ, TypeKind::Meta(MetaType::Clear)); diff --git a/src/parser/grammar/tests/mod.rs b/src/parser/grammar/tests/mod.rs index 456fe2eb..3a3ac403 100644 --- a/src/parser/grammar/tests/mod.rs +++ b/src/parser/grammar/tests/mod.rs @@ -1,6 +1,7 @@ // Copyright (c) 2025-2026 the libmagic-rs contributors // SPDX-License-Identifier: Apache-2.0 +mod hex_bytes_truncation; mod indirect_offset; mod meta_types; @@ -222,11 +223,15 @@ fn test_parse_offset_with_remaining_input() { #[test] fn test_parse_offset_edge_cases() { - // Zero with different formats + // Zero with different formats. `-0` / `-0x0` are the magic(5) + // "0 bytes from end of file" form (the EOF position, `buffer.len()`), + // NOT absolute offset 0 -- the leading `-` is significant even though + // `-0 == 0` numerically. They encode `FromEnd(0)`; unsigned `0` / `0x0` + // stay `Absolute(0)`. See gzip's `>>-0 offset >48` trailing-size gate. assert_eq!(parse_offset("0"), Ok(("", OffsetSpec::Absolute(0)))); - assert_eq!(parse_offset("-0"), Ok(("", OffsetSpec::Absolute(0)))); + assert_eq!(parse_offset("-0"), Ok(("", OffsetSpec::FromEnd(0)))); assert_eq!(parse_offset("0x0"), Ok(("", OffsetSpec::Absolute(0)))); - assert_eq!(parse_offset("-0x0"), Ok(("", OffsetSpec::Absolute(0)))); + assert_eq!(parse_offset("-0x0"), Ok(("", OffsetSpec::FromEnd(0)))); // Large offsets assert_eq!( @@ -1893,6 +1898,137 @@ fn test_parse_magic_rule_string_value() { assert_eq!(rule.message, "ZIP archive"); } +/// A bareword value on a string-family type must parse as `Value::String`, +/// not `Value::Uint`/`Value::Float` -- for BOTH equality and ordering +/// operators, since `parse_string_family_value` never inspects the +/// operator. Previously `parse_value`'s numeric branches captured +/// `0`/`0.6.1`/`20011231` as numbers, so the subsequent `String`-vs-number +/// comparison never matched and real `>0`/`000` idioms (`\b, name %s`, +/// `face %s`, version compares) silently failed. +/// +/// The `search` case deliberately uses an EQUALITY operator: `search` is a +/// pattern-bearing type, and a non-equality operator on it is an intentional +/// `UnsupportedType` fatal gap (GOTCHAS S2.4), so `search >100` -- while it +/// now correctly PARSES to `Value::String("100")` -- is not a construct that +/// evaluates. No real magic rule uses `search`/`regex` with an ordering +/// operator on a numeric bareword; the only real hit (`search/1 >\0`) is a +/// `\0`->`Value::Bytes` value on the unchanged hex branch. +#[test] +fn test_string_family_bareword_numeric_value_parses_as_string() { + // (input, expected value, expected op) + let cases: &[(&str, Value, Operator)] = &[ + // Bare equality (no operator token) -- also changed from Uint(0). + ( + "0 string 000 face %s", + Value::String("000".to_string()), + Operator::Equal, + ), + ( + "0 string >0 \\b, name %s", + Value::String("0".to_string()), + Operator::GreaterThan, + ), + ( + "0 string >0.6.1 version %s", + Value::String("0.6.1".to_string()), + Operator::GreaterThan, + ), + ( + "0 string >20011231 date %s", + Value::String("20011231".to_string()), + Operator::GreaterThan, + ), + ( + "0 string <9 low %s", + Value::String("9".to_string()), + Operator::LessThan, + ), + // search-family bareword numeric value is likewise a string. + // Equality op (not ordering): search is pattern-bearing, see S2.4. + ( + "0 search/16 100 hit %s", + Value::String("100".to_string()), + Operator::Equal, + ), + ]; + + for (input, expected_value, expected_op) in cases { + let (_, rule) = + parse_magic_rule(input).unwrap_or_else(|e| panic!("parse failed for {input:?}: {e:?}")); + assert_eq!( + rule.value, *expected_value, + "value mismatch for {input:?}: got {:?}", + rule.value + ); + assert_eq!(rule.op, *expected_op, "operator mismatch for {input:?}"); + } +} + +/// The fix must not change the two branches that were already correct for +/// string-family values: quoted strings stay `Value::String`, and +/// hex/escape byte sequences stay `Value::Bytes` (e.g. gzip's `\037\213`). +/// Hex-*letter* barewords also stay `Value::Bytes` per GOTCHAS S3.12 -- +/// only the numeric subset changes. +#[test] +fn test_string_family_value_preserves_quoted_and_hex_bytes() { + // Quoted numeric string stays a String, not a number. + let (_, rule) = parse_magic_rule("0 string \"0\" literal").unwrap(); + assert_eq!(rule.value, Value::String("0".to_string())); + + // Octal-escape byte sequence stays Bytes (gzip magic). + let (_, rule) = parse_magic_rule("0 string \\037\\213 gzip").unwrap(); + assert_eq!(rule.value, Value::Bytes(vec![0x1f, 0x8b])); + + // Mixed hex/ascii escape stays Bytes (ELF magic). + let (_, rule) = parse_magic_rule("0 string \\177ELF elf").unwrap(); + assert_eq!(rule.value, Value::Bytes(vec![0x7f, b'E', b'L', b'F'])); + + // Hex-letter bareword stays Bytes (boundary, unchanged -- GOTCHAS S3.12). + let (_, rule) = parse_magic_rule("0 string >AB thing").unwrap(); + assert_eq!(rule.value, Value::Bytes(vec![0xAB])); +} + +#[test] +fn parse_bare_string_value_high_byte_returns_bytes_not_lossy_string() { + // GOTCHAS S6.7: a bareword string value whose resolved bytes contain a + // high byte (>= 0x80) that is not valid UTF-8 must be captured as + // `Value::Bytes` holding the RAW bytes, never lossy-decoded into a + // `Value::String` (which turns the byte into U+FFFD -- 3 bytes -- and + // both inflates the pattern length and changes the byte, so the rule + // silently never matches). This is the parse-side mirror of + // `read_string_exact` (S6.4). The signature begins with literal ASCII + // (`HSP`), so it bypasses `parse_mixed_hex_ascii` (which requires a + // leading `\`) and lands in `parse_bare_string_value`. + // + // Table: (input, expected value). Covers both high-byte escape forms + // (hex `\x9b`, octal `\376`) plus the all-ASCII control that must stay + // a `Value::String`. + let cases: &[(&str, Value)] = &[ + // OS/2 INF top-level signature: HSP\x01\x9b\x00 -> raw bytes. + ( + "0 string HSP\\x01\\x9b\\x00 OS/2 INF", + Value::Bytes(vec![0x48, 0x53, 0x50, 0x01, 0x9b, 0x00]), + ), + // Octal high byte embedded after leading ASCII: AB\376 -> raw bytes. + ( + "0 string AB\\376 thing", + Value::Bytes(vec![0x41, 0x42, 0xFE]), + ), + // All-ASCII bareword (no high byte) stays a String -- %s renders. + ("0 string HSP header", Value::String("HSP".to_string())), + ]; + + for (input, expected) in cases { + let (_, rule) = + parse_magic_rule(input).unwrap_or_else(|e| panic!("parse failed for {input:?}: {e:?}")); + assert_eq!( + rule.value, *expected, + "value mismatch for {input:?}: got {:?}, expected {expected:?}", + rule.value + ); + } +} + #[test] fn test_parse_magic_rule_hex_offset() { let input = "0x10 belong 0x12345678 Test data"; @@ -2503,7 +2639,7 @@ fn test_parse_type_and_operator_regex_and_search_suffixes() { } fn sr(n: usize) -> TypeKind { TypeKind::Search { - range: NonZeroUsize::new(n).unwrap(), + range: NonZeroUsize::new(n), flags: SearchFlags::default(), } } @@ -2540,6 +2676,16 @@ fn test_parse_type_and_operator_regex_and_search_suffixes() { ("search/256", sr(256), ""), ("search/1", sr(1), ""), ("search/256 =", sr(256), "="), + // Bare `search` (no `/N`) parses with an open (`None`) range = + // scan-to-EOF, matching GNU `file`'s implementation. + ( + "search", + TypeKind::Search { + range: None, + flags: SearchFlags::default(), + }, + "", + ), ]; for &(input, ref expected_kind, expected_rest) in cases { let (rest, (kind, op, _)) = parse_type_and_operator(input).expect(input); @@ -2550,11 +2696,22 @@ fn test_parse_type_and_operator_regex_and_search_suffixes() { } #[test] -fn test_parse_type_and_operator_search_requires_range() { - // Bare `search` (no /N suffix) is a hard parse error per GNU `file`. - assert!(parse_type_and_operator("search").is_err()); - // `search/0` is also rejected -- `NonZeroUsize` makes a zero-width - // scan unrepresentable. +fn test_parse_type_and_operator_bare_search_accepted_zero_rejected() { + use crate::parser::ast::TypeKind; + // Bare `search` (no /N suffix) is ACCEPTED and parses to an open + // (`None`) range = scan-to-EOF. magic(5) documents the count as + // required, but the reference `file` binary accepts the bare form + // (`str_range == 0`); rmagic follows the implementation. Real system + // magic uses this (e.g. pdf `>8 search /Count`, `0 search + // ##fileformat=VCFv`). + let (_, (kind, _, _)) = + parse_type_and_operator("search").expect("bare search must parse (scan-to-EOF)"); + assert!( + matches!(kind, TypeKind::Search { range: None, .. }), + "bare search must yield range None, got {kind:?}" + ); + // `search/0` is still rejected -- an explicit zero-width scan is + // unrepresentable (`NonZeroUsize`). assert!(parse_type_and_operator("search/0").is_err()); } @@ -2666,7 +2823,7 @@ fn test_parse_magic_rule_regex_and_search() { assert_eq!( rule.typ, TypeKind::Search { - range: NonZeroUsize::new(256).unwrap(), + range: NonZeroUsize::new(256), flags: SearchFlags::default(), } ); diff --git a/src/parser/grammar/type_suffix.rs b/src/parser/grammar/type_suffix.rs index 168cdc58..f2371d4a 100644 --- a/src/parser/grammar/type_suffix.rs +++ b/src/parser/grammar/type_suffix.rs @@ -286,8 +286,13 @@ pub(super) fn parse_attached_operator(input: &str) -> IResult<&str, Option]` where `N` is a non-zero count. /// -/// Per GNU `file` magic(5), the range is mandatory; bare `search` and -/// `search/0` are parse errors, enforced here via `NonZeroUsize`. The +/// This helper handles ONLY the ranged form (a `/` follows the keyword). +/// `search/0` is rejected via `NonZeroUsize` (a zero-width scan is +/// unrepresentable). A bare `search` (no `/` suffix) never reaches this +/// function: the caller (`parse_type_and_operator`) accepts it directly and +/// records `range: None` = scan-to-EOF. magic(5) documents the count as +/// required, but the reference `file` binary accepts the bare form +/// (`str_range == 0`), so rmagic follows the implementation there. The /// range accepts both decimal (`search/256`) and hexadecimal /// (`search/0xffff`, `search/0x93e4f`) literals -- many real magic files /// use hex for large search windows (e.g., archive:254 scans up to diff --git a/src/parser/grammar/value.rs b/src/parser/grammar/value.rs index ede808cd..9b26e96d 100644 --- a/src/parser/grammar/value.rs +++ b/src/parser/grammar/value.rs @@ -67,6 +67,33 @@ pub(super) fn parse_mixed_hex_ascii(input: &str) -> IResult<&str, Vec> { } else if let Ok((new_remaining, hex_byte)) = parse_hex_byte_with_prefix(remaining) { bytes.push(hex_byte); remaining = new_remaining; + } else if let Some(rest) = remaining.strip_prefix('\\') { + // Unrecognized escape: GNU `file`'s getstr DROPS the backslash + // and keeps the following character literally (`\^` -> `^`, + // `\<` -> `<`, `\ ` -> a literal space that CONTINUES the token + // instead of terminating it). Recognized escapes (octal `\NNN`, + // `\xNN`, `\n`/`\r`/`\t`/`\\`/`\"`/`\'`/`\0`) were already + // consumed by the two branches above, so this fires only for a + // backslash escaping a non-special character. Without this, a + // string-family bareword beginning with a backslash (e.g. the + // sgml rule `0 string \>(" \t\n\r")(remaining) { @@ -89,6 +116,24 @@ pub(super) fn parse_mixed_hex_ascii(input: &str) -> IResult<&str, Vec> { } } +/// Returns `true` if `rest` starts at a legitimate value-token boundary: +/// end of input, whitespace, or a closing quote (`"`, matching the +/// pre-existing `parse_hex_bytes("ab\"")` contract for a value embedded +/// just before a quote). +/// +/// Used by [`parse_hex_bytes_no_prefix`] to reject a hex-digit run that +/// is immediately followed by more non-whitespace, non-hex-digit +/// characters (e.g. `4a[42`) instead of silently truncating to the +/// hex-looking prefix and leaking the remainder (`[42`) back to the +/// caller -- see the U7 audit in `grammar/tests/hex_bytes_truncation.rs` +/// for the confirmed bug this guards against. +fn is_hex_token_boundary(rest: &str) -> bool { + match rest.chars().next() { + None => true, + Some(c) => c.is_whitespace() || c == '"', + } +} + /// Parse a hex byte sequence without prefix (only if it looks like pure hex bytes) pub(super) fn parse_hex_bytes_no_prefix(input: &str) -> IResult<&str, Vec> { // Only parse as hex bytes if: @@ -96,6 +141,9 @@ pub(super) fn parse_hex_bytes_no_prefix(input: &str) -> IResult<&str, Vec> { // 2. All characters are hex digits // 3. Doesn't start with 0x (that's a number) // 4. Contains at least one non-decimal digit (a-f, A-F) + // 5. The hex-digit run is immediately followed by a legitimate token + // boundary (whitespace, a closing quote, or end of input) -- NOT + // silently truncated mid-token (see `is_hex_token_boundary`). if input.starts_with("0x") || input.starts_with('-') { return Err(nom::Err::Error(NomError::new( @@ -124,6 +172,20 @@ pub(super) fn parse_hex_bytes_no_prefix(input: &str) -> IResult<&str, Vec> { ))); } + let remaining = input.strip_prefix(hex_chars.as_str()).unwrap_or(input); + if !is_hex_token_boundary(remaining) { + // The token is not purely hex digits (e.g. `4a[42`) -- reject + // rather than truncate. Letting this succeed would silently + // shrink the parsed value to the hex-looking prefix and leak + // the rest of the token as if it were separate input (which, + // through `parse_magic_rule`, gets misinterpreted as the start + // of the rule's message text). + return Err(nom::Err::Error(NomError::new( + input, + nom::error::ErrorKind::Tag, + ))); + } + // Parse pairs of hex digits. Floor division is intended: an odd // trailing digit is handled below and only affects the capacity hint. #[allow(clippy::integer_division)] @@ -142,7 +204,6 @@ pub(super) fn parse_hex_bytes_no_prefix(input: &str) -> IResult<&str, Vec> { bytes.push(byte_val); } - let remaining = &input[hex_chars.len()..]; Ok((remaining, bytes)) } diff --git a/src/parser/hierarchy.rs b/src/parser/hierarchy.rs index 1db5db5b..f01d91ce 100644 --- a/src/parser/hierarchy.rs +++ b/src/parser/hierarchy.rs @@ -9,6 +9,19 @@ use super::preprocessing::{LineInfo, parse_magic_rule_line}; use crate::error::ParseError; use crate::parser::ast::MagicRule; +use log::warn; + +/// Count the leading `>`-continuation markers on a magic-rule line, i.e. its +/// nesting level, without fully parsing the rule. Used to drop the subtree of +/// a line that failed to parse (its deeper-indented descendants), so an +/// orphaned child cannot silently re-attach to the wrong parent. +fn leading_indent_level(content: &str) -> usize { + content + .trim_start() + .bytes() + .take_while(|&b| b == b'>') + .count() +} /// Builds a hierarchical structure from a flat list of parsed magic rules. /// @@ -39,6 +52,27 @@ use crate::parser::ast::MagicRule; /// - Any line contains invalid magic rule syntax /// - Rule parsing fails (propagated from `parse_magic_rule_line`) pub(crate) fn build_rule_hierarchy(lines: Vec) -> Result, ParseError> { + build_rule_hierarchy_impl(lines, false) +} + +/// Line-tolerant variant of [`build_rule_hierarchy`] for the **runtime** +/// loader: an unparseable rule (and its subtree) is skipped with a warning +/// instead of aborting the whole file, matching GNU `file`. This lets a real +/// system magic file that contains a construct this parser cannot yet handle +/// (e.g. the `compress` file's one `ustring` XZ rule) still contribute its +/// other rules (gzip, bzip2, ...). The strict [`build_rule_hierarchy`] is kept +/// for build-time codegen of the crate's own builtin rules, where a malformed +/// rule should fail the build. See GOTCHAS S3.11. +pub(crate) fn build_rule_hierarchy_tolerant( + lines: Vec, +) -> Result, ParseError> { + build_rule_hierarchy_impl(lines, true) +} + +fn build_rule_hierarchy_impl( + lines: Vec, + tolerant: bool, +) -> Result, ParseError> { /// Helper to pop a rule from the stack and attach it to its parent or roots fn pop_and_attach(stack: &mut Vec, roots: &mut Vec) { if let Some(completed) = stack.pop() { @@ -53,19 +87,57 @@ pub(crate) fn build_rule_hierarchy(lines: Vec) -> Result = Vec::new(); let mut roots: Vec = Vec::new(); let mut pending_strength: Option = None; + // When a rule line fails to parse we skip it AND its deeper-indented + // descendants (the subtree it heads), tracked by this level threshold. + let mut skip_subtree_deeper_than: Option = None; for line in lines { if line.is_comment { continue; } + // Drop the descendants of a rule that failed to parse: any line more + // deeply nested than the failed line belongs to its subtree. + if let Some(threshold) = skip_subtree_deeper_than { + if leading_indent_level(&line.content) > threshold { + continue; + } + // Back at the failed line's level (a sibling) or shallower -- + // resume normal processing. + skip_subtree_deeper_than = None; + } + // Handle strength directive: store modifier for next rule if line.strength_modifier.is_some() { pending_strength = line.strength_modifier; continue; } - let mut rule = parse_magic_rule_line(&line)?; + // Line-tolerant parsing (matches GNU `file`): a single unparseable rule + // no longer aborts the whole file. Skip the offending rule and its + // subtree with a warning, keeping the rest of the file's rules -- so a + // file like `compress` (whose one `ustring` XZ rule this parser cannot + // yet handle) still contributes its gzip/bzip2 detection instead of + // being dropped entirely. See GOTCHAS S3.11. + let mut rule = match parse_magic_rule_line(&line) { + Ok(rule) => rule, + Err(err) => { + if !tolerant { + // Strict mode (build-time codegen): a malformed rule fails. + return Err(err); + } + let preview: String = line.content.chars().take(80).collect(); + warn!( + "skipping unparseable magic rule at line {}: {err} (rule: {preview:?})", + line.line_number + ); + // The failed line's subtree is meaningless without it, and an + // orphaned strength directive must not leak onto a later rule. + skip_subtree_deeper_than = Some(leading_indent_level(&line.content)); + pending_strength = None; + continue; + } + }; // Apply pending strength modifier to this rule if pending_strength.is_some() { diff --git a/src/parser/loader.rs b/src/parser/loader.rs index 3e2ad5b7..3ffec727 100644 --- a/src/parser/loader.rs +++ b/src/parser/loader.rs @@ -79,6 +79,21 @@ fn read_magic_file_bounded(path: &Path) -> Result { } } +/// Whether `contents` contains at least one actual rule line -- a non-blank +/// line that is neither a comment (`#`) nor a `!:` metadata directive +/// (`!:mime`, `!:strength`, ...). Used by [`load_magic_directory`] to tell a +/// file whose rules were all skipped as unparseable (unusable) apart from a +/// genuinely empty, comment-only, or directive-only file (valid, contributes +/// no rules). `!:` directives are stripped during preprocessing and are not +/// rules, so counting them here would misclassify a directive-only file as +/// "had rules but all were skipped". +fn has_rule_lines(contents: &str) -> bool { + contents.lines().any(|line| { + let trimmed = line.trim(); + !trimmed.is_empty() && !trimmed.starts_with('#') && !trimmed.starts_with("!:") + }) +} + /// Loads and parses all magic files from a directory, merging them into a single rule set. /// /// This function reads all regular files in the specified directory, parses each as a magic file, @@ -146,6 +161,9 @@ fn read_magic_file_bounded(path: &Path) -> Result { /// # Panics /// /// This function does not panic under normal operation. +// Directory iteration + per-file parse + error aggregation runs slightly over +// the 100-line lint; splitting this module is tracked in #391. +#[allow(clippy::too_many_lines)] pub fn load_magic_directory(dir_path: &Path) -> Result { use std::fs; @@ -195,6 +213,11 @@ pub fn load_magic_directory(dir_path: &Path) -> Result let mut all_rules = Vec::new(); let mut merged_table = NameTable::empty(); let mut parse_failures: Vec<(PathBuf, ParseError)> = Vec::new(); + // Files that parsed (line-tolerantly) but contributed no usable rule or + // name despite having non-empty content -- i.e. every rule was skipped as + // unparseable. Tracked so an all-unusable directory is still reported as a + // failure even though tolerant parsing returns Ok for each such file. + let mut empty_files: Vec = Vec::new(); let mut any_success = false; let file_count = file_paths.len(); @@ -211,46 +234,69 @@ pub fn load_magic_directory(dir_path: &Path) -> Result } }; - // Parse the file - match super::parse_text_magic_file(&contents) { + // Parse the file (line-tolerant: unparseable rules are skipped with a + // warning rather than dropping the whole file). + match super::parse_text_magic_file_tolerant(&contents) { Ok(parsed) => { - any_success = true; - all_rules.extend(parsed.rules); - merged_table.merge(parsed.name_table); + if parsed.rules.is_empty() && parsed.name_table.is_empty() { + // Contributed nothing usable. If the file had actual rule + // lines (not just comments/blank lines), they were all + // skipped as unparseable -- record it so an entirely- + // unusable directory is still reported as a failure. A + // genuinely empty or comment-only file is valid and is NOT + // recorded (it simply contributes no rules). + if has_rule_lines(&contents) { + empty_files.push(path); + } + } else { + any_success = true; + all_rules.extend(parsed.rules); + merged_table.merge(parsed.name_table); + } } Err(e) => { - // Track parse failures for reporting + // Hard (preprocess-level) failure -- track for reporting. parse_failures.push((path, e)); } } } - // If all files failed to parse, return an error. - // Use `any_success` rather than `all_rules.is_empty()` so that directories - // whose files parse successfully but contain only meta-type definitions - // (e.g. a directory of pure `name`-subroutine files) are not mistaken for - // complete failure. - if !any_success && !parse_failures.is_empty() { + // A directory has loaded only if some file contributed a usable rule or + // name-table entry. `any_success` (not `all_rules.is_empty()`) so that a + // directory of pure `name`-subroutine files is not mistaken for failure. + // With line-tolerant parsing, a file whose rules were all skipped returns + // Ok with nothing; a directory where every content-bearing file is like + // that (or fails preprocessing) has loaded nothing and is reported as a + // failure -- preserving the pre-tolerance contract for a wholly-unusable + // directory. + if !any_success && (!parse_failures.is_empty() || !empty_files.is_empty()) { use std::fmt::Write; - let failure_details: Vec = parse_failures + let mut problems: Vec = parse_failures .iter() - .take(3) // Limit to first 3 failures for brevity .map(|(path, e)| format!(" - {}: {}", path.display(), e)) .collect(); + problems.extend( + empty_files + .iter() + .map(|path| format!(" - {}: no usable rules (all skipped)", path.display())), + ); - let mut message = format!("All {file_count} magic file(s) in directory failed to parse"); - if !failure_details.is_empty() { + let mut message = format!( + "All {file_count} magic file(s) in directory failed to parse or produced no usable rules" + ); + let shown = problems.iter().take(3).cloned().collect::>(); + if !shown.is_empty() { message.push_str(":\n"); - message.push_str(&failure_details.join("\n")); - if parse_failures.len() > 3 { + message.push_str(&shown.join("\n")); + if problems.len() > 3 { // fmt::Write to a String is infallible; discard the Result // rather than unwrap so the no-panic policy holds regardless. #[allow(clippy::let_underscore_must_use)] let _ = write!( message, "\n ... and {} more", - parse_failures.len().saturating_sub(3) + problems.len().saturating_sub(3) ); } } @@ -371,7 +417,7 @@ pub fn load_magic_file(path: &Path) -> Result { MagicFileFormat::Text => { // Read file contents (size-bounded) and parse as text magic file let content = read_magic_file_bounded(path)?; - super::parse_text_magic_file(&content) + super::parse_text_magic_file_tolerant(&content) } MagicFileFormat::Directory => { // Load all magic files from directory @@ -465,6 +511,48 @@ mod tests { ); } + #[test] + fn test_load_directory_all_content_bearing_but_all_rules_skipped_errors() { + use std::fs; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + + // Two files that BOTH have content (non-comment rule lines) but whose + // every rule is unparseable. Under line-tolerant parsing (GOTCHAS S3.11) + // each file returns Ok with zero rules, but the directory contributed + // nothing usable, so `load_magic_directory` must still Err -- preserving + // the pre-tolerance "all failed" contract via the `empty_files` / + // `has_rule_lines` tracking. This is distinct from a directory of + // genuinely-empty/comment-only files (test_load_directory_empty_files), + // which is a valid no-op success, and from the mixed valid+invalid case + // (test_load_directory_non_critical_error_parse), which succeeds because + // one file contributed a rule. + fs::write( + temp_dir.path().join("bad1.magic"), + "notanoffset badtype whatever\nalso not a valid rule line\n", + ) + .expect("Failed to write bad1"); + fs::write( + temp_dir.path().join("bad2.magic"), + "still invalid syntax here\n", + ) + .expect("Failed to write bad2"); + + let err = load_magic_directory(temp_dir.path()).expect_err( + "a directory whose content-bearing files all parse to zero rules must fail", + ); + let msg = err.to_string(); + assert!( + msg.contains("failed to parse"), + "error must report the all-failed contract: {msg}" + ); + assert!( + msg.contains("no usable rules (all skipped)"), + "error must attribute the content-bearing-but-all-skipped files: {msg}" + ); + } + #[test] fn test_load_directory_binary_files() { use std::fs; @@ -661,27 +749,110 @@ mod tests { } #[test] - fn test_load_magic_file_parse_error_propagation() { + fn test_load_magic_file_tolerates_unparseable_rule_and_keeps_valid_ones() { use std::fs; use tempfile::TempDir; let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let invalid_file = temp_dir.path().join("invalid.magic"); + let mixed_file = temp_dir.path().join("mixed.magic"); + + // A valid rule, an unparseable rule (missing offset), then another valid + // rule. Runtime loading is line-tolerant (GNU `file` semantics, GOTCHAS + // S3.11): the bad rule is skipped with a warning and the valid rules on + // either side survive, instead of the whole file being dropped. (This + // is what lets real system magic files keep their common-format + // detection despite a stray construct this parser cannot yet handle.) + fs::write( + &mixed_file, + "0 string GOOD1 first good rule\nstring test invalid\n0 string GOOD2 second good rule\n", + ) + .expect("Failed to write file"); - // Create file with invalid syntax (missing offset) - fs::write(&invalid_file, "string test invalid\n").expect("Failed to write invalid file"); + let parsed = load_magic_file(&mixed_file) + .expect("runtime load must tolerate an unparseable rule, not abort the whole file"); + let msgs: Vec<&str> = parsed.rules.iter().map(|r| r.message.as_str()).collect(); + assert!( + msgs.contains(&"first good rule"), + "a valid rule before the bad one must survive: {msgs:?}" + ); + assert!( + msgs.contains(&"second good rule"), + "a valid rule after the bad one must survive: {msgs:?}" + ); + assert_eq!( + parsed.rules.len(), + 2, + "the unparseable rule must be dropped, keeping exactly the two valid ones: {msgs:?}" + ); + } - // Attempt to load file with parse errors - let result = load_magic_file(&invalid_file); + #[test] + fn test_load_magic_file_drops_subtree_of_unparseable_rule_without_reattaching() { + use std::fs; + use tempfile::TempDir; - assert!(result.is_err(), "Should fail for file with parse errors"); + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let file = temp_dir.path().join("subtree.magic"); + + // A valid parent, then an UNPARSEABLE rule (no valid offset) that owns a + // `>`-indented child, then a valid sibling back at the original level. + // GOTCHAS S3.11: when the bad rule is skipped, its deeper-indented + // subtree must be dropped WITH it (`skip_subtree_deeper_than` in + // `hierarchy.rs`). The orphaned child must NOT silently re-attach to the + // previous level-0 rule (`GOOD1`) nor survive as a stray top-level rule, + // and the trailing `GOOD2` proves the skip threshold resets so a later + // sibling at the original indent still parses. + fs::write( + &file, + "0 string GOOD1 parent rule\n\ + notanoffset badtype orphan parent\n\ + >0 byte x orphaned child that must be dropped\n\ + 0 string GOOD2 sibling after the dropped subtree\n", + ) + .expect("Failed to write file"); - // Error should be a parse error (not I/O error) - let error = result.unwrap_err(); - let error_msg = format!("{error:?}"); + let parsed = load_magic_file(&file) + .expect("runtime load must tolerate the unparseable rule and its subtree"); + + let top_msgs: Vec<&str> = parsed.rules.iter().map(|r| r.message.as_str()).collect(); + assert_eq!( + parsed.rules.len(), + 2, + "exactly the two valid top-level rules survive: {top_msgs:?}" + ); + assert!( + top_msgs.contains(&"parent rule"), + "the valid rule before the bad one must survive: {top_msgs:?}" + ); + assert!( + top_msgs.contains(&"sibling after the dropped subtree"), + "the sibling after the dropped subtree must parse (threshold reset): {top_msgs:?}" + ); + + // The orphaned child must appear NOWHERE: not re-attached to the + // preceding level-0 rule, and not as any surviving rule's child. + let good1 = parsed + .rules + .iter() + .find(|r| r.message == "parent rule") + .expect("GOOD1 must be present"); + assert!( + good1.children.is_empty(), + "the dropped child must not re-attach to the previous level-0 rule: {:?}", + good1 + .children + .iter() + .map(|c| c.message.as_str()) + .collect::>() + ); + let orphan_reattached = parsed + .rules + .iter() + .flat_map(|r| r.children.iter()) + .any(|c| c.message.contains("orphaned child")); assert!( - error_msg.contains("InvalidSyntax") || error_msg.contains("syntax"), - "Error should be parse error: {error_msg}", + !orphan_reattached, + "the orphaned child of the unparseable rule must be dropped entirely" ); } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 354fa139..f460e240 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -174,7 +174,7 @@ pub struct ParsedMagic { /// Top-level rules after `Name` subroutines have been removed. pub rules: Vec, /// Extracted `name` subroutine definitions, consulted by the evaluator - /// when a rule of type `TypeKind::Meta(MetaType::Use(_))` is reached. + /// when a rule of type `TypeKind::Meta(MetaType::Use { .. })` is reached. pub(crate) name_table: name_table::NameTable, } @@ -222,6 +222,25 @@ pub fn parse_text_magic_file(input: &str) -> Result { Ok(ParsedMagic { rules, name_table }) } +/// Line-tolerant variant of [`parse_text_magic_file`] for **runtime** loading +/// of external magic files/directories. +/// +/// Unlike [`parse_text_magic_file`] (which is fail-fast, so the crate's own +/// build-time codegen rejects a malformed builtin rule), this skips an +/// unparseable rule and its subtree with a `warn!` and keeps the rest of the +/// file -- matching GNU `file`. Real system magic databases routinely mix +/// rules this parser fully supports with a handful using constructs it does +/// not yet handle (`guid`, `der`, middle-endian dates, some indirect-offset +/// forms); without tolerance, one such rule would drop the entire file -- +/// e.g. losing all of `compress`'s gzip/bzip2 detection over its lone `ustring` +/// XZ rule. See GOTCHAS S3.11. +pub(crate) fn parse_text_magic_file_tolerant(input: &str) -> Result { + let lines = preprocess_lines(input)?; + let rules = hierarchy::build_rule_hierarchy_tolerant(lines)?; + let (rules, name_table) = name_table::extract_name_table(rules); + Ok(ParsedMagic { rules, name_table }) +} + #[cfg(test)] mod unit_tests { use super::*; diff --git a/src/parser/name_table.rs b/src/parser/name_table.rs index 4f4bf1c9..79c3ecb7 100644 --- a/src/parser/name_table.rs +++ b/src/parser/name_table.rs @@ -16,11 +16,30 @@ use log::warn; use crate::parser::ast::{MagicRule, MetaType, TypeKind}; +/// One named subroutine: the `name` line's own description plus its body. +/// +/// A magic(5) `name` line can carry a description of its own (e.g. the +/// Mach-O universal subroutine `0 name mach-o \b [`, `0 name matlab4 +/// Matlab v4 mat-file`). GNU `file` emits that description when the +/// subroutine is invoked via `use`, *before* the body's matches, and +/// attaches it with no separating space (see [`extract_name_table`] and +/// the `use` dispatch in `evaluator::engine`). We keep it here so the +/// evaluator can reproduce that output; bare `name ` lines store an +/// empty string and contribute nothing. +#[derive(Debug, Clone)] +struct Subroutine { + /// The `name` line's own description text (verbatim, including any + /// leading `\b` no-separator marker). Empty for a bare `name `. + name_message: String, + /// The subroutine body -- the `name` rule's children. + rules: Arc<[MagicRule]>, +} + /// A lookup table mapping subroutine names to their child rule lists. /// /// Built by [`extract_name_table`] from a parsed magic file's top-level /// rule list. The evaluator consults this table when it encounters a -/// `TypeKind::Meta(MetaType::Use(name))` rule to retrieve the rules that +/// `TypeKind::Meta(MetaType::Use { name, .. })` rule to retrieve the rules that /// should be evaluated as if inlined at the `use` site. /// /// Subroutine bodies are stored as `Arc<[MagicRule]>` so the evaluator can @@ -29,7 +48,7 @@ use crate::parser::ast::{MagicRule, MetaType, TypeKind}; /// corpora where the same subroutine may be invoked many times per evaluation. #[derive(Debug, Default, Clone)] pub(crate) struct NameTable { - inner: HashMap>, + inner: HashMap, } impl NameTable { @@ -41,6 +60,12 @@ impl NameTable { } } + /// Returns `true` if the table holds no named subroutines. + #[must_use] + pub(crate) fn is_empty(&self) -> bool { + self.inner.is_empty() + } + /// Look up a subroutine's rule list by name. /// /// Returns an `Arc` reference so callers can clone it cheaply (reference @@ -48,23 +73,24 @@ impl NameTable { /// before mutably borrowing any surrounding context. #[must_use] pub(crate) fn get(&self, name: &str) -> Option> { - self.inner.get(name).cloned() + self.inner.get(name).map(|s| Arc::clone(&s.rules)) } - /// Sort all subroutine rule bodies in place by the provided comparator. + /// Return the `name` line's own description for a subroutine, if it + /// carries one. /// - /// Used by `MagicDatabase` after load to apply strength-based ordering to - /// each subroutine body, matching the ordering applied to top-level rules. - /// Because subroutine bodies are stored as `Arc<[MagicRule]>` (immutable - /// slices), sorting requires materializing a `Vec`, sorting it, and - /// rebuilding the `Arc`. This is a one-time cost per load, not per - /// evaluation. - pub(crate) fn sort_subroutines(&mut self, mut sort_fn: impl FnMut(&mut Vec)) { - for arc in self.inner.values_mut() { - let mut vec: Vec = arc.iter().cloned().collect(); - sort_fn(&mut vec); - *arc = Arc::from(vec); - } + /// GNU `file` emits this description when the subroutine is invoked via + /// `use`, ahead of the body's matches. Returns `None` when the name is + /// unknown or the `name` line was a bare `name ` with no description. + /// The returned string is cloned (name descriptions are short, e.g. + /// `\b [`), so the caller can drop the table borrow before mutating the + /// evaluation context. + #[must_use] + pub(crate) fn name_message(&self, name: &str) -> Option { + self.inner + .get(name) + .filter(|s| !s.name_message.is_empty()) + .map(|s| s.name_message.clone()) } /// Merge another name table into this one. @@ -73,12 +99,12 @@ impl NameTable { /// table is merged into the accumulating table. On key collisions, /// the first-seen definition is kept and a warning is emitted. pub(crate) fn merge(&mut self, other: Self) { - for (name, rules) in other.inner { + for (name, subroutine) in other.inner { if self.inner.contains_key(&name) { warn!("duplicate name definition '{name}' across magic files; keeping first"); continue; } - self.inner.insert(name, rules); + self.inner.insert(name, subroutine); } } } @@ -106,8 +132,22 @@ pub(crate) fn extract_name_table(rules: Vec) -> (Vec, Name } // Recursively scrub nested Name rules from the subroutine's // children (shouldn't appear in practice, but be defensive). + // Capture the `name` line's OWN description (`rule.message`) + // alongside the body: GNU `file` emits it when the subroutine is + // invoked via `use` (e.g. Mach-O universal `\b [`, `matlab4 + // Matlab v4 mat-file`). Dropping it here is what made rmagic omit + // those fragments. Bare `name ` lines have an empty message + // and contribute nothing. + let name = name.clone(); + let name_message = rule.message; let children = scrub_nested_names(rule.children, rule.level); - table.inner.insert(name.clone(), Arc::from(children)); + table.inner.insert( + name, + Subroutine { + name_message, + rules: Arc::from(children), + }, + ); } else { let scrubbed_children = scrub_nested_names(rule.children, rule.level); kept.push(MagicRule { @@ -186,6 +226,37 @@ mod tests { assert_eq!(subroutine[0].message, "child"); } + #[test] + fn test_extract_captures_name_line_message() { + // A `name` line with its own description (e.g. Mach-O universal + // `0 name mach-o \b [`) must have that description stored so the + // evaluator can emit it at the `use` site. A bare `name ` stores + // no message (`name_message` returns `None`). + let child = make_rule(1, TypeKind::Byte { signed: false }, "child", vec![]); + let name_rule = make_rule( + 0, + TypeKind::Meta(MetaType::Name("mach-o".to_string())), + "\\b [", + vec![child], + ); + let bare = make_rule( + 0, + TypeKind::Meta(MetaType::Name("bare".to_string())), + "", + vec![make_rule(1, TypeKind::Byte { signed: false }, "c2", vec![])], + ); + let (rules, table) = extract_name_table(vec![name_rule, bare]); + assert!(rules.is_empty()); + // Body is still reachable via `get`, unchanged. + assert_eq!(table.get("mach-o").expect("mach-o body").len(), 1); + // The name-line message is captured and returned. + assert_eq!(table.name_message("mach-o").as_deref(), Some("\\b [")); + // Bare `name ` has no message. + assert_eq!(table.name_message("bare"), None); + // Unknown name has no message. + assert_eq!(table.name_message("nope"), None); + } + #[test] fn test_extract_preserves_non_name_rules() { let byte_rule = make_rule(0, TypeKind::Byte { signed: false }, "hello", vec![]); @@ -276,72 +347,4 @@ mod tests { let subroutine = table_a.get("dup").expect("dup kept from first table"); assert_eq!(subroutine[0].message, "first-child"); } - - #[test] - fn test_sort_subroutines_reorders_rule_bodies() { - // `sort_subroutines` materializes each Arc body into a mutable - // Vec, invokes the sort closure, and rebuilds the Arc. A bug in - // that rebuild cycle (e.g., swapping Arc pointers instead of - // re-sorting) would leave the order unchanged. - let body = vec![ - make_rule(1, TypeKind::Byte { signed: false }, "c", vec![]), - make_rule(1, TypeKind::Byte { signed: false }, "a", vec![]), - make_rule(1, TypeKind::Byte { signed: false }, "b", vec![]), - ]; - let name_rule = make_rule( - 0, - TypeKind::Meta(MetaType::Name("sorted".to_string())), - "", - body, - ); - let (_, mut table) = extract_name_table(vec![name_rule]); - - table.sort_subroutines(|rules| rules.sort_by(|x, y| x.message.cmp(&y.message))); - - let after = table.get("sorted").expect("subroutine retained"); - let messages: Vec<&str> = after.iter().map(|r| r.message.as_str()).collect(); - assert_eq!(messages, vec!["a", "b", "c"]); - } - - #[test] - fn test_sort_subroutines_on_empty_table_is_noop() { - let (_, mut table) = extract_name_table(vec![]); - // The closure should never fire for an empty table. - table.sort_subroutines(|_| unreachable!("empty table must not invoke sort_fn")); - assert!(table.get("any").is_none()); - } - - #[test] - fn test_sort_subroutines_preserves_merge_policy() { - // After `sort_subroutines`, `merge` must still honor first-wins. - let first = make_rule( - 0, - TypeKind::Meta(MetaType::Name("dup".to_string())), - "", - vec![make_rule( - 1, - TypeKind::Byte { signed: false }, - "first", - vec![], - )], - ); - let second = make_rule( - 0, - TypeKind::Meta(MetaType::Name("dup".to_string())), - "", - vec![make_rule( - 1, - TypeKind::Byte { signed: false }, - "second", - vec![], - )], - ); - let (_, mut table_a) = extract_name_table(vec![first]); - table_a.sort_subroutines(|_| {}); // no-op sort to trigger rebuild - let (_, table_b) = extract_name_table(vec![second]); - table_a.merge(table_b); - - let subroutine = table_a.get("dup").expect("dup kept from first table"); - assert_eq!(subroutine[0].message, "first"); - } } diff --git a/src/tests.rs b/src/tests.rs index 08a23187..397aa5ef 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -304,9 +304,20 @@ fn test_with_builtin_rules() { zip_result.description ); - // Test unknown data falls back to "data" - let unknown_result = db.evaluate_buffer(b"random unknown content").unwrap(); - assert_eq!(unknown_result.description, "data"); + // Unmatched plain ASCII content falls back to the text/data + // classifier's "ASCII text" result, matching GNU `file` (verified: + // `file` reports "ASCII text, with no line terminators" for this + // exact buffer) -- NOT the old hardcoded "data" this test asserted + // before the text/data fallback (GOTCHAS S13.2 / issue: blank output + // for readable files) was implemented. + let unknown_text_result = db.evaluate_buffer(b"random unknown content").unwrap(); + assert_eq!(unknown_text_result.description, "ASCII text"); + + // Genuinely binary, unmatched content still falls back to "data". + let unknown_binary_result = db + .evaluate_buffer(&[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x10]) + .unwrap(); + assert_eq!(unknown_binary_result.description, "data"); } #[test] @@ -367,7 +378,13 @@ fn test_evaluation_result_confidence_from_matches() { fn test_evaluation_result_no_match_has_zero_confidence() { let db = MagicDatabase::with_builtin_rules().expect("builtin rules should load"); - let unknown_result = db.evaluate_buffer(b"random unknown content").unwrap(); + // Genuinely binary content so no built-in rule matches and the + // text/data fallback (GOTCHAS S13.2) reports "data" -- confirming + // confidence is 0.0 for a fallback-classified result, not just an + // empty-matches result. + let unknown_result = db + .evaluate_buffer(&[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x10]) + .unwrap(); assert_eq!(unknown_result.description, "data"); assert!((unknown_result.confidence - 0.0).abs() < 0.001); @@ -424,6 +441,42 @@ fn test_concatenate_messages_with_backspace() { assert_eq!(result, "ELF, 64-bit"); // No space before comma } +/// Regression: the `\b` no-separator marker reaches concatenation as the +/// LITERAL two-character sequence `\b` (backslash + 'b'), NOT a U+0008 byte, +/// because the message parser preserves description text verbatim (matching +/// GNU `file`, which keeps the desc literal and special-cases a leading `\b` +/// at print time). Real rules like msdos's `\b, for MS Windows` and the +/// Mach-O universal-binary `\b]` were rendering the literal marker into the +/// output ("PE STUB \b, for MS Windows", "...architectures: \b]") until +/// concatenation learned to strip the literal `\b` too (GOTCHAS S14.1). +#[test] +fn test_concatenate_messages_with_literal_backslash_b_marker() { + let matches = vec![ + evaluator::RuleMatch { + message: "PE STUB".to_string(), + offset: 0, + level: 0, + value: Value::Uint(0), + type_kind: TypeKind::Byte { signed: false }, + confidence: 0.3, + }, + evaluator::RuleMatch { + // Literal backslash + 'b', exactly as `parse_message` produces it. + message: "\\b, for MS Windows".to_string(), + offset: 4, + level: 1, + value: Value::Uint(0), + type_kind: TypeKind::Byte { signed: false }, + confidence: 0.5, + }, + ]; + let result = MagicDatabase::concatenate_messages(&matches); + assert_eq!( + result, "PE STUB, for MS Windows", + "literal \\b marker must suppress the space and not appear in the output" + ); +} + /// Regression test for review finding M5 / GOTCHAS S14.1: the `\b` /// (backspace) prefix must suppress the leading separator even when the /// backspace-prefixed message is the first entry. This is a degenerate diff --git a/tessl.json b/tessl.json deleted file mode 100644 index 56231b1f..00000000 --- a/tessl.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "name": "libmagic-rs", - "mode": "vendored", - "dependencies": { - "actionbook/rust-skills": { - "version": "fa60f7931223646fb71c4586b4a6c8545016076a", - "source": "https://github.com/actionbook/rust-skills", - "include": { - "skills": [ - "coding-guidelines", - "domain-cli", - "m01-ownership", - "m02-resource", - "m03-mutability", - "m04-zero-cost", - "m05-type-driven", - "m06-error-handling", - "m07-concurrency", - "m09-domain", - "m10-performance", - "m11-ecosystem", - "m12-lifecycle", - "m13-domain-error", - "m14-mental-model", - "m15-anti-pattern", - "meta-cognition-parallel", - "rust-call-graph", - "rust-code-navigator", - "rust-deps-visualizer", - "rust-learner", - "rust-refactor-helper", - "rust-skill-creator", - "rust-symbol-analyzer", - "rust-trait-explorer", - "unsafe-checker" - ] - } - }, - "pantheon-ai/github-actions-generator": { - "version": "0.1.1" - }, - "pantheon-ai/mise-complete": { - "version": "0.1.1" - }, - "pantheon-ai/dockerfile-toolkit": { - "version": "0.1.0" - }, - "pantheon-ai/moscow-prioritization": { - "version": "0.1.1" - }, - "pantheon-ai/software-design-principles": { - "version": "0.1.4" - }, - "cisco/software-security": { - "version": "1.2.5" - }, - "neonwatty/logo-designer-skill": { - "version": "8f9a4b04009c15b05eeb47b4608d5502abafa609", - "source": "https://github.com/neonwatty/logo-designer-skill" - }, - "trailofbits/skills": { - "version": "a56045e9ae00b3506cacefea0f672aab0a1a6e3c", - "source": "https://github.com/trailofbits/skills", - "include": { - "skills": [ - "ask-questions-if-underspecified", - "audit-augmentation", - "audit-context-building", - "audit-prep-assistant", - "cargo-fuzz", - "code-maturity-assessor", - "codeql", - "constant-time-analysis", - "constant-time-testing", - "coverage-analysis", - "crypto-protocol-diagram", - "designing-workflow-skills", - "devcontainer-setup", - "diagramming-code", - "differential-review", - "dimensional-analysis", - "dwarf-expert", - "fp-check", - "fuzzing-dictionary", - "fuzzing-obstacles", - "genotoxic", - "gh-cli", - "git-cleanup", - "graph-evolution", - "harness-writing", - "insecure-defaults", - "libafl", - "mermaid-to-proverif", - "mutation-testing", - "ossfuzz", - "sarif-parsing", - "seatbelt-sandboxer", - "second-opinion", - "secure-workflow-guide", - "semgrep", - "semgrep-rule-creator", - "semgrep-rule-variant-creator", - "sharp-edges", - "skill-improver", - "supply-chain-risk-auditor", - "testing-handbook-generator", - "trailmark", - "trailmark-structural", - "trailmark-summary", - "variant-analysis", - "vector-forge", - "yara-rule-authoring", - "zeroize-audit" - ] - } - }, - "tessl-labs/devops-essentials": { - "version": "0.1.1" - }, - "pantheon-ai/github-actions-validator": { - "version": "0.1.1" - }, - "pantheon-ai/acceptance-criteria": { - "version": "0.1.1" - }, - "pantheon-ai/agents-md": { - "version": "0.1.3" - } - } -} diff --git a/tests/cli_integration.rs b/tests/cli_integration.rs index 37713c28..04996031 100644 --- a/tests/cli_integration.rs +++ b/tests/cli_integration.rs @@ -79,8 +79,13 @@ fn test_builtin_format_detection() { } // PDF and GIF are not currently detected by builtin rules, so they fall - // through to the "data" fallback. We verify the CLI runs without error and - // produces either the format name or "data". + // through to the text/data fallback (GOTCHAS S13.2). `PDF_HEADER` + // (`%PDF-1.4`) and `GIF_HEADER` (`GIF89a`) are both plain ASCII, so the + // fallback classifies them as "ASCII text" (matching GNU `file`'s + // ascmagic behavior for readable content), not "data" -- "data" is + // reserved for genuinely binary content. We verify the CLI runs + // without error and produces either the format name or the text + // fallback. let fallback_cases = [ ("test.pdf", PDF_HEADER, "PDF"), ("test.gif", GIF_HEADER, "GIF"), @@ -92,7 +97,9 @@ fn test_builtin_format_detection() { .args(["--use-builtin", path_str(&test_file)]) .assert() .success() - .stdout(predicate::str::contains(format_name).or(predicate::str::contains("data"))); + .stdout( + predicate::str::contains(format_name).or(predicate::str::contains("ASCII text")), + ); } } @@ -124,7 +131,15 @@ fn test_builtin_with_json() { #[test] fn test_builtin_unknown_file_returns_data() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let test_file = create_data_file(&temp_dir, "unknown.bin", b"random data here"); + // Genuinely binary (non-ASCII, invalid-UTF-8) content so the text/data + // fallback (GOTCHAS S13.2) reports "data" -- an ASCII buffer would + // instead fall back to "ASCII text", which is what this test's name + // and assertion specifically exercise. + let test_file = create_data_file( + &temp_dir, + "unknown.bin", + &[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x80, 0x81], + ); rmagic_cmd() .args(["--use-builtin", path_str(&test_file)]) @@ -142,7 +157,10 @@ fn test_stdin_format_detection() { let cases: &[(&str, &[u8], Option<&str>)] = &[ ("ELF via stdin", ELF_HEADER, Some("ELF")), ("PNG via stdin", PNG_SIGNATURE, Some("PNG")), - ("empty stdin", b"", Some("data")), + // Empty input falls back to "empty" (GOTCHAS S13.2 text/data + // fallback), matching GNU `file`'s literal output for a + // zero-byte input -- not the old hardcoded "data". + ("empty stdin", b"", Some("empty")), ("unknown content", b"sample data", None), ]; @@ -219,21 +237,26 @@ fn test_stdin_no_false_truncation_warning() { #[test] fn test_stdin_strict_mode_with_empty_input() { - // Empty stdin in strict mode should still succeed (empty file is valid) + // Empty stdin in strict mode should still succeed (empty file is + // valid). The text/data fallback (GOTCHAS S13.2) reports "empty" for + // a zero-byte buffer, matching GNU `file` -- not the old hardcoded + // "data". rmagic_cmd() .args(["--use-builtin", "--strict", "-"]) .write_stdin(b"" as &[u8]) .assert() .success() - .stdout(predicate::str::contains("stdin: data")); + .stdout(predicate::str::contains("stdin: empty")); } #[test] fn test_stdin_non_strict_continues_on_unknown() { - // Non-strict mode should continue without error on unknown content + // Non-strict mode should continue without error on unknown content. + // Genuinely binary content so the text/data fallback reports "data" + // rather than "ASCII text". rmagic_cmd() .args(["--use-builtin", "-"]) - .write_stdin(b"random unrecognized content" as &[u8]) + .write_stdin(&[0x00u8, 0x01, 0x02, 0xFF, 0xFE, 0x80, 0x81] as &[u8]) .assert() .success() .stdout(predicate::str::contains("data")); @@ -548,7 +571,14 @@ fn test_custom_magic_file_fallback_to_data() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let magic_content = "# Test magic file\n0 byte 0xff Marker\n"; let magic_file = create_magic_file(&temp_dir, magic_content); - let data_file = create_data_file(&temp_dir, "test.bin", b"plain text"); + // Genuinely binary content (no custom rule matches, and it is not + // ASCII/UTF-8 text) so the text/data fallback (GOTCHAS S13.2) reports + // "data" as this test's name asserts. + let data_file = create_data_file( + &temp_dir, + "test.bin", + &[0x00u8, 0x01, 0x02, 0xFF, 0xFE, 0x80, 0x81], + ); rmagic_cmd() .args(["--magic-file", path_str(&magic_file), path_str(&data_file)]) @@ -590,11 +620,14 @@ fn test_empty_file() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let path = create_data_file(&temp_dir, "empty.bin", b""); + // The text/data fallback (GOTCHAS S13.2) reports "empty" for a + // zero-byte file, matching GNU `file`'s literal output -- not the + // old hardcoded "data". rmagic_cmd() .args(["--use-builtin", path_str(&path)]) .assert() .success() - .stdout(predicate::str::contains("data")); + .stdout(predicate::str::contains("empty")); } #[test] @@ -602,9 +635,11 @@ fn test_very_small_file() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let path = create_data_file(&temp_dir, "small.bin", b"x"); + // A single printable ASCII byte falls back to "ASCII text" (GOTCHAS + // S13.2), matching GNU `file` -- not "data". rmagic_cmd() .args(["--use-builtin", path_str(&path)]) .assert() .success() - .stdout(predicate::str::contains("data")); + .stdout(predicate::str::contains("ASCII text")); } diff --git a/tests/endian_flip_use_tests.rs b/tests/endian_flip_use_tests.rs new file mode 100644 index 00000000..b02120c3 --- /dev/null +++ b/tests/endian_flip_use_tests.rs @@ -0,0 +1,143 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end coverage for the magic(5) `use \^name` endian-flip flag +//! (issue #236, libmagic `softmagic.c` `cvt_flip`). +//! +//! A `use \^name` invocation flips the little/big endianness of every +//! endian-bearing read inside the invoked subroutine, and the flip +//! propagates into nested plain `use` calls. This is what lets the system +//! `images` magic parse a **big-endian** TIFF with a subroutine whose reads +//! are all declared `leshort`: +//! +//! ```text +//! 0 string MM\x00\x2a TIFF image data, big-endian +//! >(4.L) use \^tiff_ifd +//! 0 string II\x2a\x00 TIFF image data, little-endian +//! >(4.l) use tiff_ifd +//! 0 name tiff_ifd +//! >0 leshort x \b, direntries=%d +//! >2 use tiff_entry +//! ``` +//! +//! Before the fix rmagic dropped the `\^` prefix, so a big-endian TIFF read +//! `direntries` (and every `tiff_entry` tag) with little-endian byte order, +//! producing byte-swapped garbage (`direntries=22` -> `5632`) and no +//! width/height/compression fields. +//! +//! These tests use a synthetic TIFF-like structure that mirrors the real +//! magic's shape (an indirect `(4.L)`/`(4.l)` pointer into a subroutine that +//! reads `leshort`, with a nested plain `use` one level down) so the flip and +//! its propagation are pinned hermetically, independent of the system magic +//! database or any on-disk sample file. + +#![allow(clippy::expect_used)] + +use std::io::Write; + +use libmagic_rs::{EvaluationConfig, MagicDatabase}; +use tempfile::NamedTempFile; + +/// Magic mirroring the real `images` TIFF shape: a big-endian header uses +/// `\^ifd` (flip), a little-endian header uses plain `ifd` (no flip). The +/// `ifd` subroutine reads `leshort` for direntries and then invokes a nested +/// plain `use inner` that reads another `leshort` -- proving the flip reaches +/// the nested subroutine. +const TIFF_LIKE_MAGIC: &str = "0\tstring\tMM\\x00\\x2a\tBE-TIFF\n\ + >(4.L)\tuse\t\\^ifd\n\ + 0\tstring\tII\\x2a\\x00\tLE-TIFF\n\ + >(4.l)\tuse\tifd\n\ + 0\tname\tifd\n\ + >0\tleshort\tx\t\\b, direntries=%d\n\ + >2\tuse\tinner\n\ + 0\tname\tinner\n\ + >0\tleshort\tx\t\\b, tag=%d\n"; + +fn db_from_magic(magic: &str) -> MagicDatabase { + let mut f = NamedTempFile::new().expect("temp magic file"); + f.write_all(magic.as_bytes()).expect("write magic"); + f.flush().expect("flush magic"); + MagicDatabase::load_from_file_with_config(f.path(), EvaluationConfig::default()) + .expect("magic must load") +} + +/// A 12-byte big-endian TIFF-like buffer: +/// - `MM\x00\x2a` header, then a big-endian 4-byte IFD pointer (= 8) +/// - at offset 8: big-endian `direntries` = 3 (`00 03`) +/// - at offset 10: big-endian `tag` = 0x0100 = 256 (`01 00`) +/// +/// Read with little-endian byte order (the un-flipped bug) these would be +/// 0x0300 = 768 and 0x0001 = 1 respectively. +const BE_TIFF: &[u8] = &[ + 0x4D, 0x4D, 0x00, 0x2A, // "MM\0*" + 0x00, 0x00, 0x00, 0x08, // BE IFD offset = 8 + 0x00, 0x03, // BE direntries = 3 + 0x01, 0x00, // BE tag = 0x0100 = 256 +]; + +/// The little-endian mirror: `II\x2a\x00`, LE pointer, LE direntries = 3 +/// (`03 00`), LE tag = 0x0100 (`00 01`). Invoked via plain `use ifd` (no +/// flip), so the declared `leshort` reads are already correct. +const LE_TIFF: &[u8] = &[ + 0x49, 0x49, 0x2A, 0x00, // "II*\0" + 0x08, 0x00, 0x00, 0x00, // LE IFD offset = 8 + 0x03, 0x00, // LE direntries = 3 + 0x00, 0x01, // LE tag = 0x0100 = 256 +]; + +#[test] +fn flip_use_reads_big_endian_tiff_direntries_and_propagates_to_nested_use() { + let db = db_from_magic(TIFF_LIKE_MAGIC); + let result = db.evaluate_buffer(BE_TIFF).expect("evaluate BE TIFF"); + // `\^ifd` flips leshort->beshort for the direntries read, AND the nested + // `>2 use inner` inherits the flip so its `tag` leshort is also read + // big-endian. Without the fix this was `direntries=768, tag=1`. + assert_eq!( + result.description.as_str(), + "BE-TIFF, direntries=3, tag=256", + "`use \\^ifd` must read the big-endian subroutine (and its nested use) with flipped endianness" + ); +} + +#[test] +fn plain_use_leaves_little_endian_tiff_unflipped() { + // Negative control: the little-endian header uses plain `use ifd` (no + // `\^`), so the declared `leshort` reads stay little-endian and produce + // the same logical values. This guards against the flip leaking onto + // non-`\^` invocations. + let db = db_from_magic(TIFF_LIKE_MAGIC); + let result = db.evaluate_buffer(LE_TIFF).expect("evaluate LE TIFF"); + assert_eq!( + result.description.as_str(), + "LE-TIFF, direntries=3, tag=256", + "a plain `use` must NOT flip endianness" + ); +} + +#[test] +fn nested_flip_use_toggles_back_to_unflipped() { + // A `\^use` nested inside an already-flipped subroutine toggles the flip + // OFF again (libmagic's `flip = !flip`), so the innermost reads use their + // declared endianness. Here the outer BE header enters `\^outer` (flip + // on), and `outer` invokes `\^middle` (flip off again): `middle`'s + // `leshort` must read little-endian. + let magic = "0\tstring\tZZ\tROOT\n\ + >2\tuse\t\\^outer\n\ + 0\tname\touter\n\ + >0\tuse\t\\^middle\n\ + 0\tname\tmiddle\n\ + >0\tleshort\tx\t\\b, v=%d\n"; + let db = db_from_magic(magic); + // Signature "ZZ", then at offset 2 the value bytes `03 00`. With the flip + // toggled twice (on then off) the read is little-endian: 0x0003 = 3. + // A single (un-toggled) flip would read big-endian 0x0300 = 768. + let buf: &[u8] = &[0x5A, 0x5A, 0x03, 0x00]; + let result = db + .evaluate_buffer(buf) + .expect("evaluate nested-flip buffer"); + assert_eq!( + result.description.as_str(), + "ROOT, v=3", + "a `\\^use` nested inside a flipped subroutine must toggle the flip back off" + ); +} diff --git a/tests/evaluator_tests.rs b/tests/evaluator_tests.rs index 53797927..7082acc3 100644 --- a/tests/evaluator_tests.rs +++ b/tests/evaluator_tests.rs @@ -195,9 +195,11 @@ fn test_metadata_populated_for_buffer() { #[test] fn test_metadata_for_no_match() { let db = MagicDatabase::with_builtin_rules().unwrap(); + // Plain ASCII content with no rule match falls back to "ASCII text" + // (GOTCHAS S13.2), matching GNU `file` -- not the old hardcoded "data". let result = db.evaluate_buffer(b"nothing matches this").unwrap(); - assert_eq!(result.description, "data"); + assert_eq!(result.description, "ASCII text"); assert!(result.metadata.rules_evaluated > 0); } @@ -209,7 +211,10 @@ fn test_metadata_for_no_match() { fn test_evaluate_empty_buffer() { let db = MagicDatabase::with_builtin_rules().unwrap(); let result = db.evaluate_buffer(b"").unwrap(); - assert_eq!(result.description, "data"); + // The text/data fallback (GOTCHAS S13.2) reports "empty" for a + // zero-byte buffer, matching GNU `file`'s literal output -- not the + // old hardcoded "data". + assert_eq!(result.description, "empty"); } #[test] diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index dd02a97a..42e9992c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -84,10 +84,26 @@ fn test_builtin_rules_detect_gif() { #[test] fn test_builtin_rules_unknown_fallback() { let db = MagicDatabase::with_builtin_rules().unwrap(); - let result = db.evaluate_buffer(b"random bytes").unwrap(); + // Genuinely binary content: no built-in rule matches and it is not + // ASCII/UTF-8 text, so the text/data fallback (GOTCHAS S13.2) reports + // "data". `b"random bytes"` is plain ASCII and would instead fall + // back to "ASCII text" -- see `test_builtin_rules_unknown_ascii_fallback`. + let result = db + .evaluate_buffer(&[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x80, 0x81]) + .unwrap(); assert_eq!(result.description, "data"); } +/// Companion to `test_builtin_rules_unknown_fallback`: plain ASCII +/// content that no built-in rule matches falls back to "ASCII text" +/// (GOTCHAS S13.2), matching GNU `file`, not the old hardcoded "data". +#[test] +fn test_builtin_rules_unknown_ascii_fallback() { + let db = MagicDatabase::with_builtin_rules().unwrap(); + let result = db.evaluate_buffer(b"random bytes").unwrap(); + assert_eq!(result.description, "ASCII text"); +} + // ============================================================ // Load from File End-to-End // ============================================================ @@ -258,8 +274,10 @@ fn test_custom_rules_no_match_fallback() { writeln!(f, "0 string \"RARE\" Rare format").unwrap(); let db = MagicDatabase::load_from_file(&magic_path).unwrap(); + // Plain ASCII content with no rule match falls back to "ASCII text" + // (GOTCHAS S13.2), matching GNU `file` -- not the old hardcoded "data". let result = db.evaluate_buffer(b"no match here").unwrap(); - assert_eq!(result.description, "data"); + assert_eq!(result.description, "ASCII text"); } // ============================================================ @@ -430,7 +448,12 @@ fn test_reuse_database_multiple_evaluations() { let elf = db.evaluate_buffer(b"\x7fELF\x02\x01\x01\x00").unwrap(); let pdf = db.evaluate_buffer(b"%PDF-\x001.4").unwrap(); - let unknown = db.evaluate_buffer(b"nothing here").unwrap(); + // Genuinely binary content so the text/data fallback (GOTCHAS S13.2) + // reports "data" -- plain ASCII content would instead report "ASCII + // text". + let unknown = db + .evaluate_buffer(&[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x80, 0x81]) + .unwrap(); assert!(elf.description.contains("ELF")); assert!(pdf.description.contains("PDF")); diff --git a/tests/meta_types_integration.rs b/tests/meta_types_integration.rs index cf517f09..a2386879 100644 --- a/tests/meta_types_integration.rs +++ b/tests/meta_types_integration.rs @@ -154,6 +154,94 @@ fn test_default_clear_synthetic_scenario() { ); } +/// A message-bearing `clear x` directive must EMIT its own message, matching +/// libmagic's `FILE_CLEAR` (the `x` test always succeeds and `mprint` renders +/// a non-empty description). This is the mechanism behind GNU `file`'s +/// `c-lang` chain: for a plain C source file the `>>&0 clear x program text` +/// child appends "program text", producing `c program text` (rmagic +/// previously dropped it and printed only `c`). +/// +/// This also pins the coexistence verified against real `file` (file-5.41): +/// the message-bearing clear prints its message AND still resets the per-level +/// sibling-matched flag so a trailing `default` sibling fires. The child-level +/// structure mirrors c-lang -- the directives are children of a matched +/// top-level rule, so each `default`'s sibling-matched check is evaluated at +/// the child level. +#[test] +fn test_clear_with_message_emits_and_still_resets_flag() { + let temp_dir = TempDir::new().unwrap(); + let magic_path = temp_dir.path().join("clear_msg.magic"); + + // Top-level signature matches, then four children at the same level: + // default (fires -- nothing matched at the child level yet), + // clear x CLEARED-MSG (emits its message, resets the flag), + // default (fires again because clear reset the flag). + // Verified against real `file`: + // MATCH-A DEF-SKIPPED CLEARED-MSG DEF-FIRES + let mut f = fs::File::create(&magic_path).unwrap(); + writeln!(f, r"0 string ZQX9 MATCH-A").unwrap(); + writeln!(f, r">4 default x DEF-ONE").unwrap(); + writeln!(f, r">4 clear x CLEARED-MSG").unwrap(); + writeln!(f, r">4 default x DEF-TWO").unwrap(); + + // Evaluate every matching sibling (the default config stops at the first + // top-level match, which would hide the child chain). + let config = EvaluationConfig::default().with_stop_at_first_match(false); + let db = MagicDatabase::load_from_file_with_config(&magic_path, config).unwrap(); + + let buf = b"ZQX9rest-of-data"; + let result = db.evaluate_buffer(buf).unwrap(); + + // Primary regression: the clear's own message text is present. + assert!( + result.description.contains("CLEARED-MSG"), + "message-bearing clear must emit its message, got: {}", + result.description + ); + // The first child default fires (no sibling matched at the child level + // before it). + assert!( + result.description.contains("DEF-ONE"), + "first child default should fire, got: {}", + result.description + ); + // Coexistence: clear reset the flag, so the trailing default still fires. + assert!( + result.description.contains("DEF-TWO"), + "clear must reset sibling-matched so the trailing default fires, got: {}", + result.description + ); +} + +/// A bare `clear x` directive with NO message text must remain a pure +/// flag-reset: it emits no description fragment and does not advance the +/// previous-match anchor, exactly as before message-emission was added. This +/// guards the blast radius of `test_clear_with_message_emits_and_still_resets_flag` +/// -- the system magic DB contains ten message-less `clear x` directives +/// (apple, coff, elf, pmem, ...) whose output must be unchanged. +#[test] +fn test_message_less_clear_emits_nothing() { + let temp_dir = TempDir::new().unwrap(); + let magic_path = temp_dir.path().join("clear_bare.magic"); + + // A matched parent whose only child is a bare `clear x` (no message). + // The description must be exactly the parent's text with no trailing + // fragment or stray whitespace from the clear. + let mut f = fs::File::create(&magic_path).unwrap(); + writeln!(f, r"0 string ZQX9 PARENT").unwrap(); + writeln!(f, r">4 clear x").unwrap(); + + let config = EvaluationConfig::default().with_stop_at_first_match(false); + let db = MagicDatabase::load_from_file_with_config(&magic_path, config).unwrap(); + + let result = db.evaluate_buffer(b"ZQX9rest").unwrap(); + assert_eq!( + result.description, "PARENT", + "a message-less clear must contribute no text, got: {}", + result.description + ); +} + /// Synthetic end-to-end coverage of the `indirect` directive: a rule with /// `TypeKind::Meta(MetaType::Indirect)` re-applies the loaded magic /// database starting at the resolved offset. The dispatch is wired @@ -217,3 +305,173 @@ fn test_searchbug_matches_full_result_string() { .expect("evaluate_buffer on searchbug.testfile"); assert_eq!(result.description.trim(), expected.trim()); } + +/// End-to-end regression for the gzip multi-fragment description, the shape +/// that surfaced four interlocking evaluation bugs (PR #376): +/// +/// 1. `stop_at_first_match` must be TOP-LEVEL only -- every matching child +/// sibling under the matched parent must render (not truncate after the +/// first), so `default`, the `use` subroutine detail, and the trailing +/// `offset` size all appear. +/// 2. The `offset` pseudo-type must apply its comparison operator +/// (`offset >48`) against the resolved position, not treat `>48` as +/// message text. +/// 3. `-0` must resolve to the EOF position (`buffer.len()`) via +/// `FromEnd(0)`, so `>>-0 offset >48` gates on file length. +/// 4. Continuation/child rules and `name`-block bodies must stay in FILE +/// order (non-recursive strength sort), so the low-strength `default` +/// is not suppressed by a higher-strength `offset` sibling sorted ahead +/// of it, and the gzip-info detail fragments render in source order. +/// +/// Verified byte-for-byte against `file -b` on real gzip files; this test +/// pins a synthetic buffer so the parity survives without the system DB. +/// Uses the default config (`stop_at_first_match: true`) deliberately, to +/// prove children still render fully under first-match short-circuiting. +#[test] +fn test_gzip_multipart_description_end_to_end() { + let temp_dir = TempDir::new().unwrap(); + let magic_path = temp_dir.path().join("gzip.magic"); + + let mut f = fs::File::create(&magic_path).unwrap(); + // Minimal gzip shape mirroring /usr/share/file/magic/compress. + writeln!(f, r"0 string \037\213").unwrap(); + writeln!(f, r"# no FNAME/FCOMMENT bit -> binary gzip").unwrap(); + writeln!( + f, + r"# (real rule masks with &0x18; a bare `byte 0` suffices here)" + ) + .unwrap(); + writeln!(f, r"# FLG byte at offset 3").unwrap(); + writeln!(f, r">3 byte 0").unwrap(); + writeln!(f, r">>10 default x gzip compressed data").unwrap(); + writeln!(f, r">>>0 use gzip-info").unwrap(); + // -0 == EOF position; `offset >48` gates the trailing size on file length. + writeln!(f, r">>-0 offset >48").unwrap(); + writeln!(f, r">>>-4 ulelong x \b, original size modulo 2^32 %u").unwrap(); + writeln!(f, r">>-0 offset <48 \b, truncated").unwrap(); + writeln!(f, r"0 name gzip-info").unwrap(); + writeln!(f, r">9 byte 3 \b, from Unix").unwrap(); + drop(f); + + let db = MagicDatabase::load_from_file(&magic_path).unwrap(); + + // 64-byte synthetic gzip: magic, CM=deflate, FLG=0, ..., OS=3 (Unix), + // trailing little-endian original-size = 1000 in the last 4 bytes. + let mut buf = vec![0u8; 64]; + buf[0] = 0x1f; + buf[1] = 0x8b; + buf[2] = 0x08; // CM = deflate + buf[3] = 0x00; // FLG = 0 + buf[9] = 0x03; // OS = Unix + buf[60] = 0xE8; // 1000 = 0x000003E8, little-endian in last 4 bytes + buf[61] = 0x03; + + let result = db.evaluate_buffer(&buf).unwrap(); + assert_eq!( + result.description, "gzip compressed data, from Unix, original size modulo 2^32 1000", + "all four fragments must render in file order; got {:?}", + result.description + ); + + // A short (< 48-byte) buffer must take the `<48` truncated branch, not + // the size branch -- proving the FromEnd(0)+comparison gate both ways. + let mut short = vec![0u8; 20]; + short[0] = 0x1f; + short[1] = 0x8b; + short[9] = 0x03; + let short_result = db.evaluate_buffer(&short).unwrap(); + assert!( + short_result.description.contains("truncated"), + "a <48-byte gzip must hit the `offset <48` truncated branch; got {:?}", + short_result.description + ); + assert!( + !short_result.description.contains("original size"), + "the size branch must not fire for a truncated file; got {:?}", + short_result.description + ); +} + +/// A `name` subroutine's OWN description (the text on the `name` line) is +/// emitted when the subroutine is invoked via `use`, ahead of the body's +/// matches and attached with NO separating space -- matching GNU `file`. +/// +/// libmagic drops a `use`-site's own message but emits the `name` line's +/// description (verified against file-5.41). rmagic previously dropped BOTH, +/// so subroutine name-line fragments -- Mach-O universal `\b [`, `matlab4 +/// Matlab v4 mat-file`, `algol_68 Algol 68 source text` -- silently vanished. +/// +/// Buffer `ZQX9...` matches the parent `0 string ZQXNAME9`; the `>8 byte` +/// child reads the value at offset 8. Expected strings are the EXACT +/// `file -b` output for the equivalent single-file magic (see the +/// fix-system-magic-regex-graceful name-message investigation). +#[test] +fn test_use_emits_name_line_message_with_no_separator() { + // (magic body, expected description). One temp file per case keeps the + // magic isolated from stop-at-first-match cross-talk. + let cases: &[(&str, &str)] = &[ + // A: plain name message attaches with no leading space -> `ParentSUBMSG`. + ( + "0 string ZQXNAME9 Parent\n>0 use submsgtest\n0 name submsgtest SUBMSG\n>8 byte x child=%d\n", + "ParentSUBMSG child=5", + ), + // B: mach-o-style `\b [` -> the `\b` no-separator marker + literal ` [`. + ( + "0 string ZQXNAME9 Parent\n>0 use submsgtest\n0 name submsgtest \\b [\n>8 byte x child=%d\n", + "Parent [ child=5", + ), + // C: the `use` site's OWN message (USEMSG) is DROPPED; only the name + // line's NAMEMSG is emitted. + ( + "0 string ZQXNAME9 Parent\n>0 use submsgtest USEMSG\n0 name submsgtest NAMEMSG\n>8 byte x child=%d\n", + "ParentNAMEMSG child=5", + ), + // D: name message with no body still emits. + ( + "0 string ZQXNAME9 Parent\n>0 use submsgtest\n0 name submsgtest NAMEONLY\n", + "ParentNAMEONLY", + ), + ]; + + let temp_dir = TempDir::new().unwrap(); + // 9-byte buffer: "ZQXNAME9" (8 bytes) matches the parent, byte at offset 8 + // is 0x05 so `child=%d` renders `child=5`. + let buf = b"ZQXNAME9\x05"; + + for (i, (magic, expected)) in cases.iter().enumerate() { + let magic_path = temp_dir.path().join(format!("namemsg_{i}.magic")); + fs::write(&magic_path, magic).unwrap(); + let db = MagicDatabase::load_from_file(&magic_path).unwrap(); + let result = db.evaluate_buffer(buf).unwrap(); + assert_eq!( + result.description, *expected, + "case {i}: name-line message spacing must match GNU `file` for magic {magic:?}" + ); + } +} + +/// End-to-end regression for the Mach-O universal bracket detail: a nested +/// `use` chain where the outer subroutine's name line carries `\b [`, an inner +/// `use` supplies the arch label, and a `\b]` closes the bracket. +/// +/// Before the name-line-message fix, the outer `[` was dropped and the output +/// was malformed (`LABEL]`); after, the brackets balance (`[...LABEL]`), +/// matching the shape GNU `file` produces for a real fat binary. (The full +/// per-arch `:Mach-O 64-bit executable` inner classification needs the +/// separate indirect-offset cluster and is tracked in its own issue.) +#[test] +fn test_use_nested_name_message_balances_brackets() { + let temp_dir = TempDir::new().unwrap(); + let magic_path = temp_dir.path().join("nested.magic"); + let magic = "0 string ZQXNAME9 Parent:\n\ + >0 use outer\n\ + 0 name outer \\b [\n\ + >0 use inner\n\ + >8 byte x \\b]\n\ + 0 name inner INNERCPU\n"; + fs::write(&magic_path, magic).unwrap(); + let db = MagicDatabase::load_from_file(&magic_path).unwrap(); + let result = db.evaluate_buffer(b"ZQXNAME9\x05").unwrap(); + // Matches `file -b` on the equivalent single-file magic: `Parent: [INNERCPU]`. + assert_eq!(result.description, "Parent: [INNERCPU]"); +} diff --git a/tests/mime_tests.rs b/tests/mime_tests.rs index f7d30832..00de5587 100644 --- a/tests/mime_tests.rs +++ b/tests/mime_tests.rs @@ -72,10 +72,30 @@ fn test_mime_disabled_returns_none() { fn test_mime_unknown_data_returns_none() { let config = EvaluationConfig::default().with_mime_types(true); let db = MagicDatabase::with_builtin_rules_and_config(config).unwrap(); - let result = db.evaluate_buffer(b"random unknown content").unwrap(); + // Genuinely binary content so no built-in rule matches AND the + // text/data fallback (GOTCHAS S13.2) reports "data", which has no + // MIME mapping. `b"random unknown content"` is plain ASCII and would + // instead fall back to "ASCII text", which correctly maps to + // "text/plain" (matching GNU `file --mime-type`'s own output for + // ASCII text) -- see `test_mime_unknown_ascii_text_returns_text_plain`. + let result = db + .evaluate_buffer(&[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x80, 0x81]) + .unwrap(); assert_eq!(result.mime_type, None); } +/// Companion to `test_mime_unknown_data_returns_none`: plain ASCII +/// content that no built-in rule matches falls back to "ASCII text" +/// (GOTCHAS S13.2), which correctly maps to "text/plain" -- matching GNU +/// `file --mime-type`'s own output for ASCII text. +#[test] +fn test_mime_unknown_ascii_text_returns_text_plain() { + let config = EvaluationConfig::default().with_mime_types(true); + let db = MagicDatabase::with_builtin_rules_and_config(config).unwrap(); + let result = db.evaluate_buffer(b"random unknown content").unwrap(); + assert_eq!(result.mime_type.as_deref(), Some("text/plain")); +} + // ============================================================ // MimeMapper Direct Tests // ============================================================ diff --git a/tests/parser_integration_tests.rs b/tests/parser_integration_tests.rs index 6b026ee4..b03191d5 100644 --- a/tests/parser_integration_tests.rs +++ b/tests/parser_integration_tests.rs @@ -196,7 +196,7 @@ fn test_name_use_round_trip() { assert!( matches!( parsed.rules[0].typ, - TypeKind::Meta(MetaType::Use(ref n)) if n == "part2" + TypeKind::Meta(MetaType::Use { name: ref n, .. }) if n == "part2" ), "remaining top-level rule must be the use invocation" ); @@ -372,6 +372,146 @@ fn test_end_to_end_text_file_to_evaluation() { ); } +/// End-to-end proof that the string `>NUMERIC` parse fix (task #19) and the +/// string ordering-op full-field render / prefix-limited compare (task #18) +/// compose through real magic-file syntax. +/// +/// `0 string >0.6.1 version %s` is parsed straight from text (so the value +/// must survive as `Value::String("0.6.1")`, not a number), then evaluated: +/// - `0.6.2` matches (`0.6.2` > `0.6.1`) and renders the FULL field +/// (`version 0.6.2 release`), exercising the #18 full-field display read. +/// - `0.6.10` does NOT match: the comparison is prefix-limited to +/// `pattern.len()`, so the compared prefix `0.6.1` equals the pattern and +/// `>` is false -- matching real `file` (file-5.41). It falls through to +/// the ascmagic text fallback instead of a spurious `version ...`. +#[test] +fn test_string_numeric_ordering_end_to_end_composes_with_full_field_render() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let magic_file = create_test_magic_file(temp_dir.path(), "ver", "0 string >0.6.1 version %s\n"); + let db = MagicDatabase::load_from_file(&magic_file).expect("Failed to load database"); + + // 0.6.2 > 0.6.1 -> matches, renders the full string field. + let match_file = create_test_binary_file(temp_dir.path(), "v62", b"0.6.2 release"); + let matched = db + .evaluate_file(&match_file) + .expect("Failed to evaluate matching file"); + assert_eq!( + matched.description, "version 0.6.2 release", + "0.6.2 must match >0.6.1 and render the full field, got: {}", + matched.description + ); + + // 0.6.10: compared prefix `0.6.1` == pattern, so `>` is false. No match. + let nomatch_file = create_test_binary_file(temp_dir.path(), "v610", b"0.6.10 release"); + let unmatched = db + .evaluate_file(&nomatch_file) + .expect("Failed to evaluate non-matching file"); + assert!( + !unmatched.description.contains("version"), + "0.6.10 must NOT match >0.6.1 (prefix-limited compare), got: {}", + unmatched.description + ); +} + +/// GOTCHAS S6.7: a top-level `string` signature whose bareword value carries +/// a non-UTF-8 high byte (OS/2 INF's `HSP\x01\x9b\x00`, where `0x9b` is +/// invalid UTF-8) must match. Before the fix, `parse_bare_string_value` +/// lossy-decoded the value to a `Value::String` (0x9b -> U+FFFD), which both +/// inflated the pattern length (6 -> 8) and changed the byte, so the rule +/// silently never matched and the file classified as `data`. Real `file` +/// prints `OS/2 INF (My Help File)`; this pins the full output including the +/// `>107 string >0 (%s)` title child. +#[test] +fn os2_inf_high_byte_signature_matches() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let magic_file = create_test_magic_file( + temp_dir.path(), + "os2", + "0 string HSP\\x01\\x9b\\x00 OS/2 INF\n>107 string >0 (%s)\n", + ); + let db = MagicDatabase::load_from_file(&magic_file).expect("Failed to load database"); + + // Signature (6 bytes) + filler to offset 107 + NUL-terminated title. + let mut bytes = vec![0x48, 0x53, 0x50, 0x01, 0x9b, 0x00]; + bytes.resize(107, 0x00); + bytes.extend_from_slice(b"My Help File\x00"); + let file = create_test_binary_file(temp_dir.path(), "os2inf.bin", &bytes); + + let result = db + .evaluate_file(&file) + .expect("Failed to evaluate OS/2 INF file"); + assert_eq!( + result.description, "OS/2 INF (My Help File)", + "high-byte signature must match and render the title child, got: {}", + result.description + ); +} + +/// GOTCHAS S15.1: a child rule whose resolved offset lands exactly at EOF +/// (`offset == buffer.len()`) is permitted. Verified against real `file` +/// (file-5.41) with a custom magic file: a numeric child (`byte x`) at EOF is +/// DROPPED (its width-checked read fails), while a `string x` child at EOF +/// renders an EMPTY string. Before the fix, offset resolution rejected +/// `offset == len` and silently dropped BOTH children. +#[test] +fn child_at_eof_numeric_dropped_string_renders_empty() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + // Parent matches "XY" at 0; both children resolve to offset 2. "XY" is + // used (not "AB") because A/B are hex digits and a bareword of pure hex + // letters parses as `Value::Bytes`, not a string literal (GOTCHAS S3.12). + let magic_file = create_test_magic_file( + temp_dir.path(), + "eof", + "0 string XY PARENT\n>2 byte x byte=%d\n>2 string x [%s,\n", + ); + let db = MagicDatabase::load_from_file(&magic_file).expect("Failed to load database"); + + // 2-byte buffer: offset 2 == EOF. Numeric child dropped, string empty. + let at_eof = create_test_binary_file(temp_dir.path(), "twobyte", b"XY"); + let r = db.evaluate_file(&at_eof).expect("evaluate at-EOF file"); + assert_eq!( + r.description, "PARENT [,", + "at offset==EOF: byte child must drop, string child must render empty, got: {}", + r.description + ); + + // 3-byte buffer: offset 2 < EOF. Both children render ('Z' == 90). + let before_eof = create_test_binary_file(temp_dir.path(), "threebyte", b"XYZ"); + let r = db + .evaluate_file(&before_eof) + .expect("evaluate before-EOF file"); + assert_eq!( + r.description, "PARENT byte=90 [Z,", + "at offset8 string x [%s,`) start exactly at EOF. +/// Real `file` prints `LUKS encrypted file, ver 1 [,`; before the fix rmagic +/// dropped the trailing detail and printed only `LUKS encrypted file, ver 1`. +#[test] +fn luks_truncated_header_renders_empty_cipher_field() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let magic_file = create_test_magic_file( + temp_dir.path(), + "luks", + "0 string LUKS\\xba\\xbe LUKS encrypted file,\n>6 beshort x ver %d\n>8 string x [%s,\n", + ); + let db = MagicDatabase::load_from_file(&magic_file).expect("Failed to load database"); + + // 8-byte header: magic (6) + beshort version 1 (2). Offset 8 == EOF. + let bytes = vec![0x4c, 0x55, 0x4b, 0x53, 0xba, 0xbe, 0x00, 0x01]; + let file = create_test_binary_file(temp_dir.path(), "luks_trunc.img", &bytes); + let r = db.evaluate_file(&file).expect("evaluate truncated LUKS"); + assert_eq!( + r.description, "LUKS encrypted file, ver 1 [,", + "truncated LUKS must render empty cipher field `[,` like GNU file, got: {}", + r.description + ); +} + #[test] #[ignore = "Parser does not decode \\xNN escape sequences inside string values yet; rules match 'data' instead of the expected magic type. Re-enable once grammar supports hex escapes in parse_value()."] fn test_end_to_end_directory_to_evaluation() { diff --git a/tests/property_tests.proptest-regressions b/tests/property_tests.proptest-regressions index 4d459dc5..ca2665f3 100644 --- a/tests/property_tests.proptest-regressions +++ b/tests/property_tests.proptest-regressions @@ -7,3 +7,4 @@ cc f7a5c24cef58c44127f4600b0bbe65f42a90f4ac5917b1aaaec7441dd2544f52 # shrinks to rule = MagicRule { offset: Absolute(0), typ: Meta(Name("A")), op: Equal, value: Uint(0), message: "A", children: [], level: 0, strength_modifier: None }, buffer = [71, 195, 44, 100, 222, 230, 41, 133, 93, 35, 173, 30, 114, 46, 108, 185, 61, 123, 245, 13, 158, 187, 174, 211, 147, 133, 164, 5, 150, 185, 108, 124, 125, 199, 52, 155, 178, 215, 15, 213, 33, 116, 180, 122, 90, 166, 70, 85, 238, 89, 150, 107, 156, 250, 59, 237, 125, 180, 209, 174, 23, 204, 0, 132, 16, 115, 59, 180, 107, 223, 208, 101, 222, 39, 79, 24, 63, 141, 172, 233, 3, 132, 62, 189, 181, 74, 22, 79, 29, 168, 173] cc 6d92e58bff36af4467c8702355f734ddd1b7d08dd98464f2792a11905810c754 # shrinks to meta_rules = [MagicRule { offset: Relative(0), typ: Meta(Indirect), op: Equal, value: Uint(0), message: "-", children: [], level: 0, strength_modifier: None }], buffer = [0] cc fcced6ceb9872349e8c3e8d796e5c65c1e36faa3bc8b22b63b00050bc1474cae # shrinks to rule = MagicRule { offset: Relative(0), typ: Meta(Indirect), op: Equal, value: Uint(0), message: " ", children: [], level: 0, strength_modifier: None }, buffer = [0] +cc 164b431ec990cf12bf99ede572d292dcda599edb84a68593f83be5ebfddd5f50 # shrinks to rule = MagicRule { offset: Absolute(0), typ: Regex { flags: RegexFlags { case_insensitive: false, start_offset: false }, count: Default }, op: NotEqual, value: Bytes([0, 0]), message: "-", children: [], level: 0, strength_modifier: None, value_transform: None }, buffer = [74, 180, 126, 53, 219, 57, 86, 49, 164, 193, 23, 68, 123, 222, 236, 183, 16, 182, 182, 122, 152, 157, 239, 130, 9, 32, 43, 8, 155, 40, 114, 182, 96, 221, 193, 38, 243, 217, 159, 37, 96, 171, 68, 163, 158, 62, 175, 12, 10, 35, 146, 96, 206, 76] diff --git a/tests/property_tests.rs b/tests/property_tests.rs index 861c8e00..5a3e32b4 100644 --- a/tests/property_tests.rs +++ b/tests/property_tests.rs @@ -161,8 +161,11 @@ fn arb_type_kind() -> impl Strategy { }, ) }, - (1usize..=4096usize, arb_search_flags()).prop_map(|(range, flags)| TypeKind::Search { - range: ::std::num::NonZeroUsize::new(range).unwrap(), + // `0` generates the bare-`search` open window (`range: None`, + // scan-to-EOF); `1..=4096` generates a bounded `search/N`. Both must + // round-trip through codegen and never panic in evaluation. + (0usize..=4096usize, arb_search_flags()).prop_map(|(range, flags)| TypeKind::Search { + range: ::std::num::NonZeroUsize::new(range), flags, }), Just(TypeKind::Meta(MetaType::Default)), @@ -170,7 +173,12 @@ fn arb_type_kind() -> impl Strategy { Just(TypeKind::Meta(MetaType::Indirect)), Just(TypeKind::Meta(MetaType::Offset)), "[a-zA-Z_][a-zA-Z0-9_-]{0,16}".prop_map(|id| TypeKind::Meta(MetaType::Name(id))), - "[a-zA-Z_][a-zA-Z0-9_-]{0,16}".prop_map(|id| TypeKind::Meta(MetaType::Use(id))), + ("[a-zA-Z_][a-zA-Z0-9_-]{0,16}", any::()).prop_map(|(id, flip_endian)| { + TypeKind::Meta(MetaType::Use { + name: id, + flip_endian, + }) + }), ] } @@ -234,7 +242,12 @@ fn arb_meta_rule() -> impl Strategy { Just(TypeKind::Meta(MetaType::Indirect)), Just(TypeKind::Meta(MetaType::Offset)), "[a-zA-Z_][a-zA-Z0-9_-]{0,16}".prop_map(|id| TypeKind::Meta(MetaType::Name(id))), - "[a-zA-Z_][a-zA-Z0-9_-]{0,16}".prop_map(|id| TypeKind::Meta(MetaType::Use(id))), + ("[a-zA-Z_][a-zA-Z0-9_-]{0,16}", any::()).prop_map(|(id, flip_endian)| { + TypeKind::Meta(MetaType::Use { + name: id, + flip_endian, + }) + }), ]; ( arb_offset_spec(), diff --git a/tests/regex_getstr_fixtures.rs b/tests/regex_getstr_fixtures.rs new file mode 100644 index 00000000..358bd075 --- /dev/null +++ b/tests/regex_getstr_fixtures.rs @@ -0,0 +1,290 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +// Helper functions in this file (not themselves `#[test]` fns) call +// `.expect()` on evaluation results; clippy's `allow-expect-in-tests` +// (clippy.toml) only recognizes code directly inside `#[test]` bodies as +// test code, not helpers called from them. See `tests/integration_tests.rs` +// and `tests/directory_loading_tests.rs` for the same established pattern. +#![allow(clippy::expect_used)] + +//! Behavioral positive/negative fixture matrix for the getstr-resolved +//! `regex` rules in `/usr/share/file/magic/assembler` (U5 of the +//! `fix/system-magic-regex-graceful` plan). +//! +//! These tests prove the getstr-**resolved** patterns actually match the +//! right inputs and reject the wrong ones -- not merely that evaluation +//! completes without a fatal error. Per the plan's Verification Contract: +//! "Accuracy is the core value ... a non-crashing wrong answer is a +//! failure." +//! +//! # Why AST-built rules, not a parsed `.magic` file +//! +//! `parse_text_magic_file` is fail-fast (GOTCHAS S3.11): a single +//! unparseable line anywhere in a real `.magic` file aborts the entire +//! load, and the system magic DB (`/usr/share/file/magic/`) may not be +//! present in CI. So these always-run fixtures build the equivalent +//! `MagicRule` tree programmatically via the AST, mirroring +//! `tests/evaluator_tests.rs::test_regex_eol_corpus`. The real-DB, +//! real-file assertions (loading `/usr/share/file/magic/` itself and +//! diffing against GNU `file`) live in the separately gated +//! `tests/system_magic_dir.rs`. +//! +//! # Resolved pattern provenance +//! +//! The seven `assembler` rules (`grep -nE 'regex' /usr/share/file/magic/assembler`) +//! all share one pattern shape, differing only in the trailing directive +//! keyword: +//! +//! ```text +//! 0 regex \^[\040\t]{0,50}\\.KEYWORD assembler source text +//! ``` +//! +//! for `KEYWORD` in `asciiz`, `byte`, `even`, `globl`, `text`, `file`, +//! `type`. Applying the getstr escape table documented in +//! `src/parser/grammar/getstr/mod.rs` (verified against GNU `file`'s +//! `apprentice.c::getstr`): +//! +//! - `\^` -> unrecognized escape -> drop backslash -> `^` +//! - `\040` -> octal escape -> space byte (`0x20`) +//! - `\t` -> named escape -> raw tab byte (`0x09`) +//! - `\\` -> escape of backslash -> literal `\` +//! - `.` -> not preceded by an unconsumed backslash -> passthrough +//! +//! so `\\.` resolves to the two-character sequence backslash + dot +//! (`\.`), which the regex engine reads as an *escaped* (literal) dot -- +//! not the wildcard metacharacter. The full resolved pattern is +//! `^[ \t]{0,50}\.KEYWORD` (matching the plan's worked example exactly: +//! `\^[\040\t]{0,50}\\.asciiz` -> `^[ \t]{0,50}\.asciiz`). +//! +//! Rule messages for these rules are the plain literal `"assembler +//! source text"` with no `%`-format specifiers and no leading `\b` +//! (backspace), so GOTCHAS S14's printf-substitution and +//! backspace-concatenation conventions do not apply here. + +use libmagic_rs::evaluator::evaluate_rules; +use libmagic_rs::parser::ast::{RegexCount, RegexFlags}; +use libmagic_rs::{ + EvaluationConfig, EvaluationContext, MagicRule, OffsetSpec, Operator, TypeKind, Value, +}; + +/// Build a top-level `regex` rule using the getstr-resolved pattern text, +/// exactly as `src/parser/grammar/getstr/mod.rs` would produce it for a +/// magic-file `regex` rule with no flags and no explicit count. +fn assembler_regex_rule(resolved_pattern: &str) -> MagicRule { + MagicRule::new( + OffsetSpec::Absolute(0), + TypeKind::Regex { + flags: RegexFlags::default(), + count: RegexCount::Default, + }, + Operator::Equal, + Value::String(resolved_pattern.to_string()), + "assembler source text".to_string(), + ) +} + +/// Evaluate `rule` against `buffer` and report whether it produced a +/// top-level match. +fn matches(rule: &MagicRule, buffer: &[u8]) -> bool { + let config = EvaluationConfig::default(); + let mut context = EvaluationContext::new(config); + let result = evaluate_rules(std::slice::from_ref(rule), buffer, &mut context) + .expect("evaluation must not fail fatally"); + !result.is_empty() +} + +/// Getstr-resolved pattern for `\^[\040\t]{0,50}\\.KEYWORD`, given +/// `KEYWORD`. See the module doc for the escape-by-escape derivation. +fn resolved_pattern_for(keyword: &str) -> String { + format!("^[ \t]{{0,50}}\\.{keyword}") +} + +/// One positive + one negative fixture per affected `assembler` regex +/// rule (all seven `regex` rules in +/// `/usr/share/file/magic/assembler`), pinning both sides of the match +/// per the plan's accuracy mandate. +#[test] +fn test_all_affected_assembler_regex_rules_positive_and_negative() { + // (keyword, positive buffer that must match, negative buffer that + // must not match, scenario description) + let cases: &[(&str, &[u8], &[u8], &str)] = &[ + ( + "asciiz", + b"\t.asciiz \"hi\"", + b"xyz .asciiz", + "leading tab then .asciiz matches; non-whitespace prefix does not", + ), + ( + "byte", + b" .byte 0x01\n", + b"a.byte 0x01\n", + "leading spaces then .byte matches; non-whitespace prefix does not", + ), + ( + "even", + b".even\n", + b"1.even\n", + "zero leading whitespace then .even matches; digit prefix does not", + ), + ( + "globl", + b"\t\t.globl main\n", + b"foo.globl main\n", + "multiple leading tabs then .globl matches; word prefix does not", + ), + ( + "text", + b" .text\n", + b"#.text\n", + "leading spaces then .text matches; punctuation prefix does not", + ), + ( + "file", + b"\t.file \"a.s\"\n", + b"z.file \"a.s\"\n", + ".file with leading tab matches; letter prefix does not", + ), + ( + "type", + b".type foo,@function\n", + b"_type foo,@function\n", + ".type at column 0 matches; underscore prefix does not", + ), + ]; + + for (keyword, positive, negative, description) in cases { + let pattern = resolved_pattern_for(keyword); + let rule = assembler_regex_rule(&pattern); + + assert!( + matches(&rule, positive), + "keyword {keyword:?} ({description}): expected positive buffer {:?} to match \ + resolved pattern {pattern:?}", + String::from_utf8_lossy(positive) + ); + assert!( + !matches(&rule, negative), + "keyword {keyword:?} ({description}): expected negative buffer {:?} to NOT match \ + resolved pattern {pattern:?}", + String::from_utf8_lossy(negative) + ); + } +} + +/// `{0,50}` boundary: zero leading whitespace characters is within +/// bounds (the lower bound of the quantifier) and must match; 51 leading +/// spaces exceeds the upper bound and must not match anywhere in the +/// buffer (there is no embedded newline to give `^` a second anchor +/// point). +#[test] +fn test_leading_whitespace_quantifier_boundary() { + let rule = assembler_regex_rule(&resolved_pattern_for("asciiz")); + + let zero_whitespace = b".asciiz"; + assert!( + matches(&rule, zero_whitespace), + "0 leading whitespace chars is within {{0,50}}'s lower bound and must match" + ); + + let fifty_one_spaces: Vec = " " + .repeat(51) + .into_bytes() + .into_iter() + .chain(*b".asciiz") + .collect(); + assert!( + !matches(&rule, &fifty_one_spaces), + "51 leading spaces exceeds {{0,50}}'s upper bound and must not match" + ); + + // Sanity check the boundary is exercised precisely: exactly 50 + // leading spaces (the upper bound, inclusive) must still match. + let fifty_spaces: Vec = " " + .repeat(50) + .into_bytes() + .into_iter() + .chain(*b".asciiz") + .collect(); + assert!( + matches(&rule, &fifty_spaces), + "50 leading spaces is exactly {{0,50}}'s upper bound (inclusive) and must match" + ); +} + +/// The resolved pattern's `\.` must be a literal escaped dot, not the +/// regex wildcard metacharacter `.`. This is the behavioral proof that +/// the getstr resolver correctly emits the two-character `\.` sequence +/// (backslash from resolving the magic-file `\\`, followed by the +/// passthrough literal `.`) rather than a bare unescaped `.`. If the +/// resolver regressed to emitting a bare `.`, this test would start +/// passing on `Xasciiz` (since `.` as a wildcard matches any single +/// byte), so it is a real regression guard, not a tautology. +#[test] +fn test_escaped_dot_is_literal_not_wildcard() { + let rule = assembler_regex_rule(&resolved_pattern_for("asciiz")); + + assert!( + matches(&rule, b".asciiz"), + "a literal dot followed by the keyword must match" + ); + assert!( + !matches(&rule, b"Xasciiz"), + "a wildcard-eligible non-dot character in the dot's position must NOT match -- \ + if this passes, \\. regressed to an unescaped wildcard `.`" + ); +} + +/// Behavioral coverage for the KTD3 `>= 0x80` byte re-encoding path. +/// +/// None of the seven affected `assembler` rules contain an escape that +/// resolves to a byte `>= 0x80`, so per the plan's U5 requirement this is +/// a synthetic AST rule built the same way the getstr resolver would +/// build one: a resolved pattern containing a regex-native `\xHH` escape +/// (as produced by `push_resolved_byte` in +/// `src/parser/grammar/getstr/mod.rs` for a magic-file escape like +/// `\377` or `\xFF`), asserted to match a buffer containing the raw +/// byte and to reject a buffer that lacks it. +/// +/// This exercises `src/evaluator/types/regex.rs::build_regex`'s +/// `unicode(false)` setting end-to-end: with Unicode mode enabled (the +/// `regex::bytes::Regex` default), `\xff` matches the *UTF-8 encoding* +/// of U+00FF (`0xC3 0xBF`), not the raw byte `0xFF` -- silently breaking +/// this exact contract. See the regression test colocated with +/// `build_regex` (`test_read_regex_high_byte_escape_matches_raw_byte_not_utf8_encoding` +/// in `src/evaluator/types/regex.rs`) for the isolated unit-level proof; +/// this integration-level test proves the same contract holds when +/// exercised through the full `MagicRule` -> `evaluate_rules` path that +/// real magic-file rules use. +#[test] +fn test_high_byte_escape_synthetic_rule_matches_raw_byte_and_rejects_without_it() { + // Getstr-resolved form of a hypothetical `regex \xFF marker` rule: + // the parser would emit the raw byte 0xFF re-encoded as the + // regex-native `\xff` hex escape (KTD3), never as a raw 0x80+ byte + // pushed directly into the `String`. + let resolved_pattern = "marker\\xff"; + let rule = assembler_regex_rule(resolved_pattern); + + let with_high_byte: Vec = b"marker".iter().copied().chain([0xffu8]).collect(); + assert!( + matches(&rule, &with_high_byte), + "resolved pattern {resolved_pattern:?} must match a buffer containing the raw byte 0xFF" + ); + + let without_high_byte = b"markerX"; + assert!( + !matches(&rule, without_high_byte), + "resolved pattern {resolved_pattern:?} must NOT match a buffer lacking the raw byte 0xFF" + ); + + // Negative control: the UTF-8 *encoding* of U+00FF (the codepoint + // \xff would denote under Unicode-mode regex semantics) must not be + // mistaken for the raw byte -- pins the same contract from the + // opposite direction. + let utf8_encoded_codepoint: Vec = b"marker".iter().copied().chain([0xc3u8, 0xbf]).collect(); + assert!( + !matches(&rule, &utf8_encoded_codepoint), + "resolved pattern {resolved_pattern:?} must NOT match the UTF-8 encoding of U+00FF -- \ + if this passes, byte-level matching regressed back to Unicode-scalar matching" + ); +} diff --git a/tests/regex_search_corpus_tests.rs b/tests/regex_search_corpus_tests.rs index 324a7e0f..8d684572 100644 --- a/tests/regex_search_corpus_tests.rs +++ b/tests/regex_search_corpus_tests.rs @@ -76,7 +76,7 @@ fn search_rule( MagicRule::new( offset, TypeKind::Search { - range: NonZeroUsize::new(range).expect("range must be non-zero"), + range: NonZeroUsize::new(range), flags: SearchFlags::default(), }, Operator::Equal, @@ -290,16 +290,33 @@ fn test_search_parser_roundtrip_with_range() { assert_eq!(matches[0].message, "found ABC"); } -/// Parser rejects bare `search` without range even from magic-file -/// text -- the `NonZeroUsize` constraint is enforced at parse time. +/// Parser ACCEPTS bare `search` (no `/N`) and records an open (`None`) +/// range = scan-to-EOF, matching GNU `file`'s implementation (`str_range +/// == 0`). magic(5) documents the count as required, but the reference +/// binary accepts the bare form; real system magic relies on it (pdf +/// `>8 search /Count`, `0 search ##fileformat=VCFv`). `search/0` is still +/// rejected (see grammar unit tests). See GOTCHAS S2.13. #[test] -fn test_search_parser_rejects_bare_search() { - let magic = r#"0 search "ABC" bogus"#; - let result = parse_text_magic_file(magic); +fn test_search_parser_accepts_bare_search_scan_to_eof() { + use libmagic_rs::parser::ast::TypeKind; + + let magic = r#"0 search "ABC" found ABC bare"#; + let ParsedMagic { rules, .. } = + parse_text_magic_file(magic).expect("bare `search` must parse (scan-to-EOF)"); + assert_eq!(rules.len(), 1); assert!( - result.is_err(), - "bare `search` (no /N) should be a parse error" + matches!(rules[0].typ, TypeKind::Search { range: None, .. }), + "bare search must yield range None, got {:?}", + rules[0].typ ); + + // And it actually matches a pattern past any bounded window. + let mut buffer = b"prefix".to_vec(); + buffer.extend(std::iter::repeat_n(b'.', 500)); + buffer.extend_from_slice(b"ABC"); + let matches = run_rules(&rules, &buffer); + assert_eq!(matches.len(), 1, "bare search should find ABC at EOF"); + assert_eq!(matches[0].message, "found ABC bare"); } /// Parser round-trip for `regex/N` (byte-count variant). Exercises the diff --git a/tests/search_bare_range_tests.rs b/tests/search_bare_range_tests.rs new file mode 100644 index 00000000..ac208222 --- /dev/null +++ b/tests/search_bare_range_tests.rs @@ -0,0 +1,109 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end coverage for bare `search` (no `/N` range). +//! +//! magic(5) documents the `search` count as required, but the reference GNU +//! `file` binary accepts a bare `search` and treats it as `str_range == 0` -- +//! scan from the rule's offset to end-of-buffer. rmagic previously rejected +//! bare `search` at parse time, so the tolerant loader dropped the rule *and +//! its subtree*. That silently lost real detections: the PDF page-count chain +//! (`>8 search /Count` -> `>>&0 regex [0-9]+ \b, %s pages`), VCF/SAM +//! bioinformatics signatures, ICC, sfnt font name tables, and more. +//! +//! These tests pin the recovered behavior against a synthetic PDF buffer and +//! guard the negative control (a bounded `search/N` must NOT reach past its +//! window). + +#![allow(clippy::expect_used)] + +use std::io::Write; + +use libmagic_rs::{EvaluationConfig, MagicDatabase}; +use tempfile::NamedTempFile; + +/// Write `magic` to a temp file and load it into a `MagicDatabase`. +fn db_from_magic(magic: &str) -> MagicDatabase { + let mut f = NamedTempFile::new().expect("temp magic file"); + f.write_all(magic.as_bytes()).expect("write magic"); + f.flush().expect("flush magic"); + MagicDatabase::load_from_file_with_config(f.path(), EvaluationConfig::default()) + .expect("magic must load") +} + +/// The GNU `file` PDF page-count chain uses a bare `search` for `/Count`: +/// +/// ```text +/// 0 name pdf +/// >8 search /Count +/// >>&0 regex [0-9]+ \b, %s pages +/// 0 string %PDF- PDF document +/// >0 use pdf +/// ``` +/// +/// Before this fix the `>8 search /Count` line failed to parse, so the whole +/// `pdf` subroutine lost its page-count grandchild and rmagic printed only +/// `PDF document`. It now recovers `, N pages`, matching GNU `file`. +#[test] +fn bare_search_recovers_pdf_page_count() { + let magic = "0\tname\tpdf\n\ + >8\tsearch\t\t/Count\n\ + >>&0\tregex\t\t[0-9]+\t\\b, %s pages\n\ + 0\tstring\t%PDF-\tPDF document\n\ + >0\tuse\tpdf\n"; + let db = db_from_magic(magic); + + // Minimal PDF: header, then a `/Count 3` object well past offset 8. + let mut buf = b"%PDF-1.3\n".to_vec(); + buf.extend_from_slice(b"1 0 obj\n<< /Type /Pages /Count 3 /Kids [] >>\nendobj\n"); + + // This minimal magic omits the real DB's `>5 byte x \b, version %c` + // children, so no version fragment appears -- the point under test is + // the recovered `, N pages` from the bare-`search` -> regex grandchild. + let result = db.evaluate_buffer(&buf).expect("evaluate PDF buffer"); + assert_eq!( + result.description.as_str(), + "PDF document, 3 pages", + "bare `search /Count` must recover the page count via the regex grandchild" + ); +} + +/// Bare `search` scans to end-of-buffer: the pattern is found even when it +/// sits far past any typical bounded window. +#[test] +fn bare_search_scans_to_end_of_buffer() { + let magic = "0\tstring\tSTART\tmarker\n\ + >0\tsearch\t\tDEEP\t\\b, found deep\n"; + let db = db_from_magic(magic); + + let mut buf = b"START".to_vec(); + buf.extend(std::iter::repeat_n(b'.', 6000)); + buf.extend_from_slice(b"DEEP"); + + let result = db.evaluate_buffer(&buf).expect("evaluate deep buffer"); + assert_eq!( + result.description.as_str(), + "marker, found deep", + "bare search must scan the whole remaining buffer, not a bounded window" + ); +} + +/// Negative control: a bounded `search/N` must NOT reach a pattern past its +/// window. This guards against accidentally making every `search` open. +#[test] +fn bounded_search_respects_its_window() { + let magic = "0\tstring\tSTART\tmarker\n\ + >0\tsearch/16\tDEEP\t\\b, found deep\n"; + let db = db_from_magic(magic); + + let mut buf = b"START".to_vec(); + buf.extend(std::iter::repeat_n(b'.', 100)); + buf.extend_from_slice(b"DEEP"); + + let result = db.evaluate_buffer(&buf).expect("evaluate bounded buffer"); + assert_eq!( + result.description.as_str(), + "marker", + "search/16 must not find a pattern 100+ bytes past the offset" + ); +} diff --git a/tests/search_flag_conformance_tests.rs b/tests/search_flag_conformance_tests.rs index 6aff236b..b911653c 100644 --- a/tests/search_flag_conformance_tests.rs +++ b/tests/search_flag_conformance_tests.rs @@ -62,7 +62,7 @@ fn search_rule( MagicRule::new( offset, TypeKind::Search { - range: NonZeroUsize::new(range).expect("range must be non-zero"), + range: NonZeroUsize::new(range), flags, }, Operator::Equal, @@ -348,7 +348,7 @@ fn search_b_archive_epub_rule_loads_and_carries_bin_test_flag() { match &rule.typ { TypeKind::Search { range, flags } => { assert_eq!( - range.get(), + range.expect("search/100 yields a bounded Some range").get(), 100, "search range from /100 suffix should be 100" ); diff --git a/tests/system_magic_dir.rs b/tests/system_magic_dir.rs new file mode 100644 index 00000000..3cda18bb --- /dev/null +++ b/tests/system_magic_dir.rs @@ -0,0 +1,766 @@ +// Copyright (c) 2025-2026 the libmagic-rs contributors +// SPDX-License-Identifier: Apache-2.0 + +// Helper functions in this file (not themselves `#[test]` fns) call +// `.expect()` on I/O and evaluation results; clippy's `allow-expect-in-tests` +// (clippy.toml) only recognizes code directly inside `#[test]` bodies as +// test code, not helpers called from them. See `tests/integration_tests.rs` +// and `tests/directory_loading_tests.rs` for the same established pattern. +#![allow(clippy::expect_used)] + +//! Gated system-DB load test + differential parity against GNU `file` +//! (U6 of the `fix/system-magic-regex-graceful` plan). +//! +//! # Gates +//! +//! **GATE 1 (system DB present):** runs only if `/usr/share/file/magic/` +//! exists on the host. If not, the test skips cleanly via an early +//! `eprintln!` + `return` -- it does not fail, so CI hosts without the +//! macOS/Linux `file` package installed stay green. This is the floor +//! proof required by R1: loading the real system magic DB and +//! evaluating real targets must never fatally abort. +//! +//! **GATE 2 (`file` binary present):** additionally gated on the `file` +//! binary being on `PATH` (checked via `file --version`). If absent, the +//! differential-parity tests skip cleanly the same way. Where both gates +//! are satisfied, this is the authoritative arbiter of correctness (R6): +//! it compares this crate's evaluation of curated samples against GNU +//! `file`'s own output on the *specific dimension* the regex fix targets +//! (whether the `assembler` source signal is detected), not a +//! byte-for-byte whole-line comparison -- `rmagic` and GNU `file` cover +//! very different total rule sets, so unrelated-rule divergence is +//! expected and out of scope. +//! +//! # Why the library API, not the `rmagic` binary +//! +//! The differential-parity comparisons call [`libmagic_rs::MagicDatabase`] +//! directly rather than shelling out to `target/debug/rmagic`, so this +//! test has no build-order dependency on the CLI binary. `rmagic` itself +//! is just a thin wrapper over this same API (see `src/main.rs`), so +//! going through the library is equivalent for the purpose of proving +//! the regex fix's correctness, and is more robust in `cargo test` +//! invocations that do not also build binaries first. +//! +//! # Why `stop_at_first_match(false)` for the parity checks +//! +//! `EvaluationConfig::default()` sets `stop_at_first_match: true` +//! (GOTCHAS S13.2): the evaluator halts at the *first* top-level rule +//! whose match (or a descendant's) carries usable description text, in +//! strength-sorted order. The full system DB contains thousands of rules +//! across hundreds of files, and an unrelated but *message-bearing* +//! top-level rule can legitimately match a curated sample earlier in +//! strength order than the `assembler` rules, halting evaluation before +//! the assembler signal is reached. (Note: since the S13.2 fix, +//! *message-less* gating rules -- e.g. the `c-lang` `search` rules that +//! merely trigger child regex rules -- no longer shadow anything; they +//! do not stop evaluation. The remaining first-match race is purely +//! between message-bearing rules.) The parity checks therefore load with +//! `with_stop_at_first_match(false)` and search the full match list for +//! the `"assembler source text"` message, which is the behaviorally +//! correct way to ask "did the assembler signal fire anywhere," matching +//! what GNU `file`'s own multi-entry evaluation effectively does when it +//! prints `"assembler source text, ASCII text"` (two independent magic +//! entries, concatenated). The default-config tests further below +//! instead exercise the exact `stop_at_first_match: true` path the CLI +//! uses. + +use libmagic_rs::{EvaluationConfig, MagicDatabase}; +use std::path::Path; +use std::process::Command; + +/// GNU `file`'s macOS/Linux-packaged system magic directory. Hard-coded +/// per the plan (this exact path is the one the maintainer verified the +/// bug against), not read from an environment variable, since the intent +/// is to test against the specific host DB the bug report was filed +/// against. +const SYSTEM_MAGIC_DIR: &str = "/usr/share/file/magic/"; + +/// Whether `path` (a system magic directory candidate) exists AND is a +/// directory. Uses `is_dir()` rather than `exists()` so a regular file (or a +/// broken mountpoint) at the path does not pass the gate and then fail later +/// when loaded as a directory. Factored out into its own function so the +/// "clean skip" branch is exercised by an isolated unit test below, +/// independent of whether the actual host happens to have the system DB +/// installed. +fn has_system_magic_dir(path: &Path) -> bool { + path.is_dir() +} + +/// Whether the `file` binary is available on `PATH`, checked via +/// `file --version` rather than `which file` so the check works +/// identically on hosts where `which` itself might not be installed +/// (e.g., some minimal container images). +fn has_file_binary() -> bool { + Command::new("file") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) +} + +/// The "clean skip" path itself must be reachable and correct, verified +/// against a path that is guaranteed not to exist -- this does not +/// depend on the actual host's `/usr/share/file/magic/` state, so it +/// always runs (never gated) and always passes. +#[test] +fn test_skip_gate_is_reachable_for_a_missing_directory() { + // Use a TempDir and a known-nonexistent child rather than a hard-coded + // absolute path: the latter isn't guaranteed absent on every host or + // container, which could make this test flaky. The child is never + // created, so the gate must report false deterministically. + let temp = tempfile::TempDir::new().expect("create temp dir"); + let fake_path = temp.path().join("libmagic-rs-nonexistent-child"); + assert!( + !has_system_magic_dir(&fake_path), + "the gate helper must report false for a path that does not exist" + ); +} + +/// GATE 1: the system DB never fatally aborts evaluation of a real +/// target (R1, the plan's floor requirement). Per-file parse warnings +/// and non-matches are fine -- only a hard `Err` is a failure. This +/// mirrors the plan's exact repro: +/// `./target/debug/rmagic --magic-file /usr/share/file/magic/ ./Cargo.toml` +#[test] +fn test_system_magic_dir_loads_and_evaluates_cargo_toml_without_fatal_error() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + gated system-DB load test skipped cleanly" + ); + return; + } + + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + + let cargo_toml = std::fs::read("Cargo.toml").expect("Cargo.toml must be readable in CI/dev"); + let result = db + .evaluate_buffer(&cargo_toml) + .expect("evaluating Cargo.toml against the system DB must not fatally error"); + + // Not asserting on `description` content here -- Cargo.toml is a + // plain-text TOML file with no dedicated magic entry in the system + // DB, so "no confident match" is an entirely valid outcome. The + // floor requirement is solely that evaluation completes. + let _ = result; +} + +/// GATE 1 (binary target variant): if the crate's own compiled binary is +/// present (built by a prior `cargo build` in this workspace), evaluate +/// it too -- a real ELF/Mach-O binary exercises a very different set of +/// system-DB rules than a text file and is a stronger floor proof. +/// Skipped (not failed) if the binary has not been built, since this +/// test does not build it itself (see the module doc for why the +/// parity tests avoid a build-order dependency). +#[test] +fn test_system_magic_dir_loads_and_evaluates_rmagic_binary_if_present() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + gated system-DB load test skipped cleanly" + ); + return; + } + + let binary_path = Path::new("target/debug/rmagic"); + if !binary_path.exists() { + eprintln!("SKIP: target/debug/rmagic not built -- binary-target load test skipped"); + return; + } + + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + let buffer = std::fs::read(binary_path).expect("target/debug/rmagic must be readable"); + let result = db + .evaluate_buffer(&buffer) + .expect("evaluating the rmagic binary against the system DB must not fatally error"); + let _ = result; +} + +/// A curated sample and its expected assembler-detection outcome, used +/// by both the differential-parity test (against GNU `file`) and (via +/// its own positive/negative shape) as a second, real-file-backed cross +/// check on top of the AST-built fixtures in +/// `tests/regex_getstr_fixtures.rs`. +struct Sample { + filename: &'static str, + content: &'static [u8], + /// Whether both `rmagic` and GNU `file` are expected to detect the + /// `assembler` source signal for this sample. + expect_assembler_detected: bool, + description: &'static str, +} + +const SAMPLES: &[Sample] = &[ + Sample { + filename: "leading_tab_asciiz.s", + content: b"\t.asciiz \"hi\"\n", + expect_assembler_detected: true, + description: "leading tab then .asciiz -- the plan's worked-example positive case", + }, + Sample { + filename: "column_zero_globl.s", + content: b".globl main\n", + expect_assembler_detected: true, + description: "a different affected assembler keyword (.globl) at column 0", + }, + Sample { + filename: "plain_text.txt", + content: b"hello world, this is not assembler\n", + expect_assembler_detected: false, + description: "ordinary prose text with no assembler directive anywhere", + }, + Sample { + filename: "non_whitespace_prefix_asciiz.s", + content: b"xyz .asciiz\n", + expect_assembler_detected: false, + description: "non-whitespace prefix before .asciiz -- must not match (anchor requires \ + 0-50 leading whitespace chars, not arbitrary prefix text)", + }, +]; + +/// Does `message` (GNU `file`'s stdout, or one of our own rule +/// messages) carry the assembler-source signal this fix restores? +fn contains_assembler_signal(text: &str) -> bool { + text.contains("assembler source text") +} + +/// Run this crate's evaluator (via the library API, `stop_at_first_match` +/// disabled per the module doc) against `content` and report whether the +/// assembler signal fired anywhere in the match list. +fn rmagic_detects_assembler(db: &MagicDatabase, content: &[u8]) -> bool { + let result = db + .evaluate_buffer(content) + .expect("evaluation must not fatally error for a curated sample"); + result + .matches + .iter() + .any(|m| contains_assembler_signal(&m.message)) +} + +/// Run GNU `file` against a real file on disk and report whether its +/// output carries the assembler signal. +fn file_binary_detects_assembler(path: &Path) -> bool { + let output = Command::new("file") + .arg("--magic-file") + .arg(SYSTEM_MAGIC_DIR) + .arg(path) + .output() + .expect("failed to invoke the `file` binary"); + assert!( + output.status.success(), + "`file` exited non-zero for {}: stderr={}", + path.display(), + String::from_utf8_lossy(&output.stderr) + ); + contains_assembler_signal(&String::from_utf8_lossy(&output.stdout)) +} + +/// GATE 2: differential parity against GNU `file` on the specific +/// dimension this fix restores -- assembler-source detection -- for a +/// small curated sample set covering both a positive and a negative +/// case, and both the plan's worked example (`.asciiz`) and a second +/// affected keyword (`.globl`) to guard against a resolver bug that +/// happens to work for one keyword but not others. +/// +/// Per the plan's "don't leave residuals" directive: if this test ever +/// finds a genuine divergence between `rmagic` and `file`, the fix +/// belongs in `src/parser/grammar/getstr/mod.rs` (or, as this session's +/// U6 pass discovered, in `src/evaluator/types/regex.rs`'s regex-compile +/// setup -- see that file's `build_regex` doc comment for the +/// `unicode(false)` fix this test's `>= 0x80` cousin in +/// `tests/regex_getstr_fixtures.rs` is guarding), not a follow-up issue. +/// Parity (rmagic == real `file` on the same `--magic-file` DB) is the +/// host-adaptive invariant asserted on every host; the absolute +/// positive/negative expectations are additionally asserted only where the +/// host DB actually carries the assembler rules (macOS ships them as source +/// magic files; Ubuntu ships only the compiled `magic.mgc`, leaving the +/// `--magic-file` source dir empty -- see the in-body probe). +#[test] +fn test_differential_parity_against_gnu_file_for_assembler_detection() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + differential parity test skipped cleanly" + ); + return; + } + if !has_file_binary() { + eprintln!("SKIP: `file` binary not on PATH -- differential parity test skipped cleanly"); + return; + } + + // See the module doc: `stop_at_first_match(false)` is required so an + // unrelated earlier top-level match (e.g. a message-less `c-lang` + // gating `search` rule) cannot shadow the assembler rule out of the + // match list before we get to inspect it. + let config = EvaluationConfig::default().with_stop_at_first_match(false); + let db = MagicDatabase::load_from_file_with_config(system_dir, config) + .expect("loading the system magic directory must not fail"); + + let temp_dir = tempfile::TempDir::new().expect("failed to create temp dir for samples"); + + // Probe whether THIS host's `--magic-file SYSTEM_MAGIC_DIR` DB actually + // carries the assembler-source rules. This is host-dependent: macOS + // ships the *source* magic files under `/usr/share/file/magic/`, so both + // `file` and rmagic can detect assembler there; Ubuntu/Debian ship only + // the *compiled* `magic.mgc` and leave that source directory empty, so a + // `--magic-file /usr/share/file/magic/` invocation carries no assembler + // rule at all (verified: on ubuntu-latest `file --magic-file + // /usr/share/file/magic/ asm.s` prints "ASCII text", not "assembler + // source text"). The absolute positive/negative expectations only apply + // where the rules are present; parity with real `file` is asserted on + // every host regardless. + let probe_path = temp_dir.path().join("__probe_leading_tab_asciiz.s"); + std::fs::write(&probe_path, b"\t.asciiz \"hi\"\n").expect("failed to write probe sample"); + let host_db_has_assembler_rules = file_binary_detects_assembler(&probe_path); + + for sample in SAMPLES { + let sample_path = temp_dir.path().join(sample.filename); + std::fs::write(&sample_path, sample.content).expect("failed to write sample file"); + + let rmagic_result = rmagic_detects_assembler(&db, sample.content); + let file_result = file_binary_detects_assembler(&sample_path); + + // Trustworthy invariant on every host: rmagic must agree with real + // `file` on the SAME `--magic-file` DB, whatever that host DB holds. + // A genuine divergence must be diagnosed and fixed in-band (in + // `src/parser/grammar/getstr/mod.rs` or `src/evaluator/types/regex.rs`), + // not deferred. + assert_eq!( + rmagic_result, file_result, + "PARITY FAILURE for {:?} ({}): rmagic={rmagic_result} file={file_result}", + sample.filename, sample.description + ); + + // Absolute semantic check -- applicable only where the host DB + // actually carries the assembler rules (see the probe above). + if host_db_has_assembler_rules { + assert_eq!( + rmagic_result, + sample.expect_assembler_detected, + "rmagic detection mismatch for {:?} ({}): expected {}, got {}", + sample.filename, + sample.description, + sample.expect_assembler_detected, + rmagic_result + ); + } + } +} + +// ======================================================================= +// Default-config (`stop_at_first_match: true`) end-to-end coverage +// +// The tests above deliberately use `with_stop_at_first_match(false)` so +// the assembler signal can be found anywhere in the full match list +// (see the module doc). The tests below instead use +// `EvaluationConfig::default()` unmodified -- the exact configuration +// `MagicDatabase::load_from_file` uses and the CLI's default behavior -- +// to prove the actual maintainer-reported bug is fixed: a message-less +// top-level rule (e.g. a `c-lang` gating `search` rule) matching before +// the assembler/text rules used to terminate evaluation via +// `stop_at_first_match` and leave the CLI's description blank. See +// GOTCHAS S13.2 for the refined stop-at-first-match semantics +// (`has_message_bearing_match` in `src/evaluator/engine/mod.rs`) and +// `src/output/ascmagic.rs` for the text/data fallback that also +// contributes here. +// ======================================================================= + +/// A fixed, reproducible "binary blob" (not literal `/dev/urandom`, which +/// would make the test's outcome host- and run-dependent) verified against +/// the real `file` binary to produce `"data"` on this host's full system +/// magic DB. +/// +/// This is a plain linear-congruential byte stream with no special +/// constraints on any byte range. +/// +/// History: during initial implementation of the text/data fallback, this +/// exact byte stream spuriously matched `/usr/share/file/magic/ibm6000`'s +/// `AIX core file` entry -- `4 belong &0x0feeddb0` with no explicit +/// comparison and no message. That entry is GNU `file`'s magic(5) shorthand +/// for `belong &0x0feeddb0 =0x0feeddb0` ("all of the masked bits must be +/// set"), but `apply_bitwise_and` (`src/evaluator/operators/bitwise.rs`) +/// at the time implemented the bare `&` relational operator as +/// `(value & operand) != 0` ("any masked bit set") instead -- confirmed by +/// direct A/B testing against `file --magic-file .../ibm6000` on crafted +/// buffers. That operator-semantics bug has since been fixed (see GOTCHAS +/// S13.3 and S2.3): `apply_bitwise_and` now implements `(value & operand) +/// == operand`, matching libmagic's `magiccheck()`. This blob is kept +/// exactly as originally generated (rather than perturbed to dodge the old +/// bug) so this test doubles as a regression guard for that fix. +fn deterministic_binary_blob() -> Vec { + (0..64u32) + .map(|i| u8::try_from((i * 37 + 11) % 256).expect("modulo 256 always fits in u8")) + .collect() +} + +/// GOTCHAS S13.2 (the exact repro from the bug report): with the real +/// system magic DB and the library's default configuration +/// (`stop_at_first_match: true`, matching what `MagicDatabase::load_from_file` +/// and the CLI both use), an assembler-source file must no longer produce +/// a blank description. +#[test] +fn test_default_config_assembler_source_no_longer_blank() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + default-config assembler test skipped cleanly" + ); + return; + } + + // Default config: do NOT override stop_at_first_match. + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + + let result = db + .evaluate_buffer(b"\t.asciiz \"hi\"\n") + .expect("evaluation must not fatally error"); + + // The ORIGINAL bug (blank output) is host-independent -- it must never + // recur, whatever the host DB contains. This is the core regression + // guard for the maintainer-reported bug. + assert!( + !result.description.trim().is_empty(), + "description must not be blank for an assembler-source buffer, got: {:?}", + result.description + ); + + // The assembler *classification* is host-dependent (macOS ships the + // source magic rules under `/usr/share/file/magic/`; Ubuntu ships only + // the compiled `magic.mgc`, leaving that `--magic-file` dir empty). + // Assert it against what real `file` reports on the SAME DB rather than + // hardcoding a host-specific outcome. See + // `test_default_config_messageless_gate_does_not_blank_output_end_to_end` + // for the host-INDEPENDENT proof that the message-shadowing fix works. + if has_file_binary() { + let temp_dir = tempfile::TempDir::new().expect("failed to create temp dir"); + let sample_path = temp_dir.path().join("leading_tab_asciiz.s"); + std::fs::write(&sample_path, b"\t.asciiz \"hi\"\n").expect("failed to write sample"); + let file_says_assembler = file_binary_detects_assembler(&sample_path); + assert_eq!( + contains_assembler_signal(&result.description), + file_says_assembler, + "assembler-detection parity failure under default config: rmagic description={:?}, \ + GNU `file` detected assembler={file_says_assembler}", + result.description + ); + } +} + +/// Companion to the assembler case: plain ASCII text with no magic-file +/// signature anywhere must fall back to the text/data classifier's +/// `"ASCII text"` (GOTCHAS S13.2) under the default config, not a blank +/// description. +#[test] +fn test_default_config_plain_ascii_text_no_longer_blank() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + default-config plain-text test skipped cleanly" + ); + return; + } + + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + + let result = db + .evaluate_buffer(b"hello world, this is not assembler\n") + .expect("evaluation must not fatally error"); + + assert_eq!(result.description, "ASCII text"); +} + +/// Default-config regression for the message-bearing `clear` fix +/// (GOTCHAS S13.6): a C source file with `#include` must classify as +/// `c program text` -- the trailing `program text` fragment comes from +/// c-lang's `>>&0 clear x program text` child, which rmagic previously +/// dropped (printing only `c`). +/// +/// This is the CRITICAL companion to the synthetic +/// `test_clear_with_message_emits_and_still_resets_flag` in +/// `tests/meta_types_integration.rs`: that test runs with +/// `stop_at_first_match(false)` on a hand-built chain, whereas the real +/// bug -- and the CLI -- runs under the DEFAULT config +/// (`stop_at_first_match: true`) against the full c-lang chain with +/// competing top-level `pragma`/`endif` rules. Without this test a +/// refactor of stop-at-first-match could silently regress c.c back to +/// `c` while the synthetic test keeps passing. +/// +/// Host-gated on real `file` parity over the SAME `--magic-file` dir +/// (macOS ships the c-lang source rules there; a host whose +/// `/usr/share/file/magic/` lacks them makes `file` not say +/// "c program text" either, so the assertion is vacuous rather than a +/// false failure). Asserts `contains`, not `==`: real `file` also +/// appends `, ASCII text` (the deferred combined-classification garnish, +/// intentionally absent in rmagic). +#[test] +fn test_default_config_c_source_is_c_program_text() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + default-config c-source test skipped cleanly" + ); + return; + } + + // Default config: do NOT override stop_at_first_match -- this is the + // exact path the CLI and the original bug exercise. + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + + let c_source = b"#include \nint main(void){return 0;}\n"; + let result = db + .evaluate_buffer(c_source) + .expect("evaluation must not fatally error"); + + if has_file_binary() { + let temp_dir = tempfile::TempDir::new().expect("failed to create temp dir"); + let sample_path = temp_dir.path().join("sample.c"); + std::fs::write(&sample_path, c_source).expect("failed to write sample"); + + let output = Command::new("file") + .arg("--magic-file") + .arg(SYSTEM_MAGIC_DIR) + .arg(&sample_path) + .output() + .expect("failed to invoke the `file` binary"); + let file_desc = String::from_utf8_lossy(&output.stdout); + + // Only assert parity when the host's own `file` (using the SAME + // magic dir) detects the c-lang chain. This keeps the test + // host-independent: where the c-lang rules are absent, `file` + // won't say "c program text" and neither must rmagic. + if file_desc.contains("c program text") { + assert!( + result.description.contains("c program text"), + "message-bearing `clear` regression: rmagic must emit \ + \"c program text\" for a C source under the default config, \ + got: {:?} (file said: {:?})", + result.description, + file_desc.trim() + ); + } + } +} + +/// Binary content must still fall back to `"data"` under the default +/// config -- the text/data fallback must not misclassify binary content +/// as text. +#[test] +fn test_default_config_binary_blob_is_data() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + default-config binary-blob test skipped cleanly" + ); + return; + } + + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + + let result = db + .evaluate_buffer(&deterministic_binary_blob()) + .expect("evaluation must not fatally error"); + + assert_eq!(result.description, "data"); +} + +/// GATE 2 (full differential parity, default config): the exact +/// three-case set the fix's acceptance criteria call out -- an +/// assembler-source file, a plain ASCII text file, and a binary blob -- +/// each run through both `rmagic` (via the library API, default config) +/// and the real `file` binary, on the same system magic DB. Unlike +/// `test_differential_parity_against_gnu_file_for_assembler_detection` +/// above, this does NOT disable `stop_at_first_match`: it is the +/// end-to-end proof that the default configuration (what the CLI and +/// `MagicDatabase::load_from_file` actually use) matches GNU `file`, +/// not just that the assembler signal is present somewhere in an +/// exhaustive match list. +#[test] +fn test_default_config_differential_parity_three_cases() { + let system_dir = Path::new(SYSTEM_MAGIC_DIR); + if !has_system_magic_dir(system_dir) { + eprintln!( + "SKIP: {SYSTEM_MAGIC_DIR} not present on this host -- \ + default-config differential parity test skipped cleanly" + ); + return; + } + if !has_file_binary() { + eprintln!("SKIP: `file` binary not on PATH -- differential parity test skipped cleanly"); + return; + } + + let db = MagicDatabase::load_from_file(system_dir) + .expect("loading the system magic directory must not fail"); + + let temp_dir = tempfile::TempDir::new().expect("failed to create temp dir for samples"); + + // (filename, content, a predicate over (rmagic_description, file_stdout) + // proving parity on the dimension that matters for that case). + let cases: &[(&str, &[u8])] = &[ + ("assembler.s", b"\t.asciiz \"hi\"\n"), + ("plain.txt", b"hello world, this is not assembler\n"), + ]; + + for (filename, content) in cases { + let path = temp_dir.path().join(filename); + std::fs::write(&path, content).expect("failed to write sample file"); + + let rmagic_result = db + .evaluate_buffer(content) + .expect("evaluation must not fatally error"); + let file_output = Command::new("file") + .arg("--magic-file") + .arg(SYSTEM_MAGIC_DIR) + .arg(&path) + .output() + .expect("failed to invoke the `file` binary"); + assert!(file_output.status.success()); + let file_stdout = String::from_utf8_lossy(&file_output.stdout); + + assert!( + !rmagic_result.description.trim().is_empty(), + "rmagic must not produce a blank description for {filename:?}" + ); + + if *filename == "assembler.s" { + // Host-adaptive: assert rmagic AGREES with real `file` on whether + // this host's DB classifies the sample as assembler, not that it + // unconditionally does. Ubuntu's `--magic-file /usr/share/file/magic/` + // dir is empty (only compiled `magic.mgc` is shipped), so neither + // side detects assembler there; macOS's dir carries the rule, so + // both do. Either way rmagic must match `file`. + // Use the full `contains_assembler_signal` phrase, NOT a bare + // "assembler" substring -- `file`'s stdout includes the file + // PATH ("/tmp/.../assembler.s: ..."), so a bare substring would + // spuriously match the filename rather than the classification. + assert_eq!( + contains_assembler_signal(&rmagic_result.description), + contains_assembler_signal(&file_stdout), + "assembler-detection parity failure for {filename:?}: \ + rmagic={:?}, file={file_stdout:?}", + rmagic_result.description + ); + } else { + assert_eq!(rmagic_result.description, "ASCII text"); + assert!( + file_stdout.contains("ASCII text"), + "GNU `file` should also report ASCII text for {filename:?}, got: {file_stdout:?}" + ); + } + } + + // Binary blob: both must agree on "data". + let binary_path = temp_dir.path().join("blob.bin"); + let blob = deterministic_binary_blob(); + std::fs::write(&binary_path, &blob).expect("failed to write binary blob"); + + let rmagic_blob_result = db + .evaluate_buffer(&blob) + .expect("evaluation must not fatally error"); + let file_blob_output = Command::new("file") + .arg("--magic-file") + .arg(SYSTEM_MAGIC_DIR) + .arg(&binary_path) + .output() + .expect("failed to invoke the `file` binary"); + assert!(file_blob_output.status.success()); + let file_blob_stdout = String::from_utf8_lossy(&file_blob_output.stdout); + + assert_eq!(rmagic_blob_result.description, "data"); + assert!( + file_blob_stdout.contains("data"), + "GNU `file` should also report \"data\" for the binary blob, got: {file_blob_stdout:?}" + ); +} + +// ======================================================================= +// Portable, host-INDEPENDENT end-to-end guards for the stop_at_first_match +// message-shadowing fix (GOTCHAS S13.2) and the text/data fallback (S13.3). +// +// Unlike every test above, these do NOT depend on the host system magic DB +// (which differs across platforms -- macOS ships source magic files under +// `/usr/share/file/magic/`; Ubuntu ships only the compiled `magic.mgc`, +// leaving that directory empty). They build self-contained temp magic files +// so they exercise the full default-config pipeline (load -> evaluate -> +// description assembly -> message-bearing stop gate -> text/data fallback) +// identically on every CI host. These are the authoritative regression +// guards for the fix; the system-DB tests above are supplementary parity +// checks that only exercise the assembler path where the host DB carries +// the rule. +// ======================================================================= + +/// S13.3 fallback wiring: when no rule produces a description (here, a +/// single message-LESS rule matches but carries no text), the default +/// config must fall back to the text/data classifier rather than emitting +/// a blank description. Host-independent. +#[test] +fn test_default_config_no_match_falls_back_to_text_classification_end_to_end() { + let temp_dir = tempfile::TempDir::new().expect("failed to create temp dir"); + let magic_path = temp_dir.path().join("messageless_only.magic"); + // A single message-LESS rule: matches any buffer whose first byte is a + // tab (0x09), carries no description of its own. + std::fs::write(&magic_path, "0\tbyte\t0x09\n").expect("failed to write magic file"); + + let db = MagicDatabase::load_from_file(&magic_path) + .expect("loading a single-rule magic file must not fail"); + let result = db + .evaluate_buffer(b"\t.asciiz \"hi\"\n") + .expect("evaluation must not fatally error"); + + assert_eq!( + result.description, "ASCII text", + "a match with no usable description text must fall back to the text \ + classifier, not a blank description; got: {:?}", + result.description + ); +} + +/// S13.2 shadow gate (the exact shape of the maintainer-reported bug): a +/// message-LESS gating rule that matches and sorts FIRST must NOT shadow a +/// later message-BEARING rule under the default config. Two `string` rules +/// with identical strength are used so the STABLE strength sort +/// (`sort_by_cached_key`) preserves source order -- the message-less rule +/// is evaluated first. Pre-fix this halted evaluation and produced a blank +/// description (then the text fallback); post-fix evaluation continues to +/// the message-bearing rule and surfaces its text. Host-independent, so it +/// proves the fix on Ubuntu/Windows where the system DB has no assembler +/// rule to exercise. +#[test] +fn test_default_config_messageless_gate_does_not_blank_output_end_to_end() { + let temp_dir = tempfile::TempDir::new().expect("failed to create temp dir"); + let magic_path = temp_dir.path().join("gate_then_message.magic"); + // Rule 1 (message-less) and Rule 2 (message-bearing) have identical + // type/offset/operator/value, hence identical strength; the stable sort + // keeps Rule 1 first. Both match `\t.asciiz` at offset 0. + std::fs::write( + &magic_path, + "0\tstring\t\\t.asciiz\n0\tstring\t\\t.asciiz\tassembler source text\n", + ) + .expect("failed to write magic file"); + + let db = MagicDatabase::load_from_file(&magic_path) + .expect("loading the two-rule magic file must not fail"); + let result = db + .evaluate_buffer(b"\t.asciiz \"hi\"\n") + .expect("evaluation must not fatally error"); + + assert!( + result.description.contains("assembler source text"), + "the message-bearing rule's text must survive a preceding message-less \ + match under stop_at_first_match; got: {:?}", + result.description + ); +}